@qvac/inference 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (569) hide show
  1. package/NOTICE +111 -60
  2. package/README.md +4 -2
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +136 -45
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +2 -1
  13. package/dist/api/completion-stream.js +4 -1
  14. package/dist/api/decide.js +1 -1
  15. package/dist/api/delete-cache.d.ts +7 -1
  16. package/dist/api/delete-cache.js +11 -1
  17. package/dist/api/diffusion.d.ts +32 -2
  18. package/dist/api/diffusion.js +32 -2
  19. package/dist/api/download-asset.js +1 -1
  20. package/dist/api/embed.d.ts +82 -29
  21. package/dist/api/embed.js +64 -18
  22. package/dist/api/finetune.js +1 -1
  23. package/dist/api/get-model-info.d.ts +1 -1
  24. package/dist/api/index.d.ts +40 -38
  25. package/dist/api/index.js +39 -37
  26. package/dist/api/load-model.js +2 -2
  27. package/dist/api/logging-stream-registry.js +11 -7
  28. package/dist/api/logging-stream.d.ts +11 -1
  29. package/dist/api/logging-stream.js +11 -2
  30. package/dist/api/rag.d.ts +1 -1
  31. package/dist/api/rag.js +1 -1
  32. package/dist/api/registry.d.ts +18 -7
  33. package/dist/api/registry.js +37 -13
  34. package/dist/api/rerank.d.ts +17 -0
  35. package/dist/api/rerank.js +36 -0
  36. package/dist/api/subscribe-logs.js +5 -3
  37. package/dist/api/text-to-speech.d.ts +14 -5
  38. package/dist/api/text-to-speech.js +83 -29
  39. package/dist/api/transcribe.d.ts +6 -2
  40. package/dist/api/transcribe.js +7 -1
  41. package/dist/api/unload-model.js +1 -1
  42. package/dist/api/vector-index.d.ts +25 -3
  43. package/dist/api/vector-index.js +17 -2
  44. package/dist/api/video-request.d.ts +15 -15
  45. package/dist/api/video.js +1 -1
  46. package/dist/api/vla.js +1 -1
  47. package/dist/api/world-result.js +1 -1
  48. package/dist/api/world.d.ts +1 -1
  49. package/dist/api/world.js +1 -1
  50. package/dist/config/resolve-config.d.ts +1 -1
  51. package/dist/config/resolve-config.js +1 -1
  52. package/dist/constants/index.d.ts +2 -2
  53. package/dist/constants/index.js +2 -2
  54. package/dist/dispatch.d.ts +11 -4
  55. package/dist/dispatch.js +80 -18
  56. package/dist/engine.d.ts +1 -1
  57. package/dist/errors/index.d.ts +76 -0
  58. package/dist/errors/index.js +141 -1
  59. package/dist/handlers/assess-model-fit.d.ts +3 -21
  60. package/dist/handlers/assess-model-fit.js +4 -303
  61. package/dist/handlers/cancelHandler.d.ts +10 -9
  62. package/dist/handlers/cancelHandler.js +18 -14
  63. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  64. package/dist/handlers/completion-orchestrate.js +166 -24
  65. package/dist/handlers/delete-cache.d.ts +2 -1
  66. package/dist/handlers/delete-cache.js +6 -1
  67. package/dist/handlers/download-asset.js +13 -6
  68. package/dist/handlers/finetune.js +1 -1
  69. package/dist/handlers/get-loaded-model-info.js +3 -3
  70. package/dist/handlers/get-model-info.js +72 -3
  71. package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
  72. package/dist/handlers/load-model/coreml-sidecars.js +51 -91
  73. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  74. package/dist/handlers/load-model/download-lock.js +213 -0
  75. package/dist/handlers/load-model/download-manager.d.ts +11 -3
  76. package/dist/handlers/load-model/download-manager.js +62 -11
  77. package/dist/handlers/load-model/download-stats.d.ts +2 -2
  78. package/dist/handlers/load-model/download-stats.js +3 -3
  79. package/dist/handlers/load-model/handler.js +97 -41
  80. package/dist/handlers/load-model/http-verify.d.ts +1 -1
  81. package/dist/handlers/load-model/http.d.ts +1 -1
  82. package/dist/handlers/load-model/http.js +330 -208
  83. package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
  84. package/dist/handlers/load-model/hyperdrive.js +125 -52
  85. package/dist/handlers/load-model/index.d.ts +2 -2
  86. package/dist/handlers/load-model/index.js +2 -2
  87. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  88. package/dist/handlers/load-model/registry-client-options.js +4 -2
  89. package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
  90. package/dist/handlers/load-model/registry-companion-set.js +20 -2
  91. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  92. package/dist/handlers/load-model/registry-delete.js +269 -0
  93. package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
  94. package/dist/handlers/load-model/registry-download-utils.js +95 -24
  95. package/dist/handlers/load-model/registry.d.ts +22 -2
  96. package/dist/handlers/load-model/registry.js +312 -179
  97. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  98. package/dist/handlers/load-model/resolve-session.js +11 -3
  99. package/dist/handlers/load-model/resolve.d.ts +1 -1
  100. package/dist/handlers/load-model/resolve.js +9 -6
  101. package/dist/handlers/load-model/types.d.ts +2 -0
  102. package/dist/handlers/logging-stream.d.ts +2 -1
  103. package/dist/handlers/logging-stream.js +35 -7
  104. package/dist/handlers/plugin-invoke.js +152 -39
  105. package/dist/handlers/registry.d.ts +2 -2
  106. package/dist/handlers/registry.js +108 -29
  107. package/dist/handlers/rpc-server.js +7 -7
  108. package/dist/handlers/session.d.ts +11 -0
  109. package/dist/handlers/session.js +40 -0
  110. package/dist/handlers/types.d.ts +7 -1
  111. package/dist/handlers/unload-model.js +8 -3
  112. package/dist/handlers/vector-index.js +118 -19
  113. package/dist/index.d.ts +1 -1
  114. package/dist/index.js +1 -1
  115. package/dist/logging/addon.d.ts +1 -1
  116. package/dist/logging/addon.js +1 -1
  117. package/dist/logging/index.d.ts +5 -5
  118. package/dist/logging/index.js +4 -4
  119. package/dist/logging/logger.d.ts +1 -1
  120. package/dist/logging/logger.js +3 -3
  121. package/dist/logging/utils.js +11 -3
  122. package/dist/model-cache-layout.d.ts +1 -0
  123. package/dist/model-cache-layout.js +2 -0
  124. package/dist/model-fit/assess.d.ts +23 -0
  125. package/dist/model-fit/assess.js +110 -0
  126. package/dist/model-fit/describe.d.ts +12 -0
  127. package/dist/model-fit/describe.js +29 -0
  128. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
  129. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
  130. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
  131. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
  132. package/dist/model-fit/fit-load.d.ts +23 -0
  133. package/dist/model-fit/fit-load.js +29 -0
  134. package/dist/model-fit/load-check.d.ts +34 -0
  135. package/dist/model-fit/load-check.js +77 -0
  136. package/dist/model-fit/run/engine.d.ts +12 -0
  137. package/dist/model-fit/run/engine.js +9 -0
  138. package/dist/model-fit/run/run.d.ts +46 -0
  139. package/dist/model-fit/run/run.js +208 -0
  140. package/dist/model-fit/sum.d.ts +32 -0
  141. package/dist/model-fit/sum.js +60 -0
  142. package/dist/models/registry/index.d.ts +1 -1
  143. package/dist/models/registry/index.js +2 -2
  144. package/dist/models/registry/models.d.ts +1377 -130
  145. package/dist/models/registry/models.js +3622 -2312
  146. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  147. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  148. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  149. package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
  150. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  151. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  152. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  153. package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  154. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  155. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
  156. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  157. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  158. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  159. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  160. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  161. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
  162. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  163. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
  164. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
  165. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
  166. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  167. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  168. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
  169. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
  170. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
  171. package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
  172. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  173. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  174. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  175. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  176. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  177. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  178. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  179. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
  180. package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
  181. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  182. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  183. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
  184. package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
  185. package/dist/plugins/builtin/index.d.ts +16 -14
  186. package/dist/plugins/builtin/index.js +16 -14
  187. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  188. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  189. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
  190. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  191. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  192. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  193. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
  194. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  195. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
  196. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
  197. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  198. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  199. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
  200. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
  201. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
  202. package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
  203. package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
  204. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
  205. package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
  206. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  207. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
  208. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
  209. package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
  210. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  211. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  212. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
  213. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
  214. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  215. package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
  216. package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
  217. package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
  218. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
  219. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
  220. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
  221. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
  222. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
  223. package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
  224. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  225. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  226. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
  227. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
  228. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
  229. package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
  230. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  231. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  232. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  233. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  234. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
  235. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
  236. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  237. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  238. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
  239. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
  240. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
  241. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
  242. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
  243. package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
  244. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
  245. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
  246. package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
  247. package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
  248. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  249. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  250. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  251. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  252. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  253. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
  254. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
  255. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  256. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
  257. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  258. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  259. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
  260. package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
  261. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
  262. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
  263. package/dist/plugins/index.d.ts +1 -1
  264. package/dist/plugins/index.js +2 -2
  265. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  266. package/dist/plugins/ops/bci-transcribe.js +18 -4
  267. package/dist/plugins/ops/cancel.d.ts +1 -0
  268. package/dist/plugins/ops/cancel.js +11 -2
  269. package/dist/plugins/ops/embed.d.ts +5 -3
  270. package/dist/plugins/ops/embed.js +89 -34
  271. package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
  272. package/dist/plugins/ops/kv-cache-utils.js +21 -10
  273. package/dist/plugins/ops/load-model.d.ts +10 -2
  274. package/dist/plugins/ops/load-model.js +45 -19
  275. package/dist/plugins/ops/transcribe.d.ts +5 -4
  276. package/dist/plugins/ops/transcribe.js +63 -28
  277. package/dist/plugins/ops/translate.d.ts +2 -1
  278. package/dist/plugins/ops/translate.js +7 -4
  279. package/dist/plugins/ops/unload-model.d.ts +17 -1
  280. package/dist/plugins/ops/unload-model.js +32 -3
  281. package/dist/plugins/turbovec-backend.d.ts +1 -1
  282. package/dist/plugins/turbovec-backend.js +21 -6
  283. package/dist/profiling/aggregator.d.ts +1 -1
  284. package/dist/profiling/aggregator.js +1 -1
  285. package/dist/profiling/controller.d.ts +1 -1
  286. package/dist/profiling/controller.js +2 -2
  287. package/dist/profiling/events.d.ts +1 -1
  288. package/dist/profiling/events.js +2 -2
  289. package/dist/profiling/exporters.d.ts +1 -1
  290. package/dist/profiling/exporters.js +3 -3
  291. package/dist/profiling/index.d.ts +13 -13
  292. package/dist/profiling/index.js +12 -12
  293. package/dist/profiling/operation-metrics.d.ts +1 -1
  294. package/dist/profiling/operation-metrics.js +24 -21
  295. package/dist/profiling/operation-wrappers.js +3 -3
  296. package/dist/profiling/profiler.js +1 -1
  297. package/dist/rag/embedding-scope.d.ts +16 -0
  298. package/dist/rag/embedding-scope.js +81 -0
  299. package/dist/rag/handler.d.ts +25 -0
  300. package/dist/rag/handler.js +51 -17
  301. package/dist/rag/handlers/close-workspace.js +1 -1
  302. package/dist/rag/handlers/delete-embeddings.js +1 -1
  303. package/dist/rag/handlers/delete-workspace.js +1 -1
  304. package/dist/rag/handlers/ingest.js +3 -7
  305. package/dist/rag/handlers/list-workspaces.d.ts +1 -1
  306. package/dist/rag/handlers/list-workspaces.js +1 -1
  307. package/dist/rag/handlers/reindex.js +1 -1
  308. package/dist/rag/handlers/save-embeddings.js +1 -1
  309. package/dist/rag/handlers/search.d.ts +10 -1
  310. package/dist/rag/handlers/search.js +17 -8
  311. package/dist/rag/index.d.ts +11 -11
  312. package/dist/rag/index.js +11 -11
  313. package/dist/rag/rag-operation-manager.d.ts +27 -3
  314. package/dist/rag/rag-operation-manager.js +10 -27
  315. package/dist/rag/rag-workspace-manager.js +1 -1
  316. package/dist/registry.js +7 -1
  317. package/dist/resources/collector.d.ts +71 -37
  318. package/dist/resources/collector.js +9 -2
  319. package/dist/resources/instance.d.ts +1 -1
  320. package/dist/resources/instance.js +1 -1
  321. package/dist/resources/native.d.ts +1 -1
  322. package/dist/resources/normalize.d.ts +1 -1
  323. package/dist/resources/types.d.ts +2 -0
  324. package/dist/rpc/inventory.js +1 -12
  325. package/dist/runtime/env.d.ts +5 -1
  326. package/dist/runtime/env.js +6 -21
  327. package/dist/runtime/index.d.ts +8 -5
  328. package/dist/runtime/index.js +4 -2
  329. package/dist/runtime/lifecycle.d.ts +6 -4
  330. package/dist/runtime/lifecycle.js +16 -12
  331. package/dist/runtime/model-config-utils.js +8 -1
  332. package/dist/runtime/model-registry.d.ts +30 -4
  333. package/dist/runtime/model-registry.js +48 -2
  334. package/dist/runtime/model-work-lock.d.ts +13 -0
  335. package/dist/runtime/model-work-lock.js +48 -0
  336. package/dist/runtime/registry-client.d.ts +47 -1
  337. package/dist/runtime/registry-client.js +446 -88
  338. package/dist/runtime/request-context.d.ts +11 -3
  339. package/dist/runtime/request-context.js +39 -2
  340. package/dist/runtime/request-registry.d.ts +56 -8
  341. package/dist/runtime/request-registry.js +190 -72
  342. package/dist/runtime/runtime-lifecycle.js +10 -1
  343. package/dist/runtime/session-id.d.ts +18 -0
  344. package/dist/runtime/session-id.js +20 -0
  345. package/dist/runtime/session-registry.d.ts +39 -0
  346. package/dist/runtime/session-registry.js +114 -0
  347. package/dist/runtime/state.d.ts +1 -1
  348. package/dist/runtime/state.js +2 -2
  349. package/dist/runtime/vector-index-backend.d.ts +6 -2
  350. package/dist/runtime/vector-index-registry.d.ts +26 -5
  351. package/dist/runtime/vector-index-registry.js +88 -27
  352. package/dist/schemas/abort-signal.d.ts +8 -0
  353. package/dist/schemas/assess-model-fit.d.ts +138 -282
  354. package/dist/schemas/assess-model-fit.js +85 -219
  355. package/dist/schemas/audio-gen.d.ts +291 -25
  356. package/dist/schemas/audio-gen.js +130 -26
  357. package/dist/schemas/batch-completion-stream.d.ts +113 -15
  358. package/dist/schemas/batch-completion-stream.js +10 -6
  359. package/dist/schemas/bci-config.d.ts +3 -1
  360. package/dist/schemas/bci-config.js +2 -2
  361. package/dist/schemas/bci.d.ts +55 -4
  362. package/dist/schemas/bci.js +13 -18
  363. package/dist/schemas/calc-tokens.d.ts +141 -0
  364. package/dist/schemas/calc-tokens.js +53 -0
  365. package/dist/schemas/cancel.d.ts +31 -0
  366. package/dist/schemas/cancel.js +11 -2
  367. package/dist/schemas/classification.d.ts +3 -0
  368. package/dist/schemas/classification.js +6 -2
  369. package/dist/schemas/common.d.ts +2209 -348
  370. package/dist/schemas/common.js +52 -33
  371. package/dist/schemas/completion-event.d.ts +11 -2
  372. package/dist/schemas/completion-event.js +5 -2
  373. package/dist/schemas/completion-stream.d.ts +173 -19
  374. package/dist/schemas/completion-stream.js +117 -20
  375. package/dist/schemas/config.d.ts +2064 -178
  376. package/dist/schemas/config.js +21 -23
  377. package/dist/schemas/decision.d.ts +12 -2
  378. package/dist/schemas/decision.js +16 -1
  379. package/dist/schemas/download-asset.d.ts +13 -6
  380. package/dist/schemas/download-asset.js +4 -7
  381. package/dist/schemas/embed.d.ts +286 -3
  382. package/dist/schemas/embed.js +162 -16
  383. package/dist/schemas/engine-addon-map.d.ts +3 -1
  384. package/dist/schemas/engine-addon-map.js +5 -3
  385. package/dist/schemas/errors.d.ts +13 -0
  386. package/dist/schemas/errors.js +71 -1
  387. package/dist/schemas/finetune.d.ts +14 -4
  388. package/dist/schemas/finetune.js +5 -6
  389. package/dist/schemas/heartbeat.d.ts +1 -0
  390. package/dist/schemas/heartbeat.js +5 -2
  391. package/dist/schemas/image-embedding.d.ts +27 -0
  392. package/dist/schemas/image-embedding.js +107 -0
  393. package/dist/schemas/index.d.ts +55 -50
  394. package/dist/schemas/index.js +54 -49
  395. package/dist/schemas/lifecycle.d.ts +3 -0
  396. package/dist/schemas/lifecycle.js +13 -6
  397. package/dist/schemas/llamacpp-config.d.ts +443 -20
  398. package/dist/schemas/llamacpp-config.js +188 -14
  399. package/dist/schemas/load-model.d.ts +8940 -1952
  400. package/dist/schemas/load-model.js +73 -9
  401. package/dist/schemas/logging-stream.d.ts +1 -0
  402. package/dist/schemas/logging-stream.js +5 -2
  403. package/dist/schemas/mcp-adapter.d.ts +1 -1
  404. package/dist/schemas/model-config-schemas.d.ts +731 -111
  405. package/dist/schemas/model-config-schemas.js +16 -12
  406. package/dist/schemas/model-fit-policy.d.ts +2 -2
  407. package/dist/schemas/model-info.d.ts +54 -56
  408. package/dist/schemas/model-info.js +18 -7
  409. package/dist/schemas/model-ops.d.ts +12 -0
  410. package/dist/schemas/model-ops.js +33 -5
  411. package/dist/schemas/model-src-utils.d.ts +14 -6
  412. package/dist/schemas/model-src-utils.js +2 -2
  413. package/dist/schemas/model-types.d.ts +12 -4
  414. package/dist/schemas/model-types.js +2 -0
  415. package/dist/schemas/ocr.d.ts +11 -4
  416. package/dist/schemas/ocr.js +7 -3
  417. package/dist/schemas/plugin.d.ts +70 -8
  418. package/dist/schemas/plugin.js +32 -4
  419. package/dist/schemas/profiling.d.ts +32 -32
  420. package/dist/schemas/profiling.js +1 -1
  421. package/dist/schemas/rag.d.ts +9 -0
  422. package/dist/schemas/rag.js +11 -17
  423. package/dist/schemas/registry.d.ts +38 -10
  424. package/dist/schemas/registry.js +16 -7
  425. package/dist/schemas/reload-config.d.ts +45 -6
  426. package/dist/schemas/reload-config.js +13 -6
  427. package/dist/schemas/request-base.d.ts +39 -0
  428. package/dist/schemas/request-base.js +64 -0
  429. package/dist/schemas/rerank.d.ts +132 -0
  430. package/dist/schemas/rerank.js +104 -0
  431. package/dist/schemas/rpc-server.d.ts +3 -0
  432. package/dist/schemas/rpc-server.js +15 -15
  433. package/dist/schemas/sdcpp-config.d.ts +144 -18
  434. package/dist/schemas/sdcpp-config.js +123 -28
  435. package/dist/schemas/session.d.ts +37 -0
  436. package/dist/schemas/session.js +53 -0
  437. package/dist/schemas/system-resources.d.ts +364 -179
  438. package/dist/schemas/system-resources.js +8 -3
  439. package/dist/schemas/text-to-speech.d.ts +1408 -275
  440. package/dist/schemas/text-to-speech.js +547 -33
  441. package/dist/schemas/tools.d.ts +7 -0
  442. package/dist/schemas/tools.js +29 -0
  443. package/dist/schemas/transcription-config.d.ts +267 -2
  444. package/dist/schemas/transcription-config.js +138 -7
  445. package/dist/schemas/transcription.d.ts +94 -7
  446. package/dist/schemas/transcription.js +99 -22
  447. package/dist/schemas/translate.d.ts +4 -2
  448. package/dist/schemas/translate.js +6 -10
  449. package/dist/schemas/translation-config.d.ts +40 -20
  450. package/dist/schemas/translation-config.js +1 -1
  451. package/dist/schemas/tts-pcm.d.ts +35 -0
  452. package/dist/schemas/tts-pcm.js +69 -0
  453. package/dist/schemas/vector-index.d.ts +47 -1
  454. package/dist/schemas/vector-index.js +37 -10
  455. package/dist/schemas/vla.d.ts +3 -3
  456. package/dist/surface.d.ts +4 -3
  457. package/dist/surface.js +4 -3
  458. package/dist/utils/addon-responses.d.ts +71 -2
  459. package/dist/utils/aggregate-events.d.ts +1 -1
  460. package/dist/utils/aggregate-events.js +2 -2
  461. package/dist/utils/archive.d.ts +7 -0
  462. package/dist/utils/archive.js +89 -30
  463. package/dist/utils/asr-events.d.ts +5 -0
  464. package/dist/utils/asr-events.js +10 -1
  465. package/dist/utils/audio/decoder.js +3 -1
  466. package/dist/utils/audio-input.js +1 -1
  467. package/dist/utils/cache/auto-history.d.ts +2 -1
  468. package/dist/utils/cache/auto-history.js +5 -3
  469. package/dist/utils/cache/layout.d.ts +48 -0
  470. package/dist/utils/cache/layout.js +190 -0
  471. package/dist/utils/cache/paths.js +8 -10
  472. package/dist/utils/cache/types.d.ts +2 -1
  473. package/dist/utils/checksum.d.ts +7 -0
  474. package/dist/utils/checksum.js +59 -2
  475. package/dist/utils/completion-normalizer.js +287 -88
  476. package/dist/utils/gguf-architecture.d.ts +7 -0
  477. package/dist/utils/gguf-architecture.js +142 -0
  478. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  479. package/dist/utils/gguf-tensor-extractor.js +1 -50
  480. package/dist/utils/index.d.ts +13 -13
  481. package/dist/utils/index.js +13 -13
  482. package/dist/utils/mcp-adapter.d.ts +1 -1
  483. package/dist/utils/mcp-adapter.js +1 -1
  484. package/dist/utils/path-security.js +1 -1
  485. package/dist/utils/shard-utils.d.ts +0 -16
  486. package/dist/utils/shard-utils.js +14 -48
  487. package/dist/utils/tool-integration.d.ts +3 -2
  488. package/dist/utils/tool-integration.js +3 -6
  489. package/dist/utils/tools/defer.d.ts +50 -31
  490. package/dist/utils/tools/defer.js +115 -51
  491. package/dist/utils/tools/dialect.d.ts +15 -4
  492. package/dist/utils/tools/dialect.js +34 -5
  493. package/dist/utils/tools/index.d.ts +3 -3
  494. package/dist/utils/tools/index.js +3 -3
  495. package/dist/utils/tools/parser.js +8 -8
  496. package/dist/utils/tools/parsers/dsml.d.ts +1 -1
  497. package/dist/utils/tools/parsers/dsml.js +1 -1
  498. package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
  499. package/dist/utils/tools/parsers/gemma4native.js +1 -1
  500. package/dist/utils/tools/parsers/harmony.d.ts +1 -1
  501. package/dist/utils/tools/parsers/harmony.js +1 -1
  502. package/dist/utils/tools/parsers/hermes.d.ts +1 -1
  503. package/dist/utils/tools/parsers/hermes.js +1 -1
  504. package/dist/utils/tools/parsers/json.d.ts +1 -1
  505. package/dist/utils/tools/parsers/json.js +1 -1
  506. package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
  507. package/dist/utils/tools/parsers/pythonic.js +1 -1
  508. package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
  509. package/dist/utils/tools/parsers/qwen35.js +1 -1
  510. package/dist/utils/transcribe-metadata.d.ts +28 -2
  511. package/dist/utils/transcribe-metadata.js +16 -0
  512. package/dist/utils/tts-stats.d.ts +43 -1
  513. package/dist/utils/tts-stats.js +120 -22
  514. package/dist/utils/until-aborted.d.ts +7 -0
  515. package/dist/utils/until-aborted.js +34 -0
  516. package/package.json +67 -55
  517. package/dist/models/registry/resource-profiles.d.ts +0 -17
  518. package/dist/models/registry/resource-profiles.js +0 -5657
  519. package/dist/resources/model-fit/assess.d.ts +0 -72
  520. package/dist/resources/model-fit/assess.js +0 -679
  521. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
  522. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
  523. package/dist/resources/model-fit/floor.d.ts +0 -31
  524. package/dist/resources/model-fit/floor.js +0 -65
  525. package/dist/resources/model-fit/kv-cache.d.ts +0 -23
  526. package/dist/resources/model-fit/kv-cache.js +0 -85
  527. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
  528. package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
  529. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
  530. package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
  531. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
  532. package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
  533. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
  534. package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
  535. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
  536. package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
  537. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
  538. package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
  539. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
  540. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
  541. package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
  542. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
  543. package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
  544. package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
  545. package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
  546. package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
  547. package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
  548. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
  549. package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
  550. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
  551. package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
  552. package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
  553. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
  554. package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
  555. package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
  556. package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
  557. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
  558. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
  559. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
  560. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
  561. package/dist/resources/model-fit/platform.d.ts +0 -9
  562. package/dist/resources/model-fit/platform.js +0 -26
  563. package/dist/resources/model-fit/types.d.ts +0 -6
  564. package/dist/resources/model-fit/types.js +0 -1
  565. package/dist/schemas/model-resource-profile.d.ts +0 -108
  566. package/dist/schemas/model-resource-profile.js +0 -80
  567. /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
  568. /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
  569. /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
@@ -1,20 +1,26 @@
1
- import { logCacheDisabled, logCacheSave, logMessagesToAddon } from './cache-logger.js';
2
- import { extractSystemPrompt, getCurrentCacheInfo } from '../../../ops/kv-cache-utils.js';
1
+ import { logCacheDisabled, logCacheSave, logMessagesToAddon } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
2
+ import { extractSystemPrompt, getCurrentCacheInfo, resolveCacheFilePath } from '../../../../plugins/ops/kv-cache-utils.js';
3
3
  import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
4
- import { shouldCommitCachedTurn } from './kv-cache-state.js';
5
- import { createKvCacheSession, generateConfigHash } from './kv-cache-session.js';
4
+ import { shouldCommitCachedTurn } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-state.js';
5
+ import { createKvCacheSession, generateConfigHash } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
6
6
  import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
7
- import { resolveDeferredTools, toWireTool, withDeferredToolChoice } from '../../../../utils/tools/defer.js';
7
+ import { resolveDeferredTools, toWireTool } from '../../../../utils/tools/defer.js';
8
8
  import { parseToolCalls } from '../../../../utils/tools/index.js';
9
9
  import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
10
- import { buildAutoCacheSaveHistory } from '../../../../utils/index.js';
10
+ import { buildAutoCacheSaveHistory, normalizeCacheMessage } from '../../../../utils/index.js';
11
11
  import { getEngineLogger } from '../../../../logging/index.js';
12
- import { AttachmentNotFoundError } from '../../../../errors/index.js';
12
+ import { AttachmentNotFoundError, InvalidSeedFromParamsError, KvCacheSeedSharingDisabledError, KvCacheSeedSourceNotFoundError, RequestValidationFailedError } from '../../../../errors/index.js';
13
13
  import { nowMs } from '../../../../profiling/index.js';
14
14
  import { buildStreamResult } from '../../../../profiling/model-execution.js';
15
- import { normalizeCompletionStats, withEmittedTokens } from './completion-stats.js';
15
+ import { normalizeCompletionStats, withEmittedTokens } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
16
16
  import fs from 'bare-fs';
17
17
  const logger = getEngineLogger();
18
+ export function withParallelToolCalls(generationParams) {
19
+ return {
20
+ ...generationParams,
21
+ parallel_tool_calls: generationParams?.parallel_tool_calls ?? true
22
+ };
23
+ }
18
24
  function transformMessage(message) {
19
25
  const transformed = [];
20
26
  // Check if it's a tool definition (has type: "function")
@@ -28,6 +34,21 @@ function transformMessage(message) {
28
34
  return transformed;
29
35
  }
30
36
  const msg = message;
37
+ // Ordered parts reach the addon as ONE message, so text and each image share
38
+ // a turn. Attachments cannot express that: they are all sent before the
39
+ // text, which leaves the model unable to tell which image is which option.
40
+ if (Array.isArray(msg.content)) {
41
+ const parts = msg.content.map((part) => {
42
+ if (part.type === 'text')
43
+ return { type: 'text', text: part.text };
44
+ if (!fs.existsSync(part.path)) {
45
+ throw new AttachmentNotFoundError(part.path);
46
+ }
47
+ return { type: 'media', content: part.path };
48
+ });
49
+ transformed.push({ role: msg.role, content: parts });
50
+ return transformed;
51
+ }
31
52
  if (msg.attachments && msg.attachments.length > 0) {
32
53
  for (const attachment of msg.attachments) {
33
54
  if (!fs.existsSync(attachment.path)) {
@@ -134,6 +155,12 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
134
155
  ...(generationParams && { generationParams }),
135
156
  ...(cacheOptions?.cacheKey !== undefined && {
136
157
  cacheKey: cacheOptions.cacheKey
158
+ }),
159
+ ...(cacheOptions?.prefill !== undefined && {
160
+ prefill: cacheOptions.prefill
161
+ }),
162
+ ...(cacheOptions?.baseCacheKey !== undefined && {
163
+ baseCacheKey: cacheOptions.baseCacheKey
137
164
  })
138
165
  };
139
166
  const hasRunOptions = Object.keys(runOptions).length > 0;
@@ -167,23 +194,32 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
167
194
  toolCalls: toolCallsResult,
168
195
  responseText: accumulatedText,
169
196
  producedTokens,
197
+ ...(stopReason === 'eos' && { stopReason: 'eos' }),
198
+ ...(stopReason === 'antiprompt' && { stopReason: 'stopSequence' }),
170
199
  stoppedAtContextBoundary: stopReason === 'contextOverflow'
171
200
  };
172
201
  }
173
202
  export async function* completion(params, opts) {
174
- const { history: requestHistory, modelId, kvCache, tools, generationParams, responseFormat } = params;
203
+ const { history: requestHistory, modelId, kvCache, prefill, seedFrom, tools, toolSearch, generationParams, responseFormat } = params;
175
204
  const { signal, scope } = opts;
176
205
  const requestLogger = opts.logger ?? logger;
206
+ // seedFrom needs a named kvCache key other than itself: the auto key is
207
+ // never returned to the caller, so it cannot be named on either side.
208
+ if (seedFrom !== undefined && (typeof kvCache !== 'string' || seedFrom === kvCache)) {
209
+ throw new InvalidSeedFromParamsError();
210
+ }
177
211
  const modelConfig = getModelConfig(modelId);
178
212
  const history = seedConfiguredSystemPrompt(requestHistory, modelConfig);
213
+ // The addon cannot render a prompt without messages; tools alone are not enough.
214
+ if (prefill && history.length === 0) {
215
+ throw new RequestValidationFailedError('prefill needs at least one message: add a system message to history or set modelConfig.system_prompt');
216
+ }
179
217
  const toolsEnabled = modelConfig.tools === true;
180
- // With deferred tools the prompt and the parser see different lists: the
181
- // prompt carries the always-loaded ones plus `tool_search` and its catalog,
182
- // while the parser also accepts whatever earlier searches appended to the
183
- // history. `null` when nothing defers, which leaves the existing path alone.
184
- const deferred = resolveDeferredTools(tools, history);
218
+ // With deferred tools the prompt carries the always-loaded ones, `tool_search`
219
+ // and its catalog, and whatever earlier searches loaded. `null` when nothing
220
+ // defers, which leaves the existing path alone.
221
+ const deferred = resolveDeferredTools(tools, history, toolSearch);
185
222
  const toolsToRender = deferred?.toolsToRender ?? tools?.map(toWireTool);
186
- const callableTools = deferred?.callableTools ?? toolsToRender;
187
223
  const toolsActive = !!toolsToRender?.length && toolsEnabled;
188
224
  const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
189
225
  // `responseFormat` is forwarded to the addon as a per-request
@@ -203,7 +239,7 @@ export async function* completion(params, opts) {
203
239
  };
204
240
  }
205
241
  }
206
- mergedGenerationParams = withDeferredToolChoice(mergedGenerationParams, toolsActive ? deferred : null);
242
+ mergedGenerationParams = withParallelToolCalls(mergedGenerationParams);
207
243
  const model = getModel(modelId);
208
244
  // Per-request hard cancel: under continuous batching the model runs several
209
245
  // jobs at once, so an abort must cancel only THIS request's job, not the
@@ -241,7 +277,7 @@ export async function* completion(params, opts) {
241
277
  if (!kvCache) {
242
278
  logCacheDisabled();
243
279
  logMessagesToAddon(prompt, 'NO_CACHE');
244
- return yield* processModelResponse(model, prompt, callableTools, mergedGenerationParams, undefined, dialect, setActiveResponse);
280
+ return yield* processModelResponse(model, prompt, toolsToRender, mergedGenerationParams, undefined, dialect, setActiveResponse);
245
281
  }
246
282
  // ---- KV-cache path. The session owns every bookkeeping layer; the handler
247
283
  // registers one deferred unwind that `commitTurn` short-circuits. The addon
@@ -272,7 +308,8 @@ export async function* completion(params, opts) {
272
308
  turn = await session.beginTurn({
273
309
  kind: 'auto',
274
310
  configHash,
275
- history: cacheMessages,
311
+ history: prefill ? cacheMessages.map(normalizeCacheMessage) : cacheMessages,
312
+ ...(toolsActive && toolsToRender ? { tools: toolsToRender } : {}),
276
313
  signal
277
314
  });
278
315
  }
@@ -280,8 +317,35 @@ export async function* completion(params, opts) {
280
317
  // defer above so this runs before the listener detach.
281
318
  let rollbackOnUnwind = false;
282
319
  scope.defer(() => (rollbackOnUnwind ? session.rollback(turn) : session.releaseTurn(turn)));
320
+ // A seed starts a chat that has no cache yet; a chat that has one keeps it.
321
+ // The addon ignores `baseCacheKey` on a model loaded with sharing off and
322
+ // skips a missing source silently, so both are checked here first.
323
+ let baseCacheKey;
324
+ if (seedFrom !== undefined && turn.createdCache) {
325
+ if (modelConfig.cache_share_across_keys !== true) {
326
+ throw new KvCacheSeedSharingDisabledError(modelId);
327
+ }
328
+ const sourcePath = resolveCacheFilePath(modelId, configHash, seedFrom);
329
+ try {
330
+ await fs.promises.access(sourcePath);
331
+ }
332
+ catch (error) {
333
+ // Only a missing file is "not found"; a permission or I/O error keeps its cause.
334
+ if (error.code !== 'ENOENT')
335
+ throw error;
336
+ throw new KvCacheSeedSourceNotFoundError(seedFrom, modelId);
337
+ }
338
+ baseCacheKey = sourcePath;
339
+ }
340
+ else if (seedFrom !== undefined) {
341
+ requestLogger.debug(`[kv-cache] seedFrom ignored: "${kvCache}" already has a cache`);
342
+ }
283
343
  logMessagesToAddon(prompt, 'PROMPT_SEND');
284
- const result = yield* processModelResponse(model, prompt, callableTools, mergedGenerationParams, { cacheKey: turn.cachePath }, dialect, setActiveResponse);
344
+ const result = yield* processModelResponse(model, prompt, toolsToRender, mergedGenerationParams, {
345
+ cacheKey: turn.cachePath,
346
+ ...(prefill !== undefined && { prefill }),
347
+ ...(baseCacheKey !== undefined && { baseCacheKey })
348
+ }, dialect, setActiveResponse);
285
349
  if (typeof kvCache === 'string') {
286
350
  // Custom-key path: the file holds whatever the addon kept, cancelled and
287
351
  // stopped turns included.
@@ -290,7 +354,16 @@ export async function* completion(params, opts) {
290
354
  return result;
291
355
  }
292
356
  // Auto-cache path.
293
- //
357
+ if (prefill && !signal.aborted && !result.stoppedAtContextBoundary) {
358
+ await saveCacheFile(model, turn.cachePath, requestLogger);
359
+ const cacheInfo = await getCurrentCacheInfo(modelId, configHash, history.map(toCacheMessage).map(normalizeCacheMessage), toolsActive ? toolsToRender : undefined);
360
+ await session.commitTurn(turn,
361
+ // Renaming to the held path would reacquire its non-reentrant write lock.
362
+ cacheInfo.cachePath === turn.cachePath
363
+ ? { kind: 'static' }
364
+ : { kind: 'autoRename', targetCachePath: cacheInfo.cachePath });
365
+ return result;
366
+ }
294
367
  // Tool-call turns: the auto-cache key is derived from
295
368
  // `result.responseText`, which here is raw tool-call markup rather
296
369
  // than a clean assistant message. There's no safe post-response key
@@ -320,7 +393,7 @@ export async function* completion(params, opts) {
320
393
  }
321
394
  await saveCacheFile(model, turn.cachePath, requestLogger);
322
395
  const savedHistory = buildAutoCacheSaveHistory(history.map(toCacheMessage), result.responseText);
323
- const postResponseCacheInfo = await getCurrentCacheInfo(modelId, configHash, savedHistory);
396
+ const postResponseCacheInfo = await getCurrentCacheInfo(modelId, configHash, savedHistory, toolsActive ? toolsToRender : undefined);
324
397
  await session.commitTurn(turn, {
325
398
  kind: 'autoRename',
326
399
  targetCachePath: postResponseCacheInfo.cachePath
@@ -1,10 +1,11 @@
1
+ import { type PluginHandlerContext } from '../../../../schemas/plugin.js';
1
2
  import type { FinetuneRunParams, FinetuneRunRequest, FinetuneProgress, FinetuneRequest, FinetuneResult, FinetuneStatus, FinetuneGetStateRequest } from '../../../../schemas/index.js';
2
3
  type FinetuneOptions = FinetuneRunParams['options'];
3
4
  export declare function clearFinetuneRuntimeState(modelId: string): void;
4
5
  export declare function getFinetuneStateFromCheckpoints(options: FinetuneOptions): FinetuneStatus;
5
- export declare function startFinetune(request: FinetuneRunRequest, onProgress?: (progress: FinetuneProgress) => void): Promise<FinetuneResult>;
6
- export declare function pauseFinetune(modelId: string): Promise<FinetuneResult>;
7
- export declare function cancelFinetune(modelId: string): Promise<FinetuneResult>;
6
+ export declare function startFinetune(request: FinetuneRunRequest, onProgress?: (progress: FinetuneProgress) => void, context?: PluginHandlerContext): Promise<FinetuneResult>;
7
+ export declare function pauseFinetune(modelId: string, callerSessionId?: string): Promise<FinetuneResult>;
8
+ export declare function cancelFinetune(modelId: string, callerSessionId?: string): Promise<FinetuneResult>;
8
9
  export declare function getFinetuneState(params: FinetuneGetStateRequest): FinetuneResult;
9
- export declare function finetune(request: FinetuneRequest, onProgress?: (progress: FinetuneProgress) => void): Promise<FinetuneResult>;
10
+ export declare function finetune(request: FinetuneRequest, onProgress?: (progress: FinetuneProgress) => void, context?: PluginHandlerContext): Promise<FinetuneResult>;
10
11
  export {};
@@ -50,11 +50,13 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
50
50
  var e = new Error(message);
51
51
  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
52
  });
53
+ import { requestParent } from '../../../../schemas/plugin.js';
53
54
  import fs from 'bare-fs';
54
55
  import { getModel } from '../../../../runtime/model-registry.js';
55
56
  import { CompletionFailedError } from '../../../../errors/index.js';
56
57
  import { getRequestRegistry, withRequestContext } from '../../../../runtime/index.js';
57
58
  import { generateRandomRequestId } from '../../../../runtime/request-id.js';
59
+ import { OWNER_SESSION_ID, resolveSessionId } from '../../../../runtime/session-id.js';
58
60
  import { getEngineLogger } from '../../../../logging/index.js';
59
61
  const PAUSE_CHECKPOINT_PREFIX = 'pause_checkpoint_step_';
60
62
  // Ref-count, not a Set: with an exclusive writer a second finetune queues behind
@@ -94,15 +96,33 @@ const finetuneRequestIds = new Map();
94
96
  // not settled. A new finetune is rejected during this window even if the runtime ref
95
97
  // has already cleared, because handle.await() can settle before model.pause() resolves.
96
98
  const pausingModels = new Set();
99
+ // A finetune may be stopped by the session that started it, or by the owner,
100
+ // who may stop any session's finetune. Another session's pause or cancel is a
101
+ // no-op, as its registry cancel would be.
102
+ function callerMayStop(tracked, callerSessionId) {
103
+ if (callerSessionId === OWNER_SESSION_ID)
104
+ return true;
105
+ return (tracked.sessionId ?? OWNER_SESSION_ID) === callerSessionId;
106
+ }
97
107
  // Cancel the model's in-flight finetune by its tracked requestId (covers the
98
108
  // registry grant→register handoff window via the cancel-before-begin tripwire),
99
- // falling back to a broad cancel if the id is unknown.
100
- function cancelTrackedFinetune(modelId) {
101
- const requestId = finetuneRequestIds.get(modelId);
102
- if (requestId !== undefined)
103
- getRequestRegistry().cancel({ requestId });
104
- else
105
- getRequestRegistry().cancel({ modelId, kind: 'finetune' });
109
+ // falling back to a broad cancel within the caller's own session if the id is
110
+ // unknown. Returns false when the finetune belongs to another session.
111
+ function cancelTrackedFinetune(modelId, callerSessionId) {
112
+ const tracked = finetuneRequestIds.get(modelId);
113
+ if (tracked !== undefined) {
114
+ if (!callerMayStop(tracked, callerSessionId))
115
+ return false;
116
+ getRequestRegistry().cancel(tracked);
117
+ return true;
118
+ }
119
+ if (callerSessionId === OWNER_SESSION_ID) {
120
+ getRequestRegistry().cancel({ modelId, kind: 'finetune', allSessions: true });
121
+ }
122
+ else {
123
+ getRequestRegistry().cancel({ modelId, kind: 'finetune', sessionId: callerSessionId });
124
+ }
125
+ return true;
106
126
  }
107
127
  export function getFinetuneStateFromCheckpoints(options) {
108
128
  const checkpointDirectory = options.checkpointSaveDir ?? './checkpoints';
@@ -137,7 +157,7 @@ function validateExplicitFinetuneOperation(request) {
137
157
  throw new CompletionFailedError(`Model "${request.modelId}" has no paused finetune checkpoint to resume`);
138
158
  }
139
159
  }
140
- export async function startFinetune(request, onProgress) {
160
+ export async function startFinetune(request, onProgress, context) {
141
161
  const env_1 = { stack: [], error: void 0, hasError: false };
142
162
  try {
143
163
  const model = getModel(request.modelId);
@@ -156,19 +176,25 @@ export async function startFinetune(request, onProgress) {
156
176
  // observes RUNNING, not IDLE, and a concurrent start is rejected above. Released
157
177
  // on a failed begin() or on scope unwind below.
158
178
  registerRunningFinetune(request.modelId);
159
- finetuneRequestIds.set(request.modelId, requestId);
179
+ // Under an invoke the job runs as the invoke's session, the owner included,
180
+ // so the stop paths compare against that session and not one named inside
181
+ // the params.
182
+ const tracked = { requestId, sessionId: context ? context.sessionId : request.sessionId };
183
+ finetuneRequestIds.set(request.modelId, tracked);
160
184
  // Scope the run into the registry so cancel({ requestId }) and
161
185
  // cancel({ modelId, kind: "finetune" }) reach it; onAbort forwards to
162
186
  // model.cancel().
163
187
  const ctx = __addDisposableResource(env_1, await getRequestRegistry()
164
188
  .begin({
165
189
  requestId,
190
+ sessionId: request.sessionId,
191
+ ...requestParent(context),
166
192
  kind: 'finetune',
167
193
  modelId: request.modelId
168
194
  })
169
195
  .catch((err) => {
170
196
  clearFinetuneRuntimeState(request.modelId);
171
- if (finetuneRequestIds.get(request.modelId) === requestId) {
197
+ if (finetuneRequestIds.get(request.modelId) === tracked) {
172
198
  finetuneRequestIds.delete(request.modelId);
173
199
  }
174
200
  throw err;
@@ -178,7 +204,7 @@ export async function startFinetune(request, onProgress) {
178
204
  // removes the listener first.
179
205
  ctx.scope.defer(() => {
180
206
  clearFinetuneRuntimeState(request.modelId);
181
- if (finetuneRequestIds.get(request.modelId) === requestId) {
207
+ if (finetuneRequestIds.get(request.modelId) === tracked) {
182
208
  finetuneRequestIds.delete(request.modelId);
183
209
  }
184
210
  });
@@ -237,10 +263,14 @@ export async function startFinetune(request, onProgress) {
237
263
  await result_1;
238
264
  }
239
265
  }
240
- export async function pauseFinetune(modelId) {
266
+ export async function pauseFinetune(modelId, callerSessionId = OWNER_SESSION_ID) {
241
267
  // Validate the model is loaded on every path (matches main; getModel throws
242
268
  // ModelNotFoundError for an unknown/unloaded model) before reporting any state.
243
269
  const model = getModel(modelId);
270
+ const tracked = finetuneRequestIds.get(modelId);
271
+ if (tracked !== undefined && !callerMayStop(tracked, callerSessionId)) {
272
+ return { type: 'finetune', status: 'RUNNING' };
273
+ }
244
274
  if (admittedFinetunes.has(modelId)) {
245
275
  // The single finetune holds the model exclusively, so the addon-global pause is
246
276
  // safe (nothing else runs on the model). Bar a new finetune until the native
@@ -260,7 +290,7 @@ export async function pauseFinetune(modelId) {
260
290
  // pause would cancel that unrelated reader, so cancel the queued finetune by its
261
291
  // own requestId. Nothing has trained, so report CANCELLED rather than a false
262
292
  // PAUSED.
263
- cancelTrackedFinetune(modelId);
293
+ cancelTrackedFinetune(modelId, callerSessionId);
264
294
  return { type: 'finetune', status: 'CANCELLED' };
265
295
  }
266
296
  // No finetune on this model: a no-op pause, but the model was validated above.
@@ -268,8 +298,10 @@ export async function pauseFinetune(modelId) {
268
298
  }
269
299
  // Routes cancellation through the registry; the model.cancel() forward is
270
300
  // installed by startFinetune, so never call model.cancel() here.
271
- export function cancelFinetune(modelId) {
272
- cancelTrackedFinetune(modelId);
301
+ export function cancelFinetune(modelId, callerSessionId = OWNER_SESSION_ID) {
302
+ if (!cancelTrackedFinetune(modelId, callerSessionId)) {
303
+ return Promise.resolve({ type: 'finetune', status: 'RUNNING' });
304
+ }
273
305
  return Promise.resolve({
274
306
  type: 'finetune',
275
307
  status: 'CANCELLED'
@@ -282,18 +314,21 @@ export function getFinetuneState(params) {
282
314
  status: runtimeState ? 'RUNNING' : getFinetuneStateFromCheckpoints(params.options)
283
315
  };
284
316
  }
285
- export async function finetune(request, onProgress) {
317
+ export async function finetune(request, onProgress, context) {
286
318
  if (request.operation === undefined ||
287
319
  request.operation === 'start' ||
288
320
  request.operation === 'resume') {
289
- return startFinetune(request, onProgress);
321
+ return startFinetune(request, onProgress, context);
290
322
  }
323
+ // Under a plugin invoke the caller is the invoke's session; the nested
324
+ // params name whatever the caller put there.
325
+ const callerSessionId = resolveSessionId(context ?? request);
291
326
  switch (request.operation) {
292
327
  case 'getState':
293
328
  return getFinetuneState(request);
294
329
  case 'pause':
295
- return pauseFinetune(request.modelId);
330
+ return pauseFinetune(request.modelId, callerSessionId);
296
331
  case 'cancel':
297
- return cancelFinetune(request.modelId);
332
+ return cancelFinetune(request.modelId, callerSessionId);
298
333
  }
299
334
  }
@@ -1,5 +1,6 @@
1
- import { generateConfigHash } from '../../../ops/kv-cache-utils.js';
1
+ import { generateConfigHash } from '../../../../plugins/ops/kv-cache-utils.js';
2
2
  import { type CacheMessage } from '../../../../utils/index.js';
3
+ import type { Tool } from '../../../../schemas/tools.js';
3
4
  import type { Logger } from '../../../../logging/types.js';
4
5
  import { type AbortSignal } from 'bare-abort-controller';
5
6
  export interface TurnHandle {
@@ -30,6 +31,8 @@ export interface BeginAutoTurnInput {
30
31
  configHash: string;
31
32
  /** Conversation history used to compute the pre-response cache key. */
32
33
  history: CacheMessage[];
34
+ /** Rendered tools that distinguish prefixes with no assistant message. */
35
+ tools?: readonly Tool[];
33
36
  /** See `BeginCustomTurnInput.signal`. */
34
37
  signal?: AbortSignal;
35
38
  }
@@ -40,8 +43,7 @@ export interface StaticCommitResult {
40
43
  export interface AutoRenameCommitResult {
41
44
  kind: 'autoRename';
42
45
  /**
43
- * Destination path the addon's pre-response cache file should be
44
- * renamed to (computed from `cacheMessages + responseText`).
46
+ * Destination path for the history covered by the saved cache.
45
47
  */
46
48
  targetCachePath: string;
47
49
  }
@@ -1,10 +1,10 @@
1
1
  import { promises as fsPromises } from 'bare-fs';
2
2
  import path from 'bare-path';
3
- import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../ops/kv-cache-utils.js';
4
- import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../ops/kv-cache-retention.js';
3
+ import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../../plugins/ops/kv-cache-utils.js';
4
+ import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../../plugins/ops/kv-cache-retention.js';
5
5
  import { isMobile } from '../../../../runtime/state.js';
6
6
  import { getKVCacheDir } from '../../../../utils/index.js';
7
- import { logCacheSaveError, logCacheStatus } from './cache-logger.js';
7
+ import { logCacheSaveError, logCacheStatus } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
8
8
  import { getEngineLogger } from '../../../../logging/index.js';
9
9
  // Used by cross-model paths that have no `RequestContext` (e.g.
10
10
  // `deleteKvCacheState`). Per-session call sites receive a logger from
@@ -323,8 +323,9 @@ export function createKvCacheSession(modelId, options) {
323
323
  // the state lock, or commitTurn's rename could deadlock a waiter), then
324
324
  // re-check existence under the lock in case a same-file peer just renamed it.
325
325
  const discovered = await withCacheStateLock(async () => {
326
- const existingCache = await findMatchingCache(modelId, input.configHash, input.history);
327
- const cacheInfo = existingCache ?? (await getCurrentCacheInfo(modelId, input.configHash, input.history));
326
+ const existingCache = await findMatchingCache(modelId, input.configHash, input.history, input.tools);
327
+ const cacheInfo = existingCache ??
328
+ (await getCurrentCacheInfo(modelId, input.configHash, input.history, input.tools));
328
329
  return {
329
330
  cachePath: cacheInfo.cachePath,
330
331
  cacheKey: cacheInfo.cacheKey
@@ -414,8 +415,7 @@ export function createKvCacheSession(modelId, options) {
414
415
  if (state.committed || state.rolledBack)
415
416
  return;
416
417
  if (result.kind === 'static') {
417
- // Custom-key path: the addon's save wrote the new cache state at the
418
- // same path. Verify the file persisted.
418
+ // The addon's save wrote the new cache state at the same path.
419
419
  const ok = await verifySaveAndRecord(state.cachePath);
420
420
  if (!ok) {
421
421
  await runRollback(state);
@@ -424,6 +424,8 @@ export function createKvCacheSession(modelId, options) {
424
424
  state.committed = true;
425
425
  releaseCachePath(state.cachePath);
426
426
  state.releaseWriteLock();
427
+ if (state.autoCacheKey !== undefined)
428
+ scheduleAutoCacheSweep(logger);
427
429
  return;
428
430
  }
429
431
  const sourceCachePath = state.cachePath;