@qvac/inference 0.22.0 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (386) hide show
  1. package/NOTICE +59 -40
  2. package/README.md +1 -1
  3. package/dist/api/audio-gen.js +1 -1
  4. package/dist/api/completion-stream.d.ts +2 -1
  5. package/dist/api/completion-stream.js +3 -1
  6. package/dist/api/decide.js +1 -1
  7. package/dist/api/diffusion.d.ts +18 -1
  8. package/dist/api/diffusion.js +18 -1
  9. package/dist/api/download-asset.js +1 -1
  10. package/dist/api/finetune.js +1 -1
  11. package/dist/api/index.d.ts +39 -38
  12. package/dist/api/index.js +38 -37
  13. package/dist/api/load-model.js +2 -2
  14. package/dist/api/logging-stream-registry.js +11 -7
  15. package/dist/api/logging-stream.d.ts +11 -1
  16. package/dist/api/logging-stream.js +11 -2
  17. package/dist/api/registry.d.ts +2 -1
  18. package/dist/api/registry.js +5 -4
  19. package/dist/api/rerank.d.ts +17 -0
  20. package/dist/api/rerank.js +36 -0
  21. package/dist/api/subscribe-logs.js +5 -3
  22. package/dist/api/unload-model.js +1 -1
  23. package/dist/api/video.js +1 -1
  24. package/dist/api/vla.js +1 -1
  25. package/dist/api/world-result.js +1 -1
  26. package/dist/api/world.d.ts +1 -1
  27. package/dist/api/world.js +1 -1
  28. package/dist/config/resolve-config.d.ts +1 -1
  29. package/dist/config/resolve-config.js +1 -1
  30. package/dist/constants/index.d.ts +2 -2
  31. package/dist/constants/index.js +2 -2
  32. package/dist/dispatch.d.ts +3 -2
  33. package/dist/dispatch.js +16 -4
  34. package/dist/errors/index.d.ts +6 -0
  35. package/dist/errors/index.js +13 -1
  36. package/dist/handlers/assess-model-fit.d.ts +1 -1
  37. package/dist/handlers/assess-model-fit.js +4 -4
  38. package/dist/handlers/cancelHandler.js +1 -1
  39. package/dist/handlers/completion-orchestrate.js +2 -2
  40. package/dist/handlers/download-asset.js +2 -2
  41. package/dist/handlers/finetune.js +1 -1
  42. package/dist/handlers/get-loaded-model-info.js +2 -3
  43. package/dist/handlers/get-model-info.js +1 -1
  44. package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -0
  45. package/dist/handlers/load-model/coreml-sidecars.js +71 -8
  46. package/dist/handlers/load-model/download-manager.d.ts +1 -1
  47. package/dist/handlers/load-model/download-stats.d.ts +2 -2
  48. package/dist/handlers/load-model/download-stats.js +3 -3
  49. package/dist/handlers/load-model/handler.js +10 -7
  50. package/dist/handlers/load-model/http-verify.d.ts +1 -1
  51. package/dist/handlers/load-model/http.d.ts +1 -1
  52. package/dist/handlers/load-model/http.js +3 -3
  53. package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
  54. package/dist/handlers/load-model/hyperdrive.js +2 -2
  55. package/dist/handlers/load-model/index.d.ts +2 -2
  56. package/dist/handlers/load-model/index.js +2 -2
  57. package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
  58. package/dist/handlers/load-model/registry-companion-set.js +1 -1
  59. package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
  60. package/dist/handlers/load-model/registry-download-utils.js +2 -2
  61. package/dist/handlers/load-model/registry.d.ts +2 -2
  62. package/dist/handlers/load-model/registry.js +5 -5
  63. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  64. package/dist/handlers/load-model/resolve-session.js +11 -3
  65. package/dist/handlers/load-model/resolve.d.ts +1 -1
  66. package/dist/handlers/load-model/resolve.js +5 -5
  67. package/dist/handlers/logging-stream.d.ts +2 -1
  68. package/dist/handlers/logging-stream.js +23 -7
  69. package/dist/handlers/registry.js +4 -2
  70. package/dist/handlers/types.d.ts +7 -1
  71. package/dist/handlers/vector-index.js +2 -2
  72. package/dist/index.d.ts +1 -1
  73. package/dist/index.js +1 -1
  74. package/dist/logging/addon.d.ts +1 -1
  75. package/dist/logging/addon.js +1 -1
  76. package/dist/logging/index.d.ts +5 -5
  77. package/dist/logging/index.js +4 -4
  78. package/dist/logging/logger.d.ts +1 -1
  79. package/dist/logging/logger.js +3 -3
  80. package/dist/logging/utils.js +1 -1
  81. package/dist/{resources/model-fit → model-fit}/assess.d.ts +4 -4
  82. package/dist/{resources/model-fit → model-fit}/assess.js +3 -3
  83. package/dist/{resources/model-fit/native-probe/classify-fit.d.ts → model-fit/classify.d.ts} +2 -2
  84. package/dist/{resources/model-fit/native-probe/classify-fit.js → model-fit/classify.js} +4 -4
  85. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +1 -1
  86. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +5 -5
  87. package/dist/{resources/model-fit/fit-stub/project-fit-from-load.d.ts → model-fit/description/project.d.ts} +3 -3
  88. package/dist/{resources/model-fit/fit-stub/project-fit-from-load.js → model-fit/description/project.js} +2 -2
  89. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +3 -3
  90. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +6 -6
  91. package/dist/{resources/model-fit → model-fit}/floor.d.ts +2 -2
  92. package/dist/{resources/model-fit → model-fit}/floor.js +1 -1
  93. package/dist/{resources/model-fit → model-fit}/kv-cache.d.ts +1 -1
  94. package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts} +8 -10
  95. package/dist/{resources/model-fit/native-probe/advisory-fit.js → model-fit/load-check.js} +14 -40
  96. package/dist/{resources/model-fit/types.d.ts → model-fit/platform.d.ts} +8 -0
  97. package/dist/{resources/model-fit/native-probe/create-fit-request.d.ts → model-fit/request/create.d.ts} +3 -3
  98. package/dist/{resources/model-fit/native-probe/create-fit-request.js → model-fit/request/create.js} +8 -8
  99. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.d.ts +2 -2
  100. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.js +9 -1
  101. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.d.ts +1 -1
  102. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.js +1 -1
  103. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/diffusion.d.ts +1 -1
  104. package/dist/model-fit/request/engines/diffusion.js +93 -0
  105. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.d.ts +2 -2
  106. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.js +13 -10
  107. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/nmt.d.ts +1 -1
  108. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/nmt.js +1 -1
  109. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/ocr.d.ts +1 -1
  110. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/ocr.js +1 -1
  111. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.d.ts +2 -2
  112. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.js +1 -1
  113. package/dist/{resources/model-fit/native-probe/engine-fit.d.ts → model-fit/run/engine.d.ts} +3 -4
  114. package/dist/{resources/model-fit/native-probe/engine-fit.js → model-fit/run/engine.js} +1 -1
  115. package/dist/model-fit/run/run.d.ts +43 -0
  116. package/dist/model-fit/run/run.js +204 -0
  117. package/dist/models/registry/index.d.ts +1 -1
  118. package/dist/models/registry/index.js +2 -2
  119. package/dist/models/registry/models.d.ts +510 -0
  120. package/dist/models/registry/models.js +2946 -2410
  121. package/dist/models/registry/resource-profiles.js +203 -1
  122. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -2
  123. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -2
  124. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -2
  125. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +60 -54
  126. package/dist/plugins/builtin/audiogen-ggml/plugin.js +4 -4
  127. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +26 -25
  128. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +4 -4
  129. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
  130. package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
  131. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +11 -10
  132. package/dist/plugins/builtin/ggml-ocr/plugin.js +4 -4
  133. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +12 -12
  134. package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
  135. package/dist/plugins/builtin/index.d.ts +15 -14
  136. package/dist/plugins/builtin/index.js +15 -14
  137. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +5 -7
  138. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
  139. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  140. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +1 -0
  141. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +36 -19
  142. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
  143. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
  144. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +169 -78
  145. package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -17
  146. package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
  147. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
  148. package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
  149. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +1 -1
  150. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +39 -33
  151. package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
  152. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +7 -6
  153. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +16 -6
  154. package/dist/plugins/builtin/llamacpp-embedding/transform.js +6 -0
  155. package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
  156. package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
  157. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +5 -0
  158. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +117 -0
  159. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +91 -0
  160. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +45 -0
  161. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +16 -16
  162. package/dist/plugins/builtin/moss-transcribe/plugin.js +4 -4
  163. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +18 -13
  164. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -4
  165. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +29 -29
  166. package/dist/plugins/builtin/parakeet-transcription/plugin.js +4 -4
  167. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +5 -2
  168. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +168 -52
  169. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -2
  170. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +19 -8
  171. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
  172. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
  173. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +90 -34
  174. package/dist/plugins/builtin/sdcpp-generation/plugin.js +45 -45
  175. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
  176. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
  177. package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
  178. package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
  179. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +1 -1
  180. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +2 -2
  181. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +2 -2
  182. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +47 -32
  183. package/dist/plugins/builtin/tts-ggml/plugin.js +3 -3
  184. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +34 -33
  185. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +4 -4
  186. package/dist/plugins/index.d.ts +1 -1
  187. package/dist/plugins/index.js +2 -2
  188. package/dist/plugins/ops/kv-cache-utils.d.ts +4 -3
  189. package/dist/plugins/ops/kv-cache-utils.js +15 -8
  190. package/dist/plugins/ops/load-model.d.ts +8 -2
  191. package/dist/plugins/ops/load-model.js +14 -5
  192. package/dist/plugins/ops/transcribe.js +1 -1
  193. package/dist/plugins/ops/translate.js +3 -3
  194. package/dist/plugins/ops/unload-model.js +1 -1
  195. package/dist/plugins/turbovec-backend.js +5 -5
  196. package/dist/profiling/aggregator.d.ts +1 -1
  197. package/dist/profiling/aggregator.js +1 -1
  198. package/dist/profiling/controller.d.ts +1 -1
  199. package/dist/profiling/controller.js +2 -2
  200. package/dist/profiling/events.d.ts +1 -1
  201. package/dist/profiling/events.js +2 -2
  202. package/dist/profiling/exporters.d.ts +1 -1
  203. package/dist/profiling/exporters.js +3 -3
  204. package/dist/profiling/index.d.ts +13 -13
  205. package/dist/profiling/index.js +12 -12
  206. package/dist/profiling/operation-metrics.d.ts +1 -1
  207. package/dist/profiling/operation-metrics.js +2 -2
  208. package/dist/profiling/operation-wrappers.js +3 -3
  209. package/dist/profiling/profiler.js +1 -1
  210. package/dist/rag/handler.js +1 -1
  211. package/dist/rag/handlers/close-workspace.js +1 -1
  212. package/dist/rag/handlers/delete-embeddings.js +1 -1
  213. package/dist/rag/handlers/delete-workspace.js +1 -1
  214. package/dist/rag/handlers/ingest.js +1 -1
  215. package/dist/rag/handlers/list-workspaces.d.ts +1 -1
  216. package/dist/rag/handlers/list-workspaces.js +1 -1
  217. package/dist/rag/handlers/reindex.js +1 -1
  218. package/dist/rag/handlers/save-embeddings.js +1 -1
  219. package/dist/rag/handlers/search.js +1 -1
  220. package/dist/rag/index.d.ts +11 -11
  221. package/dist/rag/index.js +11 -11
  222. package/dist/rag/rag-workspace-manager.js +1 -1
  223. package/dist/registry.js +2 -1
  224. package/dist/resources/collector.d.ts +1 -1
  225. package/dist/resources/collector.js +1 -1
  226. package/dist/resources/instance.d.ts +1 -1
  227. package/dist/resources/instance.js +1 -1
  228. package/dist/resources/native.d.ts +1 -1
  229. package/dist/resources/normalize.d.ts +1 -1
  230. package/dist/rpc/inventory.js +3 -3
  231. package/dist/runtime/index.d.ts +5 -5
  232. package/dist/runtime/index.js +2 -2
  233. package/dist/runtime/lifecycle.js +7 -7
  234. package/dist/runtime/model-config-utils.js +3 -0
  235. package/dist/runtime/model-registry.d.ts +3 -1
  236. package/dist/runtime/model-registry.js +2 -1
  237. package/dist/runtime/registry-client.js +1 -1
  238. package/dist/runtime/request-context.d.ts +3 -3
  239. package/dist/runtime/request-context.js +14 -1
  240. package/dist/runtime/request-registry.d.ts +1 -1
  241. package/dist/runtime/request-registry.js +1 -1
  242. package/dist/runtime/state.d.ts +1 -1
  243. package/dist/runtime/state.js +2 -2
  244. package/dist/runtime/vector-index-backend.d.ts +2 -2
  245. package/dist/runtime/vector-index-registry.d.ts +5 -5
  246. package/dist/runtime/vector-index-registry.js +36 -13
  247. package/dist/schemas/assess-model-fit.d.ts +6 -0
  248. package/dist/schemas/assess-model-fit.js +4 -4
  249. package/dist/schemas/audio-gen.d.ts +13 -1
  250. package/dist/schemas/audio-gen.js +3 -3
  251. package/dist/schemas/batch-completion-stream.d.ts +8 -2
  252. package/dist/schemas/batch-completion-stream.js +3 -3
  253. package/dist/schemas/bci-config.d.ts +1 -0
  254. package/dist/schemas/bci-config.js +2 -2
  255. package/dist/schemas/bci.d.ts +1 -1
  256. package/dist/schemas/bci.js +2 -2
  257. package/dist/schemas/cancel.d.ts +5 -0
  258. package/dist/schemas/cancel.js +1 -0
  259. package/dist/schemas/common.d.ts +478 -100
  260. package/dist/schemas/common.js +35 -32
  261. package/dist/schemas/completion-event.d.ts +8 -2
  262. package/dist/schemas/completion-event.js +4 -1
  263. package/dist/schemas/completion-stream.d.ts +11 -2
  264. package/dist/schemas/completion-stream.js +37 -10
  265. package/dist/schemas/config.d.ts +849 -6
  266. package/dist/schemas/config.js +12 -10
  267. package/dist/schemas/decision.d.ts +8 -0
  268. package/dist/schemas/decision.js +14 -0
  269. package/dist/schemas/download-asset.d.ts +6 -4
  270. package/dist/schemas/download-asset.js +1 -1
  271. package/dist/schemas/engine-addon-map.d.ts +2 -1
  272. package/dist/schemas/engine-addon-map.js +4 -3
  273. package/dist/schemas/errors.d.ts +1 -0
  274. package/dist/schemas/errors.js +6 -1
  275. package/dist/schemas/index.d.ts +51 -50
  276. package/dist/schemas/index.js +50 -49
  277. package/dist/schemas/llamacpp-config.d.ts +182 -6
  278. package/dist/schemas/llamacpp-config.js +102 -7
  279. package/dist/schemas/load-model.d.ts +3176 -646
  280. package/dist/schemas/load-model.js +34 -1
  281. package/dist/schemas/mcp-adapter.d.ts +1 -1
  282. package/dist/schemas/model-config-schemas.d.ts +195 -1
  283. package/dist/schemas/model-config-schemas.js +14 -12
  284. package/dist/schemas/model-fit-policy.d.ts +1 -1
  285. package/dist/schemas/model-info.js +2 -2
  286. package/dist/schemas/model-resource-profile.d.ts +1 -0
  287. package/dist/schemas/model-resource-profile.js +1 -1
  288. package/dist/schemas/model-src-utils.d.ts +6 -2
  289. package/dist/schemas/model-src-utils.js +2 -2
  290. package/dist/schemas/model-types.d.ts +5 -1
  291. package/dist/schemas/model-types.js +1 -0
  292. package/dist/schemas/ocr.d.ts +2 -0
  293. package/dist/schemas/ocr.js +1 -1
  294. package/dist/schemas/plugin.d.ts +9 -3
  295. package/dist/schemas/plugin.js +6 -0
  296. package/dist/schemas/profiling.js +1 -1
  297. package/dist/schemas/rag.js +1 -1
  298. package/dist/schemas/registry.d.ts +8 -0
  299. package/dist/schemas/registry.js +2 -1
  300. package/dist/schemas/reload-config.d.ts +6 -2
  301. package/dist/schemas/reload-config.js +2 -2
  302. package/dist/schemas/rerank.d.ts +131 -0
  303. package/dist/schemas/rerank.js +102 -0
  304. package/dist/schemas/sdcpp-config.d.ts +56 -0
  305. package/dist/schemas/sdcpp-config.js +33 -15
  306. package/dist/schemas/text-to-speech.d.ts +61 -0
  307. package/dist/schemas/transcription-config.d.ts +5 -0
  308. package/dist/schemas/transcription-config.js +16 -1
  309. package/dist/schemas/transcription.d.ts +1 -1
  310. package/dist/schemas/transcription.js +1 -1
  311. package/dist/schemas/translate.d.ts +2 -2
  312. package/dist/schemas/translate.js +1 -1
  313. package/dist/schemas/translation-config.d.ts +20 -10
  314. package/dist/schemas/translation-config.js +1 -1
  315. package/dist/surface.d.ts +2 -1
  316. package/dist/surface.js +2 -1
  317. package/dist/utils/addon-responses.d.ts +2 -0
  318. package/dist/utils/aggregate-events.d.ts +1 -1
  319. package/dist/utils/aggregate-events.js +2 -2
  320. package/dist/utils/archive.js +2 -2
  321. package/dist/utils/audio/decoder.js +3 -1
  322. package/dist/utils/audio-input.js +1 -1
  323. package/dist/utils/cache/auto-history.d.ts +2 -1
  324. package/dist/utils/cache/auto-history.js +2 -2
  325. package/dist/utils/cache/paths.js +6 -6
  326. package/dist/utils/checksum.d.ts +2 -0
  327. package/dist/utils/checksum.js +41 -2
  328. package/dist/utils/completion-normalizer.js +1 -1
  329. package/dist/utils/gguf-architecture.d.ts +7 -0
  330. package/dist/utils/gguf-architecture.js +142 -0
  331. package/dist/utils/gguf-tensor-extractor.js +1 -1
  332. package/dist/utils/index.d.ts +13 -13
  333. package/dist/utils/index.js +13 -13
  334. package/dist/utils/mcp-adapter.d.ts +1 -1
  335. package/dist/utils/mcp-adapter.js +1 -1
  336. package/dist/utils/path-security.js +1 -1
  337. package/dist/utils/shard-utils.d.ts +5 -0
  338. package/dist/utils/shard-utils.js +19 -3
  339. package/dist/utils/tool-integration.d.ts +1 -0
  340. package/dist/utils/tool-integration.js +3 -6
  341. package/dist/utils/tools/defer.d.ts +16 -26
  342. package/dist/utils/tools/defer.js +50 -41
  343. package/dist/utils/tools/dialect.d.ts +15 -4
  344. package/dist/utils/tools/dialect.js +34 -5
  345. package/dist/utils/tools/index.d.ts +3 -3
  346. package/dist/utils/tools/index.js +3 -3
  347. package/dist/utils/tools/parser.js +8 -8
  348. package/dist/utils/tools/parsers/dsml.d.ts +1 -1
  349. package/dist/utils/tools/parsers/dsml.js +1 -1
  350. package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
  351. package/dist/utils/tools/parsers/gemma4native.js +1 -1
  352. package/dist/utils/tools/parsers/harmony.d.ts +1 -1
  353. package/dist/utils/tools/parsers/harmony.js +1 -1
  354. package/dist/utils/tools/parsers/hermes.d.ts +1 -1
  355. package/dist/utils/tools/parsers/hermes.js +1 -1
  356. package/dist/utils/tools/parsers/json.d.ts +1 -1
  357. package/dist/utils/tools/parsers/json.js +1 -1
  358. package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
  359. package/dist/utils/tools/parsers/pythonic.js +1 -1
  360. package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
  361. package/dist/utils/tools/parsers/qwen35.js +1 -1
  362. package/dist/utils/tts-stats.d.ts +1 -1
  363. package/dist/utils/until-aborted.d.ts +7 -0
  364. package/dist/utils/until-aborted.js +34 -0
  365. package/package.json +53 -45
  366. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
  367. package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
  368. package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
  369. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
  370. package/dist/resources/model-fit/native-probe/fit-outcome.js +0 -1
  371. package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
  372. package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
  373. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
  374. package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
  375. package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
  376. package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
  377. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
  378. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
  379. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
  380. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
  381. package/dist/resources/model-fit/platform.d.ts +0 -9
  382. package/dist/resources/model-fit/types.js +0 -1
  383. /package/dist/{resources/model-fit → model-fit}/kv-cache.js +0 -0
  384. /package/dist/{resources/model-fit → model-fit}/platform.js +0 -0
  385. /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.d.ts +0 -0
  386. /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.js +0 -0
@@ -7,7 +7,7 @@ export declare const classificationPlugin: {
7
7
  modelPath: import("zod").ZodOptional<import("zod").ZodString>;
8
8
  topK: import("zod").ZodOptional<import("zod").ZodNumber>;
9
9
  nativeLogger: import("zod").ZodOptional<import("zod").ZodBoolean>;
10
- }, import("zod/v4/core/index.js").$strip>;
10
+ }, import("zod/v4/core").$strip>;
11
11
  skipPrimaryModelPathValidation: true;
12
12
  createModel(params: CreateModelParams): PluginModelResult;
13
13
  handlers: {
@@ -19,13 +19,13 @@ export declare const classificationPlugin: {
19
19
  height: import("zod").ZodOptional<import("zod").ZodNumber>;
20
20
  channels: import("zod").ZodOptional<import("zod").ZodLiteral<3>>;
21
21
  type: import("zod").ZodLiteral<"classify">;
22
- }, import("zod/v4/core/index.js").$strip>, import("zod").ZodObject<{
22
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
23
23
  type: import("zod").ZodLiteral<"classify">;
24
24
  results: import("zod").ZodArray<import("zod").ZodObject<{
25
25
  label: import("zod").ZodString;
26
26
  confidence: import("zod").ZodNumber;
27
- }, import("zod/v4/core/index.js").$strip>>;
27
+ }, import("zod/v4/core").$strip>>;
28
28
  done: import("zod").ZodOptional<import("zod").ZodBoolean>;
29
- }, import("zod/v4/core/index.js").$strip>>;
29
+ }, import("zod/v4/core").$strip>>;
30
30
  };
31
31
  };
@@ -1,7 +1,7 @@
1
1
  import ImageClassifier from '@qvac/classification-ggml';
2
2
  import { definePlugin, defineHandler, classifyRequestSchema, classifyResponseSchema, ModelType, classificationConfigSchema, ADDON_CLASSIFICATION } from '../../../schemas/index.js';
3
3
  import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
4
- import { classify } from './ops/classify.js';
4
+ import { classify } from '../../../plugins/builtin/ggml-classification/ops/classify.js';
5
5
  export const classificationPlugin = definePlugin({
6
6
  modelType: ModelType.ggmlClassification,
7
7
  displayName: 'Image Classification (GGML)',
@@ -51,6 +51,7 @@ export declare const ocrPlugin: {
51
51
  "bci-whispercpp-transcription": "bci-whispercpp-transcription";
52
52
  "llamacpp-embedding": "llamacpp-embedding";
53
53
  "llamacpp-decision": "llamacpp-decision";
54
+ "llamacpp-rerank": "llamacpp-rerank";
54
55
  "nmtcpp-translation": "nmtcpp-translation";
55
56
  "onnx-tts": "onnx-tts";
56
57
  "tts-ggml": "tts-ggml";
@@ -68,8 +69,8 @@ export declare const ocrPlugin: {
68
69
  parakeet: "parakeet";
69
70
  classification: "classification";
70
71
  }>, import("zod").ZodLiteral<"vad">]>>;
71
- }, import("zod/v4/core/index.js").$strip>]>>;
72
- }, import("zod/v4/core/index.js").$strip>;
72
+ }, import("zod/v4/core").$strip>]>>;
73
+ }, import("zod/v4/core").$strip>;
73
74
  assessFit: typeof ocrAssessFit;
74
75
  resolveConfig(cfg: OCRConfig, ctx: ResolveContext): Promise<ResolveResult<Record<string, unknown>, "detectorModelPath">>;
75
76
  createModel(params: CreateModelParams): PluginModelResult;
@@ -79,32 +80,32 @@ export declare const ocrPlugin: {
79
80
  image: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
80
81
  type: import("zod").ZodLiteral<"base64">;
81
82
  value: import("zod").ZodString;
82
- }, import("zod/v4/core/index.js").$strip>, import("zod").ZodObject<{
83
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
83
84
  type: import("zod").ZodLiteral<"filePath">;
84
85
  value: import("zod").ZodString;
85
- }, import("zod/v4/core/index.js").$strip>], "type">;
86
+ }, import("zod/v4/core").$strip>], "type">;
86
87
  options: import("zod").ZodOptional<import("zod").ZodObject<{
87
88
  paragraph: import("zod").ZodOptional<import("zod").ZodBoolean>;
88
- }, import("zod/v4/core/index.js").$strip>>;
89
+ }, import("zod/v4/core").$strip>>;
89
90
  type: import("zod").ZodLiteral<"ocrStream">;
90
- }, import("zod/v4/core/index.js").$strip>, import("zod").ZodObject<{
91
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
91
92
  type: import("zod").ZodLiteral<"ocrStream">;
92
93
  blocks: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodObject<{
93
94
  text: import("zod").ZodString;
94
95
  bbox: import("zod").ZodOptional<import("zod").ZodTuple<[import("zod").ZodNumber, import("zod").ZodNumber, import("zod").ZodNumber, import("zod").ZodNumber], null>>;
95
96
  confidence: import("zod").ZodOptional<import("zod").ZodNumber>;
96
- }, import("zod/v4/core/index.js").$strip>>>;
97
+ }, import("zod/v4/core").$strip>>>;
97
98
  done: import("zod").ZodOptional<import("zod").ZodBoolean>;
98
99
  error: import("zod").ZodOptional<import("zod").ZodString>;
99
100
  stats: import("zod").ZodOptional<import("zod").ZodObject<{
100
101
  detectionTime: import("zod").ZodOptional<import("zod").ZodNumber>;
101
102
  recognitionTime: import("zod").ZodOptional<import("zod").ZodNumber>;
102
103
  totalTime: import("zod").ZodOptional<import("zod").ZodNumber>;
103
- }, import("zod/v4/core/index.js").$strip>>;
104
- }, import("zod/v4/core/index.js").$strip>>;
104
+ }, import("zod/v4/core").$strip>>;
105
+ }, import("zod/v4/core").$strip>>;
105
106
  };
106
107
  logging: {
107
- module: () => Promise<typeof import("@qvac/ocr-ggml/addonLogging.js")>;
108
+ module: () => Promise<typeof import("@qvac/ocr-ggml/addonLogging")>;
108
109
  namespace: "ggml-ocr";
109
110
  };
110
111
  };
@@ -2,10 +2,10 @@ import { definePlugin, defineHandler, ocrStreamRequestSchema, ocrStreamResponseS
2
2
  import { ModelLoadFailedError } from '../../../errors/index.js';
3
3
  import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
4
4
  import { OcrGgml, assessFit as ocrAssessFit } from '@qvac/ocr-ggml';
5
- import { ocr } from './ops/ocr-stream.js';
5
+ import { ocr } from '../../../plugins/builtin/ggml-ocr/ops/ocr-stream.js';
6
6
  import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
7
- import { ocrParamsFor } from './params.js';
8
- import { resolveOcrConfig } from './resolve-config.js';
7
+ import { ocrParamsFor } from '../../../plugins/builtin/ggml-ocr/params.js';
8
+ import { resolveOcrConfig } from '../../../plugins/builtin/ggml-ocr/resolve-config.js';
9
9
  import { isMobile } from '../../../runtime/state.js';
10
10
  import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
11
11
  function createOCRModel(modelId, detectorPath, recognizerPath, ocrConfig) {
@@ -83,7 +83,7 @@ export const ocrPlugin = definePlugin({
83
83
  })
84
84
  },
85
85
  logging: {
86
- module: () => import('@qvac/ocr-ggml/addonLogging.js'),
86
+ module: () => import('@qvac/ocr-ggml/addonLogging'),
87
87
  namespace: ModelType.ggmlOcr
88
88
  }
89
89
  });
@@ -20,8 +20,8 @@ export declare const vlaPlugin: {
20
20
  tag: import("zod").ZodOptional<import("zod").ZodString>;
21
21
  catId: import("zod").ZodOptional<import("zod").ZodNumber>;
22
22
  numCameras: import("zod").ZodOptional<import("zod").ZodNumber>;
23
- }, import("zod/v4/core/index.js").$strip>]>>;
24
- }, import("zod/v4/core/index.js").$strip>;
23
+ }, import("zod/v4/core").$strip>]>>;
24
+ }, import("zod/v4/core").$strip>;
25
25
  createModel(params: CreateModelParams): PluginModelResult;
26
26
  handlers: {
27
27
  vlaRun: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
@@ -34,7 +34,7 @@ export declare const vlaPlugin: {
34
34
  tokens: import("zod").ZodString;
35
35
  mask: import("zod").ZodString;
36
36
  noise: import("zod").ZodOptional<import("zod").ZodString>;
37
- }, import("zod/v4/core/index.js").$strip>, import("zod").ZodObject<{
37
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
38
38
  actions: import("zod").ZodString;
39
39
  actionDim: import("zod").ZodNumber;
40
40
  chunkSize: import("zod").ZodNumber;
@@ -47,12 +47,12 @@ export declare const vlaPlugin: {
47
47
  ode_ms: import("zod").ZodOptional<import("zod").ZodNumber>;
48
48
  total_ms: import("zod").ZodOptional<import("zod").ZodNumber>;
49
49
  backendDevice: import("zod").ZodOptional<import("zod").ZodNumber>;
50
- }, import("zod/v4/core/index.js").$strip>>;
51
- }, import("zod/v4/core/index.js").$strip>>;
50
+ }, import("zod/v4/core").$strip>>;
51
+ }, import("zod/v4/core").$strip>>;
52
52
  vlaHparams: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
53
53
  type: import("zod").ZodLiteral<"vlaHparams">;
54
54
  modelId: import("zod").ZodString;
55
- }, import("zod/v4/core/index.js").$strip>, import("zod").ZodObject<{
55
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
56
56
  hparams: import("zod").ZodObject<{
57
57
  chunkSize: import("zod").ZodNumber;
58
58
  actionDim: import("zod").ZodNumber;
@@ -72,9 +72,9 @@ export declare const vlaPlugin: {
72
72
  imagePatchElems: import("zod").ZodOptional<import("zod").ZodNumber>;
73
73
  selectedEmbodimentTag: import("zod").ZodOptional<import("zod").ZodString>;
74
74
  selectedEmbodimentCatId: import("zod").ZodOptional<import("zod").ZodNumber>;
75
- }, import("zod/v4/core/index.js").$strip>;
75
+ }, import("zod/v4/core").$strip>;
76
76
  backendName: import("zod").ZodNullable<import("zod").ZodString>;
77
- }, import("zod/v4/core/index.js").$strip>>;
77
+ }, import("zod/v4/core").$strip>>;
78
78
  vlaSetEmbodiment: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
79
79
  type: import("zod").ZodLiteral<"vlaSetEmbodiment">;
80
80
  modelId: import("zod").ZodString;
@@ -82,8 +82,8 @@ export declare const vlaPlugin: {
82
82
  tag: import("zod").ZodOptional<import("zod").ZodString>;
83
83
  catId: import("zod").ZodOptional<import("zod").ZodNumber>;
84
84
  numCameras: import("zod").ZodOptional<import("zod").ZodNumber>;
85
- }, import("zod/v4/core/index.js").$strip>]>;
86
- }, import("zod/v4/core/index.js").$strip>, import("zod").ZodObject<{
85
+ }, import("zod/v4/core").$strip>]>;
86
+ }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
87
87
  hparams: import("zod").ZodObject<{
88
88
  chunkSize: import("zod").ZodNumber;
89
89
  actionDim: import("zod").ZodNumber;
@@ -103,7 +103,7 @@ export declare const vlaPlugin: {
103
103
  imagePatchElems: import("zod").ZodOptional<import("zod").ZodNumber>;
104
104
  selectedEmbodimentTag: import("zod").ZodOptional<import("zod").ZodString>;
105
105
  selectedEmbodimentCatId: import("zod").ZodOptional<import("zod").ZodNumber>;
106
- }, import("zod/v4/core/index.js").$strip>;
107
- }, import("zod/v4/core/index.js").$strip>>;
106
+ }, import("zod/v4/core").$strip>;
107
+ }, import("zod/v4/core").$strip>>;
108
108
  };
109
109
  };
@@ -1,9 +1,9 @@
1
1
  import { VlaModel } from '@qvac/vla-ggml';
2
2
  import { definePlugin, defineHandler, vlaConfigSchema, vlaRunRequestSchema, vlaRunResponseSchema, vlaHparamsRequestSchema, vlaHparamsResponseSchema, vlaSetEmbodimentRequestSchema, vlaSetEmbodimentResponseSchema, ModelType, ADDON_VLA } from '../../../schemas/index.js';
3
3
  import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
4
- import { vlaRun } from './ops/vla-run.js';
5
- import { vlaGetHparams } from './ops/vla-hparams.js';
6
- import { vlaSetEmbodiment } from './ops/vla-set-embodiment.js';
4
+ import { vlaRun } from '../../../plugins/builtin/ggml-vla/ops/vla-run.js';
5
+ import { vlaGetHparams } from '../../../plugins/builtin/ggml-vla/ops/vla-hparams.js';
6
+ import { vlaSetEmbodiment } from '../../../plugins/builtin/ggml-vla/ops/vla-set-embodiment.js';
7
7
  // The `@qvac/vla-ggml` VlaModel exposes `load({ backend })` rather than the
8
8
  // `load(force?)` signature `PluginModel` expects. Wrap it so the plugin
9
9
  // framework can call `load()` and have the configured backend flow through.
@@ -1,14 +1,15 @@
1
- export { llmPlugin } from './llamacpp-completion/plugin.js';
2
- export { embeddingsPlugin } from './llamacpp-embedding/plugin.js';
3
- export { decisionPlugin } from './llamacpp-decision/plugin.js';
4
- export { whisperPlugin } from './whispercpp-transcription/plugin.js';
5
- export { bciPlugin } from './bci-whispercpp-transcription/plugin.js';
6
- export { parakeetPlugin } from './parakeet-transcription/plugin.js';
7
- export { nmtPlugin } from './nmtcpp-translation/plugin.js';
8
- export { ttsPlugin } from './tts-ggml/plugin.js';
9
- export { ocrPlugin } from './ggml-ocr/plugin.js';
10
- export { diffusionPlugin } from './sdcpp-generation/plugin.js';
11
- export { vlaPlugin } from './ggml-vla/plugin.js';
12
- export { classificationPlugin } from './ggml-classification/plugin.js';
13
- export { audioGenPlugin } from './audiogen-ggml/plugin.js';
14
- export { mossTranscribePlugin } from './moss-transcribe/plugin.js';
1
+ export { llmPlugin } from '../../plugins/builtin/llamacpp-completion/plugin.js';
2
+ export { embeddingsPlugin } from '../../plugins/builtin/llamacpp-embedding/plugin.js';
3
+ export { decisionPlugin } from '../../plugins/builtin/llamacpp-decision/plugin.js';
4
+ export { rerankPlugin } from '../../plugins/builtin/llamacpp-rerank/plugin.js';
5
+ export { whisperPlugin } from '../../plugins/builtin/whispercpp-transcription/plugin.js';
6
+ export { bciPlugin } from '../../plugins/builtin/bci-whispercpp-transcription/plugin.js';
7
+ export { parakeetPlugin } from '../../plugins/builtin/parakeet-transcription/plugin.js';
8
+ export { nmtPlugin } from '../../plugins/builtin/nmtcpp-translation/plugin.js';
9
+ export { ttsPlugin } from '../../plugins/builtin/tts-ggml/plugin.js';
10
+ export { ocrPlugin } from '../../plugins/builtin/ggml-ocr/plugin.js';
11
+ export { diffusionPlugin } from '../../plugins/builtin/sdcpp-generation/plugin.js';
12
+ export { vlaPlugin } from '../../plugins/builtin/ggml-vla/plugin.js';
13
+ export { classificationPlugin } from '../../plugins/builtin/ggml-classification/plugin.js';
14
+ export { audioGenPlugin } from '../../plugins/builtin/audiogen-ggml/plugin.js';
15
+ export { mossTranscribePlugin } from '../../plugins/builtin/moss-transcribe/plugin.js';
@@ -1,14 +1,15 @@
1
- export { llmPlugin } from './llamacpp-completion/plugin.js';
2
- export { embeddingsPlugin } from './llamacpp-embedding/plugin.js';
3
- export { decisionPlugin } from './llamacpp-decision/plugin.js';
4
- export { whisperPlugin } from './whispercpp-transcription/plugin.js';
5
- export { bciPlugin } from './bci-whispercpp-transcription/plugin.js';
6
- export { parakeetPlugin } from './parakeet-transcription/plugin.js';
7
- export { nmtPlugin } from './nmtcpp-translation/plugin.js';
8
- export { ttsPlugin } from './tts-ggml/plugin.js';
9
- export { ocrPlugin } from './ggml-ocr/plugin.js';
10
- export { diffusionPlugin } from './sdcpp-generation/plugin.js';
11
- export { vlaPlugin } from './ggml-vla/plugin.js';
12
- export { classificationPlugin } from './ggml-classification/plugin.js';
13
- export { audioGenPlugin } from './audiogen-ggml/plugin.js';
14
- export { mossTranscribePlugin } from './moss-transcribe/plugin.js';
1
+ export { llmPlugin } from '../../plugins/builtin/llamacpp-completion/plugin.js';
2
+ export { embeddingsPlugin } from '../../plugins/builtin/llamacpp-embedding/plugin.js';
3
+ export { decisionPlugin } from '../../plugins/builtin/llamacpp-decision/plugin.js';
4
+ export { rerankPlugin } from '../../plugins/builtin/llamacpp-rerank/plugin.js';
5
+ export { whisperPlugin } from '../../plugins/builtin/whispercpp-transcription/plugin.js';
6
+ export { bciPlugin } from '../../plugins/builtin/bci-whispercpp-transcription/plugin.js';
7
+ export { parakeetPlugin } from '../../plugins/builtin/parakeet-transcription/plugin.js';
8
+ export { nmtPlugin } from '../../plugins/builtin/nmtcpp-translation/plugin.js';
9
+ export { ttsPlugin } from '../../plugins/builtin/tts-ggml/plugin.js';
10
+ export { ocrPlugin } from '../../plugins/builtin/ggml-ocr/plugin.js';
11
+ export { diffusionPlugin } from '../../plugins/builtin/sdcpp-generation/plugin.js';
12
+ export { vlaPlugin } from '../../plugins/builtin/ggml-vla/plugin.js';
13
+ export { classificationPlugin } from '../../plugins/builtin/ggml-classification/plugin.js';
14
+ export { audioGenPlugin } from '../../plugins/builtin/audiogen-ggml/plugin.js';
15
+ export { mossTranscribePlugin } from '../../plugins/builtin/moss-transcribe/plugin.js';
@@ -3,10 +3,10 @@ import { getEngineLogger } from '../../../../logging/index.js';
3
3
  import { nowMs } from '../../../../profiling/index.js';
4
4
  import { buildStreamResult } from '../../../../profiling/model-execution.js';
5
5
  import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
6
- import { seedConfiguredSystemPrompt, transformMessages } from './completion-stream.js';
7
- import { normalizeCompletionStats } from './completion-stats.js';
6
+ import { seedConfiguredSystemPrompt, transformMessages, withParallelToolCalls } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stream.js';
7
+ import { normalizeCompletionStats } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
8
8
  import { prependToolsToHistory } from '../../../../utils/tool-integration.js';
9
- import { resolveDeferredTools, toWireTool, withDeferredToolChoice } from '../../../../utils/tools/defer.js';
9
+ import { resolveDeferredTools, toWireTool } from '../../../../utils/tools/defer.js';
10
10
  const logger = getEngineLogger();
11
11
  function runBatchModel(model, prompts) {
12
12
  const run = model.run.bind(model);
@@ -38,13 +38,11 @@ function renderPromptHistory(prompt, history, deferred, options) {
38
38
  function buildBatchPrompt(prompt, options) {
39
39
  const history = seedConfiguredSystemPrompt(prompt.history, options.modelConfig);
40
40
  const deferred = resolveDeferredTools(prompt.tools, history);
41
- const mergedGenerationParams = withDeferredToolChoice(mergeGenerationParams(prompt.generationParams, prompt.responseFormat), options.toolsEnabled ? deferred : null);
41
+ const mergedGenerationParams = withParallelToolCalls(mergeGenerationParams(prompt.generationParams, prompt.responseFormat));
42
42
  return {
43
43
  ...(prompt.id !== undefined && { id: prompt.id }),
44
44
  prompt: renderPromptHistory(prompt, history, deferred, options),
45
- ...(mergedGenerationParams && {
46
- runOptions: { generationParams: mergedGenerationParams }
47
- })
45
+ runOptions: { generationParams: mergedGenerationParams }
48
46
  };
49
47
  }
50
48
  function isBatchOutputChunk(output) {
@@ -10,6 +10,8 @@ export declare function normalizeCompletionStats(stats: LlmStats | undefined): {
10
10
  emittedTokens?: number | undefined;
11
11
  avgConcurrentSeq?: number | undefined;
12
12
  toolDefinitionsDropped?: number | undefined;
13
+ draftTokens?: number | undefined;
14
+ draftAcceptedTokens?: number | undefined;
13
15
  backendDevice?: "cpu" | "gpu" | undefined;
14
16
  } | undefined;
15
17
  /**
@@ -12,6 +12,8 @@ export function normalizeCompletionStats(stats) {
12
12
  const generatedTokens = finiteNumber(stats.generatedTokens);
13
13
  const avgConcurrentSeq = finiteNumber(stats.avgConcurrentSeq);
14
14
  const toolDefinitionsDropped = finiteNumber(stats.toolDefinitionsDropped);
15
+ const draftTokens = finiteNumber(stats.draftTokens);
16
+ const draftAcceptedTokens = finiteNumber(stats.draftAcceptedTokens);
15
17
  const normalized = {
16
18
  ...(timeToFirstToken !== undefined && { timeToFirstToken }),
17
19
  ...(tokensPerSecond !== undefined && { tokensPerSecond }),
@@ -21,6 +23,8 @@ export function normalizeCompletionStats(stats) {
21
23
  ...(generatedTokens !== undefined && { generatedTokens }),
22
24
  ...(avgConcurrentSeq !== undefined && { avgConcurrentSeq }),
23
25
  ...(toolDefinitionsDropped !== undefined && { toolDefinitionsDropped }),
26
+ ...(draftTokens !== undefined && { draftTokens }),
27
+ ...(draftAcceptedTokens !== undefined && { draftAcceptedTokens }),
24
28
  ...(stats.backendDevice !== undefined && { backendDevice: stats.backendDevice })
25
29
  };
26
30
  if (timeToFirstToken === undefined &&
@@ -31,6 +35,8 @@ export function normalizeCompletionStats(stats) {
31
35
  generatedTokens === undefined &&
32
36
  avgConcurrentSeq === undefined &&
33
37
  toolDefinitionsDropped === undefined &&
38
+ draftTokens === undefined &&
39
+ draftAcceptedTokens === undefined &&
34
40
  stats.backendDevice === undefined) {
35
41
  return undefined;
36
42
  }
@@ -26,6 +26,7 @@ interface ChatHistory {
26
26
  export type CompletionGenerationParams = GenerationParams & {
27
27
  json_schema?: string;
28
28
  };
29
+ export declare function withParallelToolCalls(generationParams: CompletionGenerationParams | undefined): CompletionGenerationParams;
29
30
  export declare function transformMessages(messages: Array<HistoryMsg | Tool>): ChatHistory[];
30
31
  type HistoryMsg = CacheMessage;
31
32
  /**
@@ -1,20 +1,26 @@
1
- import { logCacheDisabled, logCacheSave, logMessagesToAddon } from './cache-logger.js';
2
- import { extractSystemPrompt, getCurrentCacheInfo } from '../../../ops/kv-cache-utils.js';
1
+ import { logCacheDisabled, logCacheSave, logMessagesToAddon } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
2
+ import { extractSystemPrompt, getCurrentCacheInfo } from '../../../../plugins/ops/kv-cache-utils.js';
3
3
  import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
4
- import { shouldCommitCachedTurn } from './kv-cache-state.js';
5
- import { createKvCacheSession, generateConfigHash } from './kv-cache-session.js';
4
+ import { shouldCommitCachedTurn } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-state.js';
5
+ import { createKvCacheSession, generateConfigHash } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
6
6
  import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
7
- import { resolveDeferredTools, toWireTool, withDeferredToolChoice } from '../../../../utils/tools/defer.js';
7
+ import { resolveDeferredTools, toWireTool } from '../../../../utils/tools/defer.js';
8
8
  import { parseToolCalls } from '../../../../utils/tools/index.js';
9
9
  import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
10
- import { buildAutoCacheSaveHistory } from '../../../../utils/index.js';
10
+ import { buildAutoCacheSaveHistory, normalizeCacheMessage } from '../../../../utils/index.js';
11
11
  import { getEngineLogger } from '../../../../logging/index.js';
12
12
  import { AttachmentNotFoundError } from '../../../../errors/index.js';
13
13
  import { nowMs } from '../../../../profiling/index.js';
14
14
  import { buildStreamResult } from '../../../../profiling/model-execution.js';
15
- import { normalizeCompletionStats, withEmittedTokens } from './completion-stats.js';
15
+ import { normalizeCompletionStats, withEmittedTokens } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
16
16
  import fs from 'bare-fs';
17
17
  const logger = getEngineLogger();
18
+ export function withParallelToolCalls(generationParams) {
19
+ return {
20
+ ...generationParams,
21
+ parallel_tool_calls: generationParams?.parallel_tool_calls ?? true
22
+ };
23
+ }
18
24
  function transformMessage(message) {
19
25
  const transformed = [];
20
26
  // Check if it's a tool definition (has type: "function")
@@ -134,6 +140,9 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
134
140
  ...(generationParams && { generationParams }),
135
141
  ...(cacheOptions?.cacheKey !== undefined && {
136
142
  cacheKey: cacheOptions.cacheKey
143
+ }),
144
+ ...(cacheOptions?.prefill !== undefined && {
145
+ prefill: cacheOptions.prefill
137
146
  })
138
147
  };
139
148
  const hasRunOptions = Object.keys(runOptions).length > 0;
@@ -171,19 +180,17 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
171
180
  };
172
181
  }
173
182
  export async function* completion(params, opts) {
174
- const { history: requestHistory, modelId, kvCache, tools, generationParams, responseFormat } = params;
183
+ const { history: requestHistory, modelId, kvCache, prefill, tools, generationParams, responseFormat } = params;
175
184
  const { signal, scope } = opts;
176
185
  const requestLogger = opts.logger ?? logger;
177
186
  const modelConfig = getModelConfig(modelId);
178
187
  const history = seedConfiguredSystemPrompt(requestHistory, modelConfig);
179
188
  const toolsEnabled = modelConfig.tools === true;
180
- // With deferred tools the prompt and the parser see different lists: the
181
- // prompt carries the always-loaded ones plus `tool_search` and its catalog,
182
- // while the parser also accepts whatever earlier searches appended to the
183
- // history. `null` when nothing defers, which leaves the existing path alone.
189
+ // With deferred tools the prompt carries the always-loaded ones, `tool_search`
190
+ // and its catalog, and whatever earlier searches loaded. `null` when nothing
191
+ // defers, which leaves the existing path alone.
184
192
  const deferred = resolveDeferredTools(tools, history);
185
193
  const toolsToRender = deferred?.toolsToRender ?? tools?.map(toWireTool);
186
- const callableTools = deferred?.callableTools ?? toolsToRender;
187
194
  const toolsActive = !!toolsToRender?.length && toolsEnabled;
188
195
  const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
189
196
  // `responseFormat` is forwarded to the addon as a per-request
@@ -203,7 +210,7 @@ export async function* completion(params, opts) {
203
210
  };
204
211
  }
205
212
  }
206
- mergedGenerationParams = withDeferredToolChoice(mergedGenerationParams, toolsActive ? deferred : null);
213
+ mergedGenerationParams = withParallelToolCalls(mergedGenerationParams);
207
214
  const model = getModel(modelId);
208
215
  // Per-request hard cancel: under continuous batching the model runs several
209
216
  // jobs at once, so an abort must cancel only THIS request's job, not the
@@ -241,7 +248,7 @@ export async function* completion(params, opts) {
241
248
  if (!kvCache) {
242
249
  logCacheDisabled();
243
250
  logMessagesToAddon(prompt, 'NO_CACHE');
244
- return yield* processModelResponse(model, prompt, callableTools, mergedGenerationParams, undefined, dialect, setActiveResponse);
251
+ return yield* processModelResponse(model, prompt, toolsToRender, mergedGenerationParams, undefined, dialect, setActiveResponse);
245
252
  }
246
253
  // ---- KV-cache path. The session owns every bookkeeping layer; the handler
247
254
  // registers one deferred unwind that `commitTurn` short-circuits. The addon
@@ -272,7 +279,8 @@ export async function* completion(params, opts) {
272
279
  turn = await session.beginTurn({
273
280
  kind: 'auto',
274
281
  configHash,
275
- history: cacheMessages,
282
+ history: prefill ? cacheMessages.map(normalizeCacheMessage) : cacheMessages,
283
+ ...(toolsActive && toolsToRender ? { tools: toolsToRender } : {}),
276
284
  signal
277
285
  });
278
286
  }
@@ -281,7 +289,7 @@ export async function* completion(params, opts) {
281
289
  let rollbackOnUnwind = false;
282
290
  scope.defer(() => (rollbackOnUnwind ? session.rollback(turn) : session.releaseTurn(turn)));
283
291
  logMessagesToAddon(prompt, 'PROMPT_SEND');
284
- const result = yield* processModelResponse(model, prompt, callableTools, mergedGenerationParams, { cacheKey: turn.cachePath }, dialect, setActiveResponse);
292
+ const result = yield* processModelResponse(model, prompt, toolsToRender, mergedGenerationParams, { cacheKey: turn.cachePath, ...(prefill !== undefined && { prefill }) }, dialect, setActiveResponse);
285
293
  if (typeof kvCache === 'string') {
286
294
  // Custom-key path: the file holds whatever the addon kept, cancelled and
287
295
  // stopped turns included.
@@ -290,7 +298,16 @@ export async function* completion(params, opts) {
290
298
  return result;
291
299
  }
292
300
  // Auto-cache path.
293
- //
301
+ if (prefill && !signal.aborted && !result.stoppedAtContextBoundary) {
302
+ await saveCacheFile(model, turn.cachePath, requestLogger);
303
+ const cacheInfo = await getCurrentCacheInfo(modelId, configHash, history.map(toCacheMessage).map(normalizeCacheMessage), toolsActive ? toolsToRender : undefined);
304
+ await session.commitTurn(turn,
305
+ // Renaming to the held path would reacquire its non-reentrant write lock.
306
+ cacheInfo.cachePath === turn.cachePath
307
+ ? { kind: 'static' }
308
+ : { kind: 'autoRename', targetCachePath: cacheInfo.cachePath });
309
+ return result;
310
+ }
294
311
  // Tool-call turns: the auto-cache key is derived from
295
312
  // `result.responseText`, which here is raw tool-call markup rather
296
313
  // than a clean assistant message. There's no safe post-response key
@@ -320,7 +337,7 @@ export async function* completion(params, opts) {
320
337
  }
321
338
  await saveCacheFile(model, turn.cachePath, requestLogger);
322
339
  const savedHistory = buildAutoCacheSaveHistory(history.map(toCacheMessage), result.responseText);
323
- const postResponseCacheInfo = await getCurrentCacheInfo(modelId, configHash, savedHistory);
340
+ const postResponseCacheInfo = await getCurrentCacheInfo(modelId, configHash, savedHistory, toolsActive ? toolsToRender : undefined);
324
341
  await session.commitTurn(turn, {
325
342
  kind: 'autoRename',
326
343
  targetCachePath: postResponseCacheInfo.cachePath
@@ -1,5 +1,6 @@
1
- import { generateConfigHash } from '../../../ops/kv-cache-utils.js';
1
+ import { generateConfigHash } from '../../../../plugins/ops/kv-cache-utils.js';
2
2
  import { type CacheMessage } from '../../../../utils/index.js';
3
+ import type { Tool } from '../../../../schemas/tools.js';
3
4
  import type { Logger } from '../../../../logging/types.js';
4
5
  import { type AbortSignal } from 'bare-abort-controller';
5
6
  export interface TurnHandle {
@@ -30,6 +31,8 @@ export interface BeginAutoTurnInput {
30
31
  configHash: string;
31
32
  /** Conversation history used to compute the pre-response cache key. */
32
33
  history: CacheMessage[];
34
+ /** Rendered tools that distinguish prefixes with no assistant message. */
35
+ tools?: readonly Tool[];
33
36
  /** See `BeginCustomTurnInput.signal`. */
34
37
  signal?: AbortSignal;
35
38
  }
@@ -40,8 +43,7 @@ export interface StaticCommitResult {
40
43
  export interface AutoRenameCommitResult {
41
44
  kind: 'autoRename';
42
45
  /**
43
- * Destination path the addon's pre-response cache file should be
44
- * renamed to (computed from `cacheMessages + responseText`).
46
+ * Destination path for the history covered by the saved cache.
45
47
  */
46
48
  targetCachePath: string;
47
49
  }
@@ -1,10 +1,10 @@
1
1
  import { promises as fsPromises } from 'bare-fs';
2
2
  import path from 'bare-path';
3
- import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../ops/kv-cache-utils.js';
4
- import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../ops/kv-cache-retention.js';
3
+ import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../../plugins/ops/kv-cache-utils.js';
4
+ import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../../plugins/ops/kv-cache-retention.js';
5
5
  import { isMobile } from '../../../../runtime/state.js';
6
6
  import { getKVCacheDir } from '../../../../utils/index.js';
7
- import { logCacheSaveError, logCacheStatus } from './cache-logger.js';
7
+ import { logCacheSaveError, logCacheStatus } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
8
8
  import { getEngineLogger } from '../../../../logging/index.js';
9
9
  // Used by cross-model paths that have no `RequestContext` (e.g.
10
10
  // `deleteKvCacheState`). Per-session call sites receive a logger from
@@ -323,8 +323,9 @@ export function createKvCacheSession(modelId, options) {
323
323
  // the state lock, or commitTurn's rename could deadlock a waiter), then
324
324
  // re-check existence under the lock in case a same-file peer just renamed it.
325
325
  const discovered = await withCacheStateLock(async () => {
326
- const existingCache = await findMatchingCache(modelId, input.configHash, input.history);
327
- const cacheInfo = existingCache ?? (await getCurrentCacheInfo(modelId, input.configHash, input.history));
326
+ const existingCache = await findMatchingCache(modelId, input.configHash, input.history, input.tools);
327
+ const cacheInfo = existingCache ??
328
+ (await getCurrentCacheInfo(modelId, input.configHash, input.history, input.tools));
328
329
  return {
329
330
  cachePath: cacheInfo.cachePath,
330
331
  cacheKey: cacheInfo.cacheKey
@@ -414,8 +415,7 @@ export function createKvCacheSession(modelId, options) {
414
415
  if (state.committed || state.rolledBack)
415
416
  return;
416
417
  if (result.kind === 'static') {
417
- // Custom-key path: the addon's save wrote the new cache state at the
418
- // same path. Verify the file persisted.
418
+ // The addon's save wrote the new cache state at the same path.
419
419
  const ok = await verifySaveAndRecord(state.cachePath);
420
420
  if (!ok) {
421
421
  await runRollback(state);
@@ -424,6 +424,8 @@ export function createKvCacheSession(modelId, options) {
424
424
  state.committed = true;
425
425
  releaseCachePath(state.cachePath);
426
426
  state.releaseWriteLock();
427
+ if (state.autoCacheKey !== undefined)
428
+ scheduleAutoCacheSweep(logger);
427
429
  return;
428
430
  }
429
431
  const sourceCachePath = state.cachePath;