@qvac/inference 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/NOTICE +60 -28
  2. package/README.md +3 -1
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +135 -44
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +1 -1
  13. package/dist/api/completion-stream.js +2 -1
  14. package/dist/api/delete-cache.d.ts +7 -1
  15. package/dist/api/delete-cache.js +11 -1
  16. package/dist/api/diffusion.d.ts +14 -1
  17. package/dist/api/diffusion.js +14 -1
  18. package/dist/api/embed.d.ts +82 -29
  19. package/dist/api/embed.js +64 -18
  20. package/dist/api/get-model-info.d.ts +1 -1
  21. package/dist/api/index.d.ts +2 -1
  22. package/dist/api/index.js +2 -1
  23. package/dist/api/rag.d.ts +1 -1
  24. package/dist/api/rag.js +1 -1
  25. package/dist/api/registry.d.ts +16 -6
  26. package/dist/api/registry.js +32 -9
  27. package/dist/api/text-to-speech.d.ts +14 -5
  28. package/dist/api/text-to-speech.js +83 -29
  29. package/dist/api/transcribe.d.ts +6 -2
  30. package/dist/api/transcribe.js +7 -1
  31. package/dist/api/vector-index.d.ts +25 -3
  32. package/dist/api/vector-index.js +17 -2
  33. package/dist/api/video-request.d.ts +15 -15
  34. package/dist/dispatch.d.ts +10 -4
  35. package/dist/dispatch.js +65 -15
  36. package/dist/engine.d.ts +1 -1
  37. package/dist/errors/index.d.ts +70 -0
  38. package/dist/errors/index.js +128 -0
  39. package/dist/handlers/assess-model-fit.d.ts +3 -21
  40. package/dist/handlers/assess-model-fit.js +4 -303
  41. package/dist/handlers/cancelHandler.d.ts +10 -9
  42. package/dist/handlers/cancelHandler.js +17 -13
  43. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  44. package/dist/handlers/completion-orchestrate.js +164 -22
  45. package/dist/handlers/delete-cache.d.ts +2 -1
  46. package/dist/handlers/delete-cache.js +6 -1
  47. package/dist/handlers/download-asset.js +11 -4
  48. package/dist/handlers/get-loaded-model-info.js +1 -0
  49. package/dist/handlers/get-model-info.js +71 -2
  50. package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
  51. package/dist/handlers/load-model/coreml-sidecars.js +16 -119
  52. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  53. package/dist/handlers/load-model/download-lock.js +213 -0
  54. package/dist/handlers/load-model/download-manager.d.ts +10 -2
  55. package/dist/handlers/load-model/download-manager.js +62 -11
  56. package/dist/handlers/load-model/handler.js +88 -35
  57. package/dist/handlers/load-model/http.js +327 -205
  58. package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
  59. package/dist/handlers/load-model/hyperdrive.js +123 -50
  60. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  61. package/dist/handlers/load-model/registry-client-options.js +4 -2
  62. package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
  63. package/dist/handlers/load-model/registry-companion-set.js +19 -1
  64. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  65. package/dist/handlers/load-model/registry-delete.js +269 -0
  66. package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
  67. package/dist/handlers/load-model/registry-download-utils.js +93 -22
  68. package/dist/handlers/load-model/registry.d.ts +20 -0
  69. package/dist/handlers/load-model/registry.js +308 -175
  70. package/dist/handlers/load-model/resolve.js +4 -1
  71. package/dist/handlers/load-model/types.d.ts +2 -0
  72. package/dist/handlers/logging-stream.js +17 -5
  73. package/dist/handlers/plugin-invoke.js +152 -39
  74. package/dist/handlers/registry.d.ts +2 -2
  75. package/dist/handlers/registry.js +105 -28
  76. package/dist/handlers/rpc-server.js +7 -7
  77. package/dist/handlers/session.d.ts +11 -0
  78. package/dist/handlers/session.js +40 -0
  79. package/dist/handlers/unload-model.js +8 -3
  80. package/dist/handlers/vector-index.js +118 -19
  81. package/dist/index.d.ts +1 -1
  82. package/dist/index.js +1 -1
  83. package/dist/logging/utils.js +10 -2
  84. package/dist/model-cache-layout.d.ts +1 -0
  85. package/dist/model-cache-layout.js +2 -0
  86. package/dist/model-fit/assess.d.ts +18 -67
  87. package/dist/model-fit/assess.js +96 -665
  88. package/dist/model-fit/describe.d.ts +12 -0
  89. package/dist/model-fit/describe.js +29 -0
  90. package/dist/model-fit/description/fetch.d.ts +1 -1
  91. package/dist/model-fit/description/fetch.js +95 -16
  92. package/dist/model-fit/description/resolve-load.d.ts +5 -3
  93. package/dist/model-fit/description/resolve-load.js +14 -14
  94. package/dist/model-fit/fit-load.d.ts +23 -0
  95. package/dist/model-fit/fit-load.js +29 -0
  96. package/dist/model-fit/load-check.d.ts +15 -33
  97. package/dist/model-fit/load-check.js +50 -195
  98. package/dist/model-fit/run/engine.d.ts +12 -72
  99. package/dist/model-fit/run/engine.js +4 -8
  100. package/dist/model-fit/run/run.d.ts +10 -7
  101. package/dist/model-fit/run/run.js +22 -18
  102. package/dist/model-fit/sum.d.ts +32 -0
  103. package/dist/model-fit/sum.js +60 -0
  104. package/dist/models/registry/models.d.ts +865 -128
  105. package/dist/models/registry/models.js +3737 -2963
  106. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  107. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  108. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  109. package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
  110. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  111. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  112. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  113. package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  114. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  115. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
  116. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  117. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  118. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  119. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  120. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  121. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
  122. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  123. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
  124. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
  125. package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
  126. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  127. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  128. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
  129. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
  130. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
  131. package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
  132. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  133. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  134. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  135. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  136. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  137. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  138. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  139. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
  140. package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
  141. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  142. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  143. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
  144. package/dist/plugins/builtin/index.d.ts +1 -0
  145. package/dist/plugins/builtin/index.js +1 -0
  146. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  147. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  148. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
  149. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  150. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  151. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  152. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  153. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
  154. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
  155. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  156. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  157. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
  158. package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
  159. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  160. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
  161. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
  162. package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
  163. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  164. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  165. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
  166. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
  167. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  168. package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
  169. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
  170. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
  171. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
  172. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
  173. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
  174. package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
  175. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  176. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  177. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
  178. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
  179. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
  180. package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
  181. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  182. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  183. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  184. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  185. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
  186. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
  187. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  188. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  189. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
  190. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
  191. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
  192. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
  193. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
  194. package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
  195. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  196. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  197. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  198. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  199. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  200. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
  201. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
  202. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  203. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
  204. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  205. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  206. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
  207. package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
  208. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
  209. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
  210. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  211. package/dist/plugins/ops/bci-transcribe.js +18 -4
  212. package/dist/plugins/ops/cancel.d.ts +1 -0
  213. package/dist/plugins/ops/cancel.js +11 -2
  214. package/dist/plugins/ops/embed.d.ts +5 -3
  215. package/dist/plugins/ops/embed.js +89 -34
  216. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
  217. package/dist/plugins/ops/kv-cache-utils.js +6 -2
  218. package/dist/plugins/ops/load-model.d.ts +2 -0
  219. package/dist/plugins/ops/load-model.js +33 -16
  220. package/dist/plugins/ops/transcribe.d.ts +5 -4
  221. package/dist/plugins/ops/transcribe.js +62 -27
  222. package/dist/plugins/ops/translate.d.ts +2 -1
  223. package/dist/plugins/ops/translate.js +4 -1
  224. package/dist/plugins/ops/unload-model.d.ts +17 -1
  225. package/dist/plugins/ops/unload-model.js +31 -2
  226. package/dist/plugins/turbovec-backend.d.ts +1 -1
  227. package/dist/plugins/turbovec-backend.js +16 -1
  228. package/dist/profiling/operation-metrics.js +22 -19
  229. package/dist/rag/embedding-scope.d.ts +16 -0
  230. package/dist/rag/embedding-scope.js +81 -0
  231. package/dist/rag/handler.d.ts +25 -0
  232. package/dist/rag/handler.js +50 -16
  233. package/dist/rag/handlers/ingest.js +2 -6
  234. package/dist/rag/handlers/search.d.ts +10 -1
  235. package/dist/rag/handlers/search.js +16 -7
  236. package/dist/rag/rag-operation-manager.d.ts +27 -3
  237. package/dist/rag/rag-operation-manager.js +10 -27
  238. package/dist/registry.js +5 -0
  239. package/dist/resources/collector.d.ts +70 -36
  240. package/dist/resources/collector.js +8 -1
  241. package/dist/resources/types.d.ts +2 -0
  242. package/dist/rpc/inventory.js +1 -12
  243. package/dist/runtime/env.d.ts +5 -1
  244. package/dist/runtime/env.js +6 -21
  245. package/dist/runtime/index.d.ts +4 -1
  246. package/dist/runtime/index.js +2 -0
  247. package/dist/runtime/lifecycle.d.ts +6 -4
  248. package/dist/runtime/lifecycle.js +9 -5
  249. package/dist/runtime/model-config-utils.js +5 -1
  250. package/dist/runtime/model-registry.d.ts +28 -4
  251. package/dist/runtime/model-registry.js +46 -1
  252. package/dist/runtime/model-work-lock.d.ts +13 -0
  253. package/dist/runtime/model-work-lock.js +48 -0
  254. package/dist/runtime/registry-client.d.ts +47 -1
  255. package/dist/runtime/registry-client.js +446 -88
  256. package/dist/runtime/request-context.d.ts +9 -1
  257. package/dist/runtime/request-context.js +25 -1
  258. package/dist/runtime/request-registry.d.ts +55 -7
  259. package/dist/runtime/request-registry.js +189 -71
  260. package/dist/runtime/runtime-lifecycle.js +10 -1
  261. package/dist/runtime/session-id.d.ts +18 -0
  262. package/dist/runtime/session-id.js +20 -0
  263. package/dist/runtime/session-registry.d.ts +39 -0
  264. package/dist/runtime/session-registry.js +114 -0
  265. package/dist/runtime/vector-index-backend.d.ts +4 -0
  266. package/dist/runtime/vector-index-registry.d.ts +21 -0
  267. package/dist/runtime/vector-index-registry.js +56 -18
  268. package/dist/schemas/abort-signal.d.ts +8 -0
  269. package/dist/schemas/abort-signal.js +1 -0
  270. package/dist/schemas/assess-model-fit.d.ts +132 -282
  271. package/dist/schemas/assess-model-fit.js +82 -216
  272. package/dist/schemas/audio-gen.d.ts +278 -24
  273. package/dist/schemas/audio-gen.js +127 -23
  274. package/dist/schemas/batch-completion-stream.d.ts +105 -13
  275. package/dist/schemas/batch-completion-stream.js +8 -4
  276. package/dist/schemas/bci-config.d.ts +2 -1
  277. package/dist/schemas/bci.d.ts +54 -3
  278. package/dist/schemas/bci.js +11 -16
  279. package/dist/schemas/calc-tokens.d.ts +141 -0
  280. package/dist/schemas/calc-tokens.js +53 -0
  281. package/dist/schemas/cancel.d.ts +26 -0
  282. package/dist/schemas/cancel.js +10 -2
  283. package/dist/schemas/classification.d.ts +3 -0
  284. package/dist/schemas/classification.js +6 -2
  285. package/dist/schemas/common.d.ts +1831 -348
  286. package/dist/schemas/common.js +18 -2
  287. package/dist/schemas/completion-event.d.ts +3 -0
  288. package/dist/schemas/completion-event.js +1 -1
  289. package/dist/schemas/completion-stream.d.ts +162 -17
  290. package/dist/schemas/completion-stream.js +83 -13
  291. package/dist/schemas/config.d.ts +1212 -169
  292. package/dist/schemas/config.js +9 -13
  293. package/dist/schemas/decision.d.ts +4 -2
  294. package/dist/schemas/decision.js +2 -1
  295. package/dist/schemas/download-asset.d.ts +10 -5
  296. package/dist/schemas/download-asset.js +3 -6
  297. package/dist/schemas/embed.d.ts +286 -3
  298. package/dist/schemas/embed.js +162 -16
  299. package/dist/schemas/engine-addon-map.d.ts +1 -0
  300. package/dist/schemas/engine-addon-map.js +1 -0
  301. package/dist/schemas/errors.d.ts +12 -0
  302. package/dist/schemas/errors.js +66 -1
  303. package/dist/schemas/finetune.d.ts +14 -4
  304. package/dist/schemas/finetune.js +5 -6
  305. package/dist/schemas/heartbeat.d.ts +1 -0
  306. package/dist/schemas/heartbeat.js +5 -2
  307. package/dist/schemas/image-embedding.d.ts +27 -0
  308. package/dist/schemas/image-embedding.js +107 -0
  309. package/dist/schemas/index.d.ts +7 -3
  310. package/dist/schemas/index.js +7 -3
  311. package/dist/schemas/lifecycle.d.ts +3 -0
  312. package/dist/schemas/lifecycle.js +13 -6
  313. package/dist/schemas/llamacpp-config.d.ts +269 -22
  314. package/dist/schemas/llamacpp-config.js +86 -7
  315. package/dist/schemas/load-model.d.ts +6418 -1960
  316. package/dist/schemas/load-model.js +39 -8
  317. package/dist/schemas/logging-stream.d.ts +1 -0
  318. package/dist/schemas/logging-stream.js +5 -2
  319. package/dist/schemas/model-config-schemas.d.ts +542 -116
  320. package/dist/schemas/model-config-schemas.js +2 -0
  321. package/dist/schemas/model-fit-policy.d.ts +2 -2
  322. package/dist/schemas/model-info.d.ts +54 -56
  323. package/dist/schemas/model-info.js +17 -6
  324. package/dist/schemas/model-ops.d.ts +12 -0
  325. package/dist/schemas/model-ops.js +33 -5
  326. package/dist/schemas/model-src-utils.d.ts +10 -6
  327. package/dist/schemas/model-types.d.ts +8 -4
  328. package/dist/schemas/model-types.js +1 -0
  329. package/dist/schemas/ocr.d.ts +9 -4
  330. package/dist/schemas/ocr.js +6 -2
  331. package/dist/schemas/plugin.d.ts +62 -6
  332. package/dist/schemas/plugin.js +26 -4
  333. package/dist/schemas/profiling.d.ts +32 -32
  334. package/dist/schemas/rag.d.ts +9 -0
  335. package/dist/schemas/rag.js +10 -16
  336. package/dist/schemas/registry.d.ts +30 -10
  337. package/dist/schemas/registry.js +14 -6
  338. package/dist/schemas/reload-config.d.ts +41 -6
  339. package/dist/schemas/reload-config.js +11 -4
  340. package/dist/schemas/request-base.d.ts +39 -0
  341. package/dist/schemas/request-base.js +64 -0
  342. package/dist/schemas/rerank.d.ts +8 -7
  343. package/dist/schemas/rerank.js +6 -4
  344. package/dist/schemas/rpc-server.d.ts +3 -0
  345. package/dist/schemas/rpc-server.js +15 -15
  346. package/dist/schemas/sdcpp-config.d.ts +90 -20
  347. package/dist/schemas/sdcpp-config.js +96 -19
  348. package/dist/schemas/session.d.ts +37 -0
  349. package/dist/schemas/session.js +53 -0
  350. package/dist/schemas/system-resources.d.ts +364 -179
  351. package/dist/schemas/system-resources.js +8 -3
  352. package/dist/schemas/text-to-speech.d.ts +1343 -271
  353. package/dist/schemas/text-to-speech.js +547 -33
  354. package/dist/schemas/tools.d.ts +7 -0
  355. package/dist/schemas/tools.js +29 -0
  356. package/dist/schemas/transcription-config.d.ts +262 -2
  357. package/dist/schemas/transcription-config.js +122 -6
  358. package/dist/schemas/transcription.d.ts +93 -6
  359. package/dist/schemas/transcription.js +98 -21
  360. package/dist/schemas/translate.d.ts +2 -0
  361. package/dist/schemas/translate.js +5 -9
  362. package/dist/schemas/translation-config.d.ts +30 -20
  363. package/dist/schemas/tts-pcm.d.ts +35 -0
  364. package/dist/schemas/tts-pcm.js +69 -0
  365. package/dist/schemas/vector-index.d.ts +47 -1
  366. package/dist/schemas/vector-index.js +37 -10
  367. package/dist/schemas/vla.d.ts +3 -3
  368. package/dist/surface.d.ts +3 -3
  369. package/dist/surface.js +3 -3
  370. package/dist/utils/addon-responses.d.ts +69 -2
  371. package/dist/utils/archive.d.ts +7 -0
  372. package/dist/utils/archive.js +88 -29
  373. package/dist/utils/asr-events.d.ts +5 -0
  374. package/dist/utils/asr-events.js +10 -1
  375. package/dist/utils/cache/auto-history.js +3 -1
  376. package/dist/utils/cache/layout.d.ts +48 -0
  377. package/dist/utils/cache/layout.js +190 -0
  378. package/dist/utils/cache/paths.js +2 -4
  379. package/dist/utils/cache/types.d.ts +2 -1
  380. package/dist/utils/checksum.d.ts +5 -0
  381. package/dist/utils/checksum.js +18 -0
  382. package/dist/utils/completion-normalizer.js +286 -87
  383. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  384. package/dist/utils/gguf-tensor-extractor.js +1 -50
  385. package/dist/utils/shard-utils.d.ts +0 -21
  386. package/dist/utils/shard-utils.js +12 -62
  387. package/dist/utils/tool-integration.d.ts +2 -2
  388. package/dist/utils/tools/defer.d.ts +39 -10
  389. package/dist/utils/tools/defer.js +72 -17
  390. package/dist/utils/tools/index.d.ts +1 -1
  391. package/dist/utils/tools/index.js +1 -1
  392. package/dist/utils/transcribe-metadata.d.ts +28 -2
  393. package/dist/utils/transcribe-metadata.js +16 -0
  394. package/dist/utils/tts-stats.d.ts +42 -0
  395. package/dist/utils/tts-stats.js +120 -22
  396. package/package.json +43 -39
  397. package/dist/model-fit/classify.d.ts +0 -6
  398. package/dist/model-fit/classify.js +0 -166
  399. package/dist/model-fit/description/project.d.ts +0 -27
  400. package/dist/model-fit/description/project.js +0 -33
  401. package/dist/model-fit/floor.d.ts +0 -31
  402. package/dist/model-fit/floor.js +0 -65
  403. package/dist/model-fit/kv-cache.d.ts +0 -23
  404. package/dist/model-fit/kv-cache.js +0 -85
  405. package/dist/model-fit/platform.d.ts +0 -14
  406. package/dist/model-fit/platform.js +0 -26
  407. package/dist/model-fit/request/create.d.ts +0 -31
  408. package/dist/model-fit/request/create.js +0 -61
  409. package/dist/model-fit/request/engines/asr.d.ts +0 -14
  410. package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
  411. package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
  412. package/dist/model-fit/request/engines/diffusion.js +0 -93
  413. package/dist/model-fit/request/engines/llama.d.ts +0 -12
  414. package/dist/model-fit/request/engines/llama.js +0 -44
  415. package/dist/model-fit/request/engines/nmt.d.ts +0 -8
  416. package/dist/model-fit/request/engines/nmt.js +0 -11
  417. package/dist/model-fit/request/engines/ocr.d.ts +0 -9
  418. package/dist/model-fit/request/engines/ocr.js +0 -13
  419. package/dist/model-fit/request/engines/tts.d.ts +0 -10
  420. package/dist/model-fit/request/engines/tts.js +0 -152
  421. package/dist/models/registry/resource-profiles.d.ts +0 -17
  422. package/dist/models/registry/resource-profiles.js +0 -5859
  423. package/dist/schemas/model-resource-profile.d.ts +0 -109
  424. package/dist/schemas/model-resource-profile.js +0 -80
  425. /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
  426. /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
@@ -1,4 +1,5 @@
1
1
  import { type CreateModelParams, type PluginModelResult, type ResolveContext, type LlmConfigInput } from '../../../schemas/index.js';
2
+ import { llmFitRequest } from '../../../plugins/builtin/llamacpp-completion/fit.js';
2
3
  export declare const llmPlugin: {
3
4
  modelType: "llamacpp-completion";
4
5
  displayName: string;
@@ -6,10 +7,10 @@ export declare const llmPlugin: {
6
7
  loadConfigSchema: import("zod").ZodObject<{
7
8
  backend: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodEnum<{
8
9
  auto: "auto";
9
- cuda: "cuda";
10
10
  vulkan: "vulkan";
11
- metal: "metal";
12
11
  opencl: "opencl";
12
+ metal: "metal";
13
+ cuda: "cuda";
13
14
  }>>>;
14
15
  ctx_size: import("zod").ZodOptional<import("zod").ZodNumber>;
15
16
  temp: import("zod").ZodOptional<import("zod").ZodNumber>;
@@ -47,6 +48,7 @@ export declare const llmPlugin: {
47
48
  disk: "disk";
48
49
  }>>;
49
50
  cache_checkpoint_dir: import("zod").ZodOptional<import("zod").ZodString>;
51
+ cache_share_across_keys: import("zod").ZodOptional<import("zod").ZodBoolean>;
50
52
  cache_ram_mib: import("zod").ZodOptional<import("zod").ZodNumber>;
51
53
  'cache-type-k': import("zod").ZodOptional<import("zod").ZodString>;
52
54
  'cache-type-v': import("zod").ZodOptional<import("zod").ZodString>;
@@ -120,6 +122,7 @@ export declare const llmPlugin: {
120
122
  audiogen: "audiogen";
121
123
  tts: "tts";
122
124
  ocr: "ocr";
125
+ classification: "classification";
123
126
  vla: "vla";
124
127
  "llamacpp-completion": "llamacpp-completion";
125
128
  "whispercpp-transcription": "whispercpp-transcription";
@@ -136,13 +139,13 @@ export declare const llmPlugin: {
136
139
  "sdcpp-generation": "sdcpp-generation";
137
140
  "audiogen-ggml": "audiogen-ggml";
138
141
  "ggml-vla": "ggml-vla";
142
+ "ggml-image-embedding": "ggml-image-embedding";
139
143
  "ggml-classification": "ggml-classification";
140
144
  llm: "llm";
141
145
  whisper: "whisper";
142
146
  bci: "bci";
143
147
  nmt: "nmt";
144
148
  parakeet: "parakeet";
145
- classification: "classification";
146
149
  }>, import("zod").ZodLiteral<"vad">]>>;
147
150
  }, import("zod/v4/core").$strip>]>>;
148
151
  draftModelSrc: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodObject<{
@@ -162,6 +165,7 @@ export declare const llmPlugin: {
162
165
  audiogen: "audiogen";
163
166
  tts: "tts";
164
167
  ocr: "ocr";
168
+ classification: "classification";
165
169
  vla: "vla";
166
170
  "llamacpp-completion": "llamacpp-completion";
167
171
  "whispercpp-transcription": "whispercpp-transcription";
@@ -178,13 +182,13 @@ export declare const llmPlugin: {
178
182
  "sdcpp-generation": "sdcpp-generation";
179
183
  "audiogen-ggml": "audiogen-ggml";
180
184
  "ggml-vla": "ggml-vla";
185
+ "ggml-image-embedding": "ggml-image-embedding";
181
186
  "ggml-classification": "ggml-classification";
182
187
  llm: "llm";
183
188
  whisper: "whisper";
184
189
  bci: "bci";
185
190
  nmt: "nmt";
186
191
  parakeet: "parakeet";
187
- classification: "classification";
188
192
  }>, import("zod").ZodLiteral<"vad">]>>;
189
193
  }, import("zod/v4/core").$strip>]>>;
190
194
  image_tile_mode: import("zod").ZodOptional<import("zod").ZodEnum<{
@@ -198,10 +202,11 @@ export declare const llmPlugin: {
198
202
  }>>;
199
203
  'mmproj-use-gpu': import("zod").ZodOptional<import("zod").ZodBoolean>;
200
204
  }, import("zod/v4/core").$strip>;
205
+ fitRequest: typeof llmFitRequest;
201
206
  assessFit: typeof import("node_modules/@qvac/llm-llamacpp/fit").assessFit;
202
207
  resolveConfig(cfg: LlmConfigInput, ctx: ResolveContext): Promise<{
203
208
  config: {
204
- backend?: ("auto" | "cuda" | "vulkan" | "metal" | "opencl")[] | undefined;
209
+ backend?: ("auto" | "vulkan" | "opencl" | "metal" | "cuda")[] | undefined;
205
210
  ctx_size?: number | undefined;
206
211
  temp?: number | undefined;
207
212
  top_p?: number | undefined;
@@ -224,6 +229,7 @@ export declare const llmPlugin: {
224
229
  cache_checkpoints_max_bytes?: number | undefined;
225
230
  cache_checkpoint_storage?: "memory" | "disk" | undefined;
226
231
  cache_checkpoint_dir?: string | undefined;
232
+ cache_share_across_keys?: boolean | undefined;
227
233
  cache_ram_mib?: number | undefined;
228
234
  'cache-type-k'?: string | undefined;
229
235
  'cache-type-v'?: string | undefined;
@@ -268,7 +274,7 @@ export declare const llmPlugin: {
268
274
  artifacts?: never;
269
275
  } | {
270
276
  config: {
271
- backend?: ("auto" | "cuda" | "vulkan" | "metal" | "opencl")[] | undefined;
277
+ backend?: ("auto" | "vulkan" | "opencl" | "metal" | "cuda")[] | undefined;
272
278
  ctx_size?: number | undefined;
273
279
  temp?: number | undefined;
274
280
  top_p?: number | undefined;
@@ -291,6 +297,7 @@ export declare const llmPlugin: {
291
297
  cache_checkpoints_max_bytes?: number | undefined;
292
298
  cache_checkpoint_storage?: "memory" | "disk" | undefined;
293
299
  cache_checkpoint_dir?: string | undefined;
300
+ cache_share_across_keys?: boolean | undefined;
294
301
  cache_ram_mib?: number | undefined;
295
302
  'cache-type-k'?: string | undefined;
296
303
  'cache-type-v'?: string | undefined;
@@ -339,7 +346,72 @@ export declare const llmPlugin: {
339
346
  }>;
340
347
  createModel(params: CreateModelParams): PluginModelResult;
341
348
  handlers: {
342
- batchCompletionStream: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
349
+ calcTokens: import("../../../schemas/plugin.js").PluginHandlerDefinition<import("zod").ZodObject<{
350
+ sessionId: import("zod").ZodOptional<import("zod").ZodString>;
351
+ type: import("zod").ZodLiteral<"calcTokens">;
352
+ text: import("zod").ZodOptional<import("zod").ZodString>;
353
+ history: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodObject<{
354
+ role: import("zod").ZodString;
355
+ toolCalls: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodObject<{
356
+ id: import("zod").ZodOptional<import("zod").ZodString>;
357
+ name: import("zod").ZodString;
358
+ arguments: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodUnknown>;
359
+ }, import("zod/v4/core").$strip>>>;
360
+ toolCallId: import("zod").ZodOptional<import("zod").ZodString>;
361
+ toolName: import("zod").ZodOptional<import("zod").ZodString>;
362
+ content: import("zod").ZodOptional<import("zod").ZodNullable<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodArray<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
363
+ type: import("zod").ZodLiteral<"text">;
364
+ text: import("zod").ZodString;
365
+ }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
366
+ type: import("zod").ZodLiteral<"media">;
367
+ path: import("zod").ZodString;
368
+ }, import("zod/v4/core").$strict>], "type">>]>>>;
369
+ attachments: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodObject<{
370
+ path: import("zod").ZodString;
371
+ }, import("zod/v4/core").$strip>>>;
372
+ }, import("zod/v4/core").$strip>>>;
373
+ modelId: import("zod").ZodString;
374
+ tools: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodObject<{
375
+ type: import("zod").ZodLiteral<"function">;
376
+ name: import("zod").ZodString;
377
+ description: import("zod").ZodString;
378
+ deferLoading: import("zod").ZodOptional<import("zod").ZodBoolean>;
379
+ group: import("zod").ZodOptional<import("zod").ZodString>;
380
+ parameters: import("zod").ZodObject<{
381
+ type: import("zod").ZodLiteral<"object">;
382
+ properties: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodObject<{
383
+ type: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodEnum<{
384
+ string: "string";
385
+ number: "number";
386
+ boolean: "boolean";
387
+ object: "object";
388
+ null: "null";
389
+ array: "array";
390
+ integer: "integer";
391
+ }>, import("zod").ZodArray<import("zod").ZodEnum<{
392
+ string: "string";
393
+ number: "number";
394
+ boolean: "boolean";
395
+ object: "object";
396
+ null: "null";
397
+ array: "array";
398
+ integer: "integer";
399
+ }>>]>>;
400
+ description: import("zod").ZodOptional<import("zod").ZodString>;
401
+ enum: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodNumber, import("zod").ZodBoolean, import("zod").ZodNull]>>>;
402
+ }, import("zod/v4/core").$loose>>;
403
+ required: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodString>>;
404
+ }, import("zod/v4/core").$loose>;
405
+ }, import("zod/v4/core").$strip>>>;
406
+ toolSearch: import("zod").ZodOptional<import("zod").ZodObject<{
407
+ defaultLimit: import("zod").ZodOptional<import("zod").ZodNumber>;
408
+ maxLimit: import("zod").ZodOptional<import("zod").ZodNumber>;
409
+ }, import("zod/v4/core").$strict>>;
410
+ }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
411
+ type: import("zod").ZodLiteral<"calcTokens">;
412
+ tokens: import("zod").ZodNumber;
413
+ }, import("zod/v4/core").$strip>>;
414
+ batchCompletionStream: import("../../../schemas/plugin.js").PluginHandlerDefinition<import("zod").ZodObject<{
343
415
  modelId: import("zod").ZodString;
344
416
  stream: import("zod").ZodOptional<import("zod").ZodBoolean>;
345
417
  captureThinking: import("zod").ZodOptional<import("zod").ZodBoolean>;
@@ -353,12 +425,17 @@ export declare const llmPlugin: {
353
425
  gemma4: "gemma4";
354
426
  dsml: "dsml";
355
427
  }>>;
356
- requestId: import("zod").ZodOptional<import("zod").ZodString>;
357
428
  prompts: import("zod").ZodArray<import("zod").ZodObject<{
358
429
  id: import("zod").ZodOptional<import("zod").ZodString>;
359
430
  history: import("zod").ZodArray<import("zod").ZodPipe<import("zod").ZodObject<{
360
431
  role: import("zod").ZodString;
361
- content: import("zod").ZodOptional<import("zod").ZodNullable<import("zod").ZodString>>;
432
+ content: import("zod").ZodOptional<import("zod").ZodNullable<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodArray<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
433
+ type: import("zod").ZodLiteral<"text">;
434
+ text: import("zod").ZodString;
435
+ }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
436
+ type: import("zod").ZodLiteral<"media">;
437
+ path: import("zod").ZodString;
438
+ }, import("zod/v4/core").$strict>], "type">>]>>>;
362
439
  attachments: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodObject<{
363
440
  path: import("zod").ZodString;
364
441
  }, import("zod/v4/core").$strip>>>;
@@ -370,7 +447,13 @@ export declare const llmPlugin: {
370
447
  toolCallId: import("zod").ZodOptional<import("zod").ZodString>;
371
448
  toolName: import("zod").ZodOptional<import("zod").ZodString>;
372
449
  }, import("zod/v4/core").$strip>, import("zod").ZodTransform<{
373
- content: string;
450
+ content: string | ({
451
+ type: "text";
452
+ text: string;
453
+ } | {
454
+ type: "media";
455
+ path: string;
456
+ })[];
374
457
  role: string;
375
458
  attachments?: {
376
459
  path: string;
@@ -384,7 +467,13 @@ export declare const llmPlugin: {
384
467
  toolName?: string | undefined;
385
468
  }, {
386
469
  role: string;
387
- content?: string | null | undefined;
470
+ content?: string | ({
471
+ type: "text";
472
+ text: string;
473
+ } | {
474
+ type: "media";
475
+ path: string;
476
+ })[] | null | undefined;
388
477
  attachments?: {
389
478
  path: string;
390
479
  }[] | undefined;
@@ -422,6 +511,10 @@ export declare const llmPlugin: {
422
511
  strict: import("zod").ZodOptional<import("zod").ZodBoolean>;
423
512
  }, import("zod/v4/core").$strict>;
424
513
  }, import("zod/v4/core").$strict>], "type">>;
514
+ toolSearch: import("zod").ZodOptional<import("zod").ZodObject<{
515
+ defaultLimit: import("zod").ZodOptional<import("zod").ZodNumber>;
516
+ maxLimit: import("zod").ZodOptional<import("zod").ZodNumber>;
517
+ }, import("zod/v4/core").$strict>>;
425
518
  tools: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodObject<{
426
519
  type: import("zod").ZodLiteral<"function">;
427
520
  name: import("zod").ZodString;
@@ -456,6 +549,8 @@ export declare const llmPlugin: {
456
549
  }, import("zod/v4/core").$strip>>>;
457
550
  }, import("zod/v4/core").$strict>>;
458
551
  type: import("zod").ZodLiteral<"batchCompletionStream">;
552
+ sessionId: import("zod").ZodOptional<import("zod").ZodString>;
553
+ requestId: import("zod").ZodOptional<import("zod").ZodString>;
459
554
  }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
460
555
  type: import("zod").ZodLiteral<"batchCompletionStream">;
461
556
  done: import("zod").ZodOptional<import("zod").ZodBoolean>;
@@ -513,6 +608,7 @@ export declare const llmPlugin: {
513
608
  backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
514
609
  cpu: "cpu";
515
610
  gpu: "gpu";
611
+ npu: "npu";
516
612
  }>>;
517
613
  }, import("zod/v4/core").$strip>;
518
614
  }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
@@ -554,13 +650,20 @@ export declare const llmPlugin: {
554
650
  backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
555
651
  cpu: "cpu";
556
652
  gpu: "gpu";
653
+ npu: "npu";
557
654
  }>>;
558
655
  }, import("zod/v4/core").$strip>>;
559
656
  }, import("zod/v4/core").$strict>>;
560
- completionStream: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
657
+ completionStream: import("../../../schemas/plugin.js").PluginHandlerDefinition<import("zod").ZodObject<{
561
658
  history: import("zod").ZodArray<import("zod").ZodPipe<import("zod").ZodObject<{
562
659
  role: import("zod").ZodString;
563
- content: import("zod").ZodOptional<import("zod").ZodNullable<import("zod").ZodString>>;
660
+ content: import("zod").ZodOptional<import("zod").ZodNullable<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodArray<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
661
+ type: import("zod").ZodLiteral<"text">;
662
+ text: import("zod").ZodString;
663
+ }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
664
+ type: import("zod").ZodLiteral<"media">;
665
+ path: import("zod").ZodString;
666
+ }, import("zod/v4/core").$strict>], "type">>]>>>;
564
667
  attachments: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodObject<{
565
668
  path: import("zod").ZodString;
566
669
  }, import("zod/v4/core").$strip>>>;
@@ -572,7 +675,13 @@ export declare const llmPlugin: {
572
675
  toolCallId: import("zod").ZodOptional<import("zod").ZodString>;
573
676
  toolName: import("zod").ZodOptional<import("zod").ZodString>;
574
677
  }, import("zod/v4/core").$strip>, import("zod").ZodTransform<{
575
- content: string;
678
+ content: string | ({
679
+ type: "text";
680
+ text: string;
681
+ } | {
682
+ type: "media";
683
+ path: string;
684
+ })[];
576
685
  role: string;
577
686
  attachments?: {
578
687
  path: string;
@@ -586,7 +695,13 @@ export declare const llmPlugin: {
586
695
  toolName?: string | undefined;
587
696
  }, {
588
697
  role: string;
589
- content?: string | null | undefined;
698
+ content?: string | ({
699
+ type: "text";
700
+ text: string;
701
+ } | {
702
+ type: "media";
703
+ path: string;
704
+ })[] | null | undefined;
590
705
  attachments?: {
591
706
  path: string;
592
707
  }[] | undefined;
@@ -600,6 +715,7 @@ export declare const llmPlugin: {
600
715
  }>>>;
601
716
  modelId: import("zod").ZodString;
602
717
  prefill: import("zod").ZodOptional<import("zod").ZodBoolean>;
718
+ seedFrom: import("zod").ZodOptional<import("zod").ZodString>;
603
719
  tools: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodObject<{
604
720
  type: import("zod").ZodLiteral<"function">;
605
721
  name: import("zod").ZodString;
@@ -632,6 +748,10 @@ export declare const llmPlugin: {
632
748
  required: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodString>>;
633
749
  }, import("zod/v4/core").$loose>;
634
750
  }, import("zod/v4/core").$strip>>>;
751
+ toolSearch: import("zod").ZodOptional<import("zod").ZodObject<{
752
+ defaultLimit: import("zod").ZodOptional<import("zod").ZodNumber>;
753
+ maxLimit: import("zod").ZodOptional<import("zod").ZodNumber>;
754
+ }, import("zod/v4/core").$strict>>;
635
755
  stream: import("zod").ZodBoolean;
636
756
  kvCache: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodBoolean, import("zod").ZodString]>>;
637
757
  generationParams: import("zod").ZodOptional<import("zod").ZodObject<{
@@ -671,8 +791,9 @@ export declare const llmPlugin: {
671
791
  strict: import("zod").ZodOptional<import("zod").ZodBoolean>;
672
792
  }, import("zod/v4/core").$strict>;
673
793
  }, import("zod/v4/core").$strict>], "type">>;
674
- requestId: import("zod").ZodOptional<import("zod").ZodString>;
675
794
  type: import("zod").ZodLiteral<"completionStream">;
795
+ sessionId: import("zod").ZodOptional<import("zod").ZodString>;
796
+ requestId: import("zod").ZodOptional<import("zod").ZodString>;
676
797
  }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
677
798
  type: import("zod").ZodLiteral<"completionStream">;
678
799
  done: import("zod").ZodOptional<import("zod").ZodBoolean>;
@@ -727,6 +848,7 @@ export declare const llmPlugin: {
727
848
  backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
728
849
  cpu: "cpu";
729
850
  gpu: "gpu";
851
+ npu: "npu";
730
852
  }>>;
731
853
  }, import("zod/v4/core").$strip>;
732
854
  }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
@@ -753,7 +875,7 @@ export declare const llmPlugin: {
753
875
  }, import("zod/v4/core").$strip>>;
754
876
  }, import("zod/v4/core").$strict>]>>;
755
877
  }, import("zod/v4/core").$strict>>;
756
- finetune: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodUnion<readonly [import("zod").ZodObject<{
878
+ finetune: import("../../../schemas/plugin.js").PluginHandlerDefinition<import("zod").ZodUnion<readonly [import("zod").ZodObject<{
757
879
  modelId: import("zod").ZodString;
758
880
  options: import("zod").ZodObject<{
759
881
  trainDatasetDir: import("zod").ZodString;
@@ -811,10 +933,11 @@ export declare const llmPlugin: {
811
933
  }, import("zod/v4/core").$strict>;
812
934
  type: import("zod").ZodLiteral<"finetune">;
813
935
  operation: import("zod").ZodOptional<import("zod").ZodEnum<{
814
- start: "start";
815
936
  resume: "resume";
937
+ start: "start";
816
938
  }>>;
817
939
  withProgress: import("zod").ZodOptional<import("zod").ZodBoolean>;
940
+ sessionId: import("zod").ZodOptional<import("zod").ZodString>;
818
941
  requestId: import("zod").ZodOptional<import("zod").ZodString>;
819
942
  }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
820
943
  modelId: import("zod").ZodString;
@@ -874,6 +997,8 @@ export declare const llmPlugin: {
874
997
  }, import("zod/v4/core").$strict>;
875
998
  type: import("zod").ZodLiteral<"finetune">;
876
999
  operation: import("zod").ZodLiteral<"getState">;
1000
+ sessionId: import("zod").ZodOptional<import("zod").ZodString>;
1001
+ requestId: import("zod").ZodOptional<import("zod").ZodString>;
877
1002
  }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
878
1003
  modelId: import("zod").ZodString;
879
1004
  type: import("zod").ZodLiteral<"finetune">;
@@ -881,6 +1006,8 @@ export declare const llmPlugin: {
881
1006
  cancel: "cancel";
882
1007
  pause: "pause";
883
1008
  }>;
1009
+ sessionId: import("zod").ZodOptional<import("zod").ZodString>;
1010
+ requestId: import("zod").ZodOptional<import("zod").ZodString>;
884
1011
  }, import("zod/v4/core").$strict>]>, import("zod").ZodObject<{
885
1012
  type: import("zod").ZodLiteral<"finetune">;
886
1013
  status: import("zod").ZodEnum<{
@@ -904,7 +1031,7 @@ export declare const llmPlugin: {
904
1031
  epochs_completed: import("zod").ZodNumber;
905
1032
  }, import("zod/v4/core").$strict>>;
906
1033
  }, import("zod/v4/core").$strict>>;
907
- translate: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
1034
+ translate: import("../../../schemas/plugin.js").PluginHandlerDefinition<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
908
1035
  modelId: import("zod").ZodString;
909
1036
  text: import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodArray<import("zod").ZodString>]>;
910
1037
  stream: import("zod").ZodBoolean;
@@ -913,6 +1040,7 @@ export declare const llmPlugin: {
913
1040
  nmt: "nmt";
914
1041
  }>;
915
1042
  type: import("zod").ZodLiteral<"translate">;
1043
+ sessionId: import("zod").ZodOptional<import("zod").ZodString>;
916
1044
  requestId: import("zod").ZodOptional<import("zod").ZodString>;
917
1045
  }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
918
1046
  modelId: import("zod").ZodString;
@@ -926,6 +1054,7 @@ export declare const llmPlugin: {
926
1054
  to: import("zod").ZodString;
927
1055
  context: import("zod").ZodOptional<import("zod").ZodString>;
928
1056
  type: import("zod").ZodLiteral<"translate">;
1057
+ sessionId: import("zod").ZodOptional<import("zod").ZodString>;
929
1058
  requestId: import("zod").ZodOptional<import("zod").ZodString>;
930
1059
  }, import("zod/v4/core").$strip>], "modelType">, import("zod").ZodObject<{
931
1060
  type: import("zod").ZodLiteral<"translate">;
@@ -50,16 +50,19 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
50
50
  var e = new Error(message);
51
51
  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
52
  });
53
+ import { requestParent } from '../../../schemas/plugin.js';
53
54
  import LlmLlamacpp from '@qvac/llm-llamacpp';
54
- import { definePlugin, defineHandler, finetuneRequestSchema, batchCompletionStreamRequestSchema, batchCompletionStreamResponseSchema, completionStreamRequestSchema, completionStreamResponseSchema, finetuneResponseSchema, translateRequestSchema, translateResponseSchema, ModelType, llmConfigBaseSchema, ADDON_LLM } from '../../../schemas/index.js';
55
+ import { definePlugin, defineHandler, finetuneRequestSchema, batchCompletionStreamRequestSchema, batchCompletionStreamResponseSchema, completionStreamRequestSchema, completionStreamResponseSchema, calcTokensRequestSchema, calcTokensResponseSchema, finetuneResponseSchema, translateRequestSchema, translateResponseSchema, ModelType, llmConfigBaseSchema, ADDON_LLM } from '../../../schemas/index.js';
55
56
  import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
56
57
  import { getFirstShardPath } from '../../../utils/index.js';
57
58
  import { getModelParallel } from '../../../utils/config-transform.js';
58
59
  import { completion } from '../../../plugins/builtin/llamacpp-completion/ops/completion-stream.js';
59
60
  import { batchCompletion } from '../../../plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js';
60
61
  import { finetune } from '../../../plugins/builtin/llamacpp-completion/ops/finetune.js';
62
+ import { calcTokens } from '../../../plugins/builtin/llamacpp-completion/ops/calc-tokens.js';
61
63
  import { translate } from '../../../plugins/ops/translate.js';
62
64
  import { transformLlmConfig } from '../../../plugins/builtin/llamacpp-completion/transform.js';
65
+ import { llmFitRequest } from '../../../plugins/builtin/llamacpp-completion/fit.js';
63
66
  import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
64
67
  import { getModelConfig } from '../../../runtime/model-registry.js';
65
68
  import { createCompletionNormalizer } from '../../../utils/completion-normalizer.js';
@@ -119,6 +122,7 @@ export const llmPlugin = definePlugin({
119
122
  displayName: 'LLM (llama.cpp)',
120
123
  addonPackage: ADDON_LLM,
121
124
  loadConfigSchema: llmConfigBaseSchema,
125
+ fitRequest: llmFitRequest,
122
126
  assessFit: LlmLlamacpp.assessFit,
123
127
  async resolveConfig(cfg, ctx) {
124
128
  const { projectionModelSrc, draftModelSrc, ...llmConfig } = cfg;
@@ -143,13 +147,19 @@ export const llmPlugin = definePlugin({
143
147
  return { model };
144
148
  },
145
149
  handlers: {
150
+ calcTokens: defineHandler({
151
+ requestSchema: calcTokensRequestSchema,
152
+ responseSchema: calcTokensResponseSchema,
153
+ streaming: false,
154
+ handler: calcTokens
155
+ }),
146
156
  batchCompletionStream: defineHandler({
147
157
  requestSchema: batchCompletionStreamRequestSchema,
148
158
  responseSchema: batchCompletionStreamResponseSchema,
149
159
  streaming: true,
150
160
  // Request-scope: a model-wide cancel would kill concurrent batching peers.
151
161
  cancel: { scope: 'request', hard: true },
152
- handler: async function* (request) {
162
+ handler: async function* (request, _inputStream, context) {
153
163
  const env_1 = { stack: [], error: void 0, hasError: false };
154
164
  try {
155
165
  const dialect = request.toolDialect ?? detectToolDialect(request.modelId);
@@ -157,7 +167,7 @@ export const llmPlugin = definePlugin({
157
167
  const toolsActive = modelCfg.tools === true;
158
168
  // Parse against what each prompt renders: with deferred tools that
159
169
  // includes whatever an earlier `tool_search` loaded.
160
- const toolsByPosition = request.prompts.map((prompt) => resolveDeferredTools(prompt.tools, prompt.history)?.toolsToRender ??
170
+ const toolsByPosition = request.prompts.map((prompt) => resolveDeferredTools(prompt.tools, prompt.history, prompt.toolSearch)?.toolsToRender ??
161
171
  prompt.tools?.map(toWireTool) ??
162
172
  []);
163
173
  const toolsById = new Map();
@@ -183,6 +193,8 @@ export const llmPlugin = definePlugin({
183
193
  const parallel = getModelParallel(modelCfg);
184
194
  const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
185
195
  requestId: request.requestId ?? generateRandomRequestId(),
196
+ sessionId: request.sessionId,
197
+ ...requestParent(context),
186
198
  kind: 'batchCompletion',
187
199
  modelId: request.modelId,
188
200
  maxConcurrentPerModel: parallel
@@ -302,7 +314,7 @@ export const llmPlugin = definePlugin({
302
314
  streaming: true,
303
315
  // Request-scope: a model-wide cancel would kill concurrent batching peers.
304
316
  cancel: { scope: 'request', hard: true },
305
- handler: async function* (request) {
317
+ handler: async function* (request, _inputStream, context) {
306
318
  const env_2 = { stack: [], error: void 0, hasError: false };
307
319
  try {
308
320
  const filteredHistory = request.history.map((message) => ({
@@ -335,6 +347,8 @@ export const llmPlugin = definePlugin({
335
347
  const parallel = getModelParallel(modelCfg);
336
348
  const ctx = __addDisposableResource(env_2, await getRequestRegistry().begin({
337
349
  requestId: request.requestId ?? generateRandomRequestId(),
350
+ sessionId: request.sessionId,
351
+ ...requestParent(context),
338
352
  kind: 'completion',
339
353
  modelId: request.modelId,
340
354
  maxConcurrentPerModel: parallel
@@ -359,7 +373,9 @@ export const llmPlugin = definePlugin({
359
373
  modelId: request.modelId,
360
374
  kvCache: request.kvCache,
361
375
  prefill: request.prefill,
376
+ seedFrom: request.seedFrom,
362
377
  ...(toolsActive && request.tools && { tools: request.tools }),
378
+ ...(request.toolSearch && { toolSearch: request.toolSearch }),
363
379
  ...(request.generationParams && { generationParams: request.generationParams }),
364
380
  ...(toolsActive && { toolDialect: dialect }),
365
381
  ...(request.responseFormat && { responseFormat: request.responseFormat })
@@ -380,7 +396,7 @@ export const llmPlugin = definePlugin({
380
396
  }
381
397
  result = await stream.next();
382
398
  }
383
- const { modelExecutionMs, stats, toolCalls, stoppedAtContextBoundary } = result.value;
399
+ const { modelExecutionMs, stats, toolCalls, stoppedAtContextBoundary, stopReason } = result.value;
384
400
  // Cancellation rides the done path: observable via the last event's
385
401
  // stopReason; client aggregates reject with InferenceCancelledError.
386
402
  const cancelled = ctx.signal.aborted;
@@ -398,6 +414,7 @@ export const llmPlugin = definePlugin({
398
414
  const terminalEvents = normalizer.finish({
399
415
  ...(stats && { stats }),
400
416
  ...(toolCalls.length > 0 && { toolCalls }),
417
+ ...(stopReason && { stopReason }),
401
418
  ...(cancelled && { stopReason: 'cancelled' }),
402
419
  ...(lengthStop && { stopReason: 'length' })
403
420
  });
@@ -461,8 +478,8 @@ export const llmPlugin = definePlugin({
461
478
  // `cancelFinetune(modelId)` paths both flow through the
462
479
  // registry.
463
480
  cancel: { scope: 'model', hard: true },
464
- handler: function (request) {
465
- return finetune(request);
481
+ handler: function (request, _inputStream, context) {
482
+ return finetune(request, undefined, context);
466
483
  }
467
484
  }),
468
485
  translate: defineHandler({
@@ -471,8 +488,8 @@ export const llmPlugin = definePlugin({
471
488
  streaming: true,
472
489
  // LLM translate cancels its own run (request-scoped), like completion.
473
490
  cancel: { scope: 'request', hard: true },
474
- handler: async function* (request) {
475
- const stream = translate(request, request.requestId);
491
+ handler: async function* (request, _inputStream, context) {
492
+ const stream = translate(request, request.requestId, request.sessionId, context);
476
493
  try {
477
494
  let result = await stream.next();
478
495
  while (!result.done) {
@@ -1,5 +1,7 @@
1
+ import type Buffer from 'bare-buffer';
2
+ import { type PluginHandlerContext } from '../../../../schemas/plugin.js';
1
3
  import { type DecideRequest } from '../../../../schemas/index.js';
2
- export declare function decide(request: DecideRequest): Promise<{
4
+ export declare function decide(request: DecideRequest, _inputStream?: AsyncIterable<Buffer>, context?: PluginHandlerContext): Promise<{
3
5
  type: "decide";
4
6
  result: {
5
7
  model: string;
@@ -50,6 +50,7 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
50
50
  var e = new Error(message);
51
51
  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
52
  });
53
+ import { requestParent } from '../../../../schemas/plugin.js';
53
54
  import { getModel } from '../../../../runtime/model-registry.js';
54
55
  import { getRequestRegistry, withRequestContext } from '../../../../runtime/request-context.js';
55
56
  import { generateRandomRequestId } from '../../../../runtime/request-id.js';
@@ -59,13 +60,15 @@ import { nowMs } from '../../../../profiling/index.js';
59
60
  import { attachModelExecutionMs } from '../../../../profiling/model-execution.js';
60
61
  import { decisionResultSchema } from '../../../../schemas/index.js';
61
62
  import { transformDecisionRequest, transformDecisionResult } from '../../../../plugins/builtin/llamacpp-decision/helpers.js';
62
- export async function decide(request) {
63
+ export async function decide(request, _inputStream, context) {
63
64
  const env_1 = { stack: [], error: void 0, hasError: false };
64
65
  try {
65
- const { modelId, requestId } = request;
66
+ const { modelId, requestId, sessionId } = request;
66
67
  const nativeRequest = transformDecisionRequest(request);
67
68
  const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
68
69
  requestId: requestId ?? generateRandomRequestId(),
70
+ sessionId,
71
+ ...requestParent(context),
69
72
  kind: 'decision',
70
73
  modelId
71
74
  }), true);
@@ -91,8 +91,9 @@ export declare const decisionPlugin: {
91
91
  }, import("zod/v4/core").$strict>], "type">>;
92
92
  maxLen: import("zod").ZodOptional<import("zod").ZodNumber>;
93
93
  headMaxLen: import("zod").ZodOptional<import("zod").ZodNumber>;
94
- type: import("zod").ZodLiteral<"decide">;
94
+ sessionId: import("zod").ZodOptional<import("zod").ZodString>;
95
95
  requestId: import("zod").ZodOptional<import("zod").ZodString>;
96
+ type: import("zod").ZodLiteral<"decide">;
96
97
  modelId: import("zod").ZodString;
97
98
  }, import("zod/v4/core").$strict>, import("zod").ZodObject<{
98
99
  states: import("zod").ZodArray<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodNumber, import("zod").ZodBoolean, import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodUnknown>, import("zod").ZodArray<import("zod").ZodUnknown>]>>;
@@ -121,8 +122,9 @@ export declare const decisionPlugin: {
121
122
  }, import("zod/v4/core").$strict>], "type">>;
122
123
  maxLen: import("zod").ZodOptional<import("zod").ZodNumber>;
123
124
  headMaxLen: import("zod").ZodOptional<import("zod").ZodNumber>;
124
- type: import("zod").ZodLiteral<"decide">;
125
+ sessionId: import("zod").ZodOptional<import("zod").ZodString>;
125
126
  requestId: import("zod").ZodOptional<import("zod").ZodString>;
127
+ type: import("zod").ZodLiteral<"decide">;
126
128
  modelId: import("zod").ZodString;
127
129
  }, import("zod/v4/core").$strict>]>, import("zod").ZodObject<{
128
130
  type: import("zod").ZodLiteral<"decide">;
@@ -3,7 +3,7 @@ import { definePlugin, defineHandler, ModelType, ADDON_EMBEDDING, decisionConfig
3
3
  import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
4
4
  import { isMobile } from '../../../runtime/state.js';
5
5
  import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
6
- import { getModelShardFiles } from '../../../utils/shard-utils.js';
6
+ import { getFirstShardPath } from '../../../utils/shard-utils.js';
7
7
  import { decide } from './ops/decide.js';
8
8
  import { transformDecisionConfig } from './helpers.js';
9
9
  function createDecisionModel(params) {
@@ -17,7 +17,7 @@ function createDecisionModel(params) {
17
17
  const logger = createStreamLogger(params.modelId, ModelType.llamacppDecision);
18
18
  registerAddonLogger(params.modelId, ModelType.llamacppDecision, logger);
19
19
  return new LayaDecisions({
20
- files: { model: getModelShardFiles(params.modelPath) },
20
+ files: { model: [getFirstShardPath(params.modelPath)] },
21
21
  config,
22
22
  logger
23
23
  });
@@ -0,0 +1,3 @@
1
+ import type { FitLoad, FitRequestPlan } from '../../../schemas/plugin.js';
2
+ /** The embed-llamacpp fit request for a load, built from the config the load would use. */
3
+ export declare function embedFitRequest(load: FitLoad): FitRequestPlan;