@qvac/inference 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/NOTICE +60 -28
  2. package/README.md +3 -1
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +135 -44
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +1 -1
  13. package/dist/api/completion-stream.js +2 -1
  14. package/dist/api/delete-cache.d.ts +7 -1
  15. package/dist/api/delete-cache.js +11 -1
  16. package/dist/api/diffusion.d.ts +14 -1
  17. package/dist/api/diffusion.js +14 -1
  18. package/dist/api/embed.d.ts +82 -29
  19. package/dist/api/embed.js +64 -18
  20. package/dist/api/get-model-info.d.ts +1 -1
  21. package/dist/api/index.d.ts +2 -1
  22. package/dist/api/index.js +2 -1
  23. package/dist/api/rag.d.ts +1 -1
  24. package/dist/api/rag.js +1 -1
  25. package/dist/api/registry.d.ts +16 -6
  26. package/dist/api/registry.js +32 -9
  27. package/dist/api/text-to-speech.d.ts +14 -5
  28. package/dist/api/text-to-speech.js +83 -29
  29. package/dist/api/transcribe.d.ts +6 -2
  30. package/dist/api/transcribe.js +7 -1
  31. package/dist/api/vector-index.d.ts +25 -3
  32. package/dist/api/vector-index.js +17 -2
  33. package/dist/api/video-request.d.ts +15 -15
  34. package/dist/dispatch.d.ts +10 -4
  35. package/dist/dispatch.js +65 -15
  36. package/dist/engine.d.ts +1 -1
  37. package/dist/errors/index.d.ts +70 -0
  38. package/dist/errors/index.js +128 -0
  39. package/dist/handlers/assess-model-fit.d.ts +3 -21
  40. package/dist/handlers/assess-model-fit.js +4 -303
  41. package/dist/handlers/cancelHandler.d.ts +10 -9
  42. package/dist/handlers/cancelHandler.js +17 -13
  43. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  44. package/dist/handlers/completion-orchestrate.js +164 -22
  45. package/dist/handlers/delete-cache.d.ts +2 -1
  46. package/dist/handlers/delete-cache.js +6 -1
  47. package/dist/handlers/download-asset.js +11 -4
  48. package/dist/handlers/get-loaded-model-info.js +1 -0
  49. package/dist/handlers/get-model-info.js +71 -2
  50. package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
  51. package/dist/handlers/load-model/coreml-sidecars.js +16 -119
  52. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  53. package/dist/handlers/load-model/download-lock.js +213 -0
  54. package/dist/handlers/load-model/download-manager.d.ts +10 -2
  55. package/dist/handlers/load-model/download-manager.js +62 -11
  56. package/dist/handlers/load-model/handler.js +88 -35
  57. package/dist/handlers/load-model/http.js +327 -205
  58. package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
  59. package/dist/handlers/load-model/hyperdrive.js +123 -50
  60. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  61. package/dist/handlers/load-model/registry-client-options.js +4 -2
  62. package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
  63. package/dist/handlers/load-model/registry-companion-set.js +19 -1
  64. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  65. package/dist/handlers/load-model/registry-delete.js +269 -0
  66. package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
  67. package/dist/handlers/load-model/registry-download-utils.js +93 -22
  68. package/dist/handlers/load-model/registry.d.ts +20 -0
  69. package/dist/handlers/load-model/registry.js +308 -175
  70. package/dist/handlers/load-model/resolve.js +4 -1
  71. package/dist/handlers/load-model/types.d.ts +2 -0
  72. package/dist/handlers/logging-stream.js +17 -5
  73. package/dist/handlers/plugin-invoke.js +152 -39
  74. package/dist/handlers/registry.d.ts +2 -2
  75. package/dist/handlers/registry.js +105 -28
  76. package/dist/handlers/rpc-server.js +7 -7
  77. package/dist/handlers/session.d.ts +11 -0
  78. package/dist/handlers/session.js +40 -0
  79. package/dist/handlers/unload-model.js +8 -3
  80. package/dist/handlers/vector-index.js +118 -19
  81. package/dist/index.d.ts +1 -1
  82. package/dist/index.js +1 -1
  83. package/dist/logging/utils.js +10 -2
  84. package/dist/model-cache-layout.d.ts +1 -0
  85. package/dist/model-cache-layout.js +2 -0
  86. package/dist/model-fit/assess.d.ts +18 -67
  87. package/dist/model-fit/assess.js +96 -665
  88. package/dist/model-fit/describe.d.ts +12 -0
  89. package/dist/model-fit/describe.js +29 -0
  90. package/dist/model-fit/description/fetch.d.ts +1 -1
  91. package/dist/model-fit/description/fetch.js +95 -16
  92. package/dist/model-fit/description/resolve-load.d.ts +5 -3
  93. package/dist/model-fit/description/resolve-load.js +14 -14
  94. package/dist/model-fit/fit-load.d.ts +23 -0
  95. package/dist/model-fit/fit-load.js +29 -0
  96. package/dist/model-fit/load-check.d.ts +15 -33
  97. package/dist/model-fit/load-check.js +50 -195
  98. package/dist/model-fit/run/engine.d.ts +12 -72
  99. package/dist/model-fit/run/engine.js +4 -8
  100. package/dist/model-fit/run/run.d.ts +10 -7
  101. package/dist/model-fit/run/run.js +22 -18
  102. package/dist/model-fit/sum.d.ts +32 -0
  103. package/dist/model-fit/sum.js +60 -0
  104. package/dist/models/registry/models.d.ts +865 -128
  105. package/dist/models/registry/models.js +3737 -2963
  106. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  107. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  108. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  109. package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
  110. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  111. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  112. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  113. package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  114. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  115. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
  116. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  117. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  118. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  119. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  120. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  121. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
  122. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  123. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
  124. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
  125. package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
  126. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  127. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  128. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
  129. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
  130. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
  131. package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
  132. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  133. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  134. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  135. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  136. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  137. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  138. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  139. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
  140. package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
  141. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  142. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  143. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
  144. package/dist/plugins/builtin/index.d.ts +1 -0
  145. package/dist/plugins/builtin/index.js +1 -0
  146. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  147. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  148. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
  149. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  150. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  151. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  152. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  153. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
  154. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
  155. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  156. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  157. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
  158. package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
  159. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  160. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
  161. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
  162. package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
  163. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  164. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  165. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
  166. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
  167. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  168. package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
  169. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
  170. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
  171. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
  172. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
  173. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
  174. package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
  175. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  176. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  177. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
  178. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
  179. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
  180. package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
  181. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  182. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  183. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  184. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  185. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
  186. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
  187. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  188. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  189. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
  190. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
  191. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
  192. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
  193. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
  194. package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
  195. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  196. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  197. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  198. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  199. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  200. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
  201. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
  202. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  203. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
  204. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  205. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  206. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
  207. package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
  208. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
  209. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
  210. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  211. package/dist/plugins/ops/bci-transcribe.js +18 -4
  212. package/dist/plugins/ops/cancel.d.ts +1 -0
  213. package/dist/plugins/ops/cancel.js +11 -2
  214. package/dist/plugins/ops/embed.d.ts +5 -3
  215. package/dist/plugins/ops/embed.js +89 -34
  216. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
  217. package/dist/plugins/ops/kv-cache-utils.js +6 -2
  218. package/dist/plugins/ops/load-model.d.ts +2 -0
  219. package/dist/plugins/ops/load-model.js +33 -16
  220. package/dist/plugins/ops/transcribe.d.ts +5 -4
  221. package/dist/plugins/ops/transcribe.js +62 -27
  222. package/dist/plugins/ops/translate.d.ts +2 -1
  223. package/dist/plugins/ops/translate.js +4 -1
  224. package/dist/plugins/ops/unload-model.d.ts +17 -1
  225. package/dist/plugins/ops/unload-model.js +31 -2
  226. package/dist/plugins/turbovec-backend.d.ts +1 -1
  227. package/dist/plugins/turbovec-backend.js +16 -1
  228. package/dist/profiling/operation-metrics.js +22 -19
  229. package/dist/rag/embedding-scope.d.ts +16 -0
  230. package/dist/rag/embedding-scope.js +81 -0
  231. package/dist/rag/handler.d.ts +25 -0
  232. package/dist/rag/handler.js +50 -16
  233. package/dist/rag/handlers/ingest.js +2 -6
  234. package/dist/rag/handlers/search.d.ts +10 -1
  235. package/dist/rag/handlers/search.js +16 -7
  236. package/dist/rag/rag-operation-manager.d.ts +27 -3
  237. package/dist/rag/rag-operation-manager.js +10 -27
  238. package/dist/registry.js +5 -0
  239. package/dist/resources/collector.d.ts +70 -36
  240. package/dist/resources/collector.js +8 -1
  241. package/dist/resources/types.d.ts +2 -0
  242. package/dist/rpc/inventory.js +1 -12
  243. package/dist/runtime/env.d.ts +5 -1
  244. package/dist/runtime/env.js +6 -21
  245. package/dist/runtime/index.d.ts +4 -1
  246. package/dist/runtime/index.js +2 -0
  247. package/dist/runtime/lifecycle.d.ts +6 -4
  248. package/dist/runtime/lifecycle.js +9 -5
  249. package/dist/runtime/model-config-utils.js +5 -1
  250. package/dist/runtime/model-registry.d.ts +28 -4
  251. package/dist/runtime/model-registry.js +46 -1
  252. package/dist/runtime/model-work-lock.d.ts +13 -0
  253. package/dist/runtime/model-work-lock.js +48 -0
  254. package/dist/runtime/registry-client.d.ts +47 -1
  255. package/dist/runtime/registry-client.js +446 -88
  256. package/dist/runtime/request-context.d.ts +9 -1
  257. package/dist/runtime/request-context.js +25 -1
  258. package/dist/runtime/request-registry.d.ts +55 -7
  259. package/dist/runtime/request-registry.js +189 -71
  260. package/dist/runtime/runtime-lifecycle.js +10 -1
  261. package/dist/runtime/session-id.d.ts +18 -0
  262. package/dist/runtime/session-id.js +20 -0
  263. package/dist/runtime/session-registry.d.ts +39 -0
  264. package/dist/runtime/session-registry.js +114 -0
  265. package/dist/runtime/vector-index-backend.d.ts +4 -0
  266. package/dist/runtime/vector-index-registry.d.ts +21 -0
  267. package/dist/runtime/vector-index-registry.js +56 -18
  268. package/dist/schemas/abort-signal.d.ts +8 -0
  269. package/dist/schemas/abort-signal.js +1 -0
  270. package/dist/schemas/assess-model-fit.d.ts +132 -282
  271. package/dist/schemas/assess-model-fit.js +82 -216
  272. package/dist/schemas/audio-gen.d.ts +278 -24
  273. package/dist/schemas/audio-gen.js +127 -23
  274. package/dist/schemas/batch-completion-stream.d.ts +105 -13
  275. package/dist/schemas/batch-completion-stream.js +8 -4
  276. package/dist/schemas/bci-config.d.ts +2 -1
  277. package/dist/schemas/bci.d.ts +54 -3
  278. package/dist/schemas/bci.js +11 -16
  279. package/dist/schemas/calc-tokens.d.ts +141 -0
  280. package/dist/schemas/calc-tokens.js +53 -0
  281. package/dist/schemas/cancel.d.ts +26 -0
  282. package/dist/schemas/cancel.js +10 -2
  283. package/dist/schemas/classification.d.ts +3 -0
  284. package/dist/schemas/classification.js +6 -2
  285. package/dist/schemas/common.d.ts +1831 -348
  286. package/dist/schemas/common.js +18 -2
  287. package/dist/schemas/completion-event.d.ts +3 -0
  288. package/dist/schemas/completion-event.js +1 -1
  289. package/dist/schemas/completion-stream.d.ts +162 -17
  290. package/dist/schemas/completion-stream.js +83 -13
  291. package/dist/schemas/config.d.ts +1212 -169
  292. package/dist/schemas/config.js +9 -13
  293. package/dist/schemas/decision.d.ts +4 -2
  294. package/dist/schemas/decision.js +2 -1
  295. package/dist/schemas/download-asset.d.ts +10 -5
  296. package/dist/schemas/download-asset.js +3 -6
  297. package/dist/schemas/embed.d.ts +286 -3
  298. package/dist/schemas/embed.js +162 -16
  299. package/dist/schemas/engine-addon-map.d.ts +1 -0
  300. package/dist/schemas/engine-addon-map.js +1 -0
  301. package/dist/schemas/errors.d.ts +12 -0
  302. package/dist/schemas/errors.js +66 -1
  303. package/dist/schemas/finetune.d.ts +14 -4
  304. package/dist/schemas/finetune.js +5 -6
  305. package/dist/schemas/heartbeat.d.ts +1 -0
  306. package/dist/schemas/heartbeat.js +5 -2
  307. package/dist/schemas/image-embedding.d.ts +27 -0
  308. package/dist/schemas/image-embedding.js +107 -0
  309. package/dist/schemas/index.d.ts +7 -3
  310. package/dist/schemas/index.js +7 -3
  311. package/dist/schemas/lifecycle.d.ts +3 -0
  312. package/dist/schemas/lifecycle.js +13 -6
  313. package/dist/schemas/llamacpp-config.d.ts +269 -22
  314. package/dist/schemas/llamacpp-config.js +86 -7
  315. package/dist/schemas/load-model.d.ts +6418 -1960
  316. package/dist/schemas/load-model.js +39 -8
  317. package/dist/schemas/logging-stream.d.ts +1 -0
  318. package/dist/schemas/logging-stream.js +5 -2
  319. package/dist/schemas/model-config-schemas.d.ts +542 -116
  320. package/dist/schemas/model-config-schemas.js +2 -0
  321. package/dist/schemas/model-fit-policy.d.ts +2 -2
  322. package/dist/schemas/model-info.d.ts +54 -56
  323. package/dist/schemas/model-info.js +17 -6
  324. package/dist/schemas/model-ops.d.ts +12 -0
  325. package/dist/schemas/model-ops.js +33 -5
  326. package/dist/schemas/model-src-utils.d.ts +10 -6
  327. package/dist/schemas/model-types.d.ts +8 -4
  328. package/dist/schemas/model-types.js +1 -0
  329. package/dist/schemas/ocr.d.ts +9 -4
  330. package/dist/schemas/ocr.js +6 -2
  331. package/dist/schemas/plugin.d.ts +62 -6
  332. package/dist/schemas/plugin.js +26 -4
  333. package/dist/schemas/profiling.d.ts +32 -32
  334. package/dist/schemas/rag.d.ts +9 -0
  335. package/dist/schemas/rag.js +10 -16
  336. package/dist/schemas/registry.d.ts +30 -10
  337. package/dist/schemas/registry.js +14 -6
  338. package/dist/schemas/reload-config.d.ts +41 -6
  339. package/dist/schemas/reload-config.js +11 -4
  340. package/dist/schemas/request-base.d.ts +39 -0
  341. package/dist/schemas/request-base.js +64 -0
  342. package/dist/schemas/rerank.d.ts +8 -7
  343. package/dist/schemas/rerank.js +6 -4
  344. package/dist/schemas/rpc-server.d.ts +3 -0
  345. package/dist/schemas/rpc-server.js +15 -15
  346. package/dist/schemas/sdcpp-config.d.ts +90 -20
  347. package/dist/schemas/sdcpp-config.js +96 -19
  348. package/dist/schemas/session.d.ts +37 -0
  349. package/dist/schemas/session.js +53 -0
  350. package/dist/schemas/system-resources.d.ts +364 -179
  351. package/dist/schemas/system-resources.js +8 -3
  352. package/dist/schemas/text-to-speech.d.ts +1343 -271
  353. package/dist/schemas/text-to-speech.js +547 -33
  354. package/dist/schemas/tools.d.ts +7 -0
  355. package/dist/schemas/tools.js +29 -0
  356. package/dist/schemas/transcription-config.d.ts +262 -2
  357. package/dist/schemas/transcription-config.js +122 -6
  358. package/dist/schemas/transcription.d.ts +93 -6
  359. package/dist/schemas/transcription.js +98 -21
  360. package/dist/schemas/translate.d.ts +2 -0
  361. package/dist/schemas/translate.js +5 -9
  362. package/dist/schemas/translation-config.d.ts +30 -20
  363. package/dist/schemas/tts-pcm.d.ts +35 -0
  364. package/dist/schemas/tts-pcm.js +69 -0
  365. package/dist/schemas/vector-index.d.ts +47 -1
  366. package/dist/schemas/vector-index.js +37 -10
  367. package/dist/schemas/vla.d.ts +3 -3
  368. package/dist/surface.d.ts +3 -3
  369. package/dist/surface.js +3 -3
  370. package/dist/utils/addon-responses.d.ts +69 -2
  371. package/dist/utils/archive.d.ts +7 -0
  372. package/dist/utils/archive.js +88 -29
  373. package/dist/utils/asr-events.d.ts +5 -0
  374. package/dist/utils/asr-events.js +10 -1
  375. package/dist/utils/cache/auto-history.js +3 -1
  376. package/dist/utils/cache/layout.d.ts +48 -0
  377. package/dist/utils/cache/layout.js +190 -0
  378. package/dist/utils/cache/paths.js +2 -4
  379. package/dist/utils/cache/types.d.ts +2 -1
  380. package/dist/utils/checksum.d.ts +5 -0
  381. package/dist/utils/checksum.js +18 -0
  382. package/dist/utils/completion-normalizer.js +286 -87
  383. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  384. package/dist/utils/gguf-tensor-extractor.js +1 -50
  385. package/dist/utils/shard-utils.d.ts +0 -21
  386. package/dist/utils/shard-utils.js +12 -62
  387. package/dist/utils/tool-integration.d.ts +2 -2
  388. package/dist/utils/tools/defer.d.ts +39 -10
  389. package/dist/utils/tools/defer.js +72 -17
  390. package/dist/utils/tools/index.d.ts +1 -1
  391. package/dist/utils/tools/index.js +1 -1
  392. package/dist/utils/transcribe-metadata.d.ts +28 -2
  393. package/dist/utils/transcribe-metadata.js +16 -0
  394. package/dist/utils/tts-stats.d.ts +42 -0
  395. package/dist/utils/tts-stats.js +120 -22
  396. package/package.json +43 -39
  397. package/dist/model-fit/classify.d.ts +0 -6
  398. package/dist/model-fit/classify.js +0 -166
  399. package/dist/model-fit/description/project.d.ts +0 -27
  400. package/dist/model-fit/description/project.js +0 -33
  401. package/dist/model-fit/floor.d.ts +0 -31
  402. package/dist/model-fit/floor.js +0 -65
  403. package/dist/model-fit/kv-cache.d.ts +0 -23
  404. package/dist/model-fit/kv-cache.js +0 -85
  405. package/dist/model-fit/platform.d.ts +0 -14
  406. package/dist/model-fit/platform.js +0 -26
  407. package/dist/model-fit/request/create.d.ts +0 -31
  408. package/dist/model-fit/request/create.js +0 -61
  409. package/dist/model-fit/request/engines/asr.d.ts +0 -14
  410. package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
  411. package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
  412. package/dist/model-fit/request/engines/diffusion.js +0 -93
  413. package/dist/model-fit/request/engines/llama.d.ts +0 -12
  414. package/dist/model-fit/request/engines/llama.js +0 -44
  415. package/dist/model-fit/request/engines/nmt.d.ts +0 -8
  416. package/dist/model-fit/request/engines/nmt.js +0 -11
  417. package/dist/model-fit/request/engines/ocr.d.ts +0 -9
  418. package/dist/model-fit/request/engines/ocr.js +0 -13
  419. package/dist/model-fit/request/engines/tts.d.ts +0 -10
  420. package/dist/model-fit/request/engines/tts.js +0 -152
  421. package/dist/models/registry/resource-profiles.d.ts +0 -17
  422. package/dist/models/registry/resource-profiles.js +0 -5859
  423. package/dist/schemas/model-resource-profile.d.ts +0 -109
  424. package/dist/schemas/model-resource-profile.js +0 -80
  425. /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
  426. /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
@@ -0,0 +1,39 @@
1
+ import { MegaLoc } from '@qvac/classification-ggml';
2
+ import { definePlugin, defineHandler, ModelType, ADDON_CLASSIFICATION, imageEmbeddingConfigSchema, embedRequestSchema, embedResponseSchema, embedBatchRequestSchema, embedBatchResponseSchema } from '../../../schemas/index.js';
3
+ import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
4
+ import { embed, embedBatch } from './ops/embed.js';
5
+ export const imageEmbeddingPlugin = definePlugin({
6
+ modelType: ModelType.ggmlImageEmbedding,
7
+ displayName: 'Image embeddings (MegaLoc)',
8
+ addonPackage: ADDON_CLASSIFICATION,
9
+ loadConfigSchema: imageEmbeddingConfigSchema,
10
+ createModel(params) {
11
+ const config = imageEmbeddingConfigSchema.parse(params.modelConfig ?? {});
12
+ const logger = createStreamLogger(params.modelId, ModelType.ggmlImageEmbedding);
13
+ registerAddonLogger(params.modelId, ModelType.ggmlImageEmbedding, logger);
14
+ const model = new MegaLoc({
15
+ modelPath: params.modelPath,
16
+ ...(config.backend !== undefined && { backend: config.backend }),
17
+ ...(config.threads !== undefined && { threads: config.threads }),
18
+ ...(config.backendsDir !== undefined && { backendsDir: config.backendsDir }),
19
+ logger
20
+ });
21
+ return { model };
22
+ },
23
+ handlers: {
24
+ embed: defineHandler({
25
+ requestSchema: embedRequestSchema,
26
+ responseSchema: embedResponseSchema,
27
+ streaming: false,
28
+ cancel: { scope: 'none' },
29
+ handler: (request, _inputStream, context) => embed(request, context)
30
+ }),
31
+ embedBatch: defineHandler({
32
+ requestSchema: embedBatchRequestSchema,
33
+ responseSchema: embedBatchResponseSchema,
34
+ streaming: false,
35
+ cancel: { scope: 'none' },
36
+ handler: (request, _inputStream, context) => embedBatch(request, context)
37
+ })
38
+ }
39
+ });
@@ -0,0 +1,6 @@
1
+ import type { FitLoad, FitRequestPlan } from '../../../schemas/plugin.js';
2
+ /**
3
+ * The ocr fit request for a load: the parameters the load builds from its
4
+ * detector and recognizer, which is the load's primary model.
5
+ */
6
+ export declare function ocrFitRequest(load: FitLoad): FitRequestPlan;
@@ -0,0 +1,19 @@
1
+ import { ocrParamsFor } from '../../../plugins/builtin/ggml-ocr/params.js';
2
+ import { isMobile } from '../../../runtime/state.js';
3
+ import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
4
+ /**
5
+ * The ocr fit request for a load: the parameters the load builds from its
6
+ * detector and recognizer, which is the load's primary model.
7
+ */
8
+ export function ocrFitRequest(load) {
9
+ const detectorPath = load.artifacts?.['detectorModelPath'];
10
+ if (detectorPath === undefined)
11
+ return { unknown: 'the ocr detector is not resolved' };
12
+ const config = (load.modelConfig ?? {});
13
+ const params = ocrParamsFor(detectorPath, load.modelPath, config);
14
+ // A mobile load drops the multi-GPU keys before it runs.
15
+ if (isMobile())
16
+ stripMultiGpuKeys(params);
17
+ const request = params;
18
+ return { request };
19
+ }
@@ -3,7 +3,7 @@ import type { OCRConfig } from '../../../schemas/index.js';
3
3
  export declare function ocrParamsFor(detectorPath: string, recognizerPath: string, ocrConfig: OCRConfig): {
4
4
  'main-gpu'?: number | "integrated" | "dedicated";
5
5
  gpuDevice?: number;
6
- backendDevice?: "cpu" | "vulkan" | "metal" | "opencl";
6
+ backendDevice?: "cpu" | "vulkan" | "opencl" | "metal";
7
7
  nThreads?: number;
8
8
  recognizerBatchSize?: number;
9
9
  lowConfidenceThreshold?: number;
@@ -1,5 +1,6 @@
1
1
  import { type CreateModelParams, type PluginModelResult, type OCRConfig, type ResolveContext, type ResolveResult } from '../../../schemas/index.js';
2
2
  import { assessFit as ocrAssessFit } from '@qvac/ocr-ggml';
3
+ import { ocrFitRequest } from '../../../plugins/builtin/ggml-ocr/fit.js';
3
4
  export declare const ocrPlugin: {
4
5
  modelType: "ggml-ocr";
5
6
  displayName: string;
@@ -20,8 +21,8 @@ export declare const ocrPlugin: {
20
21
  backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
21
22
  cpu: "cpu";
22
23
  vulkan: "vulkan";
23
- metal: "metal";
24
24
  opencl: "opencl";
25
+ metal: "metal";
25
26
  }>>;
26
27
  gpuDevice: import("zod").ZodOptional<import("zod").ZodNumber>;
27
28
  mainGpu: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodEnum<{
@@ -45,6 +46,7 @@ export declare const ocrPlugin: {
45
46
  audiogen: "audiogen";
46
47
  tts: "tts";
47
48
  ocr: "ocr";
49
+ classification: "classification";
48
50
  vla: "vla";
49
51
  "llamacpp-completion": "llamacpp-completion";
50
52
  "whispercpp-transcription": "whispercpp-transcription";
@@ -61,21 +63,22 @@ export declare const ocrPlugin: {
61
63
  "sdcpp-generation": "sdcpp-generation";
62
64
  "audiogen-ggml": "audiogen-ggml";
63
65
  "ggml-vla": "ggml-vla";
66
+ "ggml-image-embedding": "ggml-image-embedding";
64
67
  "ggml-classification": "ggml-classification";
65
68
  llm: "llm";
66
69
  whisper: "whisper";
67
70
  bci: "bci";
68
71
  nmt: "nmt";
69
72
  parakeet: "parakeet";
70
- classification: "classification";
71
73
  }>, import("zod").ZodLiteral<"vad">]>>;
72
74
  }, import("zod/v4/core").$strip>]>>;
73
75
  }, import("zod/v4/core").$strip>;
76
+ fitRequest: typeof ocrFitRequest;
74
77
  assessFit: typeof ocrAssessFit;
75
78
  resolveConfig(cfg: OCRConfig, ctx: ResolveContext): Promise<ResolveResult<Record<string, unknown>, "detectorModelPath">>;
76
79
  createModel(params: CreateModelParams): PluginModelResult;
77
80
  handlers: {
78
- ocrStream: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
81
+ ocrStream: import("../../../schemas/plugin.js").PluginHandlerDefinition<import("zod").ZodObject<{
79
82
  modelId: import("zod").ZodString;
80
83
  image: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
81
84
  type: import("zod").ZodLiteral<"base64">;
@@ -88,8 +91,11 @@ export declare const ocrPlugin: {
88
91
  paragraph: import("zod").ZodOptional<import("zod").ZodBoolean>;
89
92
  }, import("zod/v4/core").$strip>>;
90
93
  type: import("zod").ZodLiteral<"ocrStream">;
94
+ sessionId: import("zod").ZodOptional<import("zod").ZodString>;
95
+ requestId: import("zod").ZodOptional<import("zod").ZodString>;
91
96
  }, import("zod/v4/core").$strip>, import("zod").ZodObject<{
92
97
  type: import("zod").ZodLiteral<"ocrStream">;
98
+ requestId: import("zod").ZodOptional<import("zod").ZodString>;
93
99
  blocks: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodObject<{
94
100
  text: import("zod").ZodString;
95
101
  bbox: import("zod").ZodOptional<import("zod").ZodTuple<[import("zod").ZodNumber, import("zod").ZodNumber, import("zod").ZodNumber, import("zod").ZodNumber], null>>;
@@ -1,11 +1,67 @@
1
+ var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
+ if (value !== null && value !== void 0) {
3
+ if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
+ var dispose, inner;
5
+ if (async) {
6
+ if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
+ dispose = value[Symbol.asyncDispose];
8
+ }
9
+ if (dispose === void 0) {
10
+ if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
+ dispose = value[Symbol.dispose];
12
+ if (async) inner = dispose;
13
+ }
14
+ if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
+ if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
+ env.stack.push({ value: value, dispose: dispose, async: async });
17
+ }
18
+ else if (async) {
19
+ env.stack.push({ async: true });
20
+ }
21
+ return value;
22
+ };
23
+ var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
+ return function (env) {
25
+ function fail(e) {
26
+ env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
+ env.hasError = true;
28
+ }
29
+ var r, s = 0;
30
+ function next() {
31
+ while (r = env.stack.pop()) {
32
+ try {
33
+ if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
+ if (r.dispose) {
35
+ var result = r.dispose.call(r.value);
36
+ if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
+ }
38
+ else s |= 1;
39
+ }
40
+ catch (e) {
41
+ fail(e);
42
+ }
43
+ }
44
+ if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
+ if (env.hasError) throw env.error;
46
+ }
47
+ return next();
48
+ };
49
+ })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
+ var e = new Error(message);
51
+ return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
+ });
53
+ import { requestParent } from '../../../schemas/plugin.js';
1
54
  import { definePlugin, defineHandler, ocrStreamRequestSchema, ocrStreamResponseSchema, ModelType, ocrLoadConfigSchema, ADDON_OCR } from '../../../schemas/index.js';
2
- import { ModelLoadFailedError } from '../../../errors/index.js';
55
+ import { InferenceCancelledError, ModelLoadFailedError } from '../../../errors/index.js';
56
+ import { getRequestRegistry } from '../../../runtime/index.js';
57
+ import { generateRandomRequestId } from '../../../runtime/request-id.js';
3
58
  import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
4
59
  import { OcrGgml, assessFit as ocrAssessFit } from '@qvac/ocr-ggml';
5
60
  import { ocr } from '../../../plugins/builtin/ggml-ocr/ops/ocr-stream.js';
6
61
  import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
7
62
  import { ocrParamsFor } from '../../../plugins/builtin/ggml-ocr/params.js';
8
63
  import { resolveOcrConfig } from '../../../plugins/builtin/ggml-ocr/resolve-config.js';
64
+ import { ocrFitRequest } from '../../../plugins/builtin/ggml-ocr/fit.js';
9
65
  import { isMobile } from '../../../runtime/state.js';
10
66
  import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
11
67
  function createOCRModel(modelId, detectorPath, recognizerPath, ocrConfig) {
@@ -31,6 +87,7 @@ export const ocrPlugin = definePlugin({
31
87
  displayName: 'OCR (GGML)',
32
88
  addonPackage: ADDON_OCR,
33
89
  loadConfigSchema: ocrLoadConfigSchema,
90
+ fitRequest: ocrFitRequest,
34
91
  assessFit: ocrAssessFit,
35
92
  async resolveConfig(cfg, ctx) {
36
93
  return resolveOcrConfig(cfg, ctx);
@@ -53,31 +110,60 @@ export const ocrPlugin = definePlugin({
53
110
  // GGML OCR does not expose a cancel surface — we fall back
54
111
  // to soft-cancel.
55
112
  cancel: { scope: 'none' },
56
- handler: async function* (request) {
57
- const stream = ocr({
58
- modelId: request.modelId,
59
- image: request.image,
60
- options: request.options
61
- });
113
+ handler: async function* (request, _inputStream, context) {
114
+ const env_1 = { stack: [], error: void 0, hasError: false };
62
115
  try {
63
- let result = await stream.next();
64
- while (!result.done) {
65
- yield {
116
+ // No addon abort hook: a cancel stops delivery, the native pass runs
117
+ // to completion, and the request is still visible to cancel and
118
+ // to model unload through the registry.
119
+ const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
120
+ requestId: request.requestId ?? generateRandomRequestId(),
121
+ sessionId: request.sessionId,
122
+ ...requestParent(context),
123
+ kind: 'ocr',
124
+ modelId: request.modelId
125
+ }), true);
126
+ if (ctx.signal.aborted)
127
+ throw new InferenceCancelledError(ctx.requestId);
128
+ const stream = ocr({
129
+ modelId: request.modelId,
130
+ image: request.image,
131
+ options: request.options
132
+ });
133
+ try {
134
+ let result = await stream.next();
135
+ while (!result.done) {
136
+ if (ctx.signal.aborted)
137
+ throw new InferenceCancelledError(ctx.requestId);
138
+ yield {
139
+ type: 'ocrStream',
140
+ blocks: result.value.blocks
141
+ };
142
+ result = await stream.next();
143
+ }
144
+ if (ctx.signal.aborted)
145
+ throw new InferenceCancelledError(ctx.requestId);
146
+ const { modelExecutionMs, stats } = result.value;
147
+ yield attachModelExecutionMs({
66
148
  type: 'ocrStream',
67
- blocks: result.value.blocks
68
- };
69
- result = await stream.next();
149
+ requestId: ctx.requestId,
150
+ blocks: [],
151
+ done: true,
152
+ ...(stats && { stats })
153
+ }, modelExecutionMs);
154
+ }
155
+ finally {
156
+ await stream.return?.(undefined);
70
157
  }
71
- const { modelExecutionMs, stats } = result.value;
72
- yield attachModelExecutionMs({
73
- type: 'ocrStream',
74
- blocks: [],
75
- done: true,
76
- ...(stats && { stats })
77
- }, modelExecutionMs);
158
+ }
159
+ catch (e_1) {
160
+ env_1.error = e_1;
161
+ env_1.hasError = true;
78
162
  }
79
163
  finally {
80
- await stream.return?.(undefined);
164
+ const result_1 = __disposeResources(env_1);
165
+ if (result_1)
166
+ await result_1;
81
167
  }
82
168
  }
83
169
  })
@@ -1,2 +1,2 @@
1
- import { type VlaRunRequest, type VlaRunResponse } from '../../../../schemas/index.js';
2
- export declare function vlaRun(request: VlaRunRequest): Promise<VlaRunResponse>;
1
+ import { type PluginHandlerContext, type VlaRunRequest, type VlaRunResponse } from '../../../../schemas/index.js';
2
+ export declare function vlaRun(request: VlaRunRequest, _inputStream?: AsyncIterable<unknown>, context?: PluginHandlerContext): Promise<VlaRunResponse>;
@@ -48,7 +48,7 @@ function pickStats(raw) {
48
48
  out.backendDevice = r['backendDevice'];
49
49
  return Object.keys(out).length > 0 ? out : undefined;
50
50
  }
51
- export async function vlaRun(request) {
51
+ export async function vlaRun(request, _inputStream, context) {
52
52
  const model = getModel(request.modelId);
53
53
  const images = request.images.map(f32FromBase64);
54
54
  const state = f32FromBase64(request.state);
@@ -64,7 +64,23 @@ export async function vlaRun(request) {
64
64
  mask,
65
65
  ...(noise && { noise })
66
66
  });
67
- const result = await response.await();
67
+ // The addon exposes a model-wide hard cancel; the engine registers the
68
+ // invoke, so the abort arrives on the context signal.
69
+ const signal = context?.signal;
70
+ const onAbort = () => {
71
+ response.cancel().catch(() => { });
72
+ };
73
+ if (signal?.aborted)
74
+ onAbort();
75
+ else
76
+ signal?.addEventListener('abort', onAbort, { once: true });
77
+ let result;
78
+ try {
79
+ result = await response.await();
80
+ }
81
+ finally {
82
+ signal?.removeEventListener('abort', onAbort);
83
+ }
68
84
  const hp = model.hparams;
69
85
  if (!hp) {
70
86
  throw new Error('VLA model hparams unavailable after run');
@@ -7,12 +7,12 @@ export declare const vlaPlugin: {
7
7
  backend: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodEnum<{
8
8
  cpu: "cpu";
9
9
  auto: "auto";
10
- cuda: "cuda";
11
10
  vulkan: "vulkan";
12
- metal: "metal";
13
11
  opencl: "opencl";
14
- hip: "hip";
12
+ metal: "metal";
13
+ cuda: "cuda";
15
14
  rocm: "rocm";
15
+ hip: "hip";
16
16
  sycl: "sycl";
17
17
  }>>>;
18
18
  verbosity: import("zod").ZodOptional<import("zod").ZodNumber>;
@@ -1,4 +1,5 @@
1
1
  export { llmPlugin } from '../../plugins/builtin/llamacpp-completion/plugin.js';
2
+ export { imageEmbeddingPlugin } from '../../plugins/builtin/ggml-image-embedding/plugin.js';
2
3
  export { embeddingsPlugin } from '../../plugins/builtin/llamacpp-embedding/plugin.js';
3
4
  export { decisionPlugin } from '../../plugins/builtin/llamacpp-decision/plugin.js';
4
5
  export { rerankPlugin } from '../../plugins/builtin/llamacpp-rerank/plugin.js';
@@ -1,4 +1,5 @@
1
1
  export { llmPlugin } from '../../plugins/builtin/llamacpp-completion/plugin.js';
2
+ export { imageEmbeddingPlugin } from '../../plugins/builtin/ggml-image-embedding/plugin.js';
2
3
  export { embeddingsPlugin } from '../../plugins/builtin/llamacpp-embedding/plugin.js';
3
4
  export { decisionPlugin } from '../../plugins/builtin/llamacpp-decision/plugin.js';
4
5
  export { rerankPlugin } from '../../plugins/builtin/llamacpp-rerank/plugin.js';
@@ -0,0 +1,3 @@
1
+ import type { FitLoad, FitRequestPlan } from '../../../schemas/plugin.js';
2
+ /** The llm-llamacpp fit request for a load, built from the config the load would use. */
3
+ export declare function llmFitRequest(load: FitLoad): FitRequestPlan;
@@ -0,0 +1,28 @@
1
+ import { transformLlmConfig } from '../../../plugins/builtin/llamacpp-completion/transform.js';
2
+ import { isMobile } from '../../../runtime/state.js';
3
+ import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
4
+ /** The llm-llamacpp fit request for a load, built from the config the load would use. */
5
+ export function llmFitRequest(load) {
6
+ if (load.isShardedModel) {
7
+ return { unknown: 'the fitter reads one model file, and this model is split' };
8
+ }
9
+ if (load.artifacts?.['projectionModelPath'] !== undefined) {
10
+ return { unknown: 'the fitter does not project a multimodal projector' };
11
+ }
12
+ // The fitter rejects these as unknown arguments, and none changes what it
13
+ // projects: the image limit gates images at inference, and the prompt-cache
14
+ // keys size host-RAM and disk state the fitter does not model or choose which
15
+ // conversations share it.
16
+ const { 'rpc-servers': rpcServers, 'image-max-megapixels': _imageMaxMegapixels, cache_checkpoints: _cacheCheckpoints, cache_checkpoints_max_bytes: _cacheCheckpointsMaxBytes, cache_checkpoint_storage: _cacheCheckpointStorage, cache_checkpoint_dir: _cacheCheckpointDir, cache_ram_mib: _cacheRamMib, cache_share_across_keys: _cacheShareAcrossKeys, ...fitConfig } = (load.modelConfig ?? {});
17
+ if (rpcServers !== undefined)
18
+ return { unknown: 'an RPC load places layers on remote devices' };
19
+ const config = transformLlmConfig(fitConfig);
20
+ if (isMobile())
21
+ stripMultiGpuKeys(config);
22
+ // The fit reads the draft model (DFlash model or external MTP head) like the load does.
23
+ const draftModelPath = load.artifacts?.['draftModelPath'];
24
+ if (draftModelPath)
25
+ config['spec_draft_model'] = draftModelPath;
26
+ const request = { modelPath: load.modelPath, config };
27
+ return { request };
28
+ }
@@ -37,7 +37,7 @@ function renderPromptHistory(prompt, history, deferred, options) {
37
37
  }
38
38
  function buildBatchPrompt(prompt, options) {
39
39
  const history = seedConfiguredSystemPrompt(prompt.history, options.modelConfig);
40
- const deferred = resolveDeferredTools(prompt.tools, history);
40
+ const deferred = resolveDeferredTools(prompt.tools, history, prompt.toolSearch);
41
41
  const mergedGenerationParams = withParallelToolCalls(mergeGenerationParams(prompt.generationParams, prompt.responseFormat));
42
42
  return {
43
43
  ...(prompt.id !== undefined && { id: prompt.id }),
@@ -1,13 +1,24 @@
1
1
  import { getEngineLogger } from '../../../../logging/index.js';
2
2
  const logger = getEngineLogger();
3
+ /** A one-line summary of a message's content, parts included. */
4
+ function summarizeContent(content) {
5
+ if (!content)
6
+ return '';
7
+ if (typeof content === 'string')
8
+ return content;
9
+ return content
10
+ .map((part) => (part.type === 'text' ? part.text : `<media ${part.content}>`))
11
+ .join('');
12
+ }
3
13
  function formatMessages(messages) {
4
14
  return messages
5
15
  .map((m) => {
6
16
  const role = m.role || m.type || '?';
7
- const content = m.content
8
- ? m.content.length > 40
9
- ? m.content.substring(0, 40) + '...'
10
- : m.content
17
+ const summary = summarizeContent(m.content);
18
+ const content = summary
19
+ ? summary.length > 40
20
+ ? summary.substring(0, 40) + '...'
21
+ : summary
11
22
  : m.name || '(no content)';
12
23
  return `{${role}: "${content}"}`;
13
24
  })
@@ -0,0 +1,2 @@
1
+ import type { CalcTokensRequest, CalcTokensResponse } from '../../../../schemas/index.js';
2
+ export declare function calcTokens(request: CalcTokensRequest): Promise<CalcTokensResponse>;
@@ -0,0 +1,32 @@
1
+ import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
2
+ import { seedConfiguredSystemPrompt, transformMessages } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stream.js';
3
+ import { resolveDeferredTools, toWireTool } from '../../../../utils/tools/defer.js';
4
+ export async function calcTokens(request) {
5
+ const modelConfig = getModelConfig(request.modelId);
6
+ const toolsEnabled = modelConfig.tools === true;
7
+ const rendersTools = toolsEnabled && (request.tools?.length ?? 0) > 0;
8
+ const rendersPrompt = request.history !== undefined || rendersTools;
9
+ const requestHistory = request.history?.map((message) => ({
10
+ ...message,
11
+ content: message.content ?? ''
12
+ }));
13
+ const history = rendersPrompt
14
+ ? seedConfiguredSystemPrompt(requestHistory ?? [], modelConfig)
15
+ : undefined;
16
+ const deferred = toolsEnabled
17
+ ? resolveDeferredTools(request.tools, history ?? [], request.toolSearch)
18
+ : null;
19
+ const tools = toolsEnabled
20
+ ? (deferred?.toolsToRender ?? request.tools?.map(toWireTool))
21
+ : undefined;
22
+ const model = getModel(request.modelId);
23
+ const transformedHistory = history
24
+ ? transformMessages(history)
25
+ : undefined;
26
+ const input = {
27
+ ...(request.text !== undefined && { text: request.text }),
28
+ ...(transformedHistory !== undefined && { history: transformedHistory }),
29
+ ...(tools !== undefined && tools.length > 0 && { tools })
30
+ };
31
+ return { type: 'calcTokens', tokens: await model.calcTokens(input) };
32
+ }
@@ -12,7 +12,7 @@ export declare function normalizeCompletionStats(stats: LlmStats | undefined): {
12
12
  toolDefinitionsDropped?: number | undefined;
13
13
  draftTokens?: number | undefined;
14
14
  draftAcceptedTokens?: number | undefined;
15
- backendDevice?: "cpu" | "gpu" | undefined;
15
+ backendDevice?: "cpu" | "gpu" | "npu" | undefined;
16
16
  } | undefined;
17
17
  /**
18
18
  * Attach the count of non-empty pieces the addon streamed without
@@ -1,5 +1,5 @@
1
1
  import type { AbortSignal } from 'bare-abort-controller';
2
- import type { CompletionParams, CompletionStats, GenerationParams, ResponseFormat, Tool, ToolCall, ToolDialect } from '../../../../schemas/index.js';
2
+ import type { CompletionParams, CompletionStats, GenerationParams, HistoryContent, ResponseFormat, Tool, ToolCall, ToolDialect, ToolSearchOptions } from '../../../../schemas/index.js';
3
3
  import type { DisposableScope } from '../../../../runtime/disposable-scope.js';
4
4
  import { type CacheMessage } from '../../../../utils/index.js';
5
5
  import type { Logger } from '../../../../logging/types.js';
@@ -8,10 +8,23 @@ interface CompletionResult {
8
8
  stats?: CompletionStats;
9
9
  toolCalls: ToolCall[];
10
10
  stoppedAtContextBoundary: boolean;
11
+ stopReason?: 'eos' | 'stopSequence';
11
12
  }
13
+ /**
14
+ * One ordered content part as the addon receives it. The addon's media part
15
+ * carries its path in `content`, so a caller's `{ type: 'media', path }` is
16
+ * translated here.
17
+ */
18
+ export type ChatHistoryContentPart = {
19
+ type: 'text';
20
+ text: string;
21
+ } | {
22
+ type: 'media';
23
+ content: string;
24
+ };
12
25
  interface ChatHistory {
13
26
  role?: string;
14
- content?: string;
27
+ content?: string | ChatHistoryContentPart[];
15
28
  type?: string;
16
29
  name?: string;
17
30
  description?: string;
@@ -36,10 +49,11 @@ type HistoryMsg = CacheMessage;
36
49
  */
37
50
  export declare function seedConfiguredSystemPrompt<T extends {
38
51
  role: string;
39
- content: string;
52
+ content: HistoryContent;
40
53
  }>(history: T[], modelConfig: unknown): T[];
41
54
  export declare function completion(params: CompletionParams & {
42
55
  tools?: Tool[];
56
+ toolSearch?: ToolSearchOptions | undefined;
43
57
  generationParams?: GenerationParams;
44
58
  toolDialect?: ToolDialect;
45
59
  responseFormat?: ResponseFormat;