@qvac/inference 0.18.1 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (388) hide show
  1. package/NOTICE +11 -2
  2. package/README.md +1 -5
  3. package/dist/api/assess-model-fit.d.ts +16 -0
  4. package/dist/api/assess-model-fit.js +27 -0
  5. package/dist/api/audio-gen.d.ts +4 -4
  6. package/dist/api/audio-gen.js +17 -6
  7. package/dist/api/completion-stream.d.ts +1 -1
  8. package/dist/api/completion-stream.js +1 -1
  9. package/dist/api/download-asset.d.ts +1 -1
  10. package/dist/api/download-asset.js +2 -2
  11. package/dist/api/finetune.js +1 -1
  12. package/dist/api/get-loaded-model-info.d.ts +6 -10
  13. package/dist/api/get-loaded-model-info.js +6 -10
  14. package/dist/api/heartbeat.d.ts +5 -22
  15. package/dist/api/heartbeat.js +6 -23
  16. package/dist/api/index.d.ts +36 -35
  17. package/dist/api/index.js +35 -35
  18. package/dist/api/load-model.d.ts +5 -1
  19. package/dist/api/load-model.js +3 -5
  20. package/dist/api/logging-stream-registry.js +1 -1
  21. package/dist/api/resume.d.ts +0 -3
  22. package/dist/api/resume.js +0 -3
  23. package/dist/api/subscribe-logs.js +1 -1
  24. package/dist/api/suspend.d.ts +0 -4
  25. package/dist/api/suspend.js +0 -4
  26. package/dist/api/translate.d.ts +3 -1
  27. package/dist/api/translate.js +16 -8
  28. package/dist/api/unload-model.js +3 -5
  29. package/dist/api/video-request.d.ts +207 -0
  30. package/dist/api/video-request.js +18 -0
  31. package/dist/api/video.d.ts +22 -2
  32. package/dist/api/video.js +25 -15
  33. package/dist/api/vla-helpers.d.ts +1 -2
  34. package/dist/api/vla-helpers.js +1 -2
  35. package/dist/api/vla.js +1 -1
  36. package/dist/api/world-result.d.ts +71 -0
  37. package/dist/api/world-result.js +293 -0
  38. package/dist/api/world.d.ts +143 -0
  39. package/dist/api/world.js +71 -0
  40. package/dist/config/resolve-config.d.ts +1 -1
  41. package/dist/config/resolve-config.js +1 -1
  42. package/dist/constants/index.d.ts +2 -2
  43. package/dist/constants/index.js +2 -2
  44. package/dist/dispatch.js +15 -28
  45. package/dist/errors/index.d.ts +35 -31
  46. package/dist/errors/index.js +66 -56
  47. package/dist/handlers/assess-model-fit.d.ts +10 -0
  48. package/dist/handlers/assess-model-fit.js +75 -0
  49. package/dist/handlers/cancelHandler.js +1 -1
  50. package/dist/handlers/completion-orchestrate.js +1 -1
  51. package/dist/handlers/download-asset.js +6 -5
  52. package/dist/handlers/finetune.js +1 -1
  53. package/dist/handlers/get-loaded-model-info.js +0 -12
  54. package/dist/handlers/get-model-info.js +1 -1
  55. package/dist/handlers/get-system-resources.js +8 -0
  56. package/dist/handlers/load-model/download-manager.d.ts +1 -1
  57. package/dist/handlers/load-model/download-stats.d.ts +3 -3
  58. package/dist/handlers/load-model/download-stats.js +5 -5
  59. package/dist/handlers/load-model/handler.js +6 -8
  60. package/dist/handlers/load-model/http-verify.d.ts +20 -0
  61. package/dist/handlers/load-model/http-verify.js +54 -0
  62. package/dist/handlers/load-model/http.d.ts +12 -2
  63. package/dist/handlers/load-model/http.js +119 -230
  64. package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
  65. package/dist/handlers/load-model/hyperdrive.js +1 -1
  66. package/dist/handlers/load-model/index.d.ts +2 -2
  67. package/dist/handlers/load-model/index.js +2 -2
  68. package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
  69. package/dist/handlers/load-model/registry-companion-set.js +1 -1
  70. package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
  71. package/dist/handlers/load-model/registry-download-utils.js +2 -2
  72. package/dist/handlers/load-model/registry.d.ts +2 -2
  73. package/dist/handlers/load-model/registry.js +4 -4
  74. package/dist/handlers/load-model/resolve-session.d.ts +8 -1
  75. package/dist/handlers/load-model/resolve-session.js +12 -9
  76. package/dist/handlers/load-model/resolve.d.ts +4 -4
  77. package/dist/handlers/load-model/resolve.js +18 -16
  78. package/dist/handlers/load-model/safe-fetch.d.ts +44 -0
  79. package/dist/handlers/load-model/safe-fetch.js +143 -0
  80. package/dist/handlers/load-model/types.d.ts +9 -0
  81. package/dist/handlers/plugin-dispatch.js +1 -4
  82. package/dist/handlers/plugin-invoke.js +1 -4
  83. package/dist/handlers/types.d.ts +0 -2
  84. package/dist/handlers/unload-model.js +1 -4
  85. package/dist/index.d.ts +1 -1
  86. package/dist/index.js +1 -1
  87. package/dist/logging/addon.d.ts +1 -1
  88. package/dist/logging/addon.js +1 -1
  89. package/dist/logging/index.d.ts +5 -5
  90. package/dist/logging/index.js +4 -4
  91. package/dist/logging/logger.d.ts +1 -1
  92. package/dist/logging/logger.js +2 -2
  93. package/dist/models/registry/index.d.ts +1 -1
  94. package/dist/models/registry/index.js +2 -2
  95. package/dist/models/registry/models.d.ts +250 -0
  96. package/dist/models/registry/models.js +2071 -1806
  97. package/dist/models/registry/resource-profiles.d.ts +18 -0
  98. package/dist/models/registry/resource-profiles.js +4934 -0
  99. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +14 -0
  100. package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
  101. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +47 -5
  102. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +134 -4
  103. package/dist/plugins/builtin/audiogen-ggml/plugin.js +47 -28
  104. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +1 -1
  105. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
  106. package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
  107. package/dist/plugins/builtin/ggml-ocr/plugin.js +2 -2
  108. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
  109. package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
  110. package/dist/plugins/builtin/index.d.ts +12 -12
  111. package/dist/plugins/builtin/index.js +12 -12
  112. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +2 -2
  113. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +37 -26
  114. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +12 -11
  115. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +133 -41
  116. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +7 -1
  117. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +26 -4
  118. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +19 -8
  119. package/dist/plugins/builtin/llamacpp-completion/plugin.js +10 -13
  120. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +4 -0
  121. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +13 -2
  122. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +2 -2
  123. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +2 -2
  124. package/dist/plugins/builtin/parakeet-transcription/plugin.js +3 -3
  125. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +1 -1
  126. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +32 -2
  127. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +126 -0
  128. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +820 -0
  129. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +153 -4
  130. package/dist/plugins/builtin/sdcpp-generation/plugin.js +134 -13
  131. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +1 -1
  132. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +1 -1
  133. package/dist/plugins/builtin/tts-ggml/plugin.js +2 -2
  134. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +2 -2
  135. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +3 -3
  136. package/dist/plugins/index.d.ts +1 -1
  137. package/dist/plugins/index.js +2 -2
  138. package/dist/plugins/ops/kv-cache-utils.js +1 -1
  139. package/dist/plugins/ops/load-model.js +1 -1
  140. package/dist/plugins/ops/transcribe.js +2 -2
  141. package/dist/plugins/ops/translate.js +4 -12
  142. package/dist/plugins/ops/unload-model.js +19 -21
  143. package/dist/plugins/registry.d.ts +1 -0
  144. package/dist/plugins/registry.js +12 -0
  145. package/dist/profiling/aggregator.d.ts +1 -1
  146. package/dist/profiling/aggregator.js +1 -1
  147. package/dist/profiling/controller.d.ts +1 -1
  148. package/dist/profiling/controller.js +2 -2
  149. package/dist/profiling/envelope.d.ts +0 -1
  150. package/dist/profiling/envelope.js +0 -10
  151. package/dist/profiling/events.d.ts +2 -3
  152. package/dist/profiling/events.js +2 -9
  153. package/dist/profiling/exporters.d.ts +1 -1
  154. package/dist/profiling/exporters.js +3 -3
  155. package/dist/profiling/index.d.ts +13 -14
  156. package/dist/profiling/index.js +12 -13
  157. package/dist/profiling/operation-metrics.d.ts +1 -1
  158. package/dist/profiling/operation-metrics.js +35 -2
  159. package/dist/profiling/operation-wrappers.js +12 -15
  160. package/dist/profiling/profiler.d.ts +1 -2
  161. package/dist/profiling/profiler.js +7 -26
  162. package/dist/profiling/types.d.ts +1 -1
  163. package/dist/rag/handler.js +1 -1
  164. package/dist/rag/handlers/close-workspace.js +1 -1
  165. package/dist/rag/handlers/delete-embeddings.js +1 -1
  166. package/dist/rag/handlers/delete-workspace.js +2 -6
  167. package/dist/rag/handlers/ingest.js +1 -1
  168. package/dist/rag/handlers/list-workspaces.d.ts +1 -1
  169. package/dist/rag/handlers/list-workspaces.js +1 -1
  170. package/dist/rag/handlers/reindex.js +1 -1
  171. package/dist/rag/handlers/save-embeddings.js +1 -1
  172. package/dist/rag/handlers/search.js +1 -1
  173. package/dist/rag/index.d.ts +11 -11
  174. package/dist/rag/index.js +11 -11
  175. package/dist/rag/rag-workspace-manager.d.ts +4 -2
  176. package/dist/rag/rag-workspace-manager.js +268 -15
  177. package/dist/registry.js +8 -58
  178. package/dist/resources/collector.d.ts +3 -1
  179. package/dist/resources/collector.js +91 -9
  180. package/dist/resources/instance.d.ts +1 -1
  181. package/dist/resources/instance.js +1 -1
  182. package/dist/resources/model-fit/assess.d.ts +36 -0
  183. package/dist/resources/model-fit/assess.js +578 -0
  184. package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +8 -0
  185. package/dist/resources/model-fit/calibration/darwin-arm64.js +36 -0
  186. package/dist/resources/model-fit/calibration/darwin-x64.d.ts +8 -0
  187. package/dist/resources/model-fit/calibration/darwin-x64.js +38 -0
  188. package/dist/resources/model-fit/calibration/fit.d.ts +73 -0
  189. package/dist/resources/model-fit/calibration/fit.js +152 -0
  190. package/dist/resources/model-fit/calibration/index.d.ts +37 -0
  191. package/dist/resources/model-fit/calibration/index.js +75 -0
  192. package/dist/resources/model-fit/calibration/linux-arm64.d.ts +8 -0
  193. package/dist/resources/model-fit/calibration/linux-arm64.js +38 -0
  194. package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +8 -0
  195. package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +39 -0
  196. package/dist/resources/model-fit/calibration/linux-x64.d.ts +8 -0
  197. package/dist/resources/model-fit/calibration/linux-x64.js +38 -0
  198. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +8 -0
  199. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +39 -0
  200. package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +8 -0
  201. package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +36 -0
  202. package/dist/resources/model-fit/calibration/win32-x64.d.ts +8 -0
  203. package/dist/resources/model-fit/calibration/win32-x64.js +38 -0
  204. package/dist/resources/model-fit/estimators/llm.d.ts +45 -0
  205. package/dist/resources/model-fit/estimators/llm.js +220 -0
  206. package/dist/resources/model-fit/estimators/whisper.d.ts +10 -0
  207. package/dist/resources/model-fit/estimators/whisper.js +77 -0
  208. package/dist/resources/model-fit/types.d.ts +120 -0
  209. package/dist/resources/model-fit/types.js +1 -0
  210. package/dist/resources/native.d.ts +1 -1
  211. package/dist/resources/native.js +14 -0
  212. package/dist/resources/normalize.d.ts +1 -1
  213. package/dist/resources/types.d.ts +16 -0
  214. package/dist/runtime/index.d.ts +5 -5
  215. package/dist/runtime/index.js +2 -2
  216. package/dist/runtime/lifecycle.js +5 -7
  217. package/dist/runtime/model-registry.d.ts +0 -14
  218. package/dist/runtime/model-registry.js +17 -40
  219. package/dist/runtime/registry-client.js +1 -1
  220. package/dist/runtime/request-context.d.ts +3 -3
  221. package/dist/runtime/request-context.js +13 -1
  222. package/dist/runtime/request-registry.d.ts +1 -1
  223. package/dist/runtime/request-registry.js +4 -5
  224. package/dist/runtime/state.d.ts +1 -1
  225. package/dist/runtime/state.js +18 -3
  226. package/dist/schemas/assess-model-fit.d.ts +271 -0
  227. package/dist/schemas/assess-model-fit.js +153 -0
  228. package/dist/schemas/audio-gen.d.ts +434 -4
  229. package/dist/schemas/audio-gen.js +131 -19
  230. package/dist/schemas/batch-completion-stream.d.ts +2 -2
  231. package/dist/schemas/batch-completion-stream.js +3 -3
  232. package/dist/schemas/bci-config.js +73 -27
  233. package/dist/schemas/bci.d.ts +2 -2
  234. package/dist/schemas/bci.js +5 -5
  235. package/dist/schemas/cancel.d.ts +5 -0
  236. package/dist/schemas/cancel.js +1 -0
  237. package/dist/schemas/classification.js +13 -6
  238. package/dist/schemas/common.d.ts +662 -331
  239. package/dist/schemas/common.js +33 -31
  240. package/dist/schemas/completion-event.d.ts +2 -2
  241. package/dist/schemas/completion-event.js +1 -1
  242. package/dist/schemas/completion-stream.d.ts +1 -5
  243. package/dist/schemas/completion-stream.js +4 -8
  244. package/dist/schemas/config.d.ts +640 -18
  245. package/dist/schemas/config.js +50 -8
  246. package/dist/schemas/download-asset.d.ts +19 -1
  247. package/dist/schemas/download-asset.js +20 -4
  248. package/dist/schemas/engine-addon-map.d.ts +1 -1
  249. package/dist/schemas/engine-addon-map.js +3 -3
  250. package/dist/schemas/error.d.ts +2 -2
  251. package/dist/schemas/error.js +2 -2
  252. package/dist/schemas/errors.d.ts +2 -6
  253. package/dist/schemas/errors.js +30 -40
  254. package/dist/schemas/heartbeat.d.ts +10 -0
  255. package/dist/schemas/heartbeat.js +8 -0
  256. package/dist/schemas/index.d.ts +46 -44
  257. package/dist/schemas/index.js +45 -43
  258. package/dist/schemas/llamacpp-config.d.ts +32 -10
  259. package/dist/schemas/llamacpp-config.js +11 -8
  260. package/dist/schemas/load-model.d.ts +2933 -1948
  261. package/dist/schemas/load-model.js +52 -52
  262. package/dist/schemas/mcp-adapter.d.ts +1 -1
  263. package/dist/schemas/model-config-schemas.d.ts +2354 -0
  264. package/dist/schemas/model-config-schemas.js +46 -0
  265. package/dist/schemas/model-info.d.ts +10 -20
  266. package/dist/schemas/model-info.js +4 -25
  267. package/dist/schemas/model-ops.d.ts +0 -1
  268. package/dist/schemas/model-ops.js +1 -2
  269. package/dist/schemas/model-resource-profile.d.ts +109 -0
  270. package/dist/schemas/model-resource-profile.js +83 -0
  271. package/dist/schemas/model-src-utils.d.ts +1 -0
  272. package/dist/schemas/model-src-utils.js +33 -13
  273. package/dist/schemas/ocr.js +48 -13
  274. package/dist/schemas/path-validation.js +8 -2
  275. package/dist/schemas/plugin.d.ts +14 -1
  276. package/dist/schemas/plugin.js +11 -0
  277. package/dist/schemas/profiling.d.ts +138 -43
  278. package/dist/schemas/profiling.js +1 -22
  279. package/dist/schemas/rag.js +1 -1
  280. package/dist/schemas/registry.js +1 -1
  281. package/dist/schemas/reload-config.d.ts +0 -2
  282. package/dist/schemas/reload-config.js +2 -3
  283. package/dist/schemas/sdcpp-config.d.ts +344 -4
  284. package/dist/schemas/sdcpp-config.js +504 -17
  285. package/dist/schemas/system-resources.d.ts +138 -0
  286. package/dist/schemas/system-resources.js +3 -1
  287. package/dist/schemas/text-to-speech.js +183 -76
  288. package/dist/schemas/transcription-config.js +253 -85
  289. package/dist/schemas/transcription.d.ts +3 -3
  290. package/dist/schemas/translate.d.ts +2 -2
  291. package/dist/schemas/translate.js +1 -1
  292. package/dist/schemas/translation-config.js +53 -25
  293. package/dist/schemas/vla.d.ts +1 -1
  294. package/dist/selection.d.ts +1 -5
  295. package/dist/selection.js +2 -6
  296. package/dist/surface.d.ts +1 -1
  297. package/dist/utils/aggregate-events.d.ts +1 -1
  298. package/dist/utils/aggregate-events.js +2 -2
  299. package/dist/utils/archive.js +2 -2
  300. package/dist/utils/audio-input.js +1 -1
  301. package/dist/utils/cache/auto-history.d.ts +1 -1
  302. package/dist/utils/cache/auto-history.js +1 -1
  303. package/dist/utils/cache/paths.js +5 -5
  304. package/dist/utils/completion-normalizer.js +1 -1
  305. package/dist/utils/gguf-tensor-extractor.js +1 -1
  306. package/dist/utils/image-dimensions.d.ts +26 -0
  307. package/dist/utils/image-dimensions.js +46 -0
  308. package/dist/utils/index.d.ts +13 -10
  309. package/dist/utils/index.js +13 -10
  310. package/dist/utils/mcp-adapter.d.ts +1 -1
  311. package/dist/utils/mcp-adapter.js +1 -1
  312. package/dist/utils/path-sanitize.d.ts +7 -0
  313. package/dist/utils/path-sanitize.js +36 -0
  314. package/dist/utils/path-security.d.ts +0 -7
  315. package/dist/utils/path-security.js +1 -36
  316. package/dist/utils/shard-utils.js +3 -3
  317. package/dist/utils/tool-integration.js +1 -1
  318. package/dist/utils/tools/dialect.js +1 -1
  319. package/dist/utils/tools/index.d.ts +2 -2
  320. package/dist/utils/tools/index.js +2 -2
  321. package/dist/utils/tools/parser.js +8 -8
  322. package/dist/utils/tools/parsers/dsml.d.ts +1 -1
  323. package/dist/utils/tools/parsers/dsml.js +1 -1
  324. package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
  325. package/dist/utils/tools/parsers/gemma4native.js +1 -1
  326. package/dist/utils/tools/parsers/harmony.d.ts +1 -1
  327. package/dist/utils/tools/parsers/harmony.js +1 -1
  328. package/dist/utils/tools/parsers/hermes.d.ts +1 -1
  329. package/dist/utils/tools/parsers/hermes.js +1 -1
  330. package/dist/utils/tools/parsers/json.d.ts +1 -1
  331. package/dist/utils/tools/parsers/json.js +1 -1
  332. package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
  333. package/dist/utils/tools/parsers/pythonic.js +1 -1
  334. package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
  335. package/dist/utils/tools/parsers/qwen35.js +3 -3
  336. package/dist/utils/url-security.d.ts +17 -0
  337. package/dist/utils/url-security.js +75 -0
  338. package/package.json +31 -26
  339. package/dist/api/provide.d.ts +0 -23
  340. package/dist/api/provide.js +0 -32
  341. package/dist/api/stop-provider.d.ts +0 -20
  342. package/dist/api/stop-provider.js +0 -30
  343. package/dist/p2p/cancel-delegated.d.ts +0 -3
  344. package/dist/p2p/cancel-delegated.js +0 -64
  345. package/dist/p2p/completion-stream-delegated.d.ts +0 -4
  346. package/dist/p2p/completion-stream-delegated.js +0 -51
  347. package/dist/p2p/delegate-client.d.ts +0 -9
  348. package/dist/p2p/delegate-client.js +0 -219
  349. package/dist/p2p/delegate-connect-diagnostics.d.ts +0 -8
  350. package/dist/p2p/delegate-connect-diagnostics.js +0 -38
  351. package/dist/p2p/delegate-transport.d.ts +0 -18
  352. package/dist/p2p/delegate-transport.js +0 -237
  353. package/dist/p2p/error-handlers.d.ts +0 -4
  354. package/dist/p2p/error-handlers.js +0 -31
  355. package/dist/p2p/handle-request.d.ts +0 -2
  356. package/dist/p2p/handle-request.js +0 -171
  357. package/dist/p2p/handler-utils.d.ts +0 -19
  358. package/dist/p2p/handler-utils.js +0 -180
  359. package/dist/p2p/heartbeat-delegated.d.ts +0 -3
  360. package/dist/p2p/heartbeat-delegated.js +0 -32
  361. package/dist/p2p/load-model-delegated.d.ts +0 -6
  362. package/dist/p2p/load-model-delegated.js +0 -132
  363. package/dist/p2p/progress-throttle.d.ts +0 -7
  364. package/dist/p2p/progress-throttle.js +0 -47
  365. package/dist/p2p/provideHandler/connection.d.ts +0 -2
  366. package/dist/p2p/provideHandler/connection.js +0 -38
  367. package/dist/p2p/provideHandler/handler.d.ts +0 -2
  368. package/dist/p2p/provideHandler/handler.js +0 -66
  369. package/dist/p2p/provideHandler/proxy.d.ts +0 -2
  370. package/dist/p2p/provideHandler/proxy.js +0 -24
  371. package/dist/p2p/provideHandler.d.ts +0 -1
  372. package/dist/p2p/provideHandler.js +0 -2
  373. package/dist/p2p/rpc-utils.d.ts +0 -4
  374. package/dist/p2p/rpc-utils.js +0 -8
  375. package/dist/p2p/stopProvideHandler.d.ts +0 -2
  376. package/dist/p2p/stopProvideHandler.js +0 -29
  377. package/dist/p2p/swarm.d.ts +0 -9
  378. package/dist/p2p/swarm.js +0 -103
  379. package/dist/p2p/transport-selector.d.ts +0 -2
  380. package/dist/p2p/transport-selector.js +0 -18
  381. package/dist/p2p/unload-model-delegated.d.ts +0 -2
  382. package/dist/p2p/unload-model-delegated.js +0 -31
  383. package/dist/profiling/delegation-profiler.d.ts +0 -31
  384. package/dist/profiling/delegation-profiler.js +0 -155
  385. package/dist/schemas/delegate.d.ts +0 -29
  386. package/dist/schemas/delegate.js +0 -40
  387. package/dist/schemas/provide.d.ts +0 -50
  388. package/dist/schemas/provide.js +0 -32
@@ -1,8 +1,9 @@
1
- import { logCacheDisabled, logCacheInit, logCacheSave, logMessagesToAddon } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
2
- import { extractSystemPrompt, getCurrentCacheInfo } from '../../../../plugins/ops/kv-cache-utils.js';
1
+ import { logCacheDisabled, logCacheInit, logCacheSave, logMessagesToAddon } from './cache-logger.js';
2
+ import { extractSystemPrompt, getCurrentCacheInfo } from '../../../ops/kv-cache-utils.js';
3
+ import { isAddonPreMutationRefusal, isAddonContextOverflowError } from './context-overflow.js';
3
4
  import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
4
- import { decideCachedHistorySlice, shouldCommitCachedTurn } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-state.js';
5
- import { createKvCacheSession, generateConfigHash } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
5
+ import { decideCachedHistorySlice, shouldCommitCachedTurn } from './kv-cache-state.js';
6
+ import { createKvCacheSession, generateConfigHash } from './kv-cache-session.js';
6
7
  import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
7
8
  import { parseToolCalls } from '../../../../utils/tools/index.js';
8
9
  import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
@@ -11,7 +12,7 @@ import { getEngineLogger } from '../../../../logging/index.js';
11
12
  import { AttachmentNotFoundError } from '../../../../errors/index.js';
12
13
  import { nowMs } from '../../../../profiling/index.js';
13
14
  import { buildStreamResult } from '../../../../profiling/model-execution.js';
14
- import { normalizeCompletionStats, withEmittedTokens } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
15
+ import { normalizeCompletionStats, withEmittedTokens } from './completion-stats.js';
15
16
  import fs from 'bare-fs';
16
17
  const logger = getEngineLogger();
17
18
  function transformMessage(message) {
@@ -119,7 +120,7 @@ function rendersToolBlock(messages, toolBlock) {
119
120
  * - The tool block travels only with the turn that writes it into the
120
121
  * cache; see `skipToolBlock` below.
121
122
  */
122
- function prepareMessagesForCache(session, turn, cacheExists, history, tools, toolBlockEvictable = false) {
123
+ function prepareMessagesForCache(session, turn, cacheExists, history, tools) {
123
124
  const toolBlock = tools?.length ? transformMessages(tools) : [];
124
125
  if (!(cacheExists && history.length > 0)) {
125
126
  const historyWithoutSystem = history.filter((msg) => msg.role !== 'system');
@@ -146,9 +147,8 @@ function prepareMessagesForCache(session, turn, cacheExists, history, tools, too
146
147
  // conversation. Skip it only when the prefix is known to hold a rendered
147
148
  // one: `toolBlockCached` records that a previous turn actually got it into
148
149
  // the cache, which a committed message count does not prove. A stale
149
- // boundary means we are resending the whole conversation anyway, and an
150
- // evictable block can no longer be assumed present.
151
- const skipToolBlock = turn.toolBlockCached && !clearStaleCount && !toolBlockEvictable;
150
+ // boundary means we are resending the whole conversation anyway.
151
+ const skipToolBlock = turn.toolBlockCached && !clearStaleCount;
152
152
  const blockToSend = skipToolBlock ? [] : toolBlock;
153
153
  return {
154
154
  messages: withToolBlock(transformMessages(messages), blockToSend),
@@ -208,12 +208,6 @@ export async function* completion(params, opts) {
208
208
  const modelConfig = getModelConfig(modelId);
209
209
  const toolsEnabled = modelConfig.tools === true;
210
210
  const toolsActive = !!tools?.length && toolsEnabled;
211
- // Sliding is opt-in (`n_discarded` defaults to 0). Once on, the addon's
212
- // discard window opens at the end of the primed prefix — which is where the
213
- // tool block sits, since the prime is the system prompt alone — and nothing
214
- // protects it. So while sliding is possible the block cannot be assumed to
215
- // survive, and it has to travel with every turn.
216
- const toolBlockEvictable = (modelConfig.n_discarded ?? 0) > 0;
217
211
  const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
218
212
  // `responseFormat` is forwarded to the addon as a per-request
219
213
  // `generationParams.json_schema`, which the addon converts to GBNF and
@@ -273,12 +267,11 @@ export async function* completion(params, opts) {
273
267
  logMessagesToAddon(transformedHistory, 'NO_CACHE');
274
268
  return yield* processModelResponse(model, transformedHistory, tools, mergedGenerationParams, undefined, dialect, setActiveResponse);
275
269
  }
276
- // ---- KV-cache path. The session owns all three bookkeeping layers
277
- // (on-disk `.bin`, `initializedCaches`, `cachedMessageCounts`). The
278
- // handler asks for a turn, registers rollback on the scope, and on
279
- // the happy path calls `commitTurn` which short-circuits the deferred
280
- // rollback. Cancellations / zero-token replies / rename failures all
281
- // unwind through the same `scope.defer` hook. ----
270
+ // ---- KV-cache path. The session owns every bookkeeping layer; the
271
+ // handler registers one deferred unwind (`rollback`, or the non-destructive
272
+ // `releaseTurn` on a recognised pre-mutation refusal) that `commitTurn`
273
+ // short-circuits on the happy path. Cancellations / zero-token replies /
274
+ // rename failures all still unwind destructively through the same hook. ----
282
275
  const session = createKvCacheSession(modelId, { logger: requestLogger });
283
276
  const systemPromptFromHistory = extractSystemPrompt(history);
284
277
  // The tool block is baked into the cache on the turn that first sends it and
@@ -319,16 +312,34 @@ export async function* completion(params, opts) {
319
312
  // flips the turn's internal `committed` flag so this becomes a no-op
320
313
  // on the happy path. Scope unwinding is LIFO — registered after the
321
314
  // `removeEventListener` defer above so rollback runs before the
322
- // listener detach.
323
- scope.defer(() => session.rollback(turn));
315
+ // listener detach. A thrown overflow, pre-mutation refusal, or pre-addon
316
+ // attachment failure never persists the turn; the committed cache stays valid.
317
+ let preserveCacheOnUnwind = false;
318
+ scope.defer(() => (preserveCacheOnUnwind ? session.releaseTurn(turn) : session.rollback(turn)));
324
319
  // `cacheExists` is implied by `beginTurn` — the session either found
325
320
  // an existing cache or just primed one. Pass `true` to the message
326
321
  // selector so the slicing branches engage.
327
- const payload = prepareMessagesForCache(session, turn,
328
- /* cacheExists */ true, history, toolsActive ? tools : undefined, toolBlockEvictable);
322
+ let payload;
323
+ try {
324
+ payload = prepareMessagesForCache(session, turn,
325
+ /* cacheExists */ true, history, toolsActive ? tools : undefined);
326
+ }
327
+ catch (error) {
328
+ // A missing attachment is caller input rejected before the addon runs,
329
+ // so the committed cache is untouched and must survive.
330
+ preserveCacheOnUnwind = error instanceof AttachmentNotFoundError;
331
+ throw error;
332
+ }
329
333
  const messagesToSend = payload.messages;
330
334
  logMessagesToAddon(messagesToSend, 'PROMPT_SEND');
331
- const result = yield* processModelResponse(model, messagesToSend, tools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse);
335
+ let result;
336
+ try {
337
+ result = yield* processModelResponse(model, messagesToSend, tools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse);
338
+ }
339
+ catch (error) {
340
+ preserveCacheOnUnwind = isAddonContextOverflowError(error) || isAddonPreMutationRefusal(error);
341
+ throw error;
342
+ }
332
343
  const shouldCommitTurn = shouldCommitCachedTurn({
333
344
  aborted: signal.aborted,
334
345
  producedTokens: result.producedTokens,
@@ -6,23 +6,24 @@
6
6
  * msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
7
7
  * `qvac_errors::StatusError::codeString()` and formats as
8
8
  * `"[ <addonId> :: ContextOverflow ]"`. The message carries the
9
- * C++-formatted detail (which may include the prompt/ctx sizes).
9
+ * C++-formatted detail (which may include the prompt/ctx sizes). That
10
+ * codeString survives only the synchronous throw path — asynchronous run
11
+ * errors are transported as `exception.what()` alone, with no code.
10
12
  *
11
13
  * These helpers let the plugin handler convert that addon error into a
12
14
  * typed `ContextOverflowError` and let unit tests assert the detection
13
15
  * + extraction logic without a real model load.
14
16
  */
15
17
  export declare function isAddonContextOverflowError(err: unknown): boolean;
16
- /**
17
- * Best-effort extraction of `promptTokens` / `ctxSize` from the addon
18
- * error message. The C++ paths in `TextLlmContext.cpp` and
19
- * `MtmdLlmContext.cpp` format both numbers into the message; the
20
- * `LlamaModel::processPromptImpl` fallback path emits a bare
21
- * `"<func>: context overflow\n"` with neither number. Returning
22
- * `undefined` for either field is fine — `ContextOverflowError` holds
23
- * them as optional and the message factory degrades gracefully.
24
- */
25
- export declare function parseContextOverflowMessage(message: string): {
18
+ export declare function isAddonPreMutationRefusal(err: unknown): boolean;
19
+ export type ContextOverflowSizes = {
20
+ /** The prompt alone, in tokens; unset when the guard reports KV cells. */
26
21
  promptTokens?: number;
22
+ /** Cached conversation on a warm-cache guard, in `ctxSize` units. */
23
+ cachedTokens?: number;
24
+ /** Total the failing guard reported, in `ctxSize` units; the guards trigger on `>=`, so it can equal the window. */
25
+ requiredTokens?: number;
26
+ /** Effective per-request ceiling: `ctx_size` split across slots at `parallel > 1`. */
27
27
  ctxSize?: number;
28
28
  };
29
+ export declare function parseContextOverflowMessage(message: string): ContextOverflowSizes;
@@ -6,60 +6,152 @@
6
6
  * msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
7
7
  * `qvac_errors::StatusError::codeString()` and formats as
8
8
  * `"[ <addonId> :: ContextOverflow ]"`. The message carries the
9
- * C++-formatted detail (which may include the prompt/ctx sizes).
9
+ * C++-formatted detail (which may include the prompt/ctx sizes). That
10
+ * codeString survives only the synchronous throw path — asynchronous run
11
+ * errors are transported as `exception.what()` alone, with no code.
10
12
  *
11
13
  * These helpers let the plugin handler convert that addon error into a
12
14
  * typed `ContextOverflowError` and let unit tests assert the detection
13
15
  * + extraction logic without a real model load.
14
16
  */
17
+ // The guards' emitted starts; the numeric tails vary across addon
18
+ // generations, so the forms are start-anchored and single-line.
19
+ const CONTEXT_OVERFLOW_FORMS = [
20
+ /^\[TextLlm\] context overflow at (?:batch )?prefill step[^\r\n]*$/,
21
+ /^\[MtmdLlm\] context overflow at (?:batch )?prefill step[^\r\n]*$/,
22
+ // Unreachable at the pinned addon generation (generateResponse never
23
+ // reports !ok); kept in case a patch revives the path.
24
+ /^processPromptImpl: context overflow$/,
25
+ // The batch scheduler's capacity refusals are the same out-of-context
26
+ // condition, reported under a generic status.
27
+ /^ContinuousBatchScheduler::submit: prompt of \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
28
+ /^ContinuousBatchScheduler::submit: prompt of \d+ tokens leaves no room under per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
29
+ /^ContinuousBatchScheduler::submit: prefill prompt of \d+ tokens exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
30
+ /^ContinuousBatchScheduler::submit: n_predict [1-9]\d* \+ prompt \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/
31
+ ];
32
+ // Both detectors classify raw addon errors only — an already-typed QvacError
33
+ // (numeric `code`) is deliberately rejected by the present-code gate.
15
34
  export function isAddonContextOverflowError(err) {
16
35
  if (typeof err !== 'object' || err === null)
17
36
  return false;
37
+ // A present status code is authoritative: exact ContextOverflow accepts,
38
+ // anything else (a non-string included) rejects with no message fallback.
18
39
  const code = err.code;
19
- // Anchor to the codeString tail (`[ <addonId> :: ContextOverflow ]`)
20
- // so we don't false-positive on a hypothetical sibling like
21
- // `ContextOverflowRecovered` or `PostContextOverflow` after an
22
- // addon-side rename.
23
- if (typeof code === 'string' && /::\s*ContextOverflow\s*\]/.test(code)) {
24
- return true;
40
+ if (code !== undefined) {
41
+ return typeof code === 'string' && /^\[\s*[\w.-]+\s*::\s*ContextOverflow\s*\]$/.test(code);
25
42
  }
26
- // Message-substring fallback for the bare `LlamaModel::processPromptImpl`
27
- // path, which doesn't go through `StatusError::codeString()`. Match the
28
- // two known C++-emitted formats only — broader substring would catch
29
- // wrapper / cause-chain text that mentions overflow without being one.
43
+ // The production path: the async transport strips status metadata from
44
+ // every run error, so the message alone identifies the overflow guards.
30
45
  const message = err.message;
31
- return (typeof message === 'string' &&
32
- /(?:context overflow at prefill step|processPromptImpl: context overflow)/i.test(message));
46
+ if (typeof message !== 'string')
47
+ return false;
48
+ const trimmed = message.trim();
49
+ return CONTEXT_OVERFLOW_FORMS.some((form) => form.test(trimmed));
33
50
  }
34
- /**
35
- * Best-effort extraction of `promptTokens` / `ctxSize` from the addon
36
- * error message. The C++ paths in `TextLlmContext.cpp` and
37
- * `MtmdLlmContext.cpp` format both numbers into the message; the
38
- * `LlamaModel::processPromptImpl` fallback path emits a bare
39
- * `"<func>: context overflow\n"` with neither number. Returning
40
- * `undefined` for either field is fine — `ContextOverflowError` holds
41
- * them as optional and the message factory degrades gracefully.
42
- */
43
- export function parseContextOverflowMessage(message) {
44
- // "[TextLlm] context overflow at prefill step: prompt tokens N, max context tokens M\n"
45
- // Same-clause separator (no `[^]*?` cross-newline walk) so a future
46
- // wrapper that pastes overflow text alongside unrelated numbers can't
47
- // produce a mismatched pair.
48
- const longForm = message.match(/prompt tokens (\d+)[,\s]+max context tokens (\d+)/i);
49
- if (longForm?.[1] && longForm[2]) {
50
- return {
51
- promptTokens: Number(longForm[1]),
52
- ctxSize: Number(longForm[2])
53
- };
51
+ // Refusals thrown before any decode or disk save; the async transport delivers
52
+ // exception.what() alone, so each form is matched complete and end-anchored.
53
+ const PRE_MUTATION_REFUSAL_FORMS = [
54
+ /^ContinuousBatchScheduler::submit: prompt of \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
55
+ /^ContinuousBatchScheduler::submit: prompt of \d+ tokens leaves no room under per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
56
+ /^ContinuousBatchScheduler::submit: prefill prompt of \d+ tokens exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
57
+ /^ContinuousBatchScheduler::submit: n_predict [1-9]\d* \+ prompt \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
58
+ /^ContinuousBatchScheduler::submit: failed to add to batch \(MultiRequestBatcher::AddStatus=[1-4]\)$/,
59
+ /^invalid generationParams\.json_schema: [^\r\n]*$/,
60
+ /^failed to initialise sampler with per-request generationParams \(invalid grammar or json_schema\?\)$/,
61
+ /^\[MtmdLlm\] Media buffer is empty$/,
62
+ /^\[MtmdLlm\] Filename is empty$/,
63
+ /^\[MtmdLlm\] Failed to load media from file: [^\r\n]*$/,
64
+ /^\[MtmdLlm\] preparePrefill: prompt must end with text after the last media item$/
65
+ ];
66
+ export function isAddonPreMutationRefusal(err) {
67
+ if (typeof err !== 'object' || err === null)
68
+ return false;
69
+ // A status code only exists on the synchronous throw path; when present it
70
+ // must be InvalidArgument, and its absence (the async transport) is fine.
71
+ const code = err.code;
72
+ if (code !== undefined &&
73
+ (typeof code !== 'string' || !/^\[\s*[\w.-]+\s*::\s*InvalidArgument\s*\]$/.test(code))) {
74
+ return false;
75
+ }
76
+ const message = err.message;
77
+ if (typeof message !== 'string')
78
+ return false;
79
+ const trimmed = message.trim();
80
+ return PRE_MUTATION_REFUSAL_FORMS.some((form) => form.test(trimmed));
81
+ }
82
+ // One entry per number-formatting guard, most specific first; keep in step
83
+ // with TextLlmContext.cpp / MtmdLlmContext.cpp or fields silently vanish.
84
+ const MESSAGE_PATTERNS = [
85
+ {
86
+ // "cached tokens C plus prompt tokens N exceed the max context tokens M"
87
+ pattern: /cached tokens (\d+) plus prompt tokens (\d+) exceeds? the max context tokens (\d+)/i,
88
+ map: ([cached, prompt, ctx]) => ({
89
+ promptTokens: prompt,
90
+ cachedTokens: cached,
91
+ requiredTokens: cached + prompt,
92
+ ctxSize: ctx
93
+ })
94
+ },
95
+ {
96
+ // "cached P positions / C KV cells plus ... of prompt exceed the max
97
+ // context tokens M" — KV cells, not tokens, so promptTokens stays unset.
98
+ pattern: /cached (\d+) positions \/ (\d+) KV cells plus (\d+) positions \/ (\d+) KV cells of prompt exceeds? the max context tokens (\d+)/i,
99
+ map: ([cPos, cCells, pPos, pCells, ctx]) => ({
100
+ cachedTokens: Math.max(cPos, cCells),
101
+ requiredTokens: Math.max(cPos + pPos, cCells + pCells),
102
+ ctxSize: ctx
103
+ })
104
+ },
105
+ {
106
+ // "prompt tokens N, max context tokens M"
107
+ pattern: /prompt tokens (\d+)[, \t]+max context tokens (\d+)/i,
108
+ map: ([prompt, ctx]) => ({ promptTokens: prompt, requiredTokens: prompt, ctxSize: ctx })
109
+ },
110
+ {
111
+ // "prompt spans P positions / N KV cells, max context tokens M" —
112
+ // not tokens, so promptTokens stays unset.
113
+ pattern: /prompt spans (\d+) positions \/ (\d+) KV cells,[ \t]*max context tokens (\d+)/i,
114
+ map: ([pos, cells, ctx]) => ({ requiredTokens: Math.max(pos, cells), ctxSize: ctx })
115
+ },
116
+ {
117
+ // "(N tokens, P positions, max M)" — the "tokens" figure is
118
+ // mtmd_helper_get_n_tokens, i.e. KV cells, so promptTokens stays unset.
119
+ pattern: /\((\d+)[ \t]+tokens,[ \t]*(\d+)[ \t]+positions,[ \t]*max[ \t]+(\d+)\)/i,
120
+ map: ([cells, pos, ctx]) => ({
121
+ requiredTokens: Math.max(cells, pos),
122
+ ctxSize: ctx
123
+ })
124
+ },
125
+ {
126
+ // "(N tokens, max M)", the retired short form. Both of its emitters
127
+ // format a cached total, not the prompt alone — promptTokens stays unset.
128
+ pattern: /\((\d+)[ \t]+tokens,[ \t]*max[ \t]+(\d+)\)/i,
129
+ map: ([total, ctx]) => ({ requiredTokens: total, ctxSize: ctx })
130
+ },
131
+ {
132
+ // Scheduler cap refusals: the total is cache-plus-prompt in cells or
133
+ // tokens, and the cap is the effective per-request ceiling.
134
+ pattern: /prompt of (\d+) (?:KV cells|tokens) (?:exceeds|leaves no room under) per-sequence cap (\d+)/,
135
+ map: ([total, cap]) => ({ requiredTokens: total, ctxSize: cap })
136
+ },
137
+ {
138
+ // "n_predict P + prompt N KV cells exceeds per-sequence cap M" — the
139
+ // reservation plus the prompt is the space the request needs.
140
+ pattern: /n_predict (\d+) \+ prompt (\d+) KV cells exceeds per-sequence cap (\d+)/,
141
+ map: ([predict, cells, cap]) => ({ requiredTokens: predict + cells, ctxSize: cap })
54
142
  }
55
- // "[TextLlm] context overflow at prefill step (N tokens, max M)\n"
56
- // "[MtmdLlm] context overflow at prefill step (N tokens, max M)\n"
57
- const shortForm = message.match(/\((\d+)\s+tokens,\s*max\s+(\d+)\)/i);
58
- if (shortForm?.[1] && shortForm[2]) {
59
- return {
60
- promptTokens: Number(shortForm[1]),
61
- ctxSize: Number(shortForm[2])
62
- };
143
+ ];
144
+ // Best-effort extraction of the overflow sizes; the bare processPromptImpl
145
+ // path carries no numbers, and undefined fields degrade gracefully.
146
+ export function parseContextOverflowMessage(message) {
147
+ for (const { pattern, map } of MESSAGE_PATTERNS) {
148
+ const match = message.match(pattern);
149
+ if (!match)
150
+ continue;
151
+ const numbers = match.slice(1).map(Number);
152
+ if (!numbers.every(Number.isFinite))
153
+ continue;
154
+ return map(numbers);
63
155
  }
64
156
  return {};
65
157
  }
@@ -1,4 +1,4 @@
1
- import { generateConfigHash } from '../../../../plugins/ops/kv-cache-utils.js';
1
+ import { generateConfigHash } from '../../../ops/kv-cache-utils.js';
2
2
  import { type CacheMessage } from '../../../../utils/index.js';
3
3
  import type { Logger } from '../../../../logging/types.js';
4
4
  import { type AbortSignal } from 'bare-abort-controller';
@@ -106,6 +106,12 @@ export interface KvCacheSession {
106
106
  * itself via `commitTurn`).
107
107
  */
108
108
  rollback(turn: TurnHandle): Promise<void>;
109
+ /**
110
+ * Non-destructive counterpart of `rollback`: releases locks and refs but
111
+ * keeps the committed disk cache and its recorded prefix valid for a retry.
112
+ * A cache freshly primed by this same turn rolls back instead.
113
+ */
114
+ releaseTurn(turn: TurnHandle): Promise<void>;
109
115
  /**
110
116
  * Forget the in-memory saved-message count for the turn's path
111
117
  * without unlinking the file or clearing the init flag. Used when
@@ -1,10 +1,10 @@
1
1
  import { promises as fsPromises } from 'bare-fs';
2
2
  import path from 'bare-path';
3
- import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../../plugins/ops/kv-cache-utils.js';
4
- import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../../plugins/ops/kv-cache-retention.js';
3
+ import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../ops/kv-cache-utils.js';
4
+ import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../ops/kv-cache-retention.js';
5
5
  import { isMobile } from '../../../../runtime/state.js';
6
6
  import { getKVCacheDir } from '../../../../utils/index.js';
7
- import { logCacheSaveError, logCacheStatus } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
7
+ import { logCacheSaveError, logCacheStatus } from './cache-logger.js';
8
8
  import { getEngineLogger } from '../../../../logging/index.js';
9
9
  // Used by cross-model paths that have no `RequestContext` (e.g.
10
10
  // `deleteKvCacheState`). Per-session call sites receive a logger from
@@ -218,7 +218,8 @@ export function createKvCacheSession(modelId, options) {
218
218
  ...(signal !== undefined && { signal }),
219
219
  releaseWriteLock,
220
220
  committed: false,
221
- rolledBack: false
221
+ rolledBack: false,
222
+ freshlyPrimed: false
222
223
  });
223
224
  markCachePathActive(cachePath);
224
225
  return handle;
@@ -277,6 +278,7 @@ export function createKvCacheSession(modelId, options) {
277
278
  await input.primeIfMissing(cachePath);
278
279
  await verifyPrimedFile(cachePath, logger);
279
280
  initializedCaches.add(cachePath);
281
+ turnState.get(handle).freshlyPrimed = true;
280
282
  }
281
283
  return handle;
282
284
  }
@@ -362,6 +364,7 @@ export function createKvCacheSession(modelId, options) {
362
364
  await input.primeIfMissing(cachePath);
363
365
  await verifyPrimedFile(cachePath, logger);
364
366
  initializedCaches.add(cachePath);
367
+ turnState.get(handle).freshlyPrimed = true;
365
368
  }
366
369
  return handle;
367
370
  }
@@ -485,6 +488,24 @@ export function createKvCacheSession(modelId, options) {
485
488
  return;
486
489
  await runRollback(state);
487
490
  }
491
+ async function releaseTurn(turn) {
492
+ const state = turnState.get(turn);
493
+ if (!state)
494
+ return;
495
+ if (state.committed || state.rolledBack)
496
+ return;
497
+ // A cache this same turn primed has no committed state to keep — a failed
498
+ // first turn must not leave its own prime behind.
499
+ if (state.freshlyPrimed) {
500
+ await runRollback(state);
501
+ return;
502
+ }
503
+ releaseCachePath(state.cachePath);
504
+ state.rolledBack = true;
505
+ state.releaseWriteLock();
506
+ if (state.autoCacheKey !== undefined)
507
+ scheduleAutoCacheSweep(logger);
508
+ }
488
509
  async function runRollback(state) {
489
510
  // Order matters only weakly: unlink first so a partial disk-state
490
511
  // can't be re-loaded by a sibling turn between the file delete and
@@ -520,6 +541,7 @@ export function createKvCacheSession(modelId, options) {
520
541
  beginTurn,
521
542
  commitTurn,
522
543
  rollback,
544
+ releaseTurn,
523
545
  dropStaleSavedCount
524
546
  };
525
547
  }
@@ -14,7 +14,13 @@ export declare const llmPlugin: {
14
14
  device: import("zod").ZodOptional<import("zod").ZodString>;
15
15
  predict: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodLiteral<-1>, import("zod").ZodLiteral<-2>, import("zod").ZodNumber]>>;
16
16
  system_prompt: import("zod").ZodOptional<import("zod").ZodString>;
17
- no_mmap: import("zod").ZodOptional<import("zod").ZodBoolean>;
17
+ load_mode: import("zod").ZodOptional<import("zod").ZodEnum<{
18
+ none: "none";
19
+ mmap: "mmap";
20
+ mlock: "mlock";
21
+ "mmap+mlock": "mmap+mlock";
22
+ dio: "dio";
23
+ }>>;
18
24
  verbosity: import("zod").ZodOptional<import("zod").ZodEnum<{
19
25
  readonly ERROR: 0;
20
26
  readonly WARN: 1;
@@ -25,7 +31,6 @@ export declare const llmPlugin: {
25
31
  frequency_penalty: import("zod").ZodOptional<import("zod").ZodNumber>;
26
32
  repeat_penalty: import("zod").ZodOptional<import("zod").ZodNumber>;
27
33
  stop_sequences: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodString>>;
28
- n_discarded: import("zod").ZodOptional<import("zod").ZodNumber>;
29
34
  parallel: import("zod").ZodOptional<import("zod").ZodNumber>;
30
35
  tools: import("zod").ZodOptional<import("zod").ZodBoolean>;
31
36
  'cache-type-k': import("zod").ZodOptional<import("zod").ZodString>;
@@ -38,6 +43,12 @@ export declare const llmPlugin: {
38
43
  none: "none";
39
44
  layer: "layer";
40
45
  row: "row";
46
+ tensor: "tensor";
47
+ }>>;
48
+ 'flash-attn': import("zod").ZodOptional<import("zod").ZodEnum<{
49
+ on: "on";
50
+ off: "off";
51
+ auto: "auto";
41
52
  }>>;
42
53
  'tensor-split': import("zod").ZodOptional<import("zod").ZodString>;
43
54
  openclCacheDir: import("zod").ZodOptional<import("zod").ZodString>;
@@ -104,19 +115,19 @@ export declare const llmPlugin: {
104
115
  device?: string | undefined;
105
116
  predict?: number | undefined;
106
117
  system_prompt?: string | undefined;
107
- no_mmap?: boolean | undefined;
118
+ load_mode?: "none" | "mmap" | "mlock" | "mmap+mlock" | "dio" | undefined;
108
119
  verbosity?: 0 | 3 | 2 | 1 | undefined;
109
120
  presence_penalty?: number | undefined;
110
121
  frequency_penalty?: number | undefined;
111
122
  repeat_penalty?: number | undefined;
112
123
  stop_sequences?: string[] | undefined;
113
- n_discarded?: number | undefined;
114
124
  parallel?: number | undefined;
115
125
  tools?: boolean | undefined;
116
126
  'cache-type-k'?: string | undefined;
117
127
  'cache-type-v'?: string | undefined;
118
128
  'main-gpu'?: number | "integrated" | "dedicated" | undefined;
119
- 'split-mode'?: "none" | "layer" | "row" | undefined;
129
+ 'split-mode'?: "none" | "layer" | "row" | "tensor" | undefined;
130
+ 'flash-attn'?: "on" | "off" | "auto" | undefined;
120
131
  'tensor-split'?: string | undefined;
121
132
  openclCacheDir?: string | undefined;
122
133
  reasoning_budget?: number | undefined;
@@ -137,19 +148,19 @@ export declare const llmPlugin: {
137
148
  device?: string | undefined;
138
149
  predict?: number | undefined;
139
150
  system_prompt?: string | undefined;
140
- no_mmap?: boolean | undefined;
151
+ load_mode?: "none" | "mmap" | "mlock" | "mmap+mlock" | "dio" | undefined;
141
152
  verbosity?: 0 | 3 | 2 | 1 | undefined;
142
153
  presence_penalty?: number | undefined;
143
154
  frequency_penalty?: number | undefined;
144
155
  repeat_penalty?: number | undefined;
145
156
  stop_sequences?: string[] | undefined;
146
- n_discarded?: number | undefined;
147
157
  parallel?: number | undefined;
148
158
  tools?: boolean | undefined;
149
159
  'cache-type-k'?: string | undefined;
150
160
  'cache-type-v'?: string | undefined;
151
161
  'main-gpu'?: number | "integrated" | "dedicated" | undefined;
152
- 'split-mode'?: "none" | "layer" | "row" | undefined;
162
+ 'split-mode'?: "none" | "layer" | "row" | "tensor" | undefined;
163
+ 'flash-attn'?: "on" | "off" | "auto" | undefined;
153
164
  'tensor-split'?: string | undefined;
154
165
  openclCacheDir?: string | undefined;
155
166
  reasoning_budget?: number | undefined;
@@ -56,11 +56,11 @@ import { definePlugin, defineHandler, finetuneRequestSchema, batchCompletionStre
56
56
  import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
57
57
  import { getFirstShardPath } from '../../../utils/index.js';
58
58
  import { getModelParallel } from '../../../utils/config-transform.js';
59
- import { completion } from '../../../plugins/builtin/llamacpp-completion/ops/completion-stream.js';
60
- import { batchCompletion } from '../../../plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js';
61
- import { finetune } from '../../../plugins/builtin/llamacpp-completion/ops/finetune.js';
62
- import { translate } from '../../../plugins/ops/translate.js';
63
- import { transformLlmConfig } from '../../../plugins/builtin/llamacpp-completion/transform.js';
59
+ import { completion } from './ops/completion-stream.js';
60
+ import { batchCompletion } from './ops/batch-completion-stream.js';
61
+ import { finetune } from './ops/finetune.js';
62
+ import { translate } from '../../ops/translate.js';
63
+ import { transformLlmConfig } from './transform.js';
64
64
  import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
65
65
  import { getModelConfig } from '../../../runtime/model-registry.js';
66
66
  import { createCompletionNormalizer } from '../../../utils/completion-normalizer.js';
@@ -68,8 +68,8 @@ import { detectToolDialect } from '../../../utils/tool-integration.js';
68
68
  import { getRequestRegistry, withRequestContext } from '../../../runtime/index.js';
69
69
  import { generateRandomRequestId } from '../../../runtime/request-id.js';
70
70
  import { ContextOverflowError } from '../../../errors/index.js';
71
- import { isAddonContextOverflowError, parseContextOverflowMessage } from '../../../plugins/builtin/llamacpp-completion/ops/context-overflow.js';
72
- import { stoppedByLength } from '../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
71
+ import { isAddonContextOverflowError, parseContextOverflowMessage } from './ops/context-overflow.js';
72
+ import { stoppedByLength } from './ops/completion-stats.js';
73
73
  import { isMobile } from '../../../runtime/state.js';
74
74
  import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
75
75
  function createLlmModel(modelId, modelPath, llmConfig, projectionModelPath) {
@@ -245,8 +245,7 @@ export const llmPlugin = definePlugin({
245
245
  }
246
246
  catch (err) {
247
247
  if (isAddonContextOverflowError(err)) {
248
- const { promptTokens, ctxSize } = parseContextOverflowMessage(err instanceof Error ? err.message : '');
249
- throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
248
+ throw new ContextOverflowError(parseContextOverflowMessage(err instanceof Error ? err.message : ''), request.modelId, err);
250
249
  }
251
250
  // Once the registry accepts cancellation, the request signal owns the
252
251
  // terminal outcome. The addon may reject with different shapes
@@ -404,8 +403,7 @@ export const llmPlugin = definePlugin({
404
403
  // signal, so a real overflow racing a cancel surfaces as a typed
405
404
  // ContextOverflowError.
406
405
  if (isAddonContextOverflowError(err)) {
407
- const { promptTokens, ctxSize } = parseContextOverflowMessage(err instanceof Error ? err.message : '');
408
- throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
406
+ throw new ContextOverflowError(parseContextOverflowMessage(err instanceof Error ? err.message : ''), request.modelId, err);
409
407
  }
410
408
  // Context overflow is classified above. For every other error after
411
409
  // accepted cancellation, the request signal owns the terminal outcome
@@ -483,8 +481,7 @@ export const llmPlugin = definePlugin({
483
481
  // Same addon, same overflow path as `completionStream`. Wrap so
484
482
  // translate consumers can `instanceof ContextOverflowError` too.
485
483
  if (isAddonContextOverflowError(err)) {
486
- const { promptTokens, ctxSize } = parseContextOverflowMessage(err instanceof Error ? err.message : '');
487
- throw new ContextOverflowError(promptTokens, ctxSize, request.modelId, err);
484
+ throw new ContextOverflowError(parseContextOverflowMessage(err instanceof Error ? err.message : ''), request.modelId, err);
488
485
  }
489
486
  throw err;
490
487
  }
@@ -1,3 +1,4 @@
1
+ import type { TurboVecIndexProvider } from '@qvac/rag';
1
2
  import { type CreateModelParams, type PluginModelResult } from '../../../schemas/index.js';
2
3
  export declare const embeddingsPlugin: {
3
4
  modelType: "llamacpp-embedding";
@@ -69,6 +70,9 @@ export declare const embeddingsPlugin: {
69
70
  error: import("zod").ZodOptional<import("zod").ZodString>;
70
71
  }, import("zod/v4/core").$strip>>;
71
72
  };
73
+ capabilities: {
74
+ turbovecIndexProvider: TurboVecIndexProvider;
75
+ };
72
76
  logging: {
73
77
  module: import("@qvac/embed-llamacpp/addonLogging").AddonLogging;
74
78
  namespace: "llamacpp-embedding";
@@ -1,12 +1,20 @@
1
- import EmbedLlamacpp from '@qvac/embed-llamacpp';
1
+ import EmbedLlamacpp, { IdMapIndex } from '@qvac/embed-llamacpp';
2
2
  import embedAddonLogging from '@qvac/embed-llamacpp/addonLogging';
3
3
  import { definePlugin, defineHandler, embedRequestSchema, embedResponseSchema, ModelType, embedConfigBaseSchema, ADDON_EMBEDDING } from '../../../schemas/index.js';
4
4
  import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
5
5
  import { getFirstShardPath } from '../../../utils/index.js';
6
- import { embed } from '../../../plugins/ops/embed.js';
6
+ import { embed } from '../../ops/embed.js';
7
7
  import { forwardModelExecution } from '../../../profiling/model-execution.js';
8
8
  import { isMobile } from '../../../runtime/state.js';
9
9
  import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
10
+ const turbovecIndexProvider = {
11
+ create(options) {
12
+ return new IdMapIndex(options);
13
+ },
14
+ load(snapshotPath) {
15
+ return IdMapIndex.load(snapshotPath);
16
+ }
17
+ };
10
18
  function transformEmbedConfig(embedConfig) {
11
19
  const config = {
12
20
  device: embedConfig.device,
@@ -93,6 +101,9 @@ export const embeddingsPlugin = definePlugin({
93
101
  }
94
102
  })
95
103
  },
104
+ capabilities: {
105
+ turbovecIndexProvider
106
+ },
96
107
  logging: {
97
108
  module: embedAddonLogging,
98
109
  namespace: ModelType.llamacppEmbedding