@qvac/inference 0.18.2 → 0.19.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (396) hide show
  1. package/NOTICE +69 -36
  2. package/README.md +1 -5
  3. package/dist/api/assess-model-fit.d.ts +16 -0
  4. package/dist/api/assess-model-fit.js +27 -0
  5. package/dist/api/audio-gen.d.ts +4 -4
  6. package/dist/api/audio-gen.js +17 -6
  7. package/dist/api/completion-stream.d.ts +1 -1
  8. package/dist/api/completion-stream.js +1 -1
  9. package/dist/api/delete-cache.d.ts +7 -1
  10. package/dist/api/delete-cache.js +11 -1
  11. package/dist/api/download-asset.d.ts +1 -1
  12. package/dist/api/download-asset.js +2 -2
  13. package/dist/api/finetune.js +1 -1
  14. package/dist/api/get-loaded-model-info.d.ts +6 -10
  15. package/dist/api/get-loaded-model-info.js +6 -10
  16. package/dist/api/heartbeat.d.ts +5 -22
  17. package/dist/api/heartbeat.js +6 -23
  18. package/dist/api/index.d.ts +36 -35
  19. package/dist/api/index.js +35 -35
  20. package/dist/api/load-model.d.ts +5 -1
  21. package/dist/api/load-model.js +3 -5
  22. package/dist/api/logging-stream-registry.js +1 -1
  23. package/dist/api/resume.d.ts +0 -3
  24. package/dist/api/resume.js +0 -3
  25. package/dist/api/subscribe-logs.js +1 -1
  26. package/dist/api/suspend.d.ts +0 -4
  27. package/dist/api/suspend.js +0 -4
  28. package/dist/api/translate.d.ts +3 -1
  29. package/dist/api/translate.js +16 -8
  30. package/dist/api/unload-model.js +3 -5
  31. package/dist/api/video-request.d.ts +207 -0
  32. package/dist/api/video-request.js +18 -0
  33. package/dist/api/video.d.ts +22 -2
  34. package/dist/api/video.js +25 -15
  35. package/dist/api/vla-helpers.d.ts +1 -2
  36. package/dist/api/vla-helpers.js +1 -2
  37. package/dist/api/vla.js +1 -1
  38. package/dist/api/world-result.d.ts +71 -0
  39. package/dist/api/world-result.js +293 -0
  40. package/dist/api/world.d.ts +143 -0
  41. package/dist/api/world.js +71 -0
  42. package/dist/config/resolve-config.d.ts +1 -1
  43. package/dist/config/resolve-config.js +1 -1
  44. package/dist/constants/index.d.ts +2 -2
  45. package/dist/constants/index.js +2 -2
  46. package/dist/dispatch.js +15 -28
  47. package/dist/errors/index.d.ts +35 -31
  48. package/dist/errors/index.js +66 -56
  49. package/dist/handlers/assess-model-fit.d.ts +10 -0
  50. package/dist/handlers/assess-model-fit.js +75 -0
  51. package/dist/handlers/cancelHandler.js +1 -1
  52. package/dist/handlers/completion-orchestrate.js +1 -1
  53. package/dist/handlers/delete-cache.js +3 -0
  54. package/dist/handlers/download-asset.js +6 -5
  55. package/dist/handlers/finetune.js +1 -1
  56. package/dist/handlers/get-loaded-model-info.js +0 -12
  57. package/dist/handlers/get-model-info.js +1 -1
  58. package/dist/handlers/get-system-resources.js +8 -0
  59. package/dist/handlers/load-model/download-manager.d.ts +1 -1
  60. package/dist/handlers/load-model/download-stats.d.ts +3 -3
  61. package/dist/handlers/load-model/download-stats.js +5 -5
  62. package/dist/handlers/load-model/handler.js +6 -8
  63. package/dist/handlers/load-model/http-verify.d.ts +20 -0
  64. package/dist/handlers/load-model/http-verify.js +54 -0
  65. package/dist/handlers/load-model/http.d.ts +12 -2
  66. package/dist/handlers/load-model/http.js +119 -230
  67. package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
  68. package/dist/handlers/load-model/hyperdrive.js +1 -1
  69. package/dist/handlers/load-model/index.d.ts +2 -2
  70. package/dist/handlers/load-model/index.js +2 -2
  71. package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
  72. package/dist/handlers/load-model/registry-companion-set.js +1 -1
  73. package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
  74. package/dist/handlers/load-model/registry-download-utils.js +2 -2
  75. package/dist/handlers/load-model/registry.d.ts +2 -2
  76. package/dist/handlers/load-model/registry.js +4 -4
  77. package/dist/handlers/load-model/resolve-session.d.ts +8 -1
  78. package/dist/handlers/load-model/resolve-session.js +12 -9
  79. package/dist/handlers/load-model/resolve.d.ts +4 -4
  80. package/dist/handlers/load-model/resolve.js +18 -16
  81. package/dist/handlers/load-model/safe-fetch.d.ts +44 -0
  82. package/dist/handlers/load-model/safe-fetch.js +143 -0
  83. package/dist/handlers/load-model/types.d.ts +9 -0
  84. package/dist/handlers/plugin-dispatch.js +1 -4
  85. package/dist/handlers/plugin-invoke.js +1 -4
  86. package/dist/handlers/types.d.ts +0 -2
  87. package/dist/handlers/unload-model.js +1 -4
  88. package/dist/index.d.ts +1 -1
  89. package/dist/index.js +1 -1
  90. package/dist/logging/addon.d.ts +1 -1
  91. package/dist/logging/addon.js +1 -1
  92. package/dist/logging/index.d.ts +5 -5
  93. package/dist/logging/index.js +4 -4
  94. package/dist/logging/logger.d.ts +1 -1
  95. package/dist/logging/logger.js +2 -2
  96. package/dist/models/registry/index.d.ts +1 -1
  97. package/dist/models/registry/index.js +2 -2
  98. package/dist/models/registry/models.d.ts +250 -0
  99. package/dist/models/registry/models.js +2071 -1806
  100. package/dist/models/registry/resource-profiles.d.ts +18 -0
  101. package/dist/models/registry/resource-profiles.js +4934 -0
  102. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +14 -0
  103. package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
  104. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +47 -5
  105. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +134 -4
  106. package/dist/plugins/builtin/audiogen-ggml/plugin.js +47 -28
  107. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +1 -1
  108. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
  109. package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
  110. package/dist/plugins/builtin/ggml-ocr/plugin.js +2 -2
  111. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
  112. package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
  113. package/dist/plugins/builtin/index.d.ts +12 -12
  114. package/dist/plugins/builtin/index.js +12 -12
  115. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +2 -2
  116. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
  117. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
  118. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +37 -26
  119. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +12 -11
  120. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +133 -41
  121. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +15 -4
  122. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +79 -17
  123. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -8
  124. package/dist/plugins/builtin/llamacpp-completion/plugin.js +10 -13
  125. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +4 -0
  126. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +13 -2
  127. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +2 -2
  128. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +2 -2
  129. package/dist/plugins/builtin/parakeet-transcription/plugin.js +3 -3
  130. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +1 -1
  131. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +32 -2
  132. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +126 -0
  133. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +820 -0
  134. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +153 -4
  135. package/dist/plugins/builtin/sdcpp-generation/plugin.js +134 -13
  136. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +1 -1
  137. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +1 -1
  138. package/dist/plugins/builtin/tts-ggml/plugin.js +2 -2
  139. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +2 -2
  140. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +3 -3
  141. package/dist/plugins/index.d.ts +1 -1
  142. package/dist/plugins/index.js +2 -2
  143. package/dist/plugins/ops/kv-cache-utils.js +1 -1
  144. package/dist/plugins/ops/load-model.js +1 -1
  145. package/dist/plugins/ops/transcribe.js +2 -2
  146. package/dist/plugins/ops/translate.js +4 -12
  147. package/dist/plugins/ops/unload-model.js +19 -21
  148. package/dist/plugins/registry.d.ts +1 -0
  149. package/dist/plugins/registry.js +12 -0
  150. package/dist/profiling/aggregator.d.ts +1 -1
  151. package/dist/profiling/aggregator.js +1 -1
  152. package/dist/profiling/controller.d.ts +1 -1
  153. package/dist/profiling/controller.js +2 -2
  154. package/dist/profiling/envelope.d.ts +0 -1
  155. package/dist/profiling/envelope.js +0 -10
  156. package/dist/profiling/events.d.ts +2 -3
  157. package/dist/profiling/events.js +2 -9
  158. package/dist/profiling/exporters.d.ts +1 -1
  159. package/dist/profiling/exporters.js +3 -3
  160. package/dist/profiling/index.d.ts +13 -14
  161. package/dist/profiling/index.js +12 -13
  162. package/dist/profiling/operation-metrics.d.ts +1 -1
  163. package/dist/profiling/operation-metrics.js +35 -2
  164. package/dist/profiling/operation-wrappers.js +12 -15
  165. package/dist/profiling/profiler.d.ts +1 -2
  166. package/dist/profiling/profiler.js +7 -26
  167. package/dist/profiling/types.d.ts +1 -1
  168. package/dist/rag/handler.js +1 -1
  169. package/dist/rag/handlers/close-workspace.js +1 -1
  170. package/dist/rag/handlers/delete-embeddings.js +1 -1
  171. package/dist/rag/handlers/delete-workspace.js +2 -6
  172. package/dist/rag/handlers/ingest.js +1 -1
  173. package/dist/rag/handlers/list-workspaces.d.ts +1 -1
  174. package/dist/rag/handlers/list-workspaces.js +1 -1
  175. package/dist/rag/handlers/reindex.js +1 -1
  176. package/dist/rag/handlers/save-embeddings.js +1 -1
  177. package/dist/rag/handlers/search.js +1 -1
  178. package/dist/rag/index.d.ts +11 -11
  179. package/dist/rag/index.js +11 -11
  180. package/dist/rag/rag-workspace-manager.d.ts +4 -2
  181. package/dist/rag/rag-workspace-manager.js +268 -15
  182. package/dist/registry.js +8 -58
  183. package/dist/resources/collector.d.ts +3 -1
  184. package/dist/resources/collector.js +91 -9
  185. package/dist/resources/instance.d.ts +1 -1
  186. package/dist/resources/instance.js +1 -1
  187. package/dist/resources/model-fit/assess.d.ts +36 -0
  188. package/dist/resources/model-fit/assess.js +679 -0
  189. package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +8 -0
  190. package/dist/resources/model-fit/calibration/darwin-arm64.js +36 -0
  191. package/dist/resources/model-fit/calibration/darwin-x64.d.ts +8 -0
  192. package/dist/resources/model-fit/calibration/darwin-x64.js +38 -0
  193. package/dist/resources/model-fit/calibration/fit.d.ts +73 -0
  194. package/dist/resources/model-fit/calibration/fit.js +152 -0
  195. package/dist/resources/model-fit/calibration/index.d.ts +37 -0
  196. package/dist/resources/model-fit/calibration/index.js +75 -0
  197. package/dist/resources/model-fit/calibration/linux-arm64.d.ts +8 -0
  198. package/dist/resources/model-fit/calibration/linux-arm64.js +38 -0
  199. package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +8 -0
  200. package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +39 -0
  201. package/dist/resources/model-fit/calibration/linux-x64.d.ts +8 -0
  202. package/dist/resources/model-fit/calibration/linux-x64.js +38 -0
  203. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +8 -0
  204. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +39 -0
  205. package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +8 -0
  206. package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +36 -0
  207. package/dist/resources/model-fit/calibration/win32-x64.d.ts +8 -0
  208. package/dist/resources/model-fit/calibration/win32-x64.js +38 -0
  209. package/dist/resources/model-fit/estimators/llm.d.ts +46 -0
  210. package/dist/resources/model-fit/estimators/llm.js +222 -0
  211. package/dist/resources/model-fit/estimators/whisper.d.ts +10 -0
  212. package/dist/resources/model-fit/estimators/whisper.js +77 -0
  213. package/dist/resources/model-fit/floor.d.ts +31 -0
  214. package/dist/resources/model-fit/floor.js +65 -0
  215. package/dist/resources/model-fit/types.d.ts +120 -0
  216. package/dist/resources/model-fit/types.js +1 -0
  217. package/dist/resources/native.d.ts +1 -1
  218. package/dist/resources/native.js +15 -0
  219. package/dist/resources/normalize.d.ts +1 -1
  220. package/dist/resources/types.d.ts +14 -0
  221. package/dist/runtime/index.d.ts +5 -5
  222. package/dist/runtime/index.js +2 -2
  223. package/dist/runtime/lifecycle.js +5 -7
  224. package/dist/runtime/model-registry.d.ts +0 -14
  225. package/dist/runtime/model-registry.js +17 -40
  226. package/dist/runtime/registry-client.js +1 -1
  227. package/dist/runtime/request-context.d.ts +3 -3
  228. package/dist/runtime/request-context.js +13 -1
  229. package/dist/runtime/request-registry.d.ts +1 -1
  230. package/dist/runtime/request-registry.js +4 -5
  231. package/dist/runtime/state.d.ts +1 -1
  232. package/dist/runtime/state.js +18 -3
  233. package/dist/schemas/assess-model-fit.d.ts +311 -0
  234. package/dist/schemas/assess-model-fit.js +182 -0
  235. package/dist/schemas/audio-gen.d.ts +437 -7
  236. package/dist/schemas/audio-gen.js +131 -19
  237. package/dist/schemas/batch-completion-stream.d.ts +5 -2
  238. package/dist/schemas/batch-completion-stream.js +3 -3
  239. package/dist/schemas/bci-config.js +73 -27
  240. package/dist/schemas/bci.d.ts +3 -3
  241. package/dist/schemas/bci.js +5 -5
  242. package/dist/schemas/cancel.d.ts +5 -0
  243. package/dist/schemas/cancel.js +1 -0
  244. package/dist/schemas/classification.js +13 -6
  245. package/dist/schemas/common.d.ts +695 -335
  246. package/dist/schemas/common.js +33 -31
  247. package/dist/schemas/completion-event.d.ts +5 -2
  248. package/dist/schemas/completion-event.js +2 -1
  249. package/dist/schemas/completion-stream.d.ts +3 -5
  250. package/dist/schemas/completion-stream.js +4 -8
  251. package/dist/schemas/config.d.ts +640 -18
  252. package/dist/schemas/config.js +50 -8
  253. package/dist/schemas/download-asset.d.ts +19 -1
  254. package/dist/schemas/download-asset.js +20 -4
  255. package/dist/schemas/engine-addon-map.d.ts +1 -1
  256. package/dist/schemas/engine-addon-map.js +3 -3
  257. package/dist/schemas/error.d.ts +2 -2
  258. package/dist/schemas/error.js +2 -2
  259. package/dist/schemas/errors.d.ts +2 -6
  260. package/dist/schemas/errors.js +30 -40
  261. package/dist/schemas/heartbeat.d.ts +10 -0
  262. package/dist/schemas/heartbeat.js +8 -0
  263. package/dist/schemas/index.d.ts +46 -44
  264. package/dist/schemas/index.js +45 -43
  265. package/dist/schemas/llamacpp-config.d.ts +32 -10
  266. package/dist/schemas/llamacpp-config.js +11 -8
  267. package/dist/schemas/load-model.d.ts +2902 -1857
  268. package/dist/schemas/load-model.js +52 -52
  269. package/dist/schemas/mcp-adapter.d.ts +1 -1
  270. package/dist/schemas/model-config-schemas.d.ts +2354 -0
  271. package/dist/schemas/model-config-schemas.js +46 -0
  272. package/dist/schemas/model-info.d.ts +10 -20
  273. package/dist/schemas/model-info.js +4 -25
  274. package/dist/schemas/model-ops.d.ts +3 -1
  275. package/dist/schemas/model-ops.js +7 -2
  276. package/dist/schemas/model-resource-profile.d.ts +109 -0
  277. package/dist/schemas/model-resource-profile.js +83 -0
  278. package/dist/schemas/model-src-utils.d.ts +1 -0
  279. package/dist/schemas/model-src-utils.js +33 -13
  280. package/dist/schemas/ocr.js +48 -13
  281. package/dist/schemas/path-validation.js +8 -2
  282. package/dist/schemas/plugin.d.ts +23 -10
  283. package/dist/schemas/plugin.js +11 -0
  284. package/dist/schemas/profiling.d.ts +138 -43
  285. package/dist/schemas/profiling.js +1 -22
  286. package/dist/schemas/rag.js +1 -1
  287. package/dist/schemas/registry.js +1 -1
  288. package/dist/schemas/reload-config.d.ts +0 -2
  289. package/dist/schemas/reload-config.js +2 -3
  290. package/dist/schemas/sdcpp-config.d.ts +344 -4
  291. package/dist/schemas/sdcpp-config.js +504 -17
  292. package/dist/schemas/system-resources.d.ts +138 -0
  293. package/dist/schemas/system-resources.js +3 -1
  294. package/dist/schemas/text-to-speech.js +183 -76
  295. package/dist/schemas/transcription-config.js +253 -85
  296. package/dist/schemas/transcription.d.ts +3 -3
  297. package/dist/schemas/translate.d.ts +2 -2
  298. package/dist/schemas/translate.js +1 -1
  299. package/dist/schemas/translation-config.js +53 -25
  300. package/dist/schemas/vla.d.ts +1 -1
  301. package/dist/selection.d.ts +1 -5
  302. package/dist/selection.js +2 -6
  303. package/dist/surface.d.ts +1 -1
  304. package/dist/utils/addon-responses.d.ts +1 -0
  305. package/dist/utils/aggregate-events.d.ts +1 -1
  306. package/dist/utils/aggregate-events.js +2 -2
  307. package/dist/utils/archive.js +2 -2
  308. package/dist/utils/audio-input.js +1 -1
  309. package/dist/utils/cache/auto-history.d.ts +1 -1
  310. package/dist/utils/cache/auto-history.js +1 -1
  311. package/dist/utils/cache/paths.js +5 -5
  312. package/dist/utils/completion-normalizer.js +1 -1
  313. package/dist/utils/gguf-tensor-extractor.js +1 -1
  314. package/dist/utils/image-dimensions.d.ts +26 -0
  315. package/dist/utils/image-dimensions.js +46 -0
  316. package/dist/utils/index.d.ts +13 -10
  317. package/dist/utils/index.js +13 -10
  318. package/dist/utils/mcp-adapter.d.ts +1 -1
  319. package/dist/utils/mcp-adapter.js +1 -1
  320. package/dist/utils/path-sanitize.d.ts +7 -0
  321. package/dist/utils/path-sanitize.js +36 -0
  322. package/dist/utils/path-security.d.ts +0 -7
  323. package/dist/utils/path-security.js +1 -36
  324. package/dist/utils/shard-utils.js +3 -3
  325. package/dist/utils/tool-integration.js +1 -1
  326. package/dist/utils/tools/dialect.js +1 -1
  327. package/dist/utils/tools/index.d.ts +2 -2
  328. package/dist/utils/tools/index.js +2 -2
  329. package/dist/utils/tools/parser.js +8 -8
  330. package/dist/utils/tools/parsers/dsml.d.ts +1 -1
  331. package/dist/utils/tools/parsers/dsml.js +1 -1
  332. package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
  333. package/dist/utils/tools/parsers/gemma4native.js +1 -1
  334. package/dist/utils/tools/parsers/harmony.d.ts +1 -1
  335. package/dist/utils/tools/parsers/harmony.js +1 -1
  336. package/dist/utils/tools/parsers/hermes.d.ts +1 -1
  337. package/dist/utils/tools/parsers/hermes.js +1 -1
  338. package/dist/utils/tools/parsers/json.d.ts +1 -1
  339. package/dist/utils/tools/parsers/json.js +1 -1
  340. package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
  341. package/dist/utils/tools/parsers/pythonic.js +1 -1
  342. package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
  343. package/dist/utils/tools/parsers/qwen35.js +3 -3
  344. package/dist/utils/url-security.d.ts +17 -0
  345. package/dist/utils/url-security.js +75 -0
  346. package/package.json +31 -26
  347. package/dist/api/provide.d.ts +0 -23
  348. package/dist/api/provide.js +0 -32
  349. package/dist/api/stop-provider.d.ts +0 -20
  350. package/dist/api/stop-provider.js +0 -30
  351. package/dist/p2p/cancel-delegated.d.ts +0 -3
  352. package/dist/p2p/cancel-delegated.js +0 -64
  353. package/dist/p2p/completion-stream-delegated.d.ts +0 -4
  354. package/dist/p2p/completion-stream-delegated.js +0 -51
  355. package/dist/p2p/delegate-client.d.ts +0 -9
  356. package/dist/p2p/delegate-client.js +0 -219
  357. package/dist/p2p/delegate-connect-diagnostics.d.ts +0 -8
  358. package/dist/p2p/delegate-connect-diagnostics.js +0 -38
  359. package/dist/p2p/delegate-transport.d.ts +0 -18
  360. package/dist/p2p/delegate-transport.js +0 -237
  361. package/dist/p2p/error-handlers.d.ts +0 -4
  362. package/dist/p2p/error-handlers.js +0 -31
  363. package/dist/p2p/handle-request.d.ts +0 -2
  364. package/dist/p2p/handle-request.js +0 -171
  365. package/dist/p2p/handler-utils.d.ts +0 -19
  366. package/dist/p2p/handler-utils.js +0 -180
  367. package/dist/p2p/heartbeat-delegated.d.ts +0 -3
  368. package/dist/p2p/heartbeat-delegated.js +0 -32
  369. package/dist/p2p/load-model-delegated.d.ts +0 -6
  370. package/dist/p2p/load-model-delegated.js +0 -132
  371. package/dist/p2p/progress-throttle.d.ts +0 -7
  372. package/dist/p2p/progress-throttle.js +0 -47
  373. package/dist/p2p/provideHandler/connection.d.ts +0 -2
  374. package/dist/p2p/provideHandler/connection.js +0 -38
  375. package/dist/p2p/provideHandler/handler.d.ts +0 -2
  376. package/dist/p2p/provideHandler/handler.js +0 -66
  377. package/dist/p2p/provideHandler/proxy.d.ts +0 -2
  378. package/dist/p2p/provideHandler/proxy.js +0 -24
  379. package/dist/p2p/provideHandler.d.ts +0 -1
  380. package/dist/p2p/provideHandler.js +0 -2
  381. package/dist/p2p/rpc-utils.d.ts +0 -4
  382. package/dist/p2p/rpc-utils.js +0 -8
  383. package/dist/p2p/stopProvideHandler.d.ts +0 -2
  384. package/dist/p2p/stopProvideHandler.js +0 -29
  385. package/dist/p2p/swarm.d.ts +0 -9
  386. package/dist/p2p/swarm.js +0 -103
  387. package/dist/p2p/transport-selector.d.ts +0 -2
  388. package/dist/p2p/transport-selector.js +0 -18
  389. package/dist/p2p/unload-model-delegated.d.ts +0 -2
  390. package/dist/p2p/unload-model-delegated.js +0 -31
  391. package/dist/profiling/delegation-profiler.d.ts +0 -31
  392. package/dist/profiling/delegation-profiler.js +0 -155
  393. package/dist/schemas/delegate.d.ts +0 -29
  394. package/dist/schemas/delegate.js +0 -40
  395. package/dist/schemas/provide.d.ts +0 -50
  396. package/dist/schemas/provide.js +0 -32
@@ -1,8 +1,9 @@
1
- import { logCacheDisabled, logCacheInit, logCacheSave, logMessagesToAddon } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
2
- import { extractSystemPrompt, getCurrentCacheInfo } from '../../../../plugins/ops/kv-cache-utils.js';
1
+ import { logCacheDisabled, logCacheInit, logCacheSave, logMessagesToAddon } from './cache-logger.js';
2
+ import { extractSystemPrompt, getCurrentCacheInfo } from '../../../ops/kv-cache-utils.js';
3
+ import { isAddonPreMutationRefusal, isAddonContextOverflowError } from './context-overflow.js';
3
4
  import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
4
- import { decideCachedHistorySlice, shouldCommitCachedTurn } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-state.js';
5
- import { createKvCacheSession, generateConfigHash } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
5
+ import { decideCachedHistorySlice, shouldCommitCachedTurn } from './kv-cache-state.js';
6
+ import { createKvCacheSession, generateConfigHash } from './kv-cache-session.js';
6
7
  import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
7
8
  import { parseToolCalls } from '../../../../utils/tools/index.js';
8
9
  import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
@@ -11,7 +12,7 @@ import { getEngineLogger } from '../../../../logging/index.js';
11
12
  import { AttachmentNotFoundError } from '../../../../errors/index.js';
12
13
  import { nowMs } from '../../../../profiling/index.js';
13
14
  import { buildStreamResult } from '../../../../profiling/model-execution.js';
14
- import { normalizeCompletionStats, withEmittedTokens } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
15
+ import { normalizeCompletionStats, withEmittedTokens } from './completion-stats.js';
15
16
  import fs from 'bare-fs';
16
17
  const logger = getEngineLogger();
17
18
  function transformMessage(message) {
@@ -119,7 +120,7 @@ function rendersToolBlock(messages, toolBlock) {
119
120
  * - The tool block travels only with the turn that writes it into the
120
121
  * cache; see `skipToolBlock` below.
121
122
  */
122
- function prepareMessagesForCache(session, turn, cacheExists, history, tools, toolBlockEvictable = false) {
123
+ function prepareMessagesForCache(session, turn, cacheExists, history, tools) {
123
124
  const toolBlock = tools?.length ? transformMessages(tools) : [];
124
125
  if (!(cacheExists && history.length > 0)) {
125
126
  const historyWithoutSystem = history.filter((msg) => msg.role !== 'system');
@@ -146,9 +147,8 @@ function prepareMessagesForCache(session, turn, cacheExists, history, tools, too
146
147
  // conversation. Skip it only when the prefix is known to hold a rendered
147
148
  // one: `toolBlockCached` records that a previous turn actually got it into
148
149
  // the cache, which a committed message count does not prove. A stale
149
- // boundary means we are resending the whole conversation anyway, and an
150
- // evictable block can no longer be assumed present.
151
- const skipToolBlock = turn.toolBlockCached && !clearStaleCount && !toolBlockEvictable;
150
+ // boundary means we are resending the whole conversation anyway.
151
+ const skipToolBlock = turn.toolBlockCached && !clearStaleCount;
152
152
  const blockToSend = skipToolBlock ? [] : toolBlock;
153
153
  return {
154
154
  messages: withToolBlock(transformMessages(messages), blockToSend),
@@ -208,12 +208,6 @@ export async function* completion(params, opts) {
208
208
  const modelConfig = getModelConfig(modelId);
209
209
  const toolsEnabled = modelConfig.tools === true;
210
210
  const toolsActive = !!tools?.length && toolsEnabled;
211
- // Sliding is opt-in (`n_discarded` defaults to 0). Once on, the addon's
212
- // discard window opens at the end of the primed prefix — which is where the
213
- // tool block sits, since the prime is the system prompt alone — and nothing
214
- // protects it. So while sliding is possible the block cannot be assumed to
215
- // survive, and it has to travel with every turn.
216
- const toolBlockEvictable = (modelConfig.n_discarded ?? 0) > 0;
217
211
  const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
218
212
  // `responseFormat` is forwarded to the addon as a per-request
219
213
  // `generationParams.json_schema`, which the addon converts to GBNF and
@@ -273,12 +267,11 @@ export async function* completion(params, opts) {
273
267
  logMessagesToAddon(transformedHistory, 'NO_CACHE');
274
268
  return yield* processModelResponse(model, transformedHistory, tools, mergedGenerationParams, undefined, dialect, setActiveResponse);
275
269
  }
276
- // ---- KV-cache path. The session owns all three bookkeeping layers
277
- // (on-disk `.bin`, `initializedCaches`, `cachedMessageCounts`). The
278
- // handler asks for a turn, registers rollback on the scope, and on
279
- // the happy path calls `commitTurn` which short-circuits the deferred
280
- // rollback. Cancellations / zero-token replies / rename failures all
281
- // unwind through the same `scope.defer` hook. ----
270
+ // ---- KV-cache path. The session owns every bookkeeping layer; the
271
+ // handler registers one deferred unwind (`rollback`, or the non-destructive
272
+ // `releaseTurn` on a recognised pre-mutation refusal) that `commitTurn`
273
+ // short-circuits on the happy path. Cancellations / zero-token replies /
274
+ // rename failures all still unwind destructively through the same hook. ----
282
275
  const session = createKvCacheSession(modelId, { logger: requestLogger });
283
276
  const systemPromptFromHistory = extractSystemPrompt(history);
284
277
  // The tool block is baked into the cache on the turn that first sends it and
@@ -319,16 +312,34 @@ export async function* completion(params, opts) {
319
312
  // flips the turn's internal `committed` flag so this becomes a no-op
320
313
  // on the happy path. Scope unwinding is LIFO — registered after the
321
314
  // `removeEventListener` defer above so rollback runs before the
322
- // listener detach.
323
- scope.defer(() => session.rollback(turn));
315
+ // listener detach. A thrown overflow, pre-mutation refusal, or pre-addon
316
+ // attachment failure never persists the turn; the committed cache stays valid.
317
+ let preserveCacheOnUnwind = false;
318
+ scope.defer(() => (preserveCacheOnUnwind ? session.releaseTurn(turn) : session.rollback(turn)));
324
319
  // `cacheExists` is implied by `beginTurn` — the session either found
325
320
  // an existing cache or just primed one. Pass `true` to the message
326
321
  // selector so the slicing branches engage.
327
- const payload = prepareMessagesForCache(session, turn,
328
- /* cacheExists */ true, history, toolsActive ? tools : undefined, toolBlockEvictable);
322
+ let payload;
323
+ try {
324
+ payload = prepareMessagesForCache(session, turn,
325
+ /* cacheExists */ true, history, toolsActive ? tools : undefined);
326
+ }
327
+ catch (error) {
328
+ // A missing attachment is caller input rejected before the addon runs,
329
+ // so the committed cache is untouched and must survive.
330
+ preserveCacheOnUnwind = error instanceof AttachmentNotFoundError;
331
+ throw error;
332
+ }
329
333
  const messagesToSend = payload.messages;
330
334
  logMessagesToAddon(messagesToSend, 'PROMPT_SEND');
331
- const result = yield* processModelResponse(model, messagesToSend, tools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse);
335
+ let result;
336
+ try {
337
+ result = yield* processModelResponse(model, messagesToSend, tools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse);
338
+ }
339
+ catch (error) {
340
+ preserveCacheOnUnwind = isAddonContextOverflowError(error) || isAddonPreMutationRefusal(error);
341
+ throw error;
342
+ }
332
343
  const shouldCommitTurn = shouldCommitCachedTurn({
333
344
  aborted: signal.aborted,
334
345
  producedTokens: result.producedTokens,
@@ -6,23 +6,24 @@
6
6
  * msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
7
7
  * `qvac_errors::StatusError::codeString()` and formats as
8
8
  * `"[ <addonId> :: ContextOverflow ]"`. The message carries the
9
- * C++-formatted detail (which may include the prompt/ctx sizes).
9
+ * C++-formatted detail (which may include the prompt/ctx sizes). That
10
+ * codeString survives only the synchronous throw path — asynchronous run
11
+ * errors are transported as `exception.what()` alone, with no code.
10
12
  *
11
13
  * These helpers let the plugin handler convert that addon error into a
12
14
  * typed `ContextOverflowError` and let unit tests assert the detection
13
15
  * + extraction logic without a real model load.
14
16
  */
15
17
  export declare function isAddonContextOverflowError(err: unknown): boolean;
16
- /**
17
- * Best-effort extraction of `promptTokens` / `ctxSize` from the addon
18
- * error message. The C++ paths in `TextLlmContext.cpp` and
19
- * `MtmdLlmContext.cpp` format both numbers into the message; the
20
- * `LlamaModel::processPromptImpl` fallback path emits a bare
21
- * `"<func>: context overflow\n"` with neither number. Returning
22
- * `undefined` for either field is fine — `ContextOverflowError` holds
23
- * them as optional and the message factory degrades gracefully.
24
- */
25
- export declare function parseContextOverflowMessage(message: string): {
18
+ export declare function isAddonPreMutationRefusal(err: unknown): boolean;
19
+ export type ContextOverflowSizes = {
20
+ /** The prompt alone, in tokens; unset when the guard reports KV cells. */
26
21
  promptTokens?: number;
22
+ /** Cached conversation on a warm-cache guard, in `ctxSize` units. */
23
+ cachedTokens?: number;
24
+ /** Total the failing guard reported, in `ctxSize` units; the guards trigger on `>=`, so it can equal the window. */
25
+ requiredTokens?: number;
26
+ /** Effective per-request ceiling: `ctx_size` split across slots at `parallel > 1`. */
27
27
  ctxSize?: number;
28
28
  };
29
+ export declare function parseContextOverflowMessage(message: string): ContextOverflowSizes;
@@ -6,60 +6,152 @@
6
6
  * msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
7
7
  * `qvac_errors::StatusError::codeString()` and formats as
8
8
  * `"[ <addonId> :: ContextOverflow ]"`. The message carries the
9
- * C++-formatted detail (which may include the prompt/ctx sizes).
9
+ * C++-formatted detail (which may include the prompt/ctx sizes). That
10
+ * codeString survives only the synchronous throw path — asynchronous run
11
+ * errors are transported as `exception.what()` alone, with no code.
10
12
  *
11
13
  * These helpers let the plugin handler convert that addon error into a
12
14
  * typed `ContextOverflowError` and let unit tests assert the detection
13
15
  * + extraction logic without a real model load.
14
16
  */
17
+ // The guards' emitted starts; the numeric tails vary across addon
18
+ // generations, so the forms are start-anchored and single-line.
19
+ const CONTEXT_OVERFLOW_FORMS = [
20
+ /^\[TextLlm\] context overflow at (?:batch )?prefill step[^\r\n]*$/,
21
+ /^\[MtmdLlm\] context overflow at (?:batch )?prefill step[^\r\n]*$/,
22
+ // Unreachable at the pinned addon generation (generateResponse never
23
+ // reports !ok); kept in case a patch revives the path.
24
+ /^processPromptImpl: context overflow$/,
25
+ // The batch scheduler's capacity refusals are the same out-of-context
26
+ // condition, reported under a generic status.
27
+ /^ContinuousBatchScheduler::submit: prompt of \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
28
+ /^ContinuousBatchScheduler::submit: prompt of \d+ tokens leaves no room under per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
29
+ /^ContinuousBatchScheduler::submit: prefill prompt of \d+ tokens exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
30
+ /^ContinuousBatchScheduler::submit: n_predict [1-9]\d* \+ prompt \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/
31
+ ];
32
+ // Both detectors classify raw addon errors only — an already-typed QvacError
33
+ // (numeric `code`) is deliberately rejected by the present-code gate.
15
34
  export function isAddonContextOverflowError(err) {
16
35
  if (typeof err !== 'object' || err === null)
17
36
  return false;
37
+ // A present status code is authoritative: exact ContextOverflow accepts,
38
+ // anything else (a non-string included) rejects with no message fallback.
18
39
  const code = err.code;
19
- // Anchor to the codeString tail (`[ <addonId> :: ContextOverflow ]`)
20
- // so we don't false-positive on a hypothetical sibling like
21
- // `ContextOverflowRecovered` or `PostContextOverflow` after an
22
- // addon-side rename.
23
- if (typeof code === 'string' && /::\s*ContextOverflow\s*\]/.test(code)) {
24
- return true;
40
+ if (code !== undefined) {
41
+ return typeof code === 'string' && /^\[\s*[\w.-]+\s*::\s*ContextOverflow\s*\]$/.test(code);
25
42
  }
26
- // Message-substring fallback for the bare `LlamaModel::processPromptImpl`
27
- // path, which doesn't go through `StatusError::codeString()`. Match the
28
- // two known C++-emitted formats only — broader substring would catch
29
- // wrapper / cause-chain text that mentions overflow without being one.
43
+ // The production path: the async transport strips status metadata from
44
+ // every run error, so the message alone identifies the overflow guards.
30
45
  const message = err.message;
31
- return (typeof message === 'string' &&
32
- /(?:context overflow at prefill step|processPromptImpl: context overflow)/i.test(message));
46
+ if (typeof message !== 'string')
47
+ return false;
48
+ const trimmed = message.trim();
49
+ return CONTEXT_OVERFLOW_FORMS.some((form) => form.test(trimmed));
33
50
  }
34
- /**
35
- * Best-effort extraction of `promptTokens` / `ctxSize` from the addon
36
- * error message. The C++ paths in `TextLlmContext.cpp` and
37
- * `MtmdLlmContext.cpp` format both numbers into the message; the
38
- * `LlamaModel::processPromptImpl` fallback path emits a bare
39
- * `"<func>: context overflow\n"` with neither number. Returning
40
- * `undefined` for either field is fine — `ContextOverflowError` holds
41
- * them as optional and the message factory degrades gracefully.
42
- */
43
- export function parseContextOverflowMessage(message) {
44
- // "[TextLlm] context overflow at prefill step: prompt tokens N, max context tokens M\n"
45
- // Same-clause separator (no `[^]*?` cross-newline walk) so a future
46
- // wrapper that pastes overflow text alongside unrelated numbers can't
47
- // produce a mismatched pair.
48
- const longForm = message.match(/prompt tokens (\d+)[,\s]+max context tokens (\d+)/i);
49
- if (longForm?.[1] && longForm[2]) {
50
- return {
51
- promptTokens: Number(longForm[1]),
52
- ctxSize: Number(longForm[2])
53
- };
51
+ // Refusals thrown before any decode or disk save; the async transport delivers
52
+ // exception.what() alone, so each form is matched complete and end-anchored.
53
+ const PRE_MUTATION_REFUSAL_FORMS = [
54
+ /^ContinuousBatchScheduler::submit: prompt of \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
55
+ /^ContinuousBatchScheduler::submit: prompt of \d+ tokens leaves no room under per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
56
+ /^ContinuousBatchScheduler::submit: prefill prompt of \d+ tokens exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
57
+ /^ContinuousBatchScheduler::submit: n_predict [1-9]\d* \+ prompt \d+ KV cells exceeds per-sequence cap \d+ \(ctxTotalTokens \/ n_parallel\)$/,
58
+ /^ContinuousBatchScheduler::submit: failed to add to batch \(MultiRequestBatcher::AddStatus=[1-4]\)$/,
59
+ /^invalid generationParams\.json_schema: [^\r\n]*$/,
60
+ /^failed to initialise sampler with per-request generationParams \(invalid grammar or json_schema\?\)$/,
61
+ /^\[MtmdLlm\] Media buffer is empty$/,
62
+ /^\[MtmdLlm\] Filename is empty$/,
63
+ /^\[MtmdLlm\] Failed to load media from file: [^\r\n]*$/,
64
+ /^\[MtmdLlm\] preparePrefill: prompt must end with text after the last media item$/
65
+ ];
66
+ export function isAddonPreMutationRefusal(err) {
67
+ if (typeof err !== 'object' || err === null)
68
+ return false;
69
+ // A status code only exists on the synchronous throw path; when present it
70
+ // must be InvalidArgument, and its absence (the async transport) is fine.
71
+ const code = err.code;
72
+ if (code !== undefined &&
73
+ (typeof code !== 'string' || !/^\[\s*[\w.-]+\s*::\s*InvalidArgument\s*\]$/.test(code))) {
74
+ return false;
75
+ }
76
+ const message = err.message;
77
+ if (typeof message !== 'string')
78
+ return false;
79
+ const trimmed = message.trim();
80
+ return PRE_MUTATION_REFUSAL_FORMS.some((form) => form.test(trimmed));
81
+ }
82
+ // One entry per number-formatting guard, most specific first; keep in step
83
+ // with TextLlmContext.cpp / MtmdLlmContext.cpp or fields silently vanish.
84
+ const MESSAGE_PATTERNS = [
85
+ {
86
+ // "cached tokens C plus prompt tokens N exceed the max context tokens M"
87
+ pattern: /cached tokens (\d+) plus prompt tokens (\d+) exceeds? the max context tokens (\d+)/i,
88
+ map: ([cached, prompt, ctx]) => ({
89
+ promptTokens: prompt,
90
+ cachedTokens: cached,
91
+ requiredTokens: cached + prompt,
92
+ ctxSize: ctx
93
+ })
94
+ },
95
+ {
96
+ // "cached P positions / C KV cells plus ... of prompt exceed the max
97
+ // context tokens M" — KV cells, not tokens, so promptTokens stays unset.
98
+ pattern: /cached (\d+) positions \/ (\d+) KV cells plus (\d+) positions \/ (\d+) KV cells of prompt exceeds? the max context tokens (\d+)/i,
99
+ map: ([cPos, cCells, pPos, pCells, ctx]) => ({
100
+ cachedTokens: Math.max(cPos, cCells),
101
+ requiredTokens: Math.max(cPos + pPos, cCells + pCells),
102
+ ctxSize: ctx
103
+ })
104
+ },
105
+ {
106
+ // "prompt tokens N, max context tokens M"
107
+ pattern: /prompt tokens (\d+)[, \t]+max context tokens (\d+)/i,
108
+ map: ([prompt, ctx]) => ({ promptTokens: prompt, requiredTokens: prompt, ctxSize: ctx })
109
+ },
110
+ {
111
+ // "prompt spans P positions / N KV cells, max context tokens M" —
112
+ // not tokens, so promptTokens stays unset.
113
+ pattern: /prompt spans (\d+) positions \/ (\d+) KV cells,[ \t]*max context tokens (\d+)/i,
114
+ map: ([pos, cells, ctx]) => ({ requiredTokens: Math.max(pos, cells), ctxSize: ctx })
115
+ },
116
+ {
117
+ // "(N tokens, P positions, max M)" — the "tokens" figure is
118
+ // mtmd_helper_get_n_tokens, i.e. KV cells, so promptTokens stays unset.
119
+ pattern: /\((\d+)[ \t]+tokens,[ \t]*(\d+)[ \t]+positions,[ \t]*max[ \t]+(\d+)\)/i,
120
+ map: ([cells, pos, ctx]) => ({
121
+ requiredTokens: Math.max(cells, pos),
122
+ ctxSize: ctx
123
+ })
124
+ },
125
+ {
126
+ // "(N tokens, max M)", the retired short form. Both of its emitters
127
+ // format a cached total, not the prompt alone — promptTokens stays unset.
128
+ pattern: /\((\d+)[ \t]+tokens,[ \t]*max[ \t]+(\d+)\)/i,
129
+ map: ([total, ctx]) => ({ requiredTokens: total, ctxSize: ctx })
130
+ },
131
+ {
132
+ // Scheduler cap refusals: the total is cache-plus-prompt in cells or
133
+ // tokens, and the cap is the effective per-request ceiling.
134
+ pattern: /prompt of (\d+) (?:KV cells|tokens) (?:exceeds|leaves no room under) per-sequence cap (\d+)/,
135
+ map: ([total, cap]) => ({ requiredTokens: total, ctxSize: cap })
136
+ },
137
+ {
138
+ // "n_predict P + prompt N KV cells exceeds per-sequence cap M" — the
139
+ // reservation plus the prompt is the space the request needs.
140
+ pattern: /n_predict (\d+) \+ prompt (\d+) KV cells exceeds per-sequence cap (\d+)/,
141
+ map: ([predict, cells, cap]) => ({ requiredTokens: predict + cells, ctxSize: cap })
54
142
  }
55
- // "[TextLlm] context overflow at prefill step (N tokens, max M)\n"
56
- // "[MtmdLlm] context overflow at prefill step (N tokens, max M)\n"
57
- const shortForm = message.match(/\((\d+)\s+tokens,\s*max\s+(\d+)\)/i);
58
- if (shortForm?.[1] && shortForm[2]) {
59
- return {
60
- promptTokens: Number(shortForm[1]),
61
- ctxSize: Number(shortForm[2])
62
- };
143
+ ];
144
+ // Best-effort extraction of the overflow sizes; the bare processPromptImpl
145
+ // path carries no numbers, and undefined fields degrade gracefully.
146
+ export function parseContextOverflowMessage(message) {
147
+ for (const { pattern, map } of MESSAGE_PATTERNS) {
148
+ const match = message.match(pattern);
149
+ if (!match)
150
+ continue;
151
+ const numbers = match.slice(1).map(Number);
152
+ if (!numbers.every(Number.isFinite))
153
+ continue;
154
+ return map(numbers);
63
155
  }
64
156
  return {};
65
157
  }
@@ -1,4 +1,4 @@
1
- import { generateConfigHash } from '../../../../plugins/ops/kv-cache-utils.js';
1
+ import { generateConfigHash } from '../../../ops/kv-cache-utils.js';
2
2
  import { type CacheMessage } from '../../../../utils/index.js';
3
3
  import type { Logger } from '../../../../logging/types.js';
4
4
  import { type AbortSignal } from 'bare-abort-controller';
@@ -106,6 +106,12 @@ export interface KvCacheSession {
106
106
  * itself via `commitTurn`).
107
107
  */
108
108
  rollback(turn: TurnHandle): Promise<void>;
109
+ /**
110
+ * Non-destructive counterpart of `rollback`: releases locks and refs but
111
+ * keeps the committed disk cache and its recorded prefix valid for a retry.
112
+ * A cache freshly primed by this same turn rolls back instead.
113
+ */
114
+ releaseTurn(turn: TurnHandle): Promise<void>;
109
115
  /**
110
116
  * Forget the in-memory saved-message count for the turn's path
111
117
  * without unlinking the file or clearing the init flag. Used when
@@ -126,9 +132,10 @@ export declare function createKvCacheSession(modelId: string, options?: {
126
132
  }): KvCacheSession;
127
133
  /**
128
134
  * Atomically delete every layer of KV-cache state for a
129
- * `(kvCacheKey, modelId)` pair, or wipe everything. Single entry point
130
- * — the only mutation point for cross-model state outside of
131
- * turn-scoped `commitTurn`/`rollback`.
135
+ * `(kvCacheKey, modelId)` pair, wipe everything, or reclaim the auto
136
+ * caches no turn is holding. Single entry point — the only mutation
137
+ * point for cross-model state outside of turn-scoped
138
+ * `commitTurn`/`rollback`.
132
139
  *
133
140
  * Why this isn't a method on `KvCacheSession`: deletes are
134
141
  * cross-model (`all: true` has no model; the keyed form has
@@ -155,12 +162,16 @@ export declare function createKvCacheSession(modelId: string, options?: {
155
162
  * its file splits state: the file stays on disk while its saved-count and
156
163
  * init flag are cleared, so the next turn sees the file, skips priming, and
157
164
  * reports `savedCount=0`. Callers must not delete a key that is in active use.
165
+ * The `auto` target is the exception: it skips keys an in-flight turn holds
166
+ * and clears each one under the cache-state lock.
158
167
  */
159
168
  export declare function deleteKvCacheState(target: {
160
169
  kvCacheKey: string;
161
170
  modelId?: string;
162
171
  } | {
163
172
  all: true;
173
+ } | {
174
+ auto: true;
164
175
  }): Promise<void>;
165
176
  /**
166
177
  * Test-only access to the module-scoped state. Production code reaches
@@ -1,10 +1,10 @@
1
1
  import { promises as fsPromises } from 'bare-fs';
2
2
  import path from 'bare-path';
3
- import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../../plugins/ops/kv-cache-utils.js';
4
- import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../../plugins/ops/kv-cache-retention.js';
3
+ import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../ops/kv-cache-utils.js';
4
+ import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../ops/kv-cache-retention.js';
5
5
  import { isMobile } from '../../../../runtime/state.js';
6
6
  import { getKVCacheDir } from '../../../../utils/index.js';
7
- import { logCacheSaveError, logCacheStatus } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
7
+ import { logCacheSaveError, logCacheStatus } from './cache-logger.js';
8
8
  import { getEngineLogger } from '../../../../logging/index.js';
9
9
  // Used by cross-model paths that have no `RequestContext` (e.g.
10
10
  // `deleteKvCacheState`). Per-session call sites receive a logger from
@@ -128,6 +128,25 @@ function isCacheKeyActive(cacheKey) {
128
128
  function getAutoCacheMaxBytes() {
129
129
  return isMobile() ? MOBILE_AUTO_CACHE_MAX_BYTES : DESKTOP_AUTO_CACHE_MAX_BYTES;
130
130
  }
131
+ /**
132
+ * Deletes planned auto-cache keys and reports how many went.
133
+ *
134
+ * Planning and eviction are separate steps, so a turn can take a key in
135
+ * between: each one is re-checked against the active paths under the
136
+ * cache-state lock rather than trusted from the plan.
137
+ */
138
+ async function evictPlannedAutoCaches(cacheKeys) {
139
+ let evicted = 0;
140
+ await withCacheStateLock(async () => {
141
+ for (const cacheKey of cacheKeys) {
142
+ if (isCacheKeyActive(cacheKey))
143
+ continue;
144
+ await deleteKvCacheState({ kvCacheKey: cacheKey });
145
+ evicted++;
146
+ }
147
+ });
148
+ return evicted;
149
+ }
131
150
  async function withCacheStateLock(operation) {
132
151
  const previous = cacheStateLockTail;
133
152
  let releaseLock = () => { };
@@ -156,20 +175,12 @@ async function maybeSweepAutoCaches(logger, overrides) {
156
175
  const sweep = async () => {
157
176
  try {
158
177
  const cacheKeys = await planAutoCacheEvictions({
159
- activeCachePaths: Array.from(activeCachePaths.keys()),
178
+ activeCachePaths: snapshotActivePaths(),
160
179
  maxBytes: overrides?.maxBytes ?? getAutoCacheMaxBytes(),
161
180
  maxIdleMs: overrides?.maxIdleMs ?? AUTO_CACHE_MAX_IDLE_MS,
162
181
  nowMs
163
182
  });
164
- let evictionCount = 0;
165
- await withCacheStateLock(async () => {
166
- for (const cacheKey of cacheKeys) {
167
- if (isCacheKeyActive(cacheKey))
168
- continue;
169
- await deleteKvCacheState({ kvCacheKey: cacheKey });
170
- evictionCount++;
171
- }
172
- });
183
+ const evictionCount = await evictPlannedAutoCaches(cacheKeys);
173
184
  if (evictionCount > 0) {
174
185
  logger.debug(`[kv-cache] Evicted ${evictionCount} inactive auto-cache entries`);
175
186
  }
@@ -192,6 +203,30 @@ function scheduleAutoCacheSweep(logger) {
192
203
  logger.warn(`[kv-cache] Failed to schedule auto-cache retention sweep: ${error instanceof Error ? error.message : String(error)}`);
193
204
  });
194
205
  }
206
+ /**
207
+ * Drops every auto cache no turn is holding, whatever its age or size.
208
+ *
209
+ * The on-demand counterpart to `maybeSweepAutoCaches`, which applies the
210
+ * standing quota and TTL instead. Reusing the planner keeps the "is this an
211
+ * auto cache" decision in one place: only `.auto-cache-<key>` markers are
212
+ * considered, so caller-owned named caches stay out of scope.
213
+ */
214
+ async function deleteInactiveAutoCaches() {
215
+ const cacheKeys = await planAutoCacheEvictions({
216
+ activeCachePaths: snapshotActivePaths(),
217
+ // The quota pass stops at `retainedBytes <= maxBytes`, and a total reaches
218
+ // 0 while zero-byte caches remain (an empty `.bin` from a crashed prime).
219
+ // A negative quota is unreachable, so every inactive entry is selected.
220
+ maxBytes: -1,
221
+ // No age threshold: the quota pass above already selects everything.
222
+ maxIdleMs: 0,
223
+ nowMs: Date.now()
224
+ });
225
+ const evicted = await evictPlannedAutoCaches(cacheKeys);
226
+ if (evicted > 0) {
227
+ moduleLogger.debug(`[kv-cache] Reclaimed ${evicted} inactive auto-cache entries`);
228
+ }
229
+ }
195
230
  // ----- factory -----
196
231
  /**
197
232
  * Construct a session bound to one `(modelId, turn-owning request)`
@@ -218,7 +253,8 @@ export function createKvCacheSession(modelId, options) {
218
253
  ...(signal !== undefined && { signal }),
219
254
  releaseWriteLock,
220
255
  committed: false,
221
- rolledBack: false
256
+ rolledBack: false,
257
+ freshlyPrimed: false
222
258
  });
223
259
  markCachePathActive(cachePath);
224
260
  return handle;
@@ -277,6 +313,7 @@ export function createKvCacheSession(modelId, options) {
277
313
  await input.primeIfMissing(cachePath);
278
314
  await verifyPrimedFile(cachePath, logger);
279
315
  initializedCaches.add(cachePath);
316
+ turnState.get(handle).freshlyPrimed = true;
280
317
  }
281
318
  return handle;
282
319
  }
@@ -362,6 +399,7 @@ export function createKvCacheSession(modelId, options) {
362
399
  await input.primeIfMissing(cachePath);
363
400
  await verifyPrimedFile(cachePath, logger);
364
401
  initializedCaches.add(cachePath);
402
+ turnState.get(handle).freshlyPrimed = true;
365
403
  }
366
404
  return handle;
367
405
  }
@@ -485,6 +523,24 @@ export function createKvCacheSession(modelId, options) {
485
523
  return;
486
524
  await runRollback(state);
487
525
  }
526
+ async function releaseTurn(turn) {
527
+ const state = turnState.get(turn);
528
+ if (!state)
529
+ return;
530
+ if (state.committed || state.rolledBack)
531
+ return;
532
+ // A cache this same turn primed has no committed state to keep — a failed
533
+ // first turn must not leave its own prime behind.
534
+ if (state.freshlyPrimed) {
535
+ await runRollback(state);
536
+ return;
537
+ }
538
+ releaseCachePath(state.cachePath);
539
+ state.rolledBack = true;
540
+ state.releaseWriteLock();
541
+ if (state.autoCacheKey !== undefined)
542
+ scheduleAutoCacheSweep(logger);
543
+ }
488
544
  async function runRollback(state) {
489
545
  // Order matters only weakly: unlink first so a partial disk-state
490
546
  // can't be re-loaded by a sibling turn between the file delete and
@@ -520,15 +576,17 @@ export function createKvCacheSession(modelId, options) {
520
576
  beginTurn,
521
577
  commitTurn,
522
578
  rollback,
579
+ releaseTurn,
523
580
  dropStaleSavedCount
524
581
  };
525
582
  }
526
583
  // ----- module-level administrative API -----
527
584
  /**
528
585
  * Atomically delete every layer of KV-cache state for a
529
- * `(kvCacheKey, modelId)` pair, or wipe everything. Single entry point
530
- * — the only mutation point for cross-model state outside of
531
- * turn-scoped `commitTurn`/`rollback`.
586
+ * `(kvCacheKey, modelId)` pair, wipe everything, or reclaim the auto
587
+ * caches no turn is holding. Single entry point — the only mutation
588
+ * point for cross-model state outside of turn-scoped
589
+ * `commitTurn`/`rollback`.
532
590
  *
533
591
  * Why this isn't a method on `KvCacheSession`: deletes are
534
592
  * cross-model (`all: true` has no model; the keyed form has
@@ -555,8 +613,12 @@ export function createKvCacheSession(modelId, options) {
555
613
  * its file splits state: the file stays on disk while its saved-count and
556
614
  * init flag are cleared, so the next turn sees the file, skips priming, and
557
615
  * reports `savedCount=0`. Callers must not delete a key that is in active use.
616
+ * The `auto` target is the exception: it skips keys an in-flight turn holds
617
+ * and clears each one under the cache-state lock.
558
618
  */
559
619
  export async function deleteKvCacheState(target) {
620
+ if ('auto' in target)
621
+ return deleteInactiveAutoCaches();
560
622
  if ('all' in target) {
561
623
  const removed = await deleteCacheUtil({ all: true });
562
624
  cachedPrefixes.clear();