@qvac/inference 0.21.1 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (462) hide show
  1. package/NOTICE +89 -41
  2. package/README.md +44 -1
  3. package/dist/api/assess-model-fit.d.ts +4 -4
  4. package/dist/api/assess-model-fit.js +4 -4
  5. package/dist/api/audio-gen.d.ts +4 -1
  6. package/dist/api/audio-gen.js +5 -2
  7. package/dist/api/batch-completion.js +1 -1
  8. package/dist/api/completion-stream.d.ts +2 -1
  9. package/dist/api/completion-stream.js +4 -2
  10. package/dist/api/decide.d.ts +26 -0
  11. package/dist/api/decide.js +26 -0
  12. package/dist/api/diffusion.d.ts +18 -1
  13. package/dist/api/diffusion.js +18 -1
  14. package/dist/api/download-asset.js +1 -1
  15. package/dist/api/finetune.js +1 -1
  16. package/dist/api/get-model-info.d.ts +1 -1
  17. package/dist/api/index.d.ts +39 -37
  18. package/dist/api/index.js +38 -36
  19. package/dist/api/load-model.js +2 -2
  20. package/dist/api/logging-stream-registry.js +11 -7
  21. package/dist/api/logging-stream.d.ts +11 -1
  22. package/dist/api/logging-stream.js +11 -2
  23. package/dist/api/registry.d.ts +2 -1
  24. package/dist/api/registry.js +5 -4
  25. package/dist/api/rerank.d.ts +17 -0
  26. package/dist/api/rerank.js +36 -0
  27. package/dist/api/subscribe-logs.js +5 -3
  28. package/dist/api/text-to-speech.d.ts +1 -0
  29. package/dist/api/text-to-speech.js +15 -19
  30. package/dist/api/transcribe.js +5 -2
  31. package/dist/api/unload-model.js +1 -1
  32. package/dist/api/video-request.d.ts +5 -5
  33. package/dist/api/video.d.ts +5 -4
  34. package/dist/api/video.js +6 -5
  35. package/dist/api/vla.js +1 -1
  36. package/dist/api/world-result.js +1 -1
  37. package/dist/api/world.d.ts +1 -1
  38. package/dist/api/world.js +1 -1
  39. package/dist/config/resolve-config.d.ts +1 -1
  40. package/dist/config/resolve-config.js +1 -1
  41. package/dist/constants/index.d.ts +2 -2
  42. package/dist/constants/index.js +2 -2
  43. package/dist/dispatch.d.ts +3 -2
  44. package/dist/dispatch.js +16 -4
  45. package/dist/errors/index.d.ts +9 -0
  46. package/dist/errors/index.js +18 -1
  47. package/dist/handlers/assess-model-fit.d.ts +5 -3
  48. package/dist/handlers/assess-model-fit.js +84 -32
  49. package/dist/handlers/cancelHandler.js +1 -1
  50. package/dist/handlers/completion-orchestrate.js +2 -2
  51. package/dist/handlers/download-asset.js +2 -2
  52. package/dist/handlers/finetune.js +1 -1
  53. package/dist/handlers/get-loaded-model-info.js +3 -5
  54. package/dist/handlers/get-model-info.js +1 -1
  55. package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -0
  56. package/dist/handlers/load-model/coreml-sidecars.js +71 -8
  57. package/dist/handlers/load-model/download-manager.d.ts +1 -1
  58. package/dist/handlers/load-model/download-stats.d.ts +2 -2
  59. package/dist/handlers/load-model/download-stats.js +3 -3
  60. package/dist/handlers/load-model/handler.js +10 -7
  61. package/dist/handlers/load-model/http-verify.d.ts +1 -1
  62. package/dist/handlers/load-model/http.d.ts +1 -1
  63. package/dist/handlers/load-model/http.js +3 -3
  64. package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
  65. package/dist/handlers/load-model/hyperdrive.js +2 -2
  66. package/dist/handlers/load-model/index.d.ts +2 -2
  67. package/dist/handlers/load-model/index.js +2 -2
  68. package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
  69. package/dist/handlers/load-model/registry-companion-set.js +1 -1
  70. package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
  71. package/dist/handlers/load-model/registry-download-utils.js +2 -2
  72. package/dist/handlers/load-model/registry.d.ts +2 -2
  73. package/dist/handlers/load-model/registry.js +5 -5
  74. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  75. package/dist/handlers/load-model/resolve-session.js +11 -3
  76. package/dist/handlers/load-model/resolve.d.ts +1 -1
  77. package/dist/handlers/load-model/resolve.js +5 -5
  78. package/dist/handlers/load-model/safe-fetch.js +5 -6
  79. package/dist/handlers/logging-stream.d.ts +2 -1
  80. package/dist/handlers/logging-stream.js +23 -7
  81. package/dist/handlers/registry.d.ts +3 -1
  82. package/dist/handlers/registry.js +7 -5
  83. package/dist/handlers/types.d.ts +7 -1
  84. package/dist/handlers/vector-index.js +2 -2
  85. package/dist/index.d.ts +1 -1
  86. package/dist/index.js +1 -1
  87. package/dist/logging/addon.d.ts +1 -1
  88. package/dist/logging/addon.js +1 -1
  89. package/dist/logging/index.d.ts +5 -5
  90. package/dist/logging/index.js +4 -4
  91. package/dist/logging/logger.d.ts +1 -1
  92. package/dist/logging/logger.js +2 -2
  93. package/dist/{resources/model-fit → model-fit}/assess.d.ts +18 -26
  94. package/dist/{resources/model-fit → model-fit}/assess.js +210 -274
  95. package/dist/{resources/model-fit/native-probe/classify-fit.d.ts → model-fit/classify.d.ts} +2 -2
  96. package/dist/{resources/model-fit/native-probe/classify-fit.js → model-fit/classify.js} +36 -5
  97. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +4 -2
  98. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +11 -6
  99. package/dist/{resources/model-fit/fit-stub/project-fit-from-load.d.ts → model-fit/description/project.d.ts} +3 -3
  100. package/dist/{resources/model-fit/fit-stub/project-fit-from-load.js → model-fit/description/project.js} +4 -2
  101. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +6 -4
  102. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +14 -6
  103. package/dist/model-fit/floor.d.ts +31 -0
  104. package/dist/{resources/model-fit → model-fit}/floor.js +13 -13
  105. package/dist/model-fit/kv-cache.d.ts +23 -0
  106. package/dist/model-fit/kv-cache.js +85 -0
  107. package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts} +17 -14
  108. package/dist/{resources/model-fit/native-probe/advisory-fit.js → model-fit/load-check.js} +23 -64
  109. package/dist/model-fit/platform.d.ts +14 -0
  110. package/dist/model-fit/platform.js +26 -0
  111. package/dist/{resources/model-fit/native-probe/create-fit-request.d.ts → model-fit/request/create.d.ts} +5 -2
  112. package/dist/model-fit/request/create.js +61 -0
  113. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.d.ts +5 -1
  114. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.js +32 -1
  115. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.d.ts +1 -1
  116. package/dist/model-fit/request/engines/audiogen.js +63 -0
  117. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/diffusion.d.ts +1 -1
  118. package/dist/model-fit/request/engines/diffusion.js +93 -0
  119. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.d.ts +2 -3
  120. package/dist/model-fit/request/engines/llama.js +44 -0
  121. package/dist/model-fit/request/engines/nmt.d.ts +8 -0
  122. package/dist/model-fit/request/engines/nmt.js +11 -0
  123. package/dist/model-fit/request/engines/ocr.d.ts +9 -0
  124. package/dist/model-fit/request/engines/ocr.js +13 -0
  125. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.d.ts +3 -1
  126. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.js +50 -4
  127. package/dist/{resources/model-fit/native-probe/engine-fit.d.ts → model-fit/run/engine.d.ts} +17 -4
  128. package/dist/{resources/model-fit/native-probe/engine-fit.js → model-fit/run/engine.js} +2 -2
  129. package/dist/model-fit/run/run.d.ts +43 -0
  130. package/dist/model-fit/run/run.js +204 -0
  131. package/dist/models/registry/index.d.ts +1 -1
  132. package/dist/models/registry/index.js +2 -2
  133. package/dist/models/registry/models.d.ts +1072 -815
  134. package/dist/models/registry/models.js +3704 -3405
  135. package/dist/models/registry/resource-profiles.d.ts +1 -2
  136. package/dist/models/registry/resource-profiles.js +406 -3
  137. package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
  138. package/dist/plugins/builtin/asr-ggml/config.js +6 -0
  139. package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
  140. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
  141. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +29 -11
  142. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
  143. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
  144. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +7 -28
  145. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -2
  146. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +106 -13
  147. package/dist/plugins/builtin/audiogen-ggml/plugin.js +5 -4
  148. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +20 -3
  149. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
  150. package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
  151. package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
  152. package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
  153. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -0
  154. package/dist/plugins/builtin/ggml-ocr/plugin.js +17 -39
  155. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +9 -2
  156. package/dist/plugins/builtin/ggml-vla/plugin.js +4 -4
  157. package/dist/plugins/builtin/index.d.ts +15 -12
  158. package/dist/plugins/builtin/index.js +15 -12
  159. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +5 -7
  160. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
  161. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  162. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +10 -7
  163. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +141 -204
  164. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +36 -92
  165. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +67 -216
  166. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
  167. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
  168. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +208 -19
  169. package/dist/plugins/builtin/llamacpp-completion/plugin.js +28 -20
  170. package/dist/plugins/builtin/llamacpp-completion/transform.js +5 -1
  171. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +5 -0
  172. package/dist/plugins/builtin/llamacpp-decision/helpers.js +89 -0
  173. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
  174. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
  175. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +222 -0
  176. package/dist/plugins/builtin/llamacpp-decision/plugin.js +44 -0
  177. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +9 -1
  178. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +15 -5
  179. package/dist/plugins/builtin/llamacpp-embedding/transform.js +9 -0
  180. package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
  181. package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
  182. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +5 -0
  183. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +117 -0
  184. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +91 -0
  185. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +45 -0
  186. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
  187. package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
  188. package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
  189. package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
  190. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +17 -0
  191. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +10 -74
  192. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +25 -4
  193. package/dist/plugins/builtin/parakeet-transcription/plugin.js +20 -5
  194. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +5 -2
  195. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +168 -52
  196. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -2
  197. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +28 -13
  198. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +83 -1
  199. package/dist/plugins/builtin/sdcpp-generation/plugin.js +55 -52
  200. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
  201. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
  202. package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
  203. package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
  204. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
  205. package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
  206. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +1 -1
  207. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +5 -2
  208. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +19 -7
  209. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +151 -1
  210. package/dist/plugins/builtin/tts-ggml/plugin.js +39 -2
  211. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +28 -4
  212. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -4
  213. package/dist/plugins/index.d.ts +1 -1
  214. package/dist/plugins/index.js +2 -2
  215. package/dist/plugins/ops/kv-cache-utils.d.ts +5 -4
  216. package/dist/plugins/ops/kv-cache-utils.js +25 -33
  217. package/dist/plugins/ops/load-model.d.ts +8 -2
  218. package/dist/plugins/ops/load-model.js +33 -19
  219. package/dist/plugins/ops/transcribe.js +18 -2
  220. package/dist/plugins/ops/translate.js +3 -3
  221. package/dist/plugins/ops/unload-model.js +1 -1
  222. package/dist/plugins/turbovec-backend.js +5 -5
  223. package/dist/profiling/aggregator.d.ts +1 -1
  224. package/dist/profiling/aggregator.js +1 -1
  225. package/dist/profiling/controller.d.ts +1 -1
  226. package/dist/profiling/controller.js +2 -2
  227. package/dist/profiling/events.d.ts +1 -1
  228. package/dist/profiling/events.js +2 -2
  229. package/dist/profiling/exporters.d.ts +1 -1
  230. package/dist/profiling/exporters.js +3 -3
  231. package/dist/profiling/index.d.ts +13 -13
  232. package/dist/profiling/index.js +12 -12
  233. package/dist/profiling/operation-metrics.d.ts +1 -1
  234. package/dist/profiling/operation-metrics.js +2 -2
  235. package/dist/profiling/operation-wrappers.js +3 -3
  236. package/dist/profiling/profiler.js +1 -1
  237. package/dist/rag/handler.js +1 -1
  238. package/dist/rag/handlers/close-workspace.js +1 -1
  239. package/dist/rag/handlers/delete-embeddings.js +1 -1
  240. package/dist/rag/handlers/delete-workspace.js +1 -1
  241. package/dist/rag/handlers/ingest.js +1 -1
  242. package/dist/rag/handlers/list-workspaces.d.ts +1 -1
  243. package/dist/rag/handlers/list-workspaces.js +1 -1
  244. package/dist/rag/handlers/reindex.js +1 -1
  245. package/dist/rag/handlers/save-embeddings.js +1 -1
  246. package/dist/rag/handlers/search.js +1 -1
  247. package/dist/rag/index.d.ts +11 -11
  248. package/dist/rag/index.js +11 -11
  249. package/dist/rag/rag-workspace-manager.js +1 -1
  250. package/dist/registry.js +3 -1
  251. package/dist/resources/collector.d.ts +1 -1
  252. package/dist/resources/collector.js +1 -1
  253. package/dist/resources/instance.d.ts +1 -1
  254. package/dist/resources/instance.js +1 -1
  255. package/dist/resources/native.d.ts +1 -1
  256. package/dist/resources/normalize.d.ts +1 -1
  257. package/dist/rpc/inventory.js +5 -7
  258. package/dist/runtime/env.d.ts +0 -2
  259. package/dist/runtime/env.js +2 -9
  260. package/dist/runtime/index.d.ts +5 -5
  261. package/dist/runtime/index.js +2 -2
  262. package/dist/runtime/lifecycle.js +7 -7
  263. package/dist/runtime/model-config-utils.js +9 -1
  264. package/dist/runtime/model-registry.d.ts +3 -1
  265. package/dist/runtime/model-registry.js +2 -1
  266. package/dist/runtime/registry-client.js +1 -1
  267. package/dist/runtime/request-context.d.ts +3 -3
  268. package/dist/runtime/request-context.js +22 -1
  269. package/dist/runtime/request-registry.d.ts +1 -1
  270. package/dist/runtime/request-registry.js +1 -1
  271. package/dist/runtime/state.d.ts +1 -1
  272. package/dist/runtime/state.js +7 -2
  273. package/dist/runtime/vector-index-backend.d.ts +2 -2
  274. package/dist/runtime/vector-index-registry.d.ts +5 -5
  275. package/dist/runtime/vector-index-registry.js +36 -13
  276. package/dist/schemas/assess-model-fit.d.ts +106 -43
  277. package/dist/schemas/assess-model-fit.js +69 -32
  278. package/dist/schemas/audio-gen.d.ts +199 -19
  279. package/dist/schemas/audio-gen.js +183 -51
  280. package/dist/schemas/batch-completion-stream.d.ts +210 -36
  281. package/dist/schemas/batch-completion-stream.js +3 -3
  282. package/dist/schemas/bci-config.d.ts +3 -0
  283. package/dist/schemas/bci-config.js +2 -2
  284. package/dist/schemas/bci.d.ts +24 -3
  285. package/dist/schemas/bci.js +2 -2
  286. package/dist/schemas/cancel.d.ts +10 -0
  287. package/dist/schemas/cancel.js +2 -0
  288. package/dist/schemas/common.d.ts +1465 -162
  289. package/dist/schemas/common.js +37 -31
  290. package/dist/schemas/completion-event.d.ts +8 -2
  291. package/dist/schemas/completion-event.js +4 -1
  292. package/dist/schemas/completion-stream.d.ts +243 -39
  293. package/dist/schemas/completion-stream.js +100 -29
  294. package/dist/schemas/config.d.ts +1326 -18
  295. package/dist/schemas/config.js +30 -8
  296. package/dist/schemas/decision.d.ts +610 -0
  297. package/dist/schemas/decision.js +219 -0
  298. package/dist/schemas/download-asset.d.ts +10 -4
  299. package/dist/schemas/download-asset.js +1 -1
  300. package/dist/schemas/engine-addon-map.d.ts +9 -1
  301. package/dist/schemas/engine-addon-map.js +26 -5
  302. package/dist/schemas/errors.d.ts +2 -0
  303. package/dist/schemas/errors.js +11 -1
  304. package/dist/schemas/index.d.ts +51 -48
  305. package/dist/schemas/index.js +50 -47
  306. package/dist/schemas/llamacpp-config.d.ts +222 -6
  307. package/dist/schemas/llamacpp-config.js +129 -7
  308. package/dist/schemas/load-model.d.ts +7960 -1492
  309. package/dist/schemas/load-model.js +122 -10
  310. package/dist/schemas/mcp-adapter.d.ts +1 -1
  311. package/dist/schemas/model-config-schemas.d.ts +428 -9
  312. package/dist/schemas/model-config-schemas.js +16 -11
  313. package/dist/schemas/model-fit-policy.d.ts +17 -0
  314. package/dist/schemas/model-fit-policy.js +17 -0
  315. package/dist/schemas/model-info.d.ts +20 -2
  316. package/dist/schemas/model-info.js +4 -3
  317. package/dist/schemas/model-resource-profile.d.ts +4 -4
  318. package/dist/schemas/model-resource-profile.js +2 -5
  319. package/dist/schemas/model-src-utils.d.ts +14 -2
  320. package/dist/schemas/model-src-utils.js +2 -2
  321. package/dist/schemas/model-types.d.ts +14 -1
  322. package/dist/schemas/model-types.js +4 -0
  323. package/dist/schemas/ocr.d.ts +76 -0
  324. package/dist/schemas/ocr.js +43 -1
  325. package/dist/schemas/plugin.d.ts +16 -4
  326. package/dist/schemas/plugin.js +12 -0
  327. package/dist/schemas/profiling.d.ts +4 -4
  328. package/dist/schemas/profiling.js +1 -1
  329. package/dist/schemas/rag.js +1 -1
  330. package/dist/schemas/registry.d.ts +34 -0
  331. package/dist/schemas/registry.js +5 -1
  332. package/dist/schemas/reload-config.d.ts +14 -2
  333. package/dist/schemas/reload-config.js +2 -2
  334. package/dist/schemas/rerank.d.ts +131 -0
  335. package/dist/schemas/rerank.js +102 -0
  336. package/dist/schemas/rpc-server.js +3 -3
  337. package/dist/schemas/sdcpp-config.d.ts +82 -0
  338. package/dist/schemas/sdcpp-config.js +41 -22
  339. package/dist/schemas/system-resources.d.ts +4 -4
  340. package/dist/schemas/text-to-speech.d.ts +463 -3
  341. package/dist/schemas/text-to-speech.js +129 -11
  342. package/dist/schemas/tools.d.ts +13 -4
  343. package/dist/schemas/tools.js +18 -6
  344. package/dist/schemas/transcription-config.d.ts +20 -0
  345. package/dist/schemas/transcription-config.js +54 -4
  346. package/dist/schemas/transcription.d.ts +47 -5
  347. package/dist/schemas/transcription.js +52 -5
  348. package/dist/schemas/translate.d.ts +2 -2
  349. package/dist/schemas/translate.js +1 -1
  350. package/dist/schemas/translation-config.d.ts +40 -10
  351. package/dist/schemas/translation-config.js +1 -1
  352. package/dist/schemas/vla.d.ts +9 -2
  353. package/dist/schemas/vla.js +9 -5
  354. package/dist/surface.d.ts +3 -2
  355. package/dist/surface.js +3 -2
  356. package/dist/utils/addon-responses.d.ts +3 -0
  357. package/dist/utils/aggregate-events.d.ts +1 -1
  358. package/dist/utils/aggregate-events.js +2 -2
  359. package/dist/utils/archive.js +2 -2
  360. package/dist/utils/asr-events.d.ts +1 -0
  361. package/dist/utils/asr-events.js +2 -1
  362. package/dist/utils/audio/decoder.js +3 -1
  363. package/dist/utils/audio-input.js +1 -1
  364. package/dist/utils/cache/auto-history.d.ts +2 -1
  365. package/dist/utils/cache/auto-history.js +2 -2
  366. package/dist/utils/cache/paths.js +6 -6
  367. package/dist/utils/cache/types.d.ts +7 -0
  368. package/dist/utils/checksum.d.ts +2 -0
  369. package/dist/utils/checksum.js +41 -2
  370. package/dist/utils/completion-normalizer.js +1 -1
  371. package/dist/utils/gguf-architecture.d.ts +7 -0
  372. package/dist/utils/gguf-architecture.js +142 -0
  373. package/dist/utils/gguf-tensor-extractor.js +1 -1
  374. package/dist/utils/index.d.ts +13 -13
  375. package/dist/utils/index.js +13 -13
  376. package/dist/utils/mcp-adapter.d.ts +1 -1
  377. package/dist/utils/mcp-adapter.js +13 -9
  378. package/dist/utils/path-security.js +1 -1
  379. package/dist/utils/shard-utils.d.ts +5 -0
  380. package/dist/utils/shard-utils.js +19 -3
  381. package/dist/utils/tool-helpers.d.ts +1 -1
  382. package/dist/utils/tool-helpers.js +20 -52
  383. package/dist/utils/tool-integration.d.ts +1 -0
  384. package/dist/utils/tool-integration.js +3 -6
  385. package/dist/utils/tools/defer.d.ts +16 -26
  386. package/dist/utils/tools/defer.js +50 -41
  387. package/dist/utils/tools/dialect.d.ts +15 -4
  388. package/dist/utils/tools/dialect.js +34 -5
  389. package/dist/utils/tools/index.d.ts +3 -3
  390. package/dist/utils/tools/index.js +3 -3
  391. package/dist/utils/tools/parser.js +8 -8
  392. package/dist/utils/tools/parsers/dsml.d.ts +1 -1
  393. package/dist/utils/tools/parsers/dsml.js +8 -5
  394. package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
  395. package/dist/utils/tools/parsers/gemma4native.js +1 -1
  396. package/dist/utils/tools/parsers/harmony.d.ts +1 -1
  397. package/dist/utils/tools/parsers/harmony.js +1 -1
  398. package/dist/utils/tools/parsers/hermes.d.ts +1 -1
  399. package/dist/utils/tools/parsers/hermes.js +1 -1
  400. package/dist/utils/tools/parsers/json.d.ts +1 -1
  401. package/dist/utils/tools/parsers/json.js +1 -1
  402. package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
  403. package/dist/utils/tools/parsers/pythonic.js +1 -1
  404. package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
  405. package/dist/utils/tools/parsers/qwen35.js +7 -4
  406. package/dist/utils/tools/shared.d.ts +3 -0
  407. package/dist/utils/tools/shared.js +62 -0
  408. package/dist/utils/transcribe-metadata.d.ts +8 -1
  409. package/dist/utils/transcribe-metadata.js +13 -3
  410. package/dist/utils/tts-stats.d.ts +2 -1
  411. package/dist/utils/until-aborted.d.ts +7 -0
  412. package/dist/utils/until-aborted.js +34 -0
  413. package/package.json +68 -51
  414. package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
  415. package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
  416. package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
  417. package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
  418. package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
  419. package/dist/resources/model-fit/calibration/fit.js +0 -152
  420. package/dist/resources/model-fit/calibration/index.d.ts +0 -37
  421. package/dist/resources/model-fit/calibration/index.js +0 -75
  422. package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
  423. package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
  424. package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
  425. package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
  426. package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
  427. package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
  428. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
  429. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
  430. package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
  431. package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
  432. package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
  433. package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
  434. package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
  435. package/dist/resources/model-fit/estimators/llm.js +0 -222
  436. package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
  437. package/dist/resources/model-fit/estimators/whisper.js +0 -76
  438. package/dist/resources/model-fit/floor.d.ts +0 -31
  439. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
  440. package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -41
  441. package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -51
  442. package/dist/resources/model-fit/native-probe/engines/audiogen.js +0 -35
  443. package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -55
  444. package/dist/resources/model-fit/native-probe/engines/llama.js +0 -145
  445. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
  446. package/dist/resources/model-fit/native-probe/fit-outcome.js +0 -1
  447. package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
  448. package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
  449. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
  450. package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
  451. package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
  452. package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
  453. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -22
  454. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -108
  455. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
  456. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
  457. package/dist/resources/model-fit/platform.d.ts +0 -9
  458. package/dist/resources/model-fit/platform.js +0 -28
  459. package/dist/resources/model-fit/types.d.ts +0 -120
  460. package/dist/resources/model-fit/types.js +0 -1
  461. /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.d.ts +0 -0
  462. /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.js +0 -0
@@ -1,27 +1,24 @@
1
1
  import { promises as fsPromises } from 'bare-fs';
2
2
  import path from 'bare-path';
3
- import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../ops/kv-cache-utils.js';
4
- import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../ops/kv-cache-retention.js';
3
+ import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../../plugins/ops/kv-cache-utils.js';
4
+ import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../../plugins/ops/kv-cache-retention.js';
5
5
  import { isMobile } from '../../../../runtime/state.js';
6
6
  import { getKVCacheDir } from '../../../../utils/index.js';
7
- import { logCacheSaveError, logCacheStatus } from './cache-logger.js';
7
+ import { logCacheSaveError, logCacheStatus } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
8
8
  import { getEngineLogger } from '../../../../logging/index.js';
9
- import { z } from 'zod';
10
9
  // Used by cross-model paths that have no `RequestContext` (e.g.
11
10
  // `deleteKvCacheState`). Per-session call sites receive a logger from
12
11
  // the caller — typically `withRequestContext(...)`.
13
12
  const moduleLogger = getEngineLogger();
14
13
  /**
15
- * Coordinates six KV-cache state layers:
14
+ * Coordinates four KV-cache state layers:
16
15
  *
17
- * 1. `cachedPrefixes` — saved message boundaries, and whether a tool
18
- * block was rendered into them.
19
- * 2. `initializedCaches` — caches this process has established.
20
- * 3. On-disk `.bin` files written by the addon.
21
- * 4. `<bin>.meta.json` sidecars — layer 1 persisted, so a named cache's
22
- * boundary outlives the process; see `readPrefixSidecar`.
23
- * 5. `activeCachePaths` — per-path refs that block in-flight eviction.
24
- * 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
16
+ * 1. `initializedCaches` — caches this process has established.
17
+ * 2. On-disk `.bin` files written by the addon. Each one records the tokens
18
+ * it holds, and the addon reuses the longest prefix the next prompt
19
+ * shares with it, so nothing here tracks what a file covers.
20
+ * 3. `activeCachePaths` — per-path refs that block in-flight eviction.
21
+ * 4. `.auto-cache-<key>` markers — engine-generated cache ownership.
25
22
  *
26
23
  * Every turn must finish through `commitTurn`, `rollback`, or the
27
24
  * non-destructive `releaseTurn` so all inference state stays aligned,
@@ -30,100 +27,6 @@ const moduleLogger = getEngineLogger();
30
27
  */
31
28
  // ----- module-scoped state. The session is the single mutation point
32
29
  // for the in-memory KV-cache bookkeeping. -----
33
- /** What the kv-cache file at a given path is known to hold. */
34
- const cachedPrefixSchema = z.object({
35
- /** Number of chat messages the file on disk is known to cover. */
36
- messages: z.number().int().nonnegative(),
37
- /**
38
- * Whether a static tool block was rendered into that prefix. Tracked
39
- * rather than inferred from `messages`, because a committed turn is not
40
- * proof that its tool block reached the model: the addon drops tools and
41
- * still returns a usable prompt when the chat template rejects them.
42
- */
43
- toolBlock: z.boolean()
44
- });
45
- // The sidecar is written after the addon has already saved the `.bin`, so a
46
- // crash between the two leaves a boundary that describes an older file. The
47
- // `.bin` size is a function of the tokens it holds: it is unchanged when the
48
- // addon re-saves the same state (cancel rewind, key switch) and larger once
49
- // another turn landed in it, which is the case the fingerprint must catch.
50
- const persistedPrefixSchema = cachedPrefixSchema.extend({
51
- binSize: z.number().int().nonnegative()
52
- });
53
- /**
54
- * What the kv-cache file on disk is known to cover, keyed by cache path.
55
- * Written by `commitTurn`, deleted by `rollback`, `delete`,
56
- * `dropStaleSavedCount` and the orphan clear in `beginTurn`. The same
57
- * INVARIANT that existed in `kv-cache-state.ts` still holds: an entry is
58
- * present only when the corresponding `.bin` file is considered trustworthy,
59
- * so any turn that leaves the file holding something the boundary does not
60
- * describe has to remove it.
61
- *
62
- * This map outranks the sidecar. Both are written by the same commit, so an
63
- * entry here is never the older copy, and `restorePersistedCache` reads the
64
- * file only to repopulate an empty map after a restart.
65
- */
66
- const cachedPrefixes = new Map();
67
- const PREFIX_SIDECAR_SUFFIX = '.meta.json';
68
- function prefixSidecarPath(cachePath) {
69
- return `${cachePath}${PREFIX_SIDECAR_SUFFIX}`;
70
- }
71
- async function writePrefixSidecar(cachePath, prefix, logger) {
72
- try {
73
- const { size } = await fsPromises.stat(cachePath);
74
- await fsPromises.writeFile(prefixSidecarPath(cachePath), JSON.stringify({ ...prefix, binSize: size }));
75
- }
76
- catch (error) {
77
- // Only the file goes: a sidecar left from the previous commit describes a
78
- // shorter `.bin` than the one now on disk. The in-memory boundary is the
79
- // one this process keeps slicing against and stays.
80
- await removePrefixSidecar(cachePath);
81
- logger.warn(`[kv-cache] Failed to persist saved-message boundary; this process stays warm but a restart will start the cache cold. path=${cachePath} error=${error instanceof Error ? error.message : String(error)}`);
82
- }
83
- }
84
- // Anything unreadable, malformed, or describing a `.bin` of a different size
85
- // is discarded: the cache is then used from a cold boundary, which is the
86
- // pre-sidecar behaviour. Only ever consulted for a path `cachedPrefixes` has
87
- // no entry for, so it can never contradict a live boundary. Never throws: the
88
- // caller holds the cache-path write lock and a `.bin` can vanish under it
89
- // (`deleteKvCacheState` takes no locks).
90
- async function readPrefixSidecar(cachePath) {
91
- let parsed;
92
- let binSize;
93
- try {
94
- const raw = await fsPromises.readFile(prefixSidecarPath(cachePath), 'utf8');
95
- const result = persistedPrefixSchema.safeParse(JSON.parse(raw));
96
- if (!result.success)
97
- return null;
98
- parsed = result.data;
99
- binSize = (await fsPromises.stat(cachePath)).size;
100
- }
101
- catch {
102
- return null;
103
- }
104
- if (binSize !== parsed.binSize) {
105
- // The `.bin` changed after this boundary was recorded, so the file is
106
- // stale whatever wrote it. Only the file: the caller guarantees the map
107
- // holds nothing for this path.
108
- await removePrefixSidecar(cachePath);
109
- return null;
110
- }
111
- return { messages: parsed.messages, toolBlock: parsed.toolBlock };
112
- }
113
- /** Drops the persisted boundary, leaving whatever is in memory alone. */
114
- async function removePrefixSidecar(cachePath) {
115
- try {
116
- await fsPromises.unlink(prefixSidecarPath(cachePath));
117
- }
118
- catch {
119
- // No sidecar for this path.
120
- }
121
- }
122
- /** Drops the boundary everywhere: the cache it describes is gone or untrusted. */
123
- async function forgetPrefix(cachePath) {
124
- cachedPrefixes.delete(cachePath);
125
- await removePrefixSidecar(cachePath);
126
- }
127
30
  /**
128
31
  * In-memory registry of caches established this session, recorded once a save
129
32
  * is verified. The addon defers disk writes, so the absence of a `.bin` file on
@@ -154,7 +57,7 @@ class CacheLockAbortError extends Error {
154
57
  // "session"), which name the SAME file on case-insensitive filesystems (default
155
58
  // macOS/Windows), serialise on one lock and can't interleave writes. Over-locks
156
59
  // case-only variants on case-sensitive filesystems, which is safe. NOTE: the
157
- // other per-path bookkeeping (activeCachePaths, cachedPrefixes, retention)
60
+ // other per-path bookkeeping (activeCachePaths, initializedCaches, retention)
158
61
  // stays case-sensitive, so case-only key variants are otherwise unsupported —
159
62
  // see the KV-cache system doc.
160
63
  function lockKeyFor(cachePath) {
@@ -335,20 +238,24 @@ async function deleteInactiveAutoCaches() {
335
238
  * Construct a session bound to one `(modelId, turn-owning request)`
336
239
  * scope. `options.logger` is the per-instance logger the session emits
337
240
  * through (typically `withRequestContext(getEngineLogger(), ctx)`);
338
- * falls back to the module-scoped logger when omitted.
241
+ * falls back to the module-scoped logger when omitted. `options.discardCache`
242
+ * drops the addon's copy of a conversation whose cache the session rolls
243
+ * back; without it the addon writes that conversation to the deleted path at
244
+ * its next key switch or unload.
339
245
  */
340
246
  export function createKvCacheSession(modelId, options) {
341
247
  const logger = options?.logger ?? moduleLogger;
248
+ const discardCache = options?.discardCache;
342
249
  // Per-session map: each `TurnHandle` carries an opaque entry here. A
343
250
  // WeakMap so handles drop their state once the handler scope releases
344
251
  // the reference; the module-scoped maps above survive.
345
252
  const turnState = new WeakMap();
346
253
  function makeHandle(cachePath, autoCacheKey, releaseWriteLock = () => { }, signal) {
347
- const cached = cachedPrefixes.get(cachePath);
348
254
  const handle = {
349
255
  cachePath,
350
- savedCount: cached?.messages ?? 0,
351
- toolBlockCached: cached?.toolBlock ?? false
256
+ get createdCache() {
257
+ return turnState.get(handle)?.createdByThisTurn ?? false;
258
+ }
352
259
  };
353
260
  turnState.set(handle, {
354
261
  cachePath,
@@ -389,14 +296,10 @@ export function createKvCacheSession(modelId, options) {
389
296
  // In-memory registry check first — the addon defers disk writes, so a
390
297
  // just-saved cache may not yet exist on disk. If the in-memory flag isn't
391
298
  // set, fall back to a filesystem probe so caches surviving across process
392
- // restarts still hit the reuse path. Resolved before the handle is made so
393
- // it snapshots the restored boundary.
299
+ // restarts still hit the reuse path.
394
300
  let exists = initializedCaches.has(cachePath);
395
301
  if (!exists)
396
- exists = await restorePersistedCache(cachePath);
397
- // A boundary without its `.bin` describes a cache that no longer exists.
398
- if (!exists)
399
- await forgetPrefix(cachePath);
302
+ exists = await adoptPersistedCache(cachePath);
400
303
  const handle = makeHandle(cachePath, undefined, releaseWriteLock, input.signal);
401
304
  try {
402
305
  logCacheStatus(input.customKey, exists);
@@ -420,8 +323,9 @@ export function createKvCacheSession(modelId, options) {
420
323
  // the state lock, or commitTurn's rename could deadlock a waiter), then
421
324
  // re-check existence under the lock in case a same-file peer just renamed it.
422
325
  const discovered = await withCacheStateLock(async () => {
423
- const existingCache = await findMatchingCache(modelId, input.configHash, input.history);
424
- const cacheInfo = existingCache ?? (await getCurrentCacheInfo(modelId, input.configHash, input.history));
326
+ const existingCache = await findMatchingCache(modelId, input.configHash, input.history, input.tools);
327
+ const cacheInfo = existingCache ??
328
+ (await getCurrentCacheInfo(modelId, input.configHash, input.history, input.tools));
425
329
  return {
426
330
  cachePath: cacheInfo.cachePath,
427
331
  cacheKey: cacheInfo.cacheKey
@@ -511,33 +415,31 @@ export function createKvCacheSession(modelId, options) {
511
415
  if (state.committed || state.rolledBack)
512
416
  return;
513
417
  if (result.kind === 'static') {
514
- // Custom-key path: the addon wrote the new cache state inline
515
- // at the same path. Verify the file persisted (the addon
516
- // currently swallows save errors — see TODO in
517
- // `verifySaveAndRecord`) and record the new boundary.
518
- const prefix = { messages: result.messageCount, toolBlock: result.toolBlockCached };
519
- const ok = await verifySaveAndRecord(state.cachePath, prefix);
418
+ // The addon's save wrote the new cache state at the same path.
419
+ const ok = await verifySaveAndRecord(state.cachePath);
520
420
  if (!ok) {
521
421
  await runRollback(state);
522
422
  return;
523
423
  }
524
- await writePrefixSidecar(state.cachePath, prefix, logger);
525
424
  state.committed = true;
526
425
  releaseCachePath(state.cachePath);
527
426
  state.releaseWriteLock();
427
+ if (state.autoCacheKey !== undefined)
428
+ scheduleAutoCacheSweep(logger);
528
429
  return;
529
430
  }
530
431
  const sourceCachePath = state.cachePath;
531
432
  const sourceCacheKey = state.autoCacheKey;
532
433
  const targetCacheKey = path.basename(path.dirname(path.dirname(result.targetCachePath)));
533
434
  // Hold the target's lock across the rename AND commit verification, so a peer
534
- // resolving to the same file can't observe it before its saved count is
535
- // published. Abortable: a cancel while waiting for the lock rejects here.
435
+ // resolving to the same file can't observe it before it is recorded. Abortable: a cancel while waiting for the lock rejects here.
536
436
  const releaseTargetLock = await acquireCachePathWriteLock(result.targetCachePath, state.signal);
537
437
  try {
538
- // A cancel that landed while we waited for the target lock must not commit.
438
+ // A cancel that landed while we waited for the target lock must not
439
+ // commit. The file holds what the addon kept, so it stays unless this
440
+ // turn created it.
539
441
  if (state.signal?.aborted) {
540
- await runRollback(state);
442
+ await runRelease(state);
541
443
  return;
542
444
  }
543
445
  const renamed = await withCacheStateLock(async () => {
@@ -564,7 +466,6 @@ export function createKvCacheSession(modelId, options) {
564
466
  releaseCachePath(sourceCachePath);
565
467
  state.cachePath = result.targetCachePath;
566
468
  state.autoCacheKey = targetCacheKey;
567
- await forgetPrefix(sourceCachePath);
568
469
  await pruneEmptyCacheDirectories(sourceCachePath, snapshotActivePaths());
569
470
  if (sourceCacheKey !== undefined) {
570
471
  await removeAutoCacheMarkerIfMissing(sourceCacheKey);
@@ -579,10 +480,7 @@ export function createKvCacheSession(modelId, options) {
579
480
  await runRollback(state);
580
481
  return;
581
482
  }
582
- const ok = await verifySaveAndRecord(result.targetCachePath, {
583
- messages: result.messageCount,
584
- toolBlock: result.toolBlockCached
585
- });
483
+ const ok = await verifySaveAndRecord(result.targetCachePath);
586
484
  if (!ok) {
587
485
  // Rename succeeded but the file isn't where we expected. Roll back via
588
486
  // the target path instead of the (now-empty) source.
@@ -615,6 +513,9 @@ export function createKvCacheSession(modelId, options) {
615
513
  return;
616
514
  if (state.committed || state.rolledBack)
617
515
  return;
516
+ await runRelease(state);
517
+ }
518
+ async function runRelease(state) {
618
519
  // Nothing committed exists to keep, so a failed first turn is destructive.
619
520
  if (state.createdByThisTurn) {
620
521
  await runRollback(state);
@@ -641,7 +542,14 @@ export function createKvCacheSession(modelId, options) {
641
542
  logger.warn(`[kv-cache] Failed to remove cache file during rollback; next turn may load stale KV state. path=${state.cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
642
543
  }
643
544
  }
644
- await forgetPrefix(state.cachePath);
545
+ if (discardCache) {
546
+ try {
547
+ await discardCache(state.cachePath);
548
+ }
549
+ catch (discardError) {
550
+ logger.warn(`[kv-cache] Failed to discard the addon's copy of a rolled-back cache; a later unload may write it back. path=${state.cachePath} error=${discardError instanceof Error ? discardError.message : String(discardError)}`);
551
+ }
552
+ }
645
553
  // Release before pruning so an empty parent can go; a sibling still holding
646
554
  // the path keeps it in the active snapshot and protects the directory.
647
555
  releaseCachePath(state.cachePath);
@@ -655,18 +563,11 @@ export function createKvCacheSession(modelId, options) {
655
563
  if (state.autoCacheKey !== undefined)
656
564
  scheduleAutoCacheSweep(logger);
657
565
  }
658
- async function dropStaleSavedCount(turn) {
659
- const state = turnState.get(turn);
660
- if (!state)
661
- return;
662
- await forgetPrefix(state.cachePath);
663
- }
664
566
  return {
665
567
  beginTurn,
666
568
  commitTurn,
667
569
  rollback,
668
- releaseTurn,
669
- dropStaleSavedCount
570
+ releaseTurn
670
571
  };
671
572
  }
672
573
  // ----- module-level administrative API -----
@@ -687,10 +588,8 @@ export function createKvCacheSession(modelId, options) {
687
588
  * Layers cleared, in order:
688
589
  * 1. On-disk: `deleteCache(...)` removes the matching directory
689
590
  * tree (or wipes and recreates the root for `all: true`).
690
- * 2. `cachedPrefixes`: prefix-cleanup by the removed directory
691
- * so any per-cache count under the deleted tree is forgotten.
692
- * 3. `initializedCaches`: prefix-cleanup by the same removed
693
- * directory, since it is keyed by the resolved cache path.
591
+ * 2. `initializedCaches`: prefix-cleanup by the removed directory,
592
+ * since it is keyed by the resolved cache path.
694
593
  *
695
594
  * Concurrency with in-flight turns: this delete does not take the
696
595
  * per-cache-path write locks, so it races any turn holding a `TurnHandle`
@@ -699,9 +598,8 @@ export function createKvCacheSession(modelId, options) {
699
598
  * `Map.delete` / `Set.delete` no-op on absent keys), but the layers are
700
599
  * cleared without a lock, so an interleaving that lands the delete's
701
600
  * in-memory cleanup after a concurrent turn has already renamed/committed
702
- * its file splits state: the file stays on disk while its saved-count and
703
- * init flag are cleared, so the next turn loads the file and reports
704
- * `savedCount=0`. Callers must not delete a key that is in active use.
601
+ * its file splits state: the file stays on disk while its init flag is
602
+ * cleared. Callers must not delete a key that is in active use.
705
603
  * The `auto` target is the exception: it skips keys an in-flight turn holds
706
604
  * and clears each one under the cache-state lock.
707
605
  */
@@ -710,7 +608,6 @@ export async function deleteKvCacheState(target) {
710
608
  return deleteInactiveAutoCaches();
711
609
  if ('all' in target) {
712
610
  const removed = await deleteCacheUtil({ all: true });
713
- cachedPrefixes.clear();
714
611
  initializedCaches.clear();
715
612
  // `removed` is the kv-cache root dir; surfaces it for ops
716
613
  // visibility but isn't part of the contract.
@@ -728,50 +625,40 @@ export async function deleteKvCacheState(target) {
728
625
  // onto the unrelated top-level auto marker.
729
626
  await removeAutoCacheMarker(path.relative(getKVCacheDir(), removedPath));
730
627
  }
731
- // Prefix-cleanup the in-memory counts. The on-disk directory tree
732
- // is `{kvCacheRoot}/{kvCacheKey}[/{modelId}]/`, so every entry in
733
- // `cachedPrefixes` whose key is the removed directory itself
734
- // or sits beneath it must go.
735
- clearCachedMessageCountsByPrefix(removedPath, path.sep);
736
- // initializedCaches is keyed by the resolved cachePath too, so clear it by
737
- // the same removed-directory prefix as cachedPrefixes above.
628
+ // The on-disk directory tree is `{kvCacheRoot}/{kvCacheKey}[/{modelId}]/`,
629
+ // so every entry whose path is the removed directory itself or sits beneath
630
+ // it must go.
738
631
  clearInitializedCachesByPrefix(removedPath, path.sep);
739
632
  }
740
633
  // ----- private helpers -----
741
634
  /**
742
- * Verify the addon actually persisted the cache file before recording
743
- * its message count. The addon currently swallows write errors
744
- * silently, so a missing file means the next turn must resend the full
745
- * history rather than slicing against a stale `savedCount`.
746
- *
747
- * TODO: once the addon surfaces save failures (e.g. throws
748
- * `UnableToSaveSessionFile` when `llama_state_save_file` returns
749
- * false), drop the `access()` probe and wrap the `model.run()` call in
750
- * a real try/catch that forwards the error.
635
+ * Verify the cache file exists before recording it as established. The file
636
+ * is the evidence because `saveCache` rejects both when a write fails and when
637
+ * the addon kept nothing to write.
751
638
  */
752
- async function verifySaveAndRecord(cachePath, prefix) {
639
+ async function verifySaveAndRecord(cachePath) {
753
640
  try {
754
641
  await fsPromises.access(cachePath);
755
- cachedPrefixes.set(cachePath, prefix);
756
642
  initializedCaches.add(cachePath);
757
643
  return true;
758
644
  }
759
645
  catch (err) {
760
- cachedPrefixes.delete(cachePath);
761
646
  logCacheSaveError(cachePath, err);
762
647
  return false;
763
648
  }
764
649
  }
765
650
  /**
766
- * Adopt a `.bin` left by an earlier process run: record it as established and
767
- * pick up the boundary committed alongside it. Returns whether the file exists.
768
- *
769
- * An auto-rename commits its target into `cachedPrefixes` without an init
770
- * flag, so this can run for a path the map already knows. The map wins there
771
- * and the sidecar is left unread, which keeps the two from ever having to be
772
- * reconciled.
651
+ * Adopt a `.bin` left by an earlier process run. Returns whether the file
652
+ * exists. Earlier releases kept a `<bin>.meta.json` boundary sidecar beside
653
+ * it; nothing reads one now, so it goes on first contact.
773
654
  */
774
- async function restorePersistedCache(cachePath) {
655
+ async function adoptPersistedCache(cachePath) {
656
+ try {
657
+ await fsPromises.unlink(`${cachePath}.meta.json`);
658
+ }
659
+ catch {
660
+ // No sidecar for this path.
661
+ }
775
662
  try {
776
663
  await fsPromises.access(cachePath);
777
664
  }
@@ -779,28 +666,8 @@ async function restorePersistedCache(cachePath) {
779
666
  return false;
780
667
  }
781
668
  initializedCaches.add(cachePath);
782
- if (cachedPrefixes.has(cachePath))
783
- return true;
784
- const prefix = await readPrefixSidecar(cachePath);
785
- if (prefix !== null)
786
- cachedPrefixes.set(cachePath, prefix);
787
669
  return true;
788
670
  }
789
- function clearCachedMessageCountsByPrefix(prefix, sep) {
790
- if (!prefix) {
791
- cachedPrefixes.clear();
792
- return;
793
- }
794
- for (const key of cachedPrefixes.keys()) {
795
- if (key === prefix) {
796
- cachedPrefixes.delete(key);
797
- continue;
798
- }
799
- if (!key.startsWith(prefix + sep))
800
- continue;
801
- cachedPrefixes.delete(key);
802
- }
803
- }
804
671
  function clearInitializedCachesByPrefix(prefix, sep) {
805
672
  if (!prefix) {
806
673
  initializedCaches.clear();
@@ -822,21 +689,6 @@ function clearInitializedCachesByPrefix(prefix, sep) {
822
689
  * @internal
823
690
  */
824
691
  export const __kvCacheSessionTestHooks = {
825
- getSavedCount(cachePath) {
826
- return cachedPrefixes.get(cachePath)?.messages;
827
- },
828
- setSavedCountForTest(cachePath, count) {
829
- cachedPrefixes.set(cachePath, { messages: count, toolBlock: false });
830
- },
831
- getToolBlockCachedForTest(cachePath) {
832
- return cachedPrefixes.get(cachePath)?.toolBlock ?? false;
833
- },
834
- getPrefixSidecarPathForTest(cachePath) {
835
- return prefixSidecarPath(cachePath);
836
- },
837
- readPrefixSidecarForTest(cachePath) {
838
- return readPrefixSidecar(cachePath);
839
- },
840
692
  hasInitializedPath(cachePath) {
841
693
  return initializedCaches.has(cachePath);
842
694
  },
@@ -850,7 +702,6 @@ export const __kvCacheSessionTestHooks = {
850
702
  lastAutoCacheSweepMs = value;
851
703
  },
852
704
  resetForTest() {
853
- cachedPrefixes.clear();
854
705
  initializedCaches.clear();
855
706
  activeCachePaths.clear();
856
707
  lastAutoCacheSweepMs = 0;
@@ -1,37 +1,11 @@
1
1
  /**
2
- * Pure decision helper for kv-cache history slicing used by
3
- * `completion-stream.ts` (via the slice computed from
4
- * `TurnHandle.savedCount`).
2
+ * Pure commit decision for kv-cache turns used by `completion-stream.ts`.
5
3
  *
6
4
  * This module intentionally has **no** `bare-*` imports so it can be
7
- * exercised directly from unit tests running under `bun` without
5
+ * exercised directly from unit tests running under Node without
8
6
  * pulling in the Bare runtime (which is not available in that
9
7
  * environment).
10
- *
11
- * Cancel detection flows through the per-request `AbortSignal` from
12
- * `RequestRegistry`; the `cachedMessageCounts` map lives in
13
- * `kv-cache-session.ts`, which owns all three KV-cache bookkeeping
14
- * layers (saved counts, init flags, on-disk files). Only the pure
15
- * slice-decision helper remains here.
16
8
  */
17
- export interface HistoryMessage {
18
- role: string;
19
- content: string;
20
- attachments?: {
21
- path: string;
22
- }[] | undefined;
23
- }
24
- export interface HistorySliceDecision {
25
- /** Messages to send to the model on the next turn. */
26
- messages: HistoryMessage[];
27
- /**
28
- * True when the decision path proves the current `savedCount` is stale
29
- * and the caller should drop the cached entry (via
30
- * `KvCacheSession.dropStaleSavedCount(turn)`) to avoid propagating the
31
- * bad count to the next turn.
32
- */
33
- clearStaleCount: boolean;
34
- }
35
9
  interface CacheCommitContext {
36
10
  aborted: boolean;
37
11
  producedTokens: boolean;
@@ -40,25 +14,4 @@ interface CacheCommitContext {
40
14
  stoppedAtContextBoundary: boolean;
41
15
  }
42
16
  export declare function shouldCommitCachedTurn(context: CacheCommitContext): boolean;
43
- /**
44
- * Pure slice decision for `prepareMessagesForCache`.
45
- *
46
- * Mirrors the shape of the logic in `completion-stream.ts` but without
47
- * calling `transformMessages` (which depends on `bare-fs` for
48
- * attachment probing). Kept here so the decision can be unit-tested in
49
- * isolation.
50
- *
51
- * A non-zero `savedCount` is itself the proof that a cache holds a rendered
52
- * prefix: it is recorded only after a save is verified. It means the prefix
53
- * covers that many messages, so slicing from it drops exactly what the cache
54
- * supplies — the system message among them. Without a usable boundary the
55
- * cache supplies nothing, so the whole history goes to the addon, system
56
- * message included: anything held back would simply be missing from the
57
- * prompt.
58
- *
59
- * The regression guard: a non-zero `savedCount` that slices the history down
60
- * to an empty array is stale, and the caller resends the full history rather
61
- * than handing the model an empty payload.
62
- */
63
- export declare function decideCachedHistorySlice(savedCount: number, history: HistoryMessage[]): HistorySliceDecision;
64
17
  export {};
@@ -1,18 +1,10 @@
1
1
  /**
2
- * Pure decision helper for kv-cache history slicing used by
3
- * `completion-stream.ts` (via the slice computed from
4
- * `TurnHandle.savedCount`).
2
+ * Pure commit decision for kv-cache turns used by `completion-stream.ts`.
5
3
  *
6
4
  * This module intentionally has **no** `bare-*` imports so it can be
7
- * exercised directly from unit tests running under `bun` without
5
+ * exercised directly from unit tests running under Node without
8
6
  * pulling in the Bare runtime (which is not available in that
9
7
  * environment).
10
- *
11
- * Cancel detection flows through the per-request `AbortSignal` from
12
- * `RequestRegistry`; the `cachedMessageCounts` map lives in
13
- * `kv-cache-session.ts`, which owns all three KV-cache bookkeeping
14
- * layers (saved counts, init flags, on-disk files). Only the pure
15
- * slice-decision helper remains here.
16
8
  */
17
9
  export function shouldCommitCachedTurn(context) {
18
10
  const { aborted, producedTokens, generatedTokens, predict, stoppedAtContextBoundary } = context;
@@ -22,39 +14,3 @@ export function shouldCommitCachedTurn(context) {
22
14
  generatedTokens >= predict;
23
15
  return !aborted && producedTokens && !stoppedByBudget && !stoppedAtContextBoundary;
24
16
  }
25
- /**
26
- * Pure slice decision for `prepareMessagesForCache`.
27
- *
28
- * Mirrors the shape of the logic in `completion-stream.ts` but without
29
- * calling `transformMessages` (which depends on `bare-fs` for
30
- * attachment probing). Kept here so the decision can be unit-tested in
31
- * isolation.
32
- *
33
- * A non-zero `savedCount` is itself the proof that a cache holds a rendered
34
- * prefix: it is recorded only after a save is verified. It means the prefix
35
- * covers that many messages, so slicing from it drops exactly what the cache
36
- * supplies — the system message among them. Without a usable boundary the
37
- * cache supplies nothing, so the whole history goes to the addon, system
38
- * message included: anything held back would simply be missing from the
39
- * prompt.
40
- *
41
- * The regression guard: a non-zero `savedCount` that slices the history down
42
- * to an empty array is stale, and the caller resends the full history rather
43
- * than handing the model an empty payload.
44
- */
45
- export function decideCachedHistorySlice(savedCount, history) {
46
- const hasCachedPrefix = savedCount > 0 && history.length > 0;
47
- const sliced = hasCachedPrefix && savedCount <= history.length ? history.slice(savedCount) : null;
48
- // A non-null slice that is empty means the saved count is stale: the
49
- // cached turn boundary is claiming the entire current history is
50
- // already cached, which happens when a previous turn was cancelled
51
- // mid-decode and still recorded `history.length + 1`. Treat it as a
52
- // bad state and resend the full history.
53
- const useSlice = sliced !== null && sliced.length > 0;
54
- return {
55
- messages: useSlice ? sliced : history,
56
- // Only a boundary that was actually consulted can be stale. Absent a
57
- // history to slice, the count is simply unused and left alone.
58
- clearStaleCount: hasCachedPrefix && !useSlice
59
- };
60
- }