@qvac/inference 0.0.0 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/LICENSE +179 -0
  2. package/NOTICE +928 -0
  3. package/README.md +138 -0
  4. package/dist/api/audio-gen.d.ts +18 -0
  5. package/dist/api/audio-gen.js +148 -0
  6. package/dist/api/batch-completion.d.ts +14 -0
  7. package/dist/api/batch-completion.js +332 -0
  8. package/dist/api/bci-transcribe.d.ts +56 -0
  9. package/dist/api/bci-transcribe.js +172 -0
  10. package/dist/api/cancel.d.ts +34 -0
  11. package/dist/api/cancel.js +82 -0
  12. package/dist/api/classify.d.ts +22 -0
  13. package/dist/api/classify.js +46 -0
  14. package/dist/api/completion-stream.d.ts +106 -0
  15. package/dist/api/completion-stream.js +351 -0
  16. package/dist/api/delete-cache.d.ts +29 -0
  17. package/dist/api/delete-cache.js +49 -0
  18. package/dist/api/diffusion.d.ts +92 -0
  19. package/dist/api/diffusion.js +176 -0
  20. package/dist/api/download-asset.d.ts +49 -0
  21. package/dist/api/download-asset.js +87 -0
  22. package/dist/api/embed.d.ts +41 -0
  23. package/dist/api/embed.js +29 -0
  24. package/dist/api/finetune.d.ts +89 -0
  25. package/dist/api/finetune.js +138 -0
  26. package/dist/api/get-loaded-model-info.d.ts +24 -0
  27. package/dist/api/get-loaded-model-info.js +34 -0
  28. package/dist/api/get-model-info.d.ts +43 -0
  29. package/dist/api/get-model-info.js +21 -0
  30. package/dist/api/get-system-resources.d.ts +2 -0
  31. package/dist/api/get-system-resources.js +16 -0
  32. package/dist/api/heartbeat.d.ts +29 -0
  33. package/dist/api/heartbeat.js +37 -0
  34. package/dist/api/index.d.ts +35 -0
  35. package/dist/api/index.js +35 -0
  36. package/dist/api/invoke-plugin.d.ts +30 -0
  37. package/dist/api/invoke-plugin.js +54 -0
  38. package/dist/api/load-model.d.ts +169 -0
  39. package/dist/api/load-model.js +129 -0
  40. package/dist/api/logging-stream-registry.d.ts +4 -0
  41. package/dist/api/logging-stream-registry.js +61 -0
  42. package/dist/api/logging-stream.d.ts +23 -0
  43. package/dist/api/logging-stream.js +36 -0
  44. package/dist/api/ocr.d.ts +30 -0
  45. package/dist/api/ocr.js +88 -0
  46. package/dist/api/parse-input.d.ts +2 -0
  47. package/dist/api/parse-input.js +14 -0
  48. package/dist/api/provide.d.ts +23 -0
  49. package/dist/api/provide.js +32 -0
  50. package/dist/api/rag.d.ts +251 -0
  51. package/dist/api/rag.js +485 -0
  52. package/dist/api/registry.d.ts +39 -0
  53. package/dist/api/registry.js +63 -0
  54. package/dist/api/resume.d.ts +29 -0
  55. package/dist/api/resume.js +36 -0
  56. package/dist/api/state.d.ts +22 -0
  57. package/dist/api/state.js +29 -0
  58. package/dist/api/stop-provider.d.ts +20 -0
  59. package/dist/api/stop-provider.js +30 -0
  60. package/dist/api/subscribe-logs.d.ts +27 -0
  61. package/dist/api/subscribe-logs.js +43 -0
  62. package/dist/api/suspend.d.ts +30 -0
  63. package/dist/api/suspend.js +37 -0
  64. package/dist/api/text-to-speech.d.ts +64 -0
  65. package/dist/api/text-to-speech.js +434 -0
  66. package/dist/api/transcribe.d.ts +75 -0
  67. package/dist/api/transcribe.js +256 -0
  68. package/dist/api/translate.d.ts +47 -0
  69. package/dist/api/translate.js +115 -0
  70. package/dist/api/unload-model.d.ts +17 -0
  71. package/dist/api/unload-model.js +42 -0
  72. package/dist/api/upscale.d.ts +44 -0
  73. package/dist/api/upscale.js +99 -0
  74. package/dist/api/video.d.ts +86 -0
  75. package/dist/api/video.js +176 -0
  76. package/dist/api/vla-helpers.d.ts +43 -0
  77. package/dist/api/vla-helpers.js +124 -0
  78. package/dist/api/vla.d.ts +68 -0
  79. package/dist/api/vla.js +113 -0
  80. package/dist/config/config-utils.d.ts +4 -0
  81. package/dist/config/config-utils.js +18 -0
  82. package/dist/config/resolve-config.d.ts +8 -0
  83. package/dist/config/resolve-config.js +74 -0
  84. package/dist/constants/audio.d.ts +1 -0
  85. package/dist/constants/audio.js +1 -0
  86. package/dist/constants/index.d.ts +2 -0
  87. package/dist/constants/index.js +2 -0
  88. package/dist/constants/registry.d.ts +1 -0
  89. package/dist/constants/registry.js +1 -0
  90. package/dist/dispatch.d.ts +18 -0
  91. package/dist/dispatch.js +271 -0
  92. package/dist/engine.d.ts +5 -0
  93. package/dist/engine.js +27 -0
  94. package/dist/errors/index.d.ts +416 -0
  95. package/dist/errors/index.js +681 -0
  96. package/dist/errors/options.d.ts +2 -0
  97. package/dist/errors/options.js +13 -0
  98. package/dist/handlers/cancelHandler.d.ts +29 -0
  99. package/dist/handlers/cancelHandler.js +67 -0
  100. package/dist/handlers/completion-orchestrate.d.ts +41 -0
  101. package/dist/handlers/completion-orchestrate.js +190 -0
  102. package/dist/handlers/delete-cache.d.ts +11 -0
  103. package/dist/handlers/delete-cache.js +37 -0
  104. package/dist/handlers/download-asset.d.ts +2 -0
  105. package/dist/handlers/download-asset.js +146 -0
  106. package/dist/handlers/finetune.d.ts +2 -0
  107. package/dist/handlers/finetune.js +18 -0
  108. package/dist/handlers/get-loaded-model-info.d.ts +2 -0
  109. package/dist/handlers/get-loaded-model-info.js +45 -0
  110. package/dist/handlers/get-model-info.d.ts +2 -0
  111. package/dist/handlers/get-model-info.js +173 -0
  112. package/dist/handlers/get-system-resources.d.ts +2 -0
  113. package/dist/handlers/get-system-resources.js +53 -0
  114. package/dist/handlers/heartbeat.d.ts +2 -0
  115. package/dist/handlers/heartbeat.js +3 -0
  116. package/dist/handlers/load-model/download-manager.d.ts +95 -0
  117. package/dist/handlers/load-model/download-manager.js +306 -0
  118. package/dist/handlers/load-model/download-stats.d.ts +7 -0
  119. package/dist/handlers/load-model/download-stats.js +126 -0
  120. package/dist/handlers/load-model/handler.d.ts +2 -0
  121. package/dist/handlers/load-model/handler.js +287 -0
  122. package/dist/handlers/load-model/http.d.ts +3 -0
  123. package/dist/handlers/load-model/http.js +857 -0
  124. package/dist/handlers/load-model/hyperdrive.d.ts +4 -0
  125. package/dist/handlers/load-model/hyperdrive.js +560 -0
  126. package/dist/handlers/load-model/index.d.ts +2 -0
  127. package/dist/handlers/load-model/index.js +2 -0
  128. package/dist/handlers/load-model/registry-client-options.d.ts +27 -0
  129. package/dist/handlers/load-model/registry-client-options.js +18 -0
  130. package/dist/handlers/load-model/registry-companion-set.d.ts +22 -0
  131. package/dist/handlers/load-model/registry-companion-set.js +206 -0
  132. package/dist/handlers/load-model/registry-download-utils.d.ts +23 -0
  133. package/dist/handlers/load-model/registry-download-utils.js +112 -0
  134. package/dist/handlers/load-model/registry-metadata.d.ts +10 -0
  135. package/dist/handlers/load-model/registry-metadata.js +23 -0
  136. package/dist/handlers/load-model/registry.d.ts +4 -0
  137. package/dist/handlers/load-model/registry.js +298 -0
  138. package/dist/handlers/load-model/resolve-session.d.ts +33 -0
  139. package/dist/handlers/load-model/resolve-session.js +62 -0
  140. package/dist/handlers/load-model/resolve.d.ts +5 -0
  141. package/dist/handlers/load-model/resolve.js +158 -0
  142. package/dist/handlers/load-model/types.d.ts +59 -0
  143. package/dist/handlers/load-model/types.js +28 -0
  144. package/dist/handlers/logging-stream.d.ts +2 -0
  145. package/dist/handlers/logging-stream.js +39 -0
  146. package/dist/handlers/plugin-dispatch.d.ts +3 -0
  147. package/dist/handlers/plugin-dispatch.js +63 -0
  148. package/dist/handlers/plugin-invoke.d.ts +3 -0
  149. package/dist/handlers/plugin-invoke.js +81 -0
  150. package/dist/handlers/registry.d.ts +4 -0
  151. package/dist/handlers/registry.js +156 -0
  152. package/dist/handlers/resume.d.ts +2 -0
  153. package/dist/handlers/resume.js +11 -0
  154. package/dist/handlers/state.d.ts +2 -0
  155. package/dist/handlers/state.js +7 -0
  156. package/dist/handlers/suspend.d.ts +2 -0
  157. package/dist/handlers/suspend.js +11 -0
  158. package/dist/handlers/types.d.ts +13 -0
  159. package/dist/handlers/types.js +1 -0
  160. package/dist/handlers/unload-model.d.ts +2 -0
  161. package/dist/handlers/unload-model.js +29 -0
  162. package/dist/index.d.ts +4 -0
  163. package/dist/index.js +12 -0
  164. package/dist/logging/addon.d.ts +5 -0
  165. package/dist/logging/addon.js +75 -0
  166. package/dist/logging/index.d.ts +5 -0
  167. package/dist/logging/index.js +4 -0
  168. package/dist/logging/logger.d.ts +41 -0
  169. package/dist/logging/logger.js +209 -0
  170. package/dist/logging/namespaces.d.ts +6 -0
  171. package/dist/logging/namespaces.js +7 -0
  172. package/dist/logging/types.d.ts +37 -0
  173. package/dist/logging/types.js +1 -0
  174. package/dist/logging/utils.d.ts +5 -0
  175. package/dist/logging/utils.js +113 -0
  176. package/dist/models/registry/index.d.ts +18 -0
  177. package/dist/models/registry/index.js +21 -0
  178. package/dist/models/registry/models.d.ts +22680 -0
  179. package/dist/models/registry/models.js +23908 -0
  180. package/dist/p2p/cancel-delegated.d.ts +3 -0
  181. package/dist/p2p/cancel-delegated.js +64 -0
  182. package/dist/p2p/completion-stream-delegated.d.ts +4 -0
  183. package/dist/p2p/completion-stream-delegated.js +51 -0
  184. package/dist/p2p/delegate-client.d.ts +9 -0
  185. package/dist/p2p/delegate-client.js +219 -0
  186. package/dist/p2p/delegate-connect-diagnostics.d.ts +8 -0
  187. package/dist/p2p/delegate-connect-diagnostics.js +38 -0
  188. package/dist/p2p/delegate-transport.d.ts +18 -0
  189. package/dist/p2p/delegate-transport.js +237 -0
  190. package/dist/p2p/error-handlers.d.ts +4 -0
  191. package/dist/p2p/error-handlers.js +31 -0
  192. package/dist/p2p/handle-request.d.ts +2 -0
  193. package/dist/p2p/handle-request.js +171 -0
  194. package/dist/p2p/handler-utils.d.ts +19 -0
  195. package/dist/p2p/handler-utils.js +180 -0
  196. package/dist/p2p/heartbeat-delegated.d.ts +3 -0
  197. package/dist/p2p/heartbeat-delegated.js +32 -0
  198. package/dist/p2p/load-model-delegated.d.ts +6 -0
  199. package/dist/p2p/load-model-delegated.js +132 -0
  200. package/dist/p2p/progress-throttle.d.ts +7 -0
  201. package/dist/p2p/progress-throttle.js +47 -0
  202. package/dist/p2p/provideHandler/connection.d.ts +2 -0
  203. package/dist/p2p/provideHandler/connection.js +38 -0
  204. package/dist/p2p/provideHandler/handler.d.ts +2 -0
  205. package/dist/p2p/provideHandler/handler.js +66 -0
  206. package/dist/p2p/provideHandler/proxy.d.ts +2 -0
  207. package/dist/p2p/provideHandler/proxy.js +24 -0
  208. package/dist/p2p/provideHandler.d.ts +1 -0
  209. package/dist/p2p/provideHandler.js +2 -0
  210. package/dist/p2p/rpc-utils.d.ts +4 -0
  211. package/dist/p2p/rpc-utils.js +8 -0
  212. package/dist/p2p/stopProvideHandler.d.ts +2 -0
  213. package/dist/p2p/stopProvideHandler.js +29 -0
  214. package/dist/p2p/swarm.d.ts +9 -0
  215. package/dist/p2p/swarm.js +103 -0
  216. package/dist/p2p/transport-selector.d.ts +2 -0
  217. package/dist/p2p/transport-selector.js +18 -0
  218. package/dist/p2p/unload-model-delegated.d.ts +2 -0
  219. package/dist/p2p/unload-model-delegated.js +31 -0
  220. package/dist/plugins/builtin/asr-ggml/config.d.ts +31 -0
  221. package/dist/plugins/builtin/asr-ggml/config.js +46 -0
  222. package/dist/plugins/builtin/asr-ggml/logging.d.ts +10 -0
  223. package/dist/plugins/builtin/asr-ggml/logging.js +12 -0
  224. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +17 -0
  225. package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
  226. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +2 -0
  227. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +163 -0
  228. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +212 -0
  229. package/dist/plugins/builtin/audiogen-ggml/plugin.js +58 -0
  230. package/dist/plugins/builtin/bci-whispercpp-transcription/args.d.ts +2 -0
  231. package/dist/plugins/builtin/bci-whispercpp-transcription/args.js +13 -0
  232. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +173 -0
  233. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +111 -0
  234. package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.d.ts +74 -0
  235. package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.js +11 -0
  236. package/dist/plugins/builtin/ggml-classification/ops/classify.d.ts +5 -0
  237. package/dist/plugins/builtin/ggml-classification/ops/classify.js +31 -0
  238. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +31 -0
  239. package/dist/plugins/builtin/ggml-classification/plugin.js +55 -0
  240. package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.d.ts +7 -0
  241. package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.js +119 -0
  242. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +102 -0
  243. package/dist/plugins/builtin/ggml-ocr/plugin.js +110 -0
  244. package/dist/plugins/builtin/ggml-ocr/resolve-config.d.ts +5 -0
  245. package/dist/plugins/builtin/ggml-ocr/resolve-config.js +72 -0
  246. package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.d.ts +2 -0
  247. package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.js +12 -0
  248. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -0
  249. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +78 -0
  250. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +65 -0
  251. package/dist/plugins/builtin/ggml-vla/plugin.js +57 -0
  252. package/dist/plugins/builtin/index.d.ts +11 -0
  253. package/dist/plugins/builtin/index.js +11 -0
  254. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +22 -0
  255. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +30 -0
  256. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.d.ts +31 -0
  257. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +99 -0
  258. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +6 -0
  259. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +36 -0
  260. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +20 -0
  261. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +48 -0
  262. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +45 -0
  263. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +374 -0
  264. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +28 -0
  265. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +65 -0
  266. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +10 -0
  267. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +209 -0
  268. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +163 -0
  269. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +581 -0
  270. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +56 -0
  271. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +58 -0
  272. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +671 -0
  273. package/dist/plugins/builtin/llamacpp-completion/plugin.js +481 -0
  274. package/dist/plugins/builtin/llamacpp-completion/transform.d.ts +9 -0
  275. package/dist/plugins/builtin/llamacpp-completion/transform.js +34 -0
  276. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +76 -0
  277. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +101 -0
  278. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +457 -0
  279. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +143 -0
  280. package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.d.ts +31 -0
  281. package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.js +114 -0
  282. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +133 -0
  283. package/dist/plugins/builtin/parakeet-transcription/plugin.js +130 -0
  284. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +2 -0
  285. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +64 -0
  286. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +2 -0
  287. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +41 -0
  288. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -0
  289. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +239 -0
  290. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +757 -0
  291. package/dist/plugins/builtin/sdcpp-generation/plugin.js +310 -0
  292. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +6 -0
  293. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +30 -0
  294. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +7 -0
  295. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +106 -0
  296. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +6 -0
  297. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +91 -0
  298. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +196 -0
  299. package/dist/plugins/builtin/tts-ggml/plugin.js +305 -0
  300. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +346 -0
  301. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +169 -0
  302. package/dist/plugins/index.d.ts +9 -0
  303. package/dist/plugins/index.js +12 -0
  304. package/dist/plugins/ops/bci-transcribe.d.ts +13 -0
  305. package/dist/plugins/ops/bci-transcribe.js +248 -0
  306. package/dist/plugins/ops/cancel.d.ts +25 -0
  307. package/dist/plugins/ops/cancel.js +43 -0
  308. package/dist/plugins/ops/embed.d.ts +15 -0
  309. package/dist/plugins/ops/embed.js +193 -0
  310. package/dist/plugins/ops/kv-cache-retention.d.ts +11 -0
  311. package/dist/plugins/ops/kv-cache-retention.js +121 -0
  312. package/dist/plugins/ops/kv-cache-utils.d.ts +21 -0
  313. package/dist/plugins/ops/kv-cache-utils.js +120 -0
  314. package/dist/plugins/ops/load-model.d.ts +8 -0
  315. package/dist/plugins/ops/load-model.js +82 -0
  316. package/dist/plugins/ops/transcribe.d.ts +20 -0
  317. package/dist/plugins/ops/transcribe.js +343 -0
  318. package/dist/plugins/ops/translate.d.ts +7 -0
  319. package/dist/plugins/ops/translate.js +256 -0
  320. package/dist/plugins/ops/unload-model.d.ts +2 -0
  321. package/dist/plugins/ops/unload-model.js +45 -0
  322. package/dist/plugins/registry.d.ts +9 -0
  323. package/dist/plugins/registry.js +107 -0
  324. package/dist/profiling/aggregator.d.ts +14 -0
  325. package/dist/profiling/aggregator.js +100 -0
  326. package/dist/profiling/backend-diagnostics.d.ts +8 -0
  327. package/dist/profiling/backend-diagnostics.js +36 -0
  328. package/dist/profiling/clock.d.ts +8 -0
  329. package/dist/profiling/clock.js +23 -0
  330. package/dist/profiling/controller.d.ts +34 -0
  331. package/dist/profiling/controller.js +130 -0
  332. package/dist/profiling/delegation-profiler.d.ts +31 -0
  333. package/dist/profiling/delegation-profiler.js +155 -0
  334. package/dist/profiling/envelope.d.ts +10 -0
  335. package/dist/profiling/envelope.js +45 -0
  336. package/dist/profiling/events.d.ts +19 -0
  337. package/dist/profiling/events.js +35 -0
  338. package/dist/profiling/exporters.d.ts +9 -0
  339. package/dist/profiling/exporters.js +242 -0
  340. package/dist/profiling/index.d.ts +91 -0
  341. package/dist/profiling/index.js +87 -0
  342. package/dist/profiling/model-execution.d.ts +15 -0
  343. package/dist/profiling/model-execution.js +34 -0
  344. package/dist/profiling/operation-metrics.d.ts +24 -0
  345. package/dist/profiling/operation-metrics.js +370 -0
  346. package/dist/profiling/operation-wrappers.d.ts +17 -0
  347. package/dist/profiling/operation-wrappers.js +196 -0
  348. package/dist/profiling/profiler.d.ts +27 -0
  349. package/dist/profiling/profiler.js +127 -0
  350. package/dist/profiling/ring-buffer.d.ts +18 -0
  351. package/dist/profiling/ring-buffer.js +83 -0
  352. package/dist/profiling/types.d.ts +52 -0
  353. package/dist/profiling/types.js +1 -0
  354. package/dist/rag/handler.d.ts +2 -0
  355. package/dist/rag/handler.js +272 -0
  356. package/dist/rag/handlers/chunk.d.ts +2 -0
  357. package/dist/rag/handlers/chunk.js +7 -0
  358. package/dist/rag/handlers/close-workspace.d.ts +2 -0
  359. package/dist/rag/handlers/close-workspace.js +9 -0
  360. package/dist/rag/handlers/delete-embeddings.d.ts +2 -0
  361. package/dist/rag/handlers/delete-embeddings.js +11 -0
  362. package/dist/rag/handlers/delete-workspace.d.ts +2 -0
  363. package/dist/rag/handlers/delete-workspace.js +14 -0
  364. package/dist/rag/handlers/ingest.d.ts +9 -0
  365. package/dist/rag/handlers/ingest.js +19 -0
  366. package/dist/rag/handlers/list-workspaces.d.ts +2 -0
  367. package/dist/rag/handlers/list-workspaces.js +4 -0
  368. package/dist/rag/handlers/reindex.d.ts +9 -0
  369. package/dist/rag/handlers/reindex.js +16 -0
  370. package/dist/rag/handlers/save-embeddings.d.ts +9 -0
  371. package/dist/rag/handlers/save-embeddings.js +15 -0
  372. package/dist/rag/handlers/search.d.ts +2 -0
  373. package/dist/rag/handlers/search.js +16 -0
  374. package/dist/rag/index.d.ts +11 -0
  375. package/dist/rag/index.js +11 -0
  376. package/dist/rag/rag-operation-manager.d.ts +28 -0
  377. package/dist/rag/rag-operation-manager.js +76 -0
  378. package/dist/rag/rag-workspace-manager.d.ts +14 -0
  379. package/dist/rag/rag-workspace-manager.js +143 -0
  380. package/dist/registry.d.ts +2 -0
  381. package/dist/registry.js +166 -0
  382. package/dist/resources/collector.d.ts +643 -0
  383. package/dist/resources/collector.js +226 -0
  384. package/dist/resources/instance.d.ts +4 -0
  385. package/dist/resources/instance.js +15 -0
  386. package/dist/resources/native.d.ts +2 -0
  387. package/dist/resources/native.js +35 -0
  388. package/dist/resources/normalize.d.ts +12 -0
  389. package/dist/resources/normalize.js +73 -0
  390. package/dist/resources/types.d.ts +65 -0
  391. package/dist/resources/types.js +1 -0
  392. package/dist/runtime/cache-lock.d.ts +2 -0
  393. package/dist/runtime/cache-lock.js +67 -0
  394. package/dist/runtime/disposable-scope.d.ts +35 -0
  395. package/dist/runtime/disposable-scope.js +69 -0
  396. package/dist/runtime/env.d.ts +16 -0
  397. package/dist/runtime/env.js +53 -0
  398. package/dist/runtime/index.d.ts +5 -0
  399. package/dist/runtime/index.js +2 -0
  400. package/dist/runtime/lifecycle.d.ts +20 -0
  401. package/dist/runtime/lifecycle.js +102 -0
  402. package/dist/runtime/logging-stream-registry.d.ts +27 -0
  403. package/dist/runtime/logging-stream-registry.js +133 -0
  404. package/dist/runtime/model-config-utils.d.ts +14 -0
  405. package/dist/runtime/model-config-utils.js +128 -0
  406. package/dist/runtime/model-registry.d.ts +74 -0
  407. package/dist/runtime/model-registry.js +130 -0
  408. package/dist/runtime/registry-client.d.ts +4 -0
  409. package/dist/runtime/registry-client.js +118 -0
  410. package/dist/runtime/request-context.d.ts +72 -0
  411. package/dist/runtime/request-context.js +114 -0
  412. package/dist/runtime/request-id.d.ts +17 -0
  413. package/dist/runtime/request-id.js +29 -0
  414. package/dist/runtime/request-registry.d.ts +194 -0
  415. package/dist/runtime/request-registry.js +546 -0
  416. package/dist/runtime/runtime-lifecycle.d.ts +37 -0
  417. package/dist/runtime/runtime-lifecycle.js +163 -0
  418. package/dist/runtime/state.d.ts +26 -0
  419. package/dist/runtime/state.js +138 -0
  420. package/dist/schemas/audio-gen.d.ts +252 -0
  421. package/dist/schemas/audio-gen.js +69 -0
  422. package/dist/schemas/batch-completion-stream.d.ts +467 -0
  423. package/dist/schemas/batch-completion-stream.js +123 -0
  424. package/dist/schemas/bci-config.d.ts +74 -0
  425. package/dist/schemas/bci-config.js +58 -0
  426. package/dist/schemas/bci.d.ts +178 -0
  427. package/dist/schemas/bci.js +89 -0
  428. package/dist/schemas/cancel.d.ts +203 -0
  429. package/dist/schemas/cancel.js +135 -0
  430. package/dist/schemas/classification.d.ts +52 -0
  431. package/dist/schemas/classification.js +33 -0
  432. package/dist/schemas/common.d.ts +6420 -0
  433. package/dist/schemas/common.js +133 -0
  434. package/dist/schemas/completion-event.d.ts +248 -0
  435. package/dist/schemas/completion-event.js +94 -0
  436. package/dist/schemas/completion-stream.d.ts +475 -0
  437. package/dist/schemas/completion-stream.js +236 -0
  438. package/dist/schemas/config.d.ts +5022 -0
  439. package/dist/schemas/config.js +169 -0
  440. package/dist/schemas/delegate.d.ts +29 -0
  441. package/dist/schemas/delegate.js +40 -0
  442. package/dist/schemas/download-asset.d.ts +225 -0
  443. package/dist/schemas/download-asset.js +71 -0
  444. package/dist/schemas/embed.d.ts +41 -0
  445. package/dist/schemas/embed.js +35 -0
  446. package/dist/schemas/engine-addon-map.d.ts +20 -0
  447. package/dist/schemas/engine-addon-map.js +78 -0
  448. package/dist/schemas/error.d.ts +40 -0
  449. package/dist/schemas/error.js +65 -0
  450. package/dist/schemas/errors.d.ts +119 -0
  451. package/dist/schemas/errors.js +615 -0
  452. package/dist/schemas/finetune.d.ts +621 -0
  453. package/dist/schemas/finetune.js +163 -0
  454. package/dist/schemas/index.d.ts +45 -0
  455. package/dist/schemas/index.js +45 -0
  456. package/dist/schemas/json-schema.d.ts +15 -0
  457. package/dist/schemas/json-schema.js +2 -0
  458. package/dist/schemas/lifecycle.d.ts +38 -0
  459. package/dist/schemas/lifecycle.js +21 -0
  460. package/dist/schemas/llamacpp-config.d.ts +403 -0
  461. package/dist/schemas/llamacpp-config.js +133 -0
  462. package/dist/schemas/load-model.d.ts +25635 -0
  463. package/dist/schemas/load-model.js +629 -0
  464. package/dist/schemas/logging-stream.d.ts +33 -0
  465. package/dist/schemas/logging-stream.js +17 -0
  466. package/dist/schemas/mcp-adapter.d.ts +46 -0
  467. package/dist/schemas/mcp-adapter.js +1 -0
  468. package/dist/schemas/model-file.d.ts +27 -0
  469. package/dist/schemas/model-file.js +29 -0
  470. package/dist/schemas/model-info.d.ts +197 -0
  471. package/dist/schemas/model-info.js +144 -0
  472. package/dist/schemas/model-ops.d.ts +36 -0
  473. package/dist/schemas/model-ops.js +40 -0
  474. package/dist/schemas/model-src-utils.d.ts +192 -0
  475. package/dist/schemas/model-src-utils.js +61 -0
  476. package/dist/schemas/model-types.d.ts +275 -0
  477. package/dist/schemas/model-types.js +223 -0
  478. package/dist/schemas/ocr.d.ts +138 -0
  479. package/dist/schemas/ocr.js +59 -0
  480. package/dist/schemas/path-validation.d.ts +9 -0
  481. package/dist/schemas/path-validation.js +22 -0
  482. package/dist/schemas/plugin.d.ts +341 -0
  483. package/dist/schemas/plugin.js +241 -0
  484. package/dist/schemas/profiling.d.ts +695 -0
  485. package/dist/schemas/profiling.js +100 -0
  486. package/dist/schemas/provide.d.ts +50 -0
  487. package/dist/schemas/provide.js +32 -0
  488. package/dist/schemas/rag.d.ts +301 -0
  489. package/dist/schemas/rag.js +211 -0
  490. package/dist/schemas/registry.d.ts +453 -0
  491. package/dist/schemas/registry.js +104 -0
  492. package/dist/schemas/reload-config.d.ts +585 -0
  493. package/dist/schemas/reload-config.js +42 -0
  494. package/dist/schemas/runtime-context.d.ts +18 -0
  495. package/dist/schemas/runtime-context.js +7 -0
  496. package/dist/schemas/sdcpp-config.d.ts +1126 -0
  497. package/dist/schemas/sdcpp-config.js +748 -0
  498. package/dist/schemas/system-resources.d.ts +3966 -0
  499. package/dist/schemas/system-resources.js +149 -0
  500. package/dist/schemas/text-to-speech.d.ts +1352 -0
  501. package/dist/schemas/text-to-speech.js +364 -0
  502. package/dist/schemas/tools.d.ts +80 -0
  503. package/dist/schemas/tools.js +52 -0
  504. package/dist/schemas/transcription-config.d.ts +161 -0
  505. package/dist/schemas/transcription-config.js +156 -0
  506. package/dist/schemas/transcription.d.ts +261 -0
  507. package/dist/schemas/transcription.js +145 -0
  508. package/dist/schemas/translate.d.ts +138 -0
  509. package/dist/schemas/translate.js +103 -0
  510. package/dist/schemas/translation-config.d.ts +1031 -0
  511. package/dist/schemas/translation-config.js +170 -0
  512. package/dist/schemas/vla.d.ts +110 -0
  513. package/dist/schemas/vla.js +151 -0
  514. package/dist/selection.d.ts +8 -0
  515. package/dist/selection.js +14 -0
  516. package/dist/surface.d.ts +18 -0
  517. package/dist/surface.js +30 -0
  518. package/dist/utils/addon-responses.d.ts +58 -0
  519. package/dist/utils/addon-responses.js +1 -0
  520. package/dist/utils/aggregate-events.d.ts +25 -0
  521. package/dist/utils/aggregate-events.js +73 -0
  522. package/dist/utils/archive.d.ts +4 -0
  523. package/dist/utils/archive.js +180 -0
  524. package/dist/utils/asr-events.d.ts +14 -0
  525. package/dist/utils/asr-events.js +30 -0
  526. package/dist/utils/audio/decoder.d.ts +4 -0
  527. package/dist/utils/audio/decoder.js +80 -0
  528. package/dist/utils/audio-input.d.ts +9 -0
  529. package/dist/utils/audio-input.js +34 -0
  530. package/dist/utils/cache/auto-history.d.ts +3 -0
  531. package/dist/utils/cache/auto-history.js +26 -0
  532. package/dist/utils/cache/paths.d.ts +42 -0
  533. package/dist/utils/cache/paths.js +133 -0
  534. package/dist/utils/cache/types.d.ts +7 -0
  535. package/dist/utils/cache/types.js +1 -0
  536. package/dist/utils/cache-normalize.d.ts +9 -0
  537. package/dist/utils/cache-normalize.js +20 -0
  538. package/dist/utils/checksum.d.ts +3 -0
  539. package/dist/utils/checksum.js +29 -0
  540. package/dist/utils/clear-storage.d.ts +4 -0
  541. package/dist/utils/clear-storage.js +14 -0
  542. package/dist/utils/completion-normalizer.d.ts +17 -0
  543. package/dist/utils/completion-normalizer.js +453 -0
  544. package/dist/utils/config-transform.d.ts +2 -0
  545. package/dist/utils/config-transform.js +21 -0
  546. package/dist/utils/decorate-promise.d.ts +33 -0
  547. package/dist/utils/decorate-promise.js +35 -0
  548. package/dist/utils/encoding.d.ts +16 -0
  549. package/dist/utils/encoding.js +27 -0
  550. package/dist/utils/expand-gguf-shards.d.ts +2 -0
  551. package/dist/utils/expand-gguf-shards.js +22 -0
  552. package/dist/utils/formatting.d.ts +18 -0
  553. package/dist/utils/formatting.js +56 -0
  554. package/dist/utils/gguf-tensor-extractor.d.ts +14 -0
  555. package/dist/utils/gguf-tensor-extractor.js +311 -0
  556. package/dist/utils/global-singleton.d.ts +1 -0
  557. package/dist/utils/global-singleton.js +10 -0
  558. package/dist/utils/index.d.ts +11 -0
  559. package/dist/utils/index.js +11 -0
  560. package/dist/utils/load-model-validation.d.ts +6 -0
  561. package/dist/utils/load-model-validation.js +18 -0
  562. package/dist/utils/mcp-adapter.d.ts +11 -0
  563. package/dist/utils/mcp-adapter.js +96 -0
  564. package/dist/utils/multi-gpu-mobile.d.ts +4 -0
  565. package/dist/utils/multi-gpu-mobile.js +10 -0
  566. package/dist/utils/object.d.ts +1 -0
  567. package/dist/utils/object.js +3 -0
  568. package/dist/utils/path-security.d.ts +22 -0
  569. package/dist/utils/path-security.js +69 -0
  570. package/dist/utils/qvac-paths.d.ts +1 -0
  571. package/dist/utils/qvac-paths.js +5 -0
  572. package/dist/utils/response-format.d.ts +2 -0
  573. package/dist/utils/response-format.js +20 -0
  574. package/dist/utils/shard-utils.d.ts +47 -0
  575. package/dist/utils/shard-utils.js +123 -0
  576. package/dist/utils/tool-helpers.d.ts +24 -0
  577. package/dist/utils/tool-helpers.js +111 -0
  578. package/dist/utils/tool-integration.d.ts +23 -0
  579. package/dist/utils/tool-integration.js +29 -0
  580. package/dist/utils/tools/dialect.d.ts +7 -0
  581. package/dist/utils/tools/dialect.js +20 -0
  582. package/dist/utils/tools/index.d.ts +2 -0
  583. package/dist/utils/tools/index.js +2 -0
  584. package/dist/utils/tools/parser.d.ts +5 -0
  585. package/dist/utils/tools/parser.js +69 -0
  586. package/dist/utils/tools/parsers/dsml.d.ts +3 -0
  587. package/dist/utils/tools/parsers/dsml.js +177 -0
  588. package/dist/utils/tools/parsers/gemma4native.d.ts +3 -0
  589. package/dist/utils/tools/parsers/gemma4native.js +65 -0
  590. package/dist/utils/tools/parsers/harmony.d.ts +3 -0
  591. package/dist/utils/tools/parsers/harmony.js +57 -0
  592. package/dist/utils/tools/parsers/hermes.d.ts +3 -0
  593. package/dist/utils/tools/parsers/hermes.js +122 -0
  594. package/dist/utils/tools/parsers/json.d.ts +5 -0
  595. package/dist/utils/tools/parsers/json.js +115 -0
  596. package/dist/utils/tools/parsers/pythonic.d.ts +3 -0
  597. package/dist/utils/tools/parsers/pythonic.js +267 -0
  598. package/dist/utils/tools/parsers/qwen35.d.ts +3 -0
  599. package/dist/utils/tools/parsers/qwen35.js +125 -0
  600. package/dist/utils/tools/shared.d.ts +17 -0
  601. package/dist/utils/tools/shared.js +52 -0
  602. package/dist/utils/transcribe-metadata.d.ts +20 -0
  603. package/dist/utils/transcribe-metadata.js +28 -0
  604. package/dist/utils/tts-stats.d.ts +25 -0
  605. package/dist/utils/tts-stats.js +16 -0
  606. package/dist/utils/withTimeout.d.ts +17 -0
  607. package/dist/utils/withTimeout.js +47 -0
  608. package/dist/utils/zod-error.d.ts +2 -0
  609. package/dist/utils/zod-error.js +4 -0
  610. package/package.json +266 -1
@@ -0,0 +1,581 @@
1
+ import { promises as fsPromises } from 'bare-fs';
2
+ import path from 'bare-path';
3
+ import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../../plugins/ops/kv-cache-utils.js';
4
+ import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../../plugins/ops/kv-cache-retention.js';
5
+ import { isMobile } from '../../../../runtime/state.js';
6
+ import { getKVCacheDir } from '../../../../utils/index.js';
7
+ import { logCacheSaveError, logCacheStatus } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
8
+ import { getEngineLogger } from '../../../../logging/index.js';
9
+ // Used by cross-model paths that have no `RequestContext` (e.g.
10
+ // `deleteKvCacheState`). Per-session call sites receive a logger from
11
+ // the caller — typically `withRequestContext(...)`.
12
+ const moduleLogger = getEngineLogger();
13
+ /**
14
+ * Coordinates five KV-cache state layers:
15
+ *
16
+ * 1. `cachedMessageCounts` — saved message boundaries.
17
+ * 2. `initializedCaches` — caches primed in this process.
18
+ * 3. On-disk `.bin` files written by the addon.
19
+ * 4. `activeCachePaths` — per-path refs that block in-flight eviction.
20
+ * 5. `.auto-cache-<key>` markers — engine-generated cache ownership.
21
+ *
22
+ * Every turn must finish through `commitTurn` or `rollback` so all
23
+ * inference state stays aligned, the active-path ref is released, and
24
+ * marker metadata follows the cache directory lifecycle.
25
+ */
26
+ // ----- module-scoped state. The session is the single mutation point
27
+ // for the in-memory KV-cache bookkeeping. -----
28
+ /**
29
+ * Number of chat messages the kv-cache file on disk is known to cover,
30
+ * keyed by cache path. Written by `commitTurn`, read by `getSavedCount`,
31
+ * deleted by `rollback` / `delete` / `dropStaleSavedCount`. The same
32
+ * INVARIANT that existed in `kv-cache-state.ts` still holds: an entry is
33
+ * present only when the corresponding `.bin` file is considered
34
+ * trustworthy. Cancelled or zero-token turns must remove the entry so
35
+ * the next-turn slice doesn't read a stale boundary.
36
+ */
37
+ const cachedMessageCounts = new Map();
38
+ /**
39
+ * In-memory registry of caches initialized this session. The addon
40
+ * defers disk writes, so the absence of a `.bin` file on disk isn't
41
+ * proof that the cache hasn't been primed in this process. Keyed
42
+ * by `${modelId}:${configHash}:${cacheKey}`, so on-disk caches from
43
+ * older process runs still hit the lazy-load path in `beginTurn`.
44
+ */
45
+ const initializedCaches = new Set();
46
+ const activeCachePaths = new Map();
47
+ const DESKTOP_AUTO_CACHE_MAX_BYTES = 4 * 1024 * 1024 * 1024;
48
+ const MOBILE_AUTO_CACHE_MAX_BYTES = 512 * 1024 * 1024;
49
+ const AUTO_CACHE_MAX_IDLE_MS = 24 * 60 * 60 * 1000;
50
+ const AUTO_CACHE_SWEEP_INTERVAL_MS = 5 * 60 * 1000;
51
+ let lastAutoCacheSweepMs = 0;
52
+ let autoCacheSweepInFlight = null;
53
+ let cacheStateLockTail = Promise.resolve();
54
+ function initRegistryKey(modelId, configHash, cacheKey) {
55
+ return `${modelId}:${configHash}:${cacheKey}`;
56
+ }
57
+ function markCachePathActive(cachePath) {
58
+ activeCachePaths.set(cachePath, (activeCachePaths.get(cachePath) ?? 0) + 1);
59
+ }
60
+ function releaseCachePath(cachePath) {
61
+ const count = activeCachePaths.get(cachePath);
62
+ if (count === undefined)
63
+ return;
64
+ if (count === 1) {
65
+ activeCachePaths.delete(cachePath);
66
+ return;
67
+ }
68
+ activeCachePaths.set(cachePath, count - 1);
69
+ }
70
+ function isCacheKeyActive(cacheKey) {
71
+ const cacheDirectory = path.join(getKVCacheDir(), cacheKey);
72
+ return Array.from(activeCachePaths.keys()).some((cachePath) => isCachePathWithinDirectory(cacheDirectory, cachePath));
73
+ }
74
+ function getAutoCacheMaxBytes() {
75
+ return isMobile() ? MOBILE_AUTO_CACHE_MAX_BYTES : DESKTOP_AUTO_CACHE_MAX_BYTES;
76
+ }
77
+ async function withCacheStateLock(operation) {
78
+ const previous = cacheStateLockTail;
79
+ let releaseLock = () => { };
80
+ cacheStateLockTail = new Promise((resolve) => {
81
+ releaseLock = resolve;
82
+ });
83
+ await previous;
84
+ try {
85
+ return await operation();
86
+ }
87
+ finally {
88
+ releaseLock();
89
+ }
90
+ }
91
+ async function maybeSweepAutoCaches(logger, overrides) {
92
+ const nowMs = overrides?.nowMs ?? Date.now();
93
+ if (autoCacheSweepInFlight !== null) {
94
+ await autoCacheSweepInFlight;
95
+ if (!overrides?.force)
96
+ return;
97
+ return maybeSweepAutoCaches(logger, overrides);
98
+ }
99
+ if (!overrides?.force && nowMs - lastAutoCacheSweepMs < AUTO_CACHE_SWEEP_INTERVAL_MS)
100
+ return;
101
+ lastAutoCacheSweepMs = nowMs;
102
+ const sweep = async () => {
103
+ try {
104
+ const cacheKeys = await planAutoCacheEvictions({
105
+ activeCachePaths: Array.from(activeCachePaths.keys()),
106
+ maxBytes: overrides?.maxBytes ?? getAutoCacheMaxBytes(),
107
+ maxIdleMs: overrides?.maxIdleMs ?? AUTO_CACHE_MAX_IDLE_MS,
108
+ nowMs
109
+ });
110
+ let evictionCount = 0;
111
+ await withCacheStateLock(async () => {
112
+ for (const cacheKey of cacheKeys) {
113
+ if (isCacheKeyActive(cacheKey))
114
+ continue;
115
+ await deleteKvCacheState({ kvCacheKey: cacheKey });
116
+ evictionCount++;
117
+ }
118
+ });
119
+ if (evictionCount > 0) {
120
+ logger.debug(`[kv-cache] Evicted ${evictionCount} inactive auto-cache entries`);
121
+ }
122
+ }
123
+ catch (error) {
124
+ logger.warn(`[kv-cache] Auto-cache retention sweep failed: ${error instanceof Error ? error.message : String(error)}`);
125
+ }
126
+ };
127
+ const sweepPromise = sweep();
128
+ autoCacheSweepInFlight = sweepPromise;
129
+ try {
130
+ await sweepPromise;
131
+ }
132
+ finally {
133
+ autoCacheSweepInFlight = null;
134
+ }
135
+ }
136
+ function scheduleAutoCacheSweep(logger) {
137
+ void maybeSweepAutoCaches(logger).catch((error) => {
138
+ logger.warn(`[kv-cache] Failed to schedule auto-cache retention sweep: ${error instanceof Error ? error.message : String(error)}`);
139
+ });
140
+ }
141
+ // ----- factory -----
142
+ /**
143
+ * Construct a session bound to one `(modelId, turn-owning request)`
144
+ * scope. `options.logger` is the per-instance logger the session emits
145
+ * through (typically `withRequestContext(getEngineLogger(), ctx)`);
146
+ * falls back to the module-scoped logger when omitted.
147
+ */
148
+ export function createKvCacheSession(modelId, options) {
149
+ const logger = options?.logger ?? moduleLogger;
150
+ // Per-session map: each `TurnHandle` carries an opaque entry here. A
151
+ // WeakMap so handles drop their state once the handler scope releases
152
+ // the reference; the module-scoped maps above survive.
153
+ const turnState = new WeakMap();
154
+ function makeHandle(cachePath, registryKey, autoCacheKey) {
155
+ const handle = {
156
+ cachePath,
157
+ savedCount: cachedMessageCounts.get(cachePath) ?? 0
158
+ };
159
+ turnState.set(handle, {
160
+ cachePath,
161
+ registryKey,
162
+ ...(autoCacheKey !== undefined && { autoCacheKey }),
163
+ committed: false,
164
+ rolledBack: false
165
+ });
166
+ markCachePathActive(cachePath);
167
+ return handle;
168
+ }
169
+ async function beginCustom(input) {
170
+ const cachePath = await getCacheFilePath(modelId, input.configHash, input.customKey);
171
+ const registryKey = initRegistryKey(modelId, input.configHash, input.customKey);
172
+ const handle = makeHandle(cachePath, registryKey);
173
+ try {
174
+ // In-memory registry check first — the addon defers disk writes, so
175
+ // a freshly-primed cache may not yet exist on disk. If the
176
+ // in-memory flag isn't set, fall back to a filesystem probe so
177
+ // caches surviving across process restarts still hit the reuse path.
178
+ let exists = initializedCaches.has(registryKey);
179
+ if (!exists) {
180
+ try {
181
+ await fsPromises.access(cachePath);
182
+ exists = true;
183
+ initializedCaches.add(registryKey);
184
+ }
185
+ catch {
186
+ exists = false;
187
+ }
188
+ }
189
+ logCacheStatus(input.customKey, exists);
190
+ if (!exists) {
191
+ await input.primeIfMissing(cachePath);
192
+ await verifyPrimedFile(cachePath, logger);
193
+ initializedCaches.add(registryKey);
194
+ }
195
+ return handle;
196
+ }
197
+ catch (error) {
198
+ releaseCachePath(cachePath);
199
+ throw error;
200
+ }
201
+ }
202
+ async function beginAuto(input) {
203
+ // The pre-response cache key is derived from
204
+ // `history.slice(0, -1)` — `findMatchingCache` does that
205
+ // internally. The post-response key (used after a successful turn)
206
+ // is computed by the caller and passed to `commitTurn` as
207
+ // `targetCachePath`.
208
+ const setup = await withCacheStateLock(async () => {
209
+ const existingCache = await findMatchingCache(modelId, input.configHash, input.history);
210
+ const cacheInfo = existingCache ?? (await getCurrentCacheInfo(modelId, input.configHash, input.history));
211
+ const registryKey = initRegistryKey(modelId, input.configHash, cacheInfo.cacheKey);
212
+ return {
213
+ cacheExists: existingCache !== null,
214
+ cachePath: cacheInfo.cachePath,
215
+ cacheKey: cacheInfo.cacheKey,
216
+ registryKey,
217
+ handle: makeHandle(cacheInfo.cachePath, registryKey, cacheInfo.cacheKey)
218
+ };
219
+ });
220
+ const { cacheExists, cachePath, cacheKey, registryKey, handle } = setup;
221
+ logCacheStatus('auto', cacheExists);
222
+ try {
223
+ if (!cacheExists) {
224
+ await input.primeIfMissing(cachePath);
225
+ await verifyPrimedFile(cachePath, logger);
226
+ initializedCaches.add(registryKey);
227
+ }
228
+ return handle;
229
+ }
230
+ catch (error) {
231
+ releaseCachePath(cachePath);
232
+ await pruneEmptyCacheDirectories(cachePath);
233
+ await removeAutoCacheMarkerIfMissing(cacheKey);
234
+ throw error;
235
+ }
236
+ }
237
+ async function beginTurn(input) {
238
+ if (input.kind === 'custom')
239
+ return beginCustom(input);
240
+ return beginAuto(input);
241
+ }
242
+ async function commitTurn(turn, result) {
243
+ const state = turnState.get(turn);
244
+ if (!state) {
245
+ // Handle from a different session or already torn down. Treat as
246
+ // no-op — caller shouldn't be reaching into another session's
247
+ // state, but failing loudly here punishes the rollback-after-end
248
+ // path more than it helps.
249
+ return;
250
+ }
251
+ if (state.committed || state.rolledBack)
252
+ return;
253
+ if (result.kind === 'static') {
254
+ // Custom-key path: the addon wrote the new cache state inline
255
+ // at the same path. Verify the file persisted (the addon
256
+ // currently swallows save errors — see TODO in
257
+ // `verifySaveAndRecord`) and record the new boundary.
258
+ const ok = await verifySaveAndRecord(state.cachePath, result.messageCount);
259
+ if (!ok) {
260
+ // The expected save didn't land — treat the turn as a rollback
261
+ // so the next turn re-primes cleanly.
262
+ await runRollback(state);
263
+ return;
264
+ }
265
+ state.committed = true;
266
+ releaseCachePath(state.cachePath);
267
+ return;
268
+ }
269
+ const sourceCachePath = state.cachePath;
270
+ const sourceCacheKey = state.autoCacheKey;
271
+ const targetCacheKey = path.basename(path.dirname(path.dirname(result.targetCachePath)));
272
+ const renamed = await withCacheStateLock(async () => {
273
+ markCachePathActive(result.targetCachePath);
274
+ await fsPromises.mkdir(path.dirname(result.targetCachePath), { recursive: true });
275
+ await markAutoCacheKey(targetCacheKey);
276
+ if (!(await renameCacheFile(sourceCachePath, result.targetCachePath))) {
277
+ releaseCachePath(result.targetCachePath);
278
+ await pruneEmptyCacheDirectories(result.targetCachePath);
279
+ await removeAutoCacheMarkerIfMissing(targetCacheKey);
280
+ return false;
281
+ }
282
+ releaseCachePath(sourceCachePath);
283
+ state.cachePath = result.targetCachePath;
284
+ state.autoCacheKey = targetCacheKey;
285
+ await pruneEmptyCacheDirectories(sourceCachePath);
286
+ if (sourceCacheKey !== undefined) {
287
+ await removeAutoCacheMarkerIfMissing(sourceCacheKey);
288
+ }
289
+ cachedMessageCounts.delete(sourceCachePath);
290
+ initializedCaches.delete(state.registryKey);
291
+ return true;
292
+ });
293
+ if (!renamed) {
294
+ logger.warn(`[kv-cache] Auto cache rename failed; rolling back. from=${sourceCachePath} to=${result.targetCachePath}`);
295
+ await runRollback(state);
296
+ return;
297
+ }
298
+ const ok = await verifySaveAndRecord(result.targetCachePath, result.messageCount);
299
+ if (!ok) {
300
+ // Rename succeeded but the file isn't where we expected. Roll
301
+ // back via the target path instead of the (now-empty) source.
302
+ await runRollback(state);
303
+ return;
304
+ }
305
+ // Successful auto-rename. The handle's `cachePath` field still
306
+ // points at the (now-gone) source path — that's fine, the handle
307
+ // is committed and won't roll back. Future turns compute fresh
308
+ // paths.
309
+ state.committed = true;
310
+ releaseCachePath(state.cachePath);
311
+ scheduleAutoCacheSweep(logger);
312
+ }
313
+ async function rollback(turn) {
314
+ const state = turnState.get(turn);
315
+ if (!state)
316
+ return;
317
+ if (state.committed || state.rolledBack)
318
+ return;
319
+ await runRollback(state);
320
+ }
321
+ async function runRollback(state) {
322
+ // Order matters only weakly: unlink first so a partial disk-state
323
+ // can't be re-loaded by a sibling turn between the file delete and
324
+ // the in-memory clear. In practice handlers serialise per model;
325
+ // the order is belt-and-suspenders.
326
+ try {
327
+ await fsPromises.unlink(state.cachePath);
328
+ }
329
+ catch (unlinkError) {
330
+ logger.warn(`[kv-cache] Failed to remove cache file during rollback; next turn may load stale KV state. path=${state.cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
331
+ }
332
+ await pruneEmptyCacheDirectories(state.cachePath);
333
+ if (state.autoCacheKey !== undefined) {
334
+ await removeAutoCacheMarkerIfMissing(state.autoCacheKey);
335
+ }
336
+ initializedCaches.delete(state.registryKey);
337
+ cachedMessageCounts.delete(state.cachePath);
338
+ releaseCachePath(state.cachePath);
339
+ state.rolledBack = true;
340
+ if (state.autoCacheKey !== undefined)
341
+ scheduleAutoCacheSweep(logger);
342
+ }
343
+ function dropStaleSavedCount(turn) {
344
+ const state = turnState.get(turn);
345
+ if (!state)
346
+ return;
347
+ cachedMessageCounts.delete(state.cachePath);
348
+ }
349
+ return {
350
+ beginTurn,
351
+ commitTurn,
352
+ rollback,
353
+ dropStaleSavedCount
354
+ };
355
+ }
356
+ // ----- module-level administrative API -----
357
+ /**
358
+ * Atomically delete every layer of KV-cache state for a
359
+ * `(kvCacheKey, modelId)` pair, or wipe everything. Single entry point
360
+ * — the only mutation point for cross-model state outside of
361
+ * turn-scoped `commitTurn`/`rollback`.
362
+ *
363
+ * Why this isn't a method on `KvCacheSession`: deletes are
364
+ * cross-model (`all: true` has no model; the keyed form has
365
+ * `modelId` optional in the request). A session, by contrast, is
366
+ * created with a *fixed* `modelId` for the duration of a turn. Making
367
+ * delete a method would force callers to materialise an irrelevant
368
+ * session for cross-model administrative cleanups.
369
+ *
370
+ * Layers cleared, in order:
371
+ * 1. On-disk: `deleteCache(...)` removes the matching directory
372
+ * tree (or wipes and recreates the root for `all: true`).
373
+ * 2. `cachedMessageCounts`: prefix-cleanup by the removed directory
374
+ * so any per-cache count under the deleted tree is forgotten.
375
+ * 3. `initializedCaches`: scope clear by `(kvCacheKey[, modelId])`,
376
+ * matching the on-disk scope.
377
+ *
378
+ * Concurrency with in-flight turns: this delete is wire-async with
379
+ * respect to any turn currently holding a `TurnHandle` for the same
380
+ * cache key. Worst case the on-disk `.bin` is removed while a turn is
381
+ * mid-write; the turn's eventual `commitTurn(...)` then fails the
382
+ * `verifySaveAndRecord` probe (file gone) and rolls back idempotently.
383
+ * No coordination primitive is needed because every layer's mutation
384
+ * is idempotent (`unlink` no-ops if missing, `Map.delete` / `Set.delete`
385
+ * no-op on absent keys).
386
+ */
387
+ export async function deleteKvCacheState(target) {
388
+ if ('all' in target) {
389
+ const removed = await deleteCacheUtil({ all: true });
390
+ cachedMessageCounts.clear();
391
+ initializedCaches.clear();
392
+ // `removed` is the kv-cache root dir; surfaces it for ops
393
+ // visibility but isn't part of the contract.
394
+ moduleLogger.debug(`[kv-cache] Cleared all caches under ${removed}`);
395
+ return;
396
+ }
397
+ const removedPath = await deleteCacheUtil({
398
+ kvCacheKey: target.kvCacheKey,
399
+ ...(target.modelId !== undefined && { modelId: target.modelId })
400
+ });
401
+ if (target.modelId === undefined) {
402
+ await removeAutoCacheMarker(target.kvCacheKey);
403
+ }
404
+ // Prefix-cleanup the in-memory counts. The on-disk directory tree
405
+ // is `{kvCacheRoot}/{kvCacheKey}[/{modelId}]/`, so every entry in
406
+ // `cachedMessageCounts` whose key is the removed directory itself
407
+ // or sits beneath it must go.
408
+ clearCachedMessageCountsByPrefix(removedPath, path.sep);
409
+ // The in-memory init-set keys are
410
+ // `${modelId}:${configHash}:${kvCacheKey}` — clear by the user-
411
+ // facing kvCacheKey (and optionally narrow by modelId).
412
+ clearInitializedCachesByScope({
413
+ cacheKey: target.kvCacheKey,
414
+ ...(target.modelId !== undefined && { modelId: target.modelId })
415
+ });
416
+ }
417
+ // ----- private helpers -----
418
+ /**
419
+ * Verify that the addon actually persisted a usable cache file after a
420
+ * prime. Mirrors the `verifySaveAndRecord` access-probe used at commit
421
+ * time, applied at prime time so the session doesn't mark a cache
422
+ * `initializedCaches.add(...)` against a path that's missing or empty
423
+ * on disk.
424
+ *
425
+ * Failure modes this catches:
426
+ *
427
+ * - The addon's `model.run({ saveSessionPath })` was interrupted
428
+ * before the save call ran (e.g. signal abort during prefill); the
429
+ * prime closure resolves cleanly because addon save errors are not
430
+ * propagated, but no file is on disk.
431
+ * - The addon's `llama_state_save_file` was called but produced an
432
+ * empty file (out-of-space / fs error swallowed by the addon).
433
+ *
434
+ * Failure modes this does **NOT** catch:
435
+ *
436
+ * - A partial-but-nonzero file written by the addon (e.g. header +
437
+ * truncated KV state). Catching this requires either an
438
+ * addon-side change (have `CacheManager::writeCacheFile` check the
439
+ * return value of `llama_state_save_file` and throw on failure) or
440
+ * a structural hash check we can't currently compute
441
+ * engine-side. Filed as a follow-up — see `cache-api.md` in the addon
442
+ * repo / tracking ticket.
443
+ *
444
+ * On failure we best-effort `unlink` an empty leftover file (so the
445
+ * next existence probe doesn't trust it) and throw — the handler in
446
+ * `completion-stream.ts` lets the error propagate up and no
447
+ * `initializedCaches` entry is recorded.
448
+ */
449
+ async function verifyPrimedFile(cachePath, logger) {
450
+ let stats;
451
+ try {
452
+ stats = await fsPromises.stat(cachePath);
453
+ }
454
+ catch (statError) {
455
+ // ENOENT is the common case here — addon prime returned without
456
+ // calling save (most often: signal abort during prefill).
457
+ await pruneEmptyCacheDirectories(cachePath);
458
+ throw new Error(`[kv-cache] prime closure resolved but no cache file was written. path=${cachePath} cause=${statError instanceof Error ? statError.message : String(statError)}`);
459
+ }
460
+ if (stats.size === 0) {
461
+ // Best-effort cleanup so a future probe doesn't trust the empty
462
+ // file. Unlink failure is non-fatal — we still throw on the
463
+ // primary "prime didn't persist" condition.
464
+ try {
465
+ await fsPromises.unlink(cachePath);
466
+ await pruneEmptyCacheDirectories(cachePath);
467
+ }
468
+ catch (unlinkError) {
469
+ logger.warn(`[kv-cache] Failed to remove empty primed cache file. path=${cachePath} error=${unlinkError instanceof Error ? unlinkError.message : String(unlinkError)}`);
470
+ }
471
+ throw new Error(`[kv-cache] prime closure resolved but cache file is empty. path=${cachePath}`);
472
+ }
473
+ }
474
+ /**
475
+ * Verify the addon actually persisted the cache file before recording
476
+ * its message count. The addon currently swallows write errors
477
+ * silently, so a missing file means the next turn must resend the full
478
+ * history rather than slicing against a stale `savedCount`.
479
+ *
480
+ * TODO: once the addon surfaces save failures (e.g. throws
481
+ * `UnableToSaveSessionFile` when `llama_state_save_file` returns
482
+ * false), drop the `access()` probe and wrap the `model.run()` call in
483
+ * a real try/catch that forwards the error.
484
+ */
485
+ async function verifySaveAndRecord(cachePath, messageCount) {
486
+ try {
487
+ await fsPromises.access(cachePath);
488
+ cachedMessageCounts.set(cachePath, messageCount);
489
+ return true;
490
+ }
491
+ catch (err) {
492
+ cachedMessageCounts.delete(cachePath);
493
+ logCacheSaveError(cachePath, err);
494
+ return false;
495
+ }
496
+ }
497
+ function clearCachedMessageCountsByPrefix(prefix, sep) {
498
+ if (!prefix) {
499
+ cachedMessageCounts.clear();
500
+ return;
501
+ }
502
+ for (const key of cachedMessageCounts.keys()) {
503
+ if (key === prefix) {
504
+ cachedMessageCounts.delete(key);
505
+ continue;
506
+ }
507
+ if (!key.startsWith(prefix + sep))
508
+ continue;
509
+ cachedMessageCounts.delete(key);
510
+ }
511
+ }
512
+ function clearInitializedCachesByScope(scope) {
513
+ if (scope.cacheKey === undefined && scope.modelId === undefined) {
514
+ initializedCaches.clear();
515
+ return;
516
+ }
517
+ for (const key of initializedCaches) {
518
+ const firstSep = key.indexOf(':');
519
+ const secondSep = key.indexOf(':', firstSep + 1);
520
+ if (firstSep === -1 || secondSep === -1)
521
+ continue;
522
+ const entryModelId = key.slice(0, firstSep);
523
+ const entryCacheKey = key.slice(secondSep + 1);
524
+ if (scope.cacheKey !== undefined && entryCacheKey !== scope.cacheKey) {
525
+ continue;
526
+ }
527
+ if (scope.modelId !== undefined && entryModelId !== scope.modelId) {
528
+ continue;
529
+ }
530
+ initializedCaches.delete(key);
531
+ }
532
+ }
533
+ // ----- test-only escape hatches -----
534
+ /**
535
+ * Test-only access to the module-scoped state. Production code reaches
536
+ * for cache state exclusively through the session API; the unit suite
537
+ * for `kv-cache-session.test.ts` needs to seed and inspect raw state
538
+ * to assert the rollback / commit invariants. Not part of the public
539
+ * surface.
540
+ *
541
+ * @internal
542
+ */
543
+ export const __kvCacheSessionTestHooks = {
544
+ getSavedCount(cachePath) {
545
+ return cachedMessageCounts.get(cachePath);
546
+ },
547
+ setSavedCountForTest(cachePath, count) {
548
+ cachedMessageCounts.set(cachePath, count);
549
+ },
550
+ hasInitializedKey(modelId, configHash, cacheKey) {
551
+ return initializedCaches.has(initRegistryKey(modelId, configHash, cacheKey));
552
+ },
553
+ markInitializedForTest(modelId, configHash, cacheKey) {
554
+ initializedCaches.add(initRegistryKey(modelId, configHash, cacheKey));
555
+ },
556
+ getLastAutoCacheSweepMsForTest() {
557
+ return lastAutoCacheSweepMs;
558
+ },
559
+ setLastAutoCacheSweepMsForTest(value) {
560
+ lastAutoCacheSweepMs = value;
561
+ },
562
+ resetForTest() {
563
+ cachedMessageCounts.clear();
564
+ initializedCaches.clear();
565
+ activeCachePaths.clear();
566
+ lastAutoCacheSweepMs = 0;
567
+ autoCacheSweepInFlight = null;
568
+ cacheStateLockTail = Promise.resolve();
569
+ },
570
+ waitForAutoCacheSweepForTest() {
571
+ return autoCacheSweepInFlight ?? Promise.resolve();
572
+ },
573
+ sweepAutoCachesForTest(options) {
574
+ return maybeSweepAutoCaches(moduleLogger, { ...options, force: true });
575
+ }
576
+ };
577
+ // Re-export `generateConfigHash` from the path utilities so callers of
578
+ // the session can compute the hash without separately importing
579
+ // `kv-cache-utils`. The function itself stays in `kv-cache-utils.ts`
580
+ // (pure, no state) — only the re-export lives here.
581
+ export { generateConfigHash };
@@ -0,0 +1,56 @@
1
+ /**
2
+ * Pure decision helper for kv-cache history slicing used by
3
+ * `completion-stream.ts` (via the slice computed from
4
+ * `TurnHandle.savedCount`).
5
+ *
6
+ * This module intentionally has **no** `bare-*` imports so it can be
7
+ * exercised directly from unit tests running under `bun` without
8
+ * pulling in the Bare runtime (which is not available in that
9
+ * environment).
10
+ *
11
+ * Cancel detection flows through the per-request `AbortSignal` from
12
+ * `RequestRegistry`; the `cachedMessageCounts` map lives in
13
+ * `kv-cache-session.ts`, which owns all three KV-cache bookkeeping
14
+ * layers (saved counts, init flags, on-disk files). Only the pure
15
+ * slice-decision helper remains here.
16
+ */
17
+ export interface HistoryMessage {
18
+ role: string;
19
+ content: string;
20
+ attachments?: {
21
+ path: string;
22
+ }[] | undefined;
23
+ }
24
+ export interface HistorySliceDecision {
25
+ /** Messages to send to the model on the next turn. */
26
+ messages: HistoryMessage[];
27
+ /**
28
+ * True when the decision path proves the current `savedCount` is stale
29
+ * and the caller should drop the cached entry (via
30
+ * `KvCacheSession.dropStaleSavedCount(turn)`) to avoid propagating the
31
+ * bad count to the next turn.
32
+ */
33
+ clearStaleCount: boolean;
34
+ }
35
+ interface CacheCommitContext {
36
+ aborted: boolean;
37
+ producedTokens: boolean;
38
+ generatedTokens?: number | undefined;
39
+ predict?: number | undefined;
40
+ }
41
+ export declare function shouldCommitCachedTurn(context: CacheCommitContext): boolean;
42
+ /**
43
+ * Pure slice decision for `prepareMessagesForCache`.
44
+ *
45
+ * Mirrors the shape of the logic in `completion-stream.ts` but without
46
+ * calling `transformMessages` (which depends on `bare-fs` for
47
+ * attachment probing). Kept here so the decision can be unit-tested in
48
+ * isolation.
49
+ *
50
+ * The key regression guard: when a non-zero `savedCount` would slice
51
+ * the history down to an empty array, it is treated as stale — the
52
+ * caller falls back to sending the system-stripped full history rather
53
+ * than handing the model an empty payload.
54
+ */
55
+ export declare function decideCachedHistorySlice(savedCount: number, cacheExists: boolean, history: HistoryMessage[]): HistorySliceDecision;
56
+ export {};
@@ -0,0 +1,58 @@
1
+ /**
2
+ * Pure decision helper for kv-cache history slicing used by
3
+ * `completion-stream.ts` (via the slice computed from
4
+ * `TurnHandle.savedCount`).
5
+ *
6
+ * This module intentionally has **no** `bare-*` imports so it can be
7
+ * exercised directly from unit tests running under `bun` without
8
+ * pulling in the Bare runtime (which is not available in that
9
+ * environment).
10
+ *
11
+ * Cancel detection flows through the per-request `AbortSignal` from
12
+ * `RequestRegistry`; the `cachedMessageCounts` map lives in
13
+ * `kv-cache-session.ts`, which owns all three KV-cache bookkeeping
14
+ * layers (saved counts, init flags, on-disk files). Only the pure
15
+ * slice-decision helper remains here.
16
+ */
17
+ export function shouldCommitCachedTurn(context) {
18
+ const { aborted, producedTokens, generatedTokens, predict } = context;
19
+ const stoppedByBudget = predict !== undefined &&
20
+ predict > 0 &&
21
+ generatedTokens !== undefined &&
22
+ generatedTokens >= predict;
23
+ return !aborted && producedTokens && !stoppedByBudget;
24
+ }
25
+ /**
26
+ * Pure slice decision for `prepareMessagesForCache`.
27
+ *
28
+ * Mirrors the shape of the logic in `completion-stream.ts` but without
29
+ * calling `transformMessages` (which depends on `bare-fs` for
30
+ * attachment probing). Kept here so the decision can be unit-tested in
31
+ * isolation.
32
+ *
33
+ * The key regression guard: when a non-zero `savedCount` would slice
34
+ * the history down to an empty array, it is treated as stale — the
35
+ * caller falls back to sending the system-stripped full history rather
36
+ * than handing the model an empty payload.
37
+ */
38
+ export function decideCachedHistorySlice(savedCount, cacheExists, history) {
39
+ if (!cacheExists || history.length === 0) {
40
+ return {
41
+ messages: history.filter((msg) => msg.role !== 'system'),
42
+ clearStaleCount: false
43
+ };
44
+ }
45
+ const canSlice = savedCount > 0 && savedCount <= history.length;
46
+ const sliced = canSlice ? history.slice(savedCount) : null;
47
+ // A non-null slice that is empty means the saved count is stale: the
48
+ // cached turn boundary is claiming the entire current history is
49
+ // already cached, which happens when a previous turn was cancelled
50
+ // mid-decode and still recorded `history.length + 1`. Treat it as a
51
+ // bad state and resend the full (system-stripped) history.
52
+ const useSlice = sliced !== null && sliced.length > 0;
53
+ const messages = useSlice ? sliced : history.filter((msg) => msg.role !== 'system');
54
+ return {
55
+ messages,
56
+ clearStaleCount: !useSlice && savedCount > 0
57
+ };
58
+ }