@qvac/inference 0.21.1 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (462) hide show
  1. package/NOTICE +89 -41
  2. package/README.md +44 -1
  3. package/dist/api/assess-model-fit.d.ts +4 -4
  4. package/dist/api/assess-model-fit.js +4 -4
  5. package/dist/api/audio-gen.d.ts +4 -1
  6. package/dist/api/audio-gen.js +5 -2
  7. package/dist/api/batch-completion.js +1 -1
  8. package/dist/api/completion-stream.d.ts +2 -1
  9. package/dist/api/completion-stream.js +4 -2
  10. package/dist/api/decide.d.ts +26 -0
  11. package/dist/api/decide.js +26 -0
  12. package/dist/api/diffusion.d.ts +18 -1
  13. package/dist/api/diffusion.js +18 -1
  14. package/dist/api/download-asset.js +1 -1
  15. package/dist/api/finetune.js +1 -1
  16. package/dist/api/get-model-info.d.ts +1 -1
  17. package/dist/api/index.d.ts +39 -37
  18. package/dist/api/index.js +38 -36
  19. package/dist/api/load-model.js +2 -2
  20. package/dist/api/logging-stream-registry.js +11 -7
  21. package/dist/api/logging-stream.d.ts +11 -1
  22. package/dist/api/logging-stream.js +11 -2
  23. package/dist/api/registry.d.ts +2 -1
  24. package/dist/api/registry.js +5 -4
  25. package/dist/api/rerank.d.ts +17 -0
  26. package/dist/api/rerank.js +36 -0
  27. package/dist/api/subscribe-logs.js +5 -3
  28. package/dist/api/text-to-speech.d.ts +1 -0
  29. package/dist/api/text-to-speech.js +15 -19
  30. package/dist/api/transcribe.js +5 -2
  31. package/dist/api/unload-model.js +1 -1
  32. package/dist/api/video-request.d.ts +5 -5
  33. package/dist/api/video.d.ts +5 -4
  34. package/dist/api/video.js +6 -5
  35. package/dist/api/vla.js +1 -1
  36. package/dist/api/world-result.js +1 -1
  37. package/dist/api/world.d.ts +1 -1
  38. package/dist/api/world.js +1 -1
  39. package/dist/config/resolve-config.d.ts +1 -1
  40. package/dist/config/resolve-config.js +1 -1
  41. package/dist/constants/index.d.ts +2 -2
  42. package/dist/constants/index.js +2 -2
  43. package/dist/dispatch.d.ts +3 -2
  44. package/dist/dispatch.js +16 -4
  45. package/dist/errors/index.d.ts +9 -0
  46. package/dist/errors/index.js +18 -1
  47. package/dist/handlers/assess-model-fit.d.ts +5 -3
  48. package/dist/handlers/assess-model-fit.js +84 -32
  49. package/dist/handlers/cancelHandler.js +1 -1
  50. package/dist/handlers/completion-orchestrate.js +2 -2
  51. package/dist/handlers/download-asset.js +2 -2
  52. package/dist/handlers/finetune.js +1 -1
  53. package/dist/handlers/get-loaded-model-info.js +3 -5
  54. package/dist/handlers/get-model-info.js +1 -1
  55. package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -0
  56. package/dist/handlers/load-model/coreml-sidecars.js +71 -8
  57. package/dist/handlers/load-model/download-manager.d.ts +1 -1
  58. package/dist/handlers/load-model/download-stats.d.ts +2 -2
  59. package/dist/handlers/load-model/download-stats.js +3 -3
  60. package/dist/handlers/load-model/handler.js +10 -7
  61. package/dist/handlers/load-model/http-verify.d.ts +1 -1
  62. package/dist/handlers/load-model/http.d.ts +1 -1
  63. package/dist/handlers/load-model/http.js +3 -3
  64. package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
  65. package/dist/handlers/load-model/hyperdrive.js +2 -2
  66. package/dist/handlers/load-model/index.d.ts +2 -2
  67. package/dist/handlers/load-model/index.js +2 -2
  68. package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
  69. package/dist/handlers/load-model/registry-companion-set.js +1 -1
  70. package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
  71. package/dist/handlers/load-model/registry-download-utils.js +2 -2
  72. package/dist/handlers/load-model/registry.d.ts +2 -2
  73. package/dist/handlers/load-model/registry.js +5 -5
  74. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  75. package/dist/handlers/load-model/resolve-session.js +11 -3
  76. package/dist/handlers/load-model/resolve.d.ts +1 -1
  77. package/dist/handlers/load-model/resolve.js +5 -5
  78. package/dist/handlers/load-model/safe-fetch.js +5 -6
  79. package/dist/handlers/logging-stream.d.ts +2 -1
  80. package/dist/handlers/logging-stream.js +23 -7
  81. package/dist/handlers/registry.d.ts +3 -1
  82. package/dist/handlers/registry.js +7 -5
  83. package/dist/handlers/types.d.ts +7 -1
  84. package/dist/handlers/vector-index.js +2 -2
  85. package/dist/index.d.ts +1 -1
  86. package/dist/index.js +1 -1
  87. package/dist/logging/addon.d.ts +1 -1
  88. package/dist/logging/addon.js +1 -1
  89. package/dist/logging/index.d.ts +5 -5
  90. package/dist/logging/index.js +4 -4
  91. package/dist/logging/logger.d.ts +1 -1
  92. package/dist/logging/logger.js +2 -2
  93. package/dist/{resources/model-fit → model-fit}/assess.d.ts +18 -26
  94. package/dist/{resources/model-fit → model-fit}/assess.js +210 -274
  95. package/dist/{resources/model-fit/native-probe/classify-fit.d.ts → model-fit/classify.d.ts} +2 -2
  96. package/dist/{resources/model-fit/native-probe/classify-fit.js → model-fit/classify.js} +36 -5
  97. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +4 -2
  98. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +11 -6
  99. package/dist/{resources/model-fit/fit-stub/project-fit-from-load.d.ts → model-fit/description/project.d.ts} +3 -3
  100. package/dist/{resources/model-fit/fit-stub/project-fit-from-load.js → model-fit/description/project.js} +4 -2
  101. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +6 -4
  102. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +14 -6
  103. package/dist/model-fit/floor.d.ts +31 -0
  104. package/dist/{resources/model-fit → model-fit}/floor.js +13 -13
  105. package/dist/model-fit/kv-cache.d.ts +23 -0
  106. package/dist/model-fit/kv-cache.js +85 -0
  107. package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts} +17 -14
  108. package/dist/{resources/model-fit/native-probe/advisory-fit.js → model-fit/load-check.js} +23 -64
  109. package/dist/model-fit/platform.d.ts +14 -0
  110. package/dist/model-fit/platform.js +26 -0
  111. package/dist/{resources/model-fit/native-probe/create-fit-request.d.ts → model-fit/request/create.d.ts} +5 -2
  112. package/dist/model-fit/request/create.js +61 -0
  113. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.d.ts +5 -1
  114. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.js +32 -1
  115. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.d.ts +1 -1
  116. package/dist/model-fit/request/engines/audiogen.js +63 -0
  117. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/diffusion.d.ts +1 -1
  118. package/dist/model-fit/request/engines/diffusion.js +93 -0
  119. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.d.ts +2 -3
  120. package/dist/model-fit/request/engines/llama.js +44 -0
  121. package/dist/model-fit/request/engines/nmt.d.ts +8 -0
  122. package/dist/model-fit/request/engines/nmt.js +11 -0
  123. package/dist/model-fit/request/engines/ocr.d.ts +9 -0
  124. package/dist/model-fit/request/engines/ocr.js +13 -0
  125. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.d.ts +3 -1
  126. package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.js +50 -4
  127. package/dist/{resources/model-fit/native-probe/engine-fit.d.ts → model-fit/run/engine.d.ts} +17 -4
  128. package/dist/{resources/model-fit/native-probe/engine-fit.js → model-fit/run/engine.js} +2 -2
  129. package/dist/model-fit/run/run.d.ts +43 -0
  130. package/dist/model-fit/run/run.js +204 -0
  131. package/dist/models/registry/index.d.ts +1 -1
  132. package/dist/models/registry/index.js +2 -2
  133. package/dist/models/registry/models.d.ts +1072 -815
  134. package/dist/models/registry/models.js +3704 -3405
  135. package/dist/models/registry/resource-profiles.d.ts +1 -2
  136. package/dist/models/registry/resource-profiles.js +406 -3
  137. package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
  138. package/dist/plugins/builtin/asr-ggml/config.js +6 -0
  139. package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
  140. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
  141. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +29 -11
  142. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
  143. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
  144. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +7 -28
  145. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -2
  146. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +106 -13
  147. package/dist/plugins/builtin/audiogen-ggml/plugin.js +5 -4
  148. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +20 -3
  149. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
  150. package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
  151. package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
  152. package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
  153. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -0
  154. package/dist/plugins/builtin/ggml-ocr/plugin.js +17 -39
  155. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +9 -2
  156. package/dist/plugins/builtin/ggml-vla/plugin.js +4 -4
  157. package/dist/plugins/builtin/index.d.ts +15 -12
  158. package/dist/plugins/builtin/index.js +15 -12
  159. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +5 -7
  160. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
  161. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  162. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +10 -7
  163. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +141 -204
  164. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +36 -92
  165. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +67 -216
  166. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
  167. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
  168. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +208 -19
  169. package/dist/plugins/builtin/llamacpp-completion/plugin.js +28 -20
  170. package/dist/plugins/builtin/llamacpp-completion/transform.js +5 -1
  171. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +5 -0
  172. package/dist/plugins/builtin/llamacpp-decision/helpers.js +89 -0
  173. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
  174. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
  175. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +222 -0
  176. package/dist/plugins/builtin/llamacpp-decision/plugin.js +44 -0
  177. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +9 -1
  178. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +15 -5
  179. package/dist/plugins/builtin/llamacpp-embedding/transform.js +9 -0
  180. package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
  181. package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
  182. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +5 -0
  183. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +117 -0
  184. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +91 -0
  185. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +45 -0
  186. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
  187. package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
  188. package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
  189. package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
  190. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +17 -0
  191. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +10 -74
  192. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +25 -4
  193. package/dist/plugins/builtin/parakeet-transcription/plugin.js +20 -5
  194. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +5 -2
  195. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +168 -52
  196. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -2
  197. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +28 -13
  198. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +83 -1
  199. package/dist/plugins/builtin/sdcpp-generation/plugin.js +55 -52
  200. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
  201. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
  202. package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
  203. package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
  204. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
  205. package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
  206. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +1 -1
  207. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +5 -2
  208. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +19 -7
  209. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +151 -1
  210. package/dist/plugins/builtin/tts-ggml/plugin.js +39 -2
  211. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +28 -4
  212. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -4
  213. package/dist/plugins/index.d.ts +1 -1
  214. package/dist/plugins/index.js +2 -2
  215. package/dist/plugins/ops/kv-cache-utils.d.ts +5 -4
  216. package/dist/plugins/ops/kv-cache-utils.js +25 -33
  217. package/dist/plugins/ops/load-model.d.ts +8 -2
  218. package/dist/plugins/ops/load-model.js +33 -19
  219. package/dist/plugins/ops/transcribe.js +18 -2
  220. package/dist/plugins/ops/translate.js +3 -3
  221. package/dist/plugins/ops/unload-model.js +1 -1
  222. package/dist/plugins/turbovec-backend.js +5 -5
  223. package/dist/profiling/aggregator.d.ts +1 -1
  224. package/dist/profiling/aggregator.js +1 -1
  225. package/dist/profiling/controller.d.ts +1 -1
  226. package/dist/profiling/controller.js +2 -2
  227. package/dist/profiling/events.d.ts +1 -1
  228. package/dist/profiling/events.js +2 -2
  229. package/dist/profiling/exporters.d.ts +1 -1
  230. package/dist/profiling/exporters.js +3 -3
  231. package/dist/profiling/index.d.ts +13 -13
  232. package/dist/profiling/index.js +12 -12
  233. package/dist/profiling/operation-metrics.d.ts +1 -1
  234. package/dist/profiling/operation-metrics.js +2 -2
  235. package/dist/profiling/operation-wrappers.js +3 -3
  236. package/dist/profiling/profiler.js +1 -1
  237. package/dist/rag/handler.js +1 -1
  238. package/dist/rag/handlers/close-workspace.js +1 -1
  239. package/dist/rag/handlers/delete-embeddings.js +1 -1
  240. package/dist/rag/handlers/delete-workspace.js +1 -1
  241. package/dist/rag/handlers/ingest.js +1 -1
  242. package/dist/rag/handlers/list-workspaces.d.ts +1 -1
  243. package/dist/rag/handlers/list-workspaces.js +1 -1
  244. package/dist/rag/handlers/reindex.js +1 -1
  245. package/dist/rag/handlers/save-embeddings.js +1 -1
  246. package/dist/rag/handlers/search.js +1 -1
  247. package/dist/rag/index.d.ts +11 -11
  248. package/dist/rag/index.js +11 -11
  249. package/dist/rag/rag-workspace-manager.js +1 -1
  250. package/dist/registry.js +3 -1
  251. package/dist/resources/collector.d.ts +1 -1
  252. package/dist/resources/collector.js +1 -1
  253. package/dist/resources/instance.d.ts +1 -1
  254. package/dist/resources/instance.js +1 -1
  255. package/dist/resources/native.d.ts +1 -1
  256. package/dist/resources/normalize.d.ts +1 -1
  257. package/dist/rpc/inventory.js +5 -7
  258. package/dist/runtime/env.d.ts +0 -2
  259. package/dist/runtime/env.js +2 -9
  260. package/dist/runtime/index.d.ts +5 -5
  261. package/dist/runtime/index.js +2 -2
  262. package/dist/runtime/lifecycle.js +7 -7
  263. package/dist/runtime/model-config-utils.js +9 -1
  264. package/dist/runtime/model-registry.d.ts +3 -1
  265. package/dist/runtime/model-registry.js +2 -1
  266. package/dist/runtime/registry-client.js +1 -1
  267. package/dist/runtime/request-context.d.ts +3 -3
  268. package/dist/runtime/request-context.js +22 -1
  269. package/dist/runtime/request-registry.d.ts +1 -1
  270. package/dist/runtime/request-registry.js +1 -1
  271. package/dist/runtime/state.d.ts +1 -1
  272. package/dist/runtime/state.js +7 -2
  273. package/dist/runtime/vector-index-backend.d.ts +2 -2
  274. package/dist/runtime/vector-index-registry.d.ts +5 -5
  275. package/dist/runtime/vector-index-registry.js +36 -13
  276. package/dist/schemas/assess-model-fit.d.ts +106 -43
  277. package/dist/schemas/assess-model-fit.js +69 -32
  278. package/dist/schemas/audio-gen.d.ts +199 -19
  279. package/dist/schemas/audio-gen.js +183 -51
  280. package/dist/schemas/batch-completion-stream.d.ts +210 -36
  281. package/dist/schemas/batch-completion-stream.js +3 -3
  282. package/dist/schemas/bci-config.d.ts +3 -0
  283. package/dist/schemas/bci-config.js +2 -2
  284. package/dist/schemas/bci.d.ts +24 -3
  285. package/dist/schemas/bci.js +2 -2
  286. package/dist/schemas/cancel.d.ts +10 -0
  287. package/dist/schemas/cancel.js +2 -0
  288. package/dist/schemas/common.d.ts +1465 -162
  289. package/dist/schemas/common.js +37 -31
  290. package/dist/schemas/completion-event.d.ts +8 -2
  291. package/dist/schemas/completion-event.js +4 -1
  292. package/dist/schemas/completion-stream.d.ts +243 -39
  293. package/dist/schemas/completion-stream.js +100 -29
  294. package/dist/schemas/config.d.ts +1326 -18
  295. package/dist/schemas/config.js +30 -8
  296. package/dist/schemas/decision.d.ts +610 -0
  297. package/dist/schemas/decision.js +219 -0
  298. package/dist/schemas/download-asset.d.ts +10 -4
  299. package/dist/schemas/download-asset.js +1 -1
  300. package/dist/schemas/engine-addon-map.d.ts +9 -1
  301. package/dist/schemas/engine-addon-map.js +26 -5
  302. package/dist/schemas/errors.d.ts +2 -0
  303. package/dist/schemas/errors.js +11 -1
  304. package/dist/schemas/index.d.ts +51 -48
  305. package/dist/schemas/index.js +50 -47
  306. package/dist/schemas/llamacpp-config.d.ts +222 -6
  307. package/dist/schemas/llamacpp-config.js +129 -7
  308. package/dist/schemas/load-model.d.ts +7960 -1492
  309. package/dist/schemas/load-model.js +122 -10
  310. package/dist/schemas/mcp-adapter.d.ts +1 -1
  311. package/dist/schemas/model-config-schemas.d.ts +428 -9
  312. package/dist/schemas/model-config-schemas.js +16 -11
  313. package/dist/schemas/model-fit-policy.d.ts +17 -0
  314. package/dist/schemas/model-fit-policy.js +17 -0
  315. package/dist/schemas/model-info.d.ts +20 -2
  316. package/dist/schemas/model-info.js +4 -3
  317. package/dist/schemas/model-resource-profile.d.ts +4 -4
  318. package/dist/schemas/model-resource-profile.js +2 -5
  319. package/dist/schemas/model-src-utils.d.ts +14 -2
  320. package/dist/schemas/model-src-utils.js +2 -2
  321. package/dist/schemas/model-types.d.ts +14 -1
  322. package/dist/schemas/model-types.js +4 -0
  323. package/dist/schemas/ocr.d.ts +76 -0
  324. package/dist/schemas/ocr.js +43 -1
  325. package/dist/schemas/plugin.d.ts +16 -4
  326. package/dist/schemas/plugin.js +12 -0
  327. package/dist/schemas/profiling.d.ts +4 -4
  328. package/dist/schemas/profiling.js +1 -1
  329. package/dist/schemas/rag.js +1 -1
  330. package/dist/schemas/registry.d.ts +34 -0
  331. package/dist/schemas/registry.js +5 -1
  332. package/dist/schemas/reload-config.d.ts +14 -2
  333. package/dist/schemas/reload-config.js +2 -2
  334. package/dist/schemas/rerank.d.ts +131 -0
  335. package/dist/schemas/rerank.js +102 -0
  336. package/dist/schemas/rpc-server.js +3 -3
  337. package/dist/schemas/sdcpp-config.d.ts +82 -0
  338. package/dist/schemas/sdcpp-config.js +41 -22
  339. package/dist/schemas/system-resources.d.ts +4 -4
  340. package/dist/schemas/text-to-speech.d.ts +463 -3
  341. package/dist/schemas/text-to-speech.js +129 -11
  342. package/dist/schemas/tools.d.ts +13 -4
  343. package/dist/schemas/tools.js +18 -6
  344. package/dist/schemas/transcription-config.d.ts +20 -0
  345. package/dist/schemas/transcription-config.js +54 -4
  346. package/dist/schemas/transcription.d.ts +47 -5
  347. package/dist/schemas/transcription.js +52 -5
  348. package/dist/schemas/translate.d.ts +2 -2
  349. package/dist/schemas/translate.js +1 -1
  350. package/dist/schemas/translation-config.d.ts +40 -10
  351. package/dist/schemas/translation-config.js +1 -1
  352. package/dist/schemas/vla.d.ts +9 -2
  353. package/dist/schemas/vla.js +9 -5
  354. package/dist/surface.d.ts +3 -2
  355. package/dist/surface.js +3 -2
  356. package/dist/utils/addon-responses.d.ts +3 -0
  357. package/dist/utils/aggregate-events.d.ts +1 -1
  358. package/dist/utils/aggregate-events.js +2 -2
  359. package/dist/utils/archive.js +2 -2
  360. package/dist/utils/asr-events.d.ts +1 -0
  361. package/dist/utils/asr-events.js +2 -1
  362. package/dist/utils/audio/decoder.js +3 -1
  363. package/dist/utils/audio-input.js +1 -1
  364. package/dist/utils/cache/auto-history.d.ts +2 -1
  365. package/dist/utils/cache/auto-history.js +2 -2
  366. package/dist/utils/cache/paths.js +6 -6
  367. package/dist/utils/cache/types.d.ts +7 -0
  368. package/dist/utils/checksum.d.ts +2 -0
  369. package/dist/utils/checksum.js +41 -2
  370. package/dist/utils/completion-normalizer.js +1 -1
  371. package/dist/utils/gguf-architecture.d.ts +7 -0
  372. package/dist/utils/gguf-architecture.js +142 -0
  373. package/dist/utils/gguf-tensor-extractor.js +1 -1
  374. package/dist/utils/index.d.ts +13 -13
  375. package/dist/utils/index.js +13 -13
  376. package/dist/utils/mcp-adapter.d.ts +1 -1
  377. package/dist/utils/mcp-adapter.js +13 -9
  378. package/dist/utils/path-security.js +1 -1
  379. package/dist/utils/shard-utils.d.ts +5 -0
  380. package/dist/utils/shard-utils.js +19 -3
  381. package/dist/utils/tool-helpers.d.ts +1 -1
  382. package/dist/utils/tool-helpers.js +20 -52
  383. package/dist/utils/tool-integration.d.ts +1 -0
  384. package/dist/utils/tool-integration.js +3 -6
  385. package/dist/utils/tools/defer.d.ts +16 -26
  386. package/dist/utils/tools/defer.js +50 -41
  387. package/dist/utils/tools/dialect.d.ts +15 -4
  388. package/dist/utils/tools/dialect.js +34 -5
  389. package/dist/utils/tools/index.d.ts +3 -3
  390. package/dist/utils/tools/index.js +3 -3
  391. package/dist/utils/tools/parser.js +8 -8
  392. package/dist/utils/tools/parsers/dsml.d.ts +1 -1
  393. package/dist/utils/tools/parsers/dsml.js +8 -5
  394. package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
  395. package/dist/utils/tools/parsers/gemma4native.js +1 -1
  396. package/dist/utils/tools/parsers/harmony.d.ts +1 -1
  397. package/dist/utils/tools/parsers/harmony.js +1 -1
  398. package/dist/utils/tools/parsers/hermes.d.ts +1 -1
  399. package/dist/utils/tools/parsers/hermes.js +1 -1
  400. package/dist/utils/tools/parsers/json.d.ts +1 -1
  401. package/dist/utils/tools/parsers/json.js +1 -1
  402. package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
  403. package/dist/utils/tools/parsers/pythonic.js +1 -1
  404. package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
  405. package/dist/utils/tools/parsers/qwen35.js +7 -4
  406. package/dist/utils/tools/shared.d.ts +3 -0
  407. package/dist/utils/tools/shared.js +62 -0
  408. package/dist/utils/transcribe-metadata.d.ts +8 -1
  409. package/dist/utils/transcribe-metadata.js +13 -3
  410. package/dist/utils/tts-stats.d.ts +2 -1
  411. package/dist/utils/until-aborted.d.ts +7 -0
  412. package/dist/utils/until-aborted.js +34 -0
  413. package/package.json +68 -51
  414. package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
  415. package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
  416. package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
  417. package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
  418. package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
  419. package/dist/resources/model-fit/calibration/fit.js +0 -152
  420. package/dist/resources/model-fit/calibration/index.d.ts +0 -37
  421. package/dist/resources/model-fit/calibration/index.js +0 -75
  422. package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
  423. package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
  424. package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
  425. package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
  426. package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
  427. package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
  428. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
  429. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
  430. package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
  431. package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
  432. package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
  433. package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
  434. package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
  435. package/dist/resources/model-fit/estimators/llm.js +0 -222
  436. package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
  437. package/dist/resources/model-fit/estimators/whisper.js +0 -76
  438. package/dist/resources/model-fit/floor.d.ts +0 -31
  439. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
  440. package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -41
  441. package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -51
  442. package/dist/resources/model-fit/native-probe/engines/audiogen.js +0 -35
  443. package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -55
  444. package/dist/resources/model-fit/native-probe/engines/llama.js +0 -145
  445. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
  446. package/dist/resources/model-fit/native-probe/fit-outcome.js +0 -1
  447. package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
  448. package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
  449. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
  450. package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
  451. package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
  452. package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
  453. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -22
  454. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -108
  455. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
  456. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
  457. package/dist/resources/model-fit/platform.d.ts +0 -9
  458. package/dist/resources/model-fit/platform.js +0 -28
  459. package/dist/resources/model-fit/types.d.ts +0 -120
  460. package/dist/resources/model-fit/types.js +0 -1
  461. /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.d.ts +0 -0
  462. /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.js +0 -0
@@ -1,21 +1,26 @@
1
- import { logCacheDisabled, logCacheSave, logMessagesToAddon } from './cache-logger.js';
2
- import { extractSystemPrompt, getCurrentCacheInfo } from '../../../ops/kv-cache-utils.js';
1
+ import { logCacheDisabled, logCacheSave, logMessagesToAddon } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
2
+ import { extractSystemPrompt, getCurrentCacheInfo } from '../../../../plugins/ops/kv-cache-utils.js';
3
3
  import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
4
- import { decideCachedHistorySlice, shouldCommitCachedTurn } from './kv-cache-state.js';
5
- import { createKvCacheSession, generateConfigHash } from './kv-cache-session.js';
4
+ import { shouldCommitCachedTurn } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-state.js';
5
+ import { createKvCacheSession, generateConfigHash } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
6
6
  import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
7
- import { resolveDeferredTools, toWireTool, withDeferredToolChoice } from '../../../../utils/tools/defer.js';
7
+ import { resolveDeferredTools, toWireTool } from '../../../../utils/tools/defer.js';
8
8
  import { parseToolCalls } from '../../../../utils/tools/index.js';
9
9
  import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
10
- import { toolChoiceDemandsCall } from '../../../../schemas/completion-stream.js';
11
- import { buildAutoCacheSaveHistory } from '../../../../utils/index.js';
10
+ import { buildAutoCacheSaveHistory, normalizeCacheMessage } from '../../../../utils/index.js';
12
11
  import { getEngineLogger } from '../../../../logging/index.js';
13
12
  import { AttachmentNotFoundError } from '../../../../errors/index.js';
14
13
  import { nowMs } from '../../../../profiling/index.js';
15
14
  import { buildStreamResult } from '../../../../profiling/model-execution.js';
16
- import { normalizeCompletionStats, withEmittedTokens } from './completion-stats.js';
15
+ import { normalizeCompletionStats, withEmittedTokens } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
17
16
  import fs from 'bare-fs';
18
17
  const logger = getEngineLogger();
18
+ export function withParallelToolCalls(generationParams) {
19
+ return {
20
+ ...generationParams,
21
+ parallel_tool_calls: generationParams?.parallel_tool_calls ?? true
22
+ };
23
+ }
19
24
  function transformMessage(message) {
20
25
  const transformed = [];
21
26
  // Check if it's a tool definition (has type: "function")
@@ -43,7 +48,16 @@ function transformMessage(message) {
43
48
  }
44
49
  transformed.push({
45
50
  role: msg.role,
46
- content: msg.content
51
+ content: msg.content,
52
+ ...(msg.toolCalls && {
53
+ tool_calls: msg.toolCalls.map((call) => ({
54
+ ...(call.id !== undefined && { id: call.id }),
55
+ name: call.name,
56
+ arguments: call.arguments
57
+ }))
58
+ }),
59
+ ...(msg.toolCallId !== undefined && { tool_call_id: msg.toolCallId }),
60
+ ...(msg.toolName !== undefined && { name: msg.toolName })
47
61
  });
48
62
  return transformed;
49
63
  }
@@ -57,6 +71,18 @@ export function transformMessages(messages) {
57
71
  }
58
72
  return transformed;
59
73
  }
74
+ // The fields that identify a turn, so two tool turns with equal `content`
75
+ // never share a cache key.
76
+ function toCacheMessage(msg) {
77
+ return {
78
+ role: msg.role,
79
+ content: msg.content,
80
+ attachments: msg.attachments ?? undefined,
81
+ ...(msg.toolCalls && { toolCalls: msg.toolCalls }),
82
+ ...(msg.toolCallId !== undefined && { toolCallId: msg.toolCallId }),
83
+ ...(msg.toolName !== undefined && { toolName: msg.toolName })
84
+ };
85
+ }
60
86
  /**
61
87
  * Put the model's configured system prompt in front of a history that carries
62
88
  * none. `transform.ts` strips `system_prompt` from the config handed to the
@@ -69,98 +95,54 @@ export function seedConfiguredSystemPrompt(history, modelConfig) {
69
95
  // Every caller's message type makes everything past role and content optional.
70
96
  return [{ role: 'system', content: configured }, ...history];
71
97
  }
72
- /**
73
- * Attach the tool block to a turn payload, mirroring the no-kv-cache path
74
- * (`prependToolsToHistory`): after a system message when the payload carries
75
- * one, ahead of everything otherwise.
76
- */
77
- function withToolBlock(messages, toolBlock) {
78
- if (toolBlock.length === 0)
79
- return messages;
80
- const systemIndex = messages.findIndex((msg) => msg.role === 'system');
81
- if (systemIndex === -1)
82
- return [...toolBlock, ...messages];
83
- return [...messages.slice(0, systemIndex + 1), ...toolBlock, ...messages.slice(systemIndex + 1)];
84
- }
85
- /**
86
- * Fallback guess for whether a payload carrying this tool block gets it in
87
- * front of the model, used only when the addon does not report
88
- * `toolDefinitionsDropped`. Qwen-family templates anchor their tool section on
89
- * the last user query and raise without one, and the addon answers that by
90
- * re-rendering with tools stripped, so require a user message before believing
91
- * the block landed.
92
- */
93
- function rendersToolBlock(messages, toolBlock) {
94
- if (toolBlock.length === 0)
98
+ // The addon refuses to save a key it holds nothing for — a cold turn it rolled
99
+ // back — with `InvalidArgument`. As with its other status errors, an
100
+ // asynchronous rejection carries the message alone.
101
+ function isNothingCachedRejection(error) {
102
+ if (typeof error !== 'object' || error === null)
95
103
  return false;
96
- return messages.some((msg) => msg.role === 'user');
97
- }
98
- /**
99
- * Settle whether the committed prefix holds the full tool block. The addon's
100
- * `toolDefinitionsDropped` is the template's own word on whether the block it
101
- * was handed reached the model; without it (a payload with no block, or a
102
- * stand-in model) the payload's estimate stands.
103
- */
104
- function resolveToolBlockCached(payload, stats) {
105
- const dropped = stats?.toolDefinitionsDropped;
106
- if (!payload.toolBlockSent || dropped === undefined)
107
- return payload.toolBlockCached;
108
- return payload.prefixHoldsBlock || (payload.toolBlockFull && dropped === 0);
104
+ const code = error.code;
105
+ if (code !== undefined) {
106
+ return typeof code === 'string' && /^\[\s*[\w.-]+\s*::\s*InvalidArgument\s*\]$/.test(code);
107
+ }
108
+ const message = error.message;
109
+ return typeof message === 'string' && /no conversation is cached/.test(message);
109
110
  }
110
111
  /**
111
- * Pick the messages that need to reach the model for the next turn.
112
- *
113
- * The cache holds whatever a committed turn sent, the tool block included, so
114
- * the block travels with a turn rather than being written into the prefix on
115
- * its own, and only with the turn that writes it into the cache — see
116
- * `skipToolBlock` below.
112
+ * Write the conversation the addon holds for `cachePath` to that file. The
113
+ * session commits on the file, and a saved conversation is one the addon drops
114
+ * once `deleteCache` removes the file. A cold turn the addon rolled back has
115
+ * nothing to write, and the commit then finds no file. Any other rejection is
116
+ * a write that failed: the file is behind the conversation, which the next
117
+ * turn re-decodes.
117
118
  */
118
- async function prepareMessagesForCache(session, turn, history, tools, toolChoice) {
119
- const toolBlock = tools?.length ? transformMessages(tools) : [];
120
- // Slice from the turn's `savedCount` so callers can
121
- // stage multiple messages between completions. `decideCachedHistorySlice`
122
- // also guards against the QVAC-17780 stale-count regression: if the
123
- // saved boundary would slice the history down to an empty payload
124
- // (e.g. after a cancelled mid-decode), it falls back to the full
125
- // history and signals the caller to drop the bad entry.
126
- // The session owns the entry; `dropStaleSavedCount` clears it in memory
127
- // and on disk without touching the cache file (the file is still
128
- // trustworthy — only the boundary count is wrong).
129
- const { messages, clearStaleCount } = decideCachedHistorySlice(turn.savedCount, history);
130
- if (clearStaleCount) {
131
- await session.dropStaleSavedCount(turn);
119
+ async function saveCacheFile(model, cachePath, log) {
120
+ const saveCache = model.saveCache;
121
+ if (!saveCache) {
122
+ log.debug(`[kv-cache] model has no saveCache; nothing written for ${cachePath}`);
123
+ return;
124
+ }
125
+ try {
126
+ await saveCache.call(model, cachePath);
127
+ logCacheSave(cachePath);
128
+ }
129
+ catch (error) {
130
+ const detail = error instanceof Error ? error.message : String(error);
131
+ if (isNothingCachedRejection(error)) {
132
+ log.debug(`[kv-cache] saveCache had nothing to write for ${cachePath}: ${detail}`);
133
+ return;
134
+ }
135
+ log.warn(`[kv-cache] saveCache failed; the file is behind the conversation and the next turn re-decodes what it lacks. path=${cachePath} error=${detail}`);
132
136
  }
133
- // The block is never trimmed back out of the cache, so re-sending it every
134
- // turn would leave one copy per turn and grow the prefix with the
135
- // conversation. Skip it only when the prefix is known to hold a rendered
136
- // one: `toolBlockCached` records that a previous turn actually got it into
137
- // the cache, which a committed message count does not prove. A stale
138
- // boundary means we are resending the whole conversation anyway.
139
- const prefixHoldsBlock = turn.toolBlockCached && !clearStaleCount;
140
- // The addon arms the tool-call grammar only for a payload that carries
141
- // tools, and `required` / a named tool cannot be honoured without it. Such a
142
- // turn resends the block even into a prefix that holds one; the second copy
143
- // in the cache is the price of the guarantee.
144
- const demandsCall = toolChoiceDemandsCall(toolChoice) && toolBlock.length > 0;
145
- const skipToolBlock = prefixHoldsBlock && !demandsCall;
146
- const blockToSend = skipToolBlock ? [] : toolBlock;
147
- const toolBlockFull = blockToSend.length > 0 && (!demandsCall || toolChoice === 'required');
148
- return {
149
- messages: withToolBlock(transformMessages(messages), blockToSend),
150
- toolBlockSent: blockToSend.length > 0,
151
- toolBlockFull,
152
- prefixHoldsBlock,
153
- toolBlockCached: prefixHoldsBlock || (toolBlockFull && rendersToolBlock(messages, blockToSend))
154
- };
155
137
  }
156
- async function* processModelResponse(model, messagesToSend, tools, generationParams, cacheOptions, dialect, onResponse, onRunSettled) {
138
+ async function* processModelResponse(model, messagesToSend, tools, generationParams, cacheOptions, dialect, onResponse) {
157
139
  const runOptions = {
158
140
  ...(generationParams && { generationParams }),
159
141
  ...(cacheOptions?.cacheKey !== undefined && {
160
142
  cacheKey: cacheOptions.cacheKey
161
143
  }),
162
- ...(cacheOptions?.saveCacheToDisk !== undefined && {
163
- saveCacheToDisk: cacheOptions.saveCacheToDisk
144
+ ...(cacheOptions?.prefill !== undefined && {
145
+ prefill: cacheOptions.prefill
164
146
  })
165
147
  };
166
148
  const hasRunOptions = Object.keys(runOptions).length > 0;
@@ -182,11 +164,6 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
182
164
  yield { token: tokenStr };
183
165
  }
184
166
  const modelExecutionMs = nowMs() - modelStart;
185
- // The addon has finished, and saved the cache file if it was going to.
186
- onRunSettled?.();
187
- if (cacheOptions?.saveCacheToDisk && cacheOptions.cacheKey) {
188
- logCacheSave(cacheOptions.cacheKey);
189
- }
190
167
  if (tools && tools.length > 0) {
191
168
  const { toolCalls } = parseToolCalls(accumulatedText, tools, dialect);
192
169
  toolCallsResult = toolCalls;
@@ -199,24 +176,21 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
199
176
  toolCalls: toolCallsResult,
200
177
  responseText: accumulatedText,
201
178
  producedTokens,
202
- stoppedAtContextBoundary: stopReason === 'contextOverflow',
203
- generationFinished: stopReason !== 'none'
179
+ stoppedAtContextBoundary: stopReason === 'contextOverflow'
204
180
  };
205
181
  }
206
182
  export async function* completion(params, opts) {
207
- const { history: requestHistory, modelId, kvCache, tools, generationParams, responseFormat } = params;
183
+ const { history: requestHistory, modelId, kvCache, prefill, tools, generationParams, responseFormat } = params;
208
184
  const { signal, scope } = opts;
209
185
  const requestLogger = opts.logger ?? logger;
210
186
  const modelConfig = getModelConfig(modelId);
211
187
  const history = seedConfiguredSystemPrompt(requestHistory, modelConfig);
212
188
  const toolsEnabled = modelConfig.tools === true;
213
- // With deferred tools the prompt and the parser see different lists: the
214
- // prompt carries the always-loaded ones plus `tool_search` and its catalog,
215
- // while the parser also accepts whatever earlier searches appended to the
216
- // history. `null` when nothing defers, which leaves the existing path alone.
189
+ // With deferred tools the prompt carries the always-loaded ones, `tool_search`
190
+ // and its catalog, and whatever earlier searches loaded. `null` when nothing
191
+ // defers, which leaves the existing path alone.
217
192
  const deferred = resolveDeferredTools(tools, history);
218
193
  const toolsToRender = deferred?.toolsToRender ?? tools?.map(toWireTool);
219
- const callableTools = deferred?.callableTools ?? toolsToRender;
220
194
  const toolsActive = !!toolsToRender?.length && toolsEnabled;
221
195
  const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
222
196
  // `responseFormat` is forwarded to the addon as a per-request
@@ -236,7 +210,7 @@ export async function* completion(params, opts) {
236
210
  };
237
211
  }
238
212
  }
239
- mergedGenerationParams = withDeferredToolChoice(mergedGenerationParams, toolsActive ? deferred : null);
213
+ mergedGenerationParams = withParallelToolCalls(mergedGenerationParams);
240
214
  const model = getModel(modelId);
241
215
  // Per-request hard cancel: under continuous batching the model runs several
242
216
  // jobs at once, so an abort must cancel only THIS request's job, not the
@@ -267,30 +241,30 @@ export async function* completion(params, opts) {
267
241
  // Drop the ref so a late abort can't cancel an already-settled response.
268
242
  activeResponse = null;
269
243
  });
244
+ // The cached and uncached paths send the same prompt: the whole conversation
245
+ // and the tools, every turn. With a cache key the addon keeps the longest
246
+ // prefix it already holds and decodes only the rest.
247
+ const prompt = transformMessages(toolsActive && toolsToRender ? prependToolsToHistory(history, toolsToRender) : history);
270
248
  if (!kvCache) {
271
- // KV-cache disabled — straight passthrough, no session involvement.
272
- let historyWithTools = history;
273
- if (toolsActive && toolsToRender) {
274
- historyWithTools = prependToolsToHistory(history, toolsToRender);
275
- }
276
- const transformedHistory = transformMessages(historyWithTools);
277
249
  logCacheDisabled();
278
- logMessagesToAddon(transformedHistory, 'NO_CACHE');
279
- return yield* processModelResponse(model, transformedHistory, callableTools, mergedGenerationParams, undefined, dialect, setActiveResponse);
250
+ logMessagesToAddon(prompt, 'NO_CACHE');
251
+ return yield* processModelResponse(model, prompt, toolsToRender, mergedGenerationParams, undefined, dialect, setActiveResponse);
280
252
  }
281
253
  // ---- KV-cache path. The session owns every bookkeeping layer; the handler
282
- // registers one deferred unwind that `commitTurn` short-circuits on the happy
283
- // path. It is the non-destructive `releaseTurn` when the committed file is
284
- // known to be intact — a throw before the addon run settled, or a cancel the
285
- // addon rewound — and the destructive `rollback` for everything else,
286
- // including zero-token replies, budget and context stops, and rename
287
- // failures. ----
288
- const session = createKvCacheSession(modelId, { logger: requestLogger });
289
- const systemPromptFromHistory = extractSystemPrompt(history);
290
- // The tool block is baked into the cache on the turn that first sends it and
291
- // never trimmed, so a late or changed tool set has to land on a fresh cache
292
- // rather than a warm prefix holding the old block.
293
- const configHash = generateConfigHash(systemPromptFromHistory, toolsActive ? toolsToRender : undefined);
254
+ // registers one deferred unwind that `commitTurn` short-circuits. The addon
255
+ // leaves the file consistent whatever the outcome — it commits or rolls back
256
+ // the request itself — so the unwind keeps it (`releaseTurn`) except where
257
+ // the file has no key to live under. A rollback also drops the addon's copy
258
+ // of the conversation, which a later key switch or unload would otherwise
259
+ // write back to the deleted path. ----
260
+ const discardCache = model.discardCache;
261
+ const session = createKvCacheSession(modelId, {
262
+ logger: requestLogger,
263
+ ...(discardCache && {
264
+ discardCache: (cachePath) => discardCache.call(model, cachePath)
265
+ })
266
+ });
267
+ const configHash = generateConfigHash(extractSystemPrompt(history));
294
268
  let turn;
295
269
  if (typeof kvCache === 'string') {
296
270
  turn = await session.beginTurn({
@@ -301,109 +275,72 @@ export async function* completion(params, opts) {
301
275
  });
302
276
  }
303
277
  else {
304
- const cacheMessages = history.map((msg) => ({
305
- role: msg.role,
306
- content: msg.content,
307
- attachments: msg.attachments ?? undefined
308
- }));
278
+ const cacheMessages = history.map(toCacheMessage);
309
279
  turn = await session.beginTurn({
310
280
  kind: 'auto',
311
281
  configHash,
312
- history: cacheMessages,
282
+ history: prefill ? cacheMessages.map(normalizeCacheMessage) : cacheMessages,
283
+ ...(toolsActive && toolsToRender ? { tools: toolsToRender } : {}),
313
284
  signal
314
285
  });
315
286
  }
316
- // Single cleanup hook for every non-success exit path. `commitTurn`
317
- // flips the turn's internal `committed` flag so this becomes a no-op
318
- // on the happy path. Scope unwinding is LIFO — registered after the
319
- // `removeEventListener` defer above so rollback runs before the
320
- // listener detach. `preserveCacheOnUnwind` selects the non-destructive
321
- // `releaseTurn` when the file on disk is known to still hold the last
322
- // committed turn; `releaseTurn` still drops a cache this turn created.
323
- let preserveCacheOnUnwind = false;
324
- scope.defer(() => (preserveCacheOnUnwind ? session.releaseTurn(turn) : session.rollback(turn)));
325
- let payload;
326
- try {
327
- payload = await prepareMessagesForCache(session, turn, history, toolsActive ? toolsToRender : undefined, mergedGenerationParams?.tool_choice);
328
- }
329
- catch (error) {
330
- // A missing attachment is caller input rejected before the addon runs,
331
- // so the committed cache is untouched and must survive.
332
- preserveCacheOnUnwind = error instanceof AttachmentNotFoundError;
333
- throw error;
334
- }
335
- const messagesToSend = payload.messages;
336
- logMessagesToAddon(messagesToSend, 'PROMPT_SEND');
337
- let result;
338
- let addonRunSettled = false;
339
- try {
340
- result = yield* processModelResponse(model, messagesToSend, callableTools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse, () => {
341
- addonRunSettled = true;
342
- });
343
- }
344
- catch (error) {
345
- // The addon writes the cache file only after a run completes and skips the
346
- // save on every error path, so a run that threw left the committed file as
347
- // it was. An engine-side throw after the run settled is the opposite: the
348
- // file already holds this turn while no boundary was recorded for it.
349
- preserveCacheOnUnwind = !addonRunSettled;
350
- throw error;
351
- }
352
- const shouldCommitTurn = shouldCommitCachedTurn({
353
- aborted: signal.aborted,
354
- producedTokens: result.producedTokens,
355
- generatedTokens: result.stats?.generatedTokens,
356
- predict: mergedGenerationParams?.predict ?? modelConfig.predict,
357
- stoppedAtContextBoundary: result.stoppedAtContextBoundary
358
- });
359
- // A cancelled run is rewound by the addon to the pre-request state before the
360
- // file is re-saved, so the committed cache is intact. Every other non-commit
361
- // finish (zero tokens, budget or context stop) was saved as-is and must go.
362
- // An abort that landed after the addon reported a stop reason is the latter.
363
- if (!shouldCommitTurn) {
364
- preserveCacheOnUnwind = signal.aborted && !result.generationFinished;
365
- }
287
+ // Scope unwinding is LIFO — registered after the `removeEventListener`
288
+ // defer above so this runs before the listener detach.
289
+ let rollbackOnUnwind = false;
290
+ scope.defer(() => (rollbackOnUnwind ? session.rollback(turn) : session.releaseTurn(turn)));
291
+ logMessagesToAddon(prompt, 'PROMPT_SEND');
292
+ const result = yield* processModelResponse(model, prompt, toolsToRender, mergedGenerationParams, { cacheKey: turn.cachePath, ...(prefill !== undefined && { prefill }) }, dialect, setActiveResponse);
366
293
  if (typeof kvCache === 'string') {
367
- // Custom-key path: the addon wrote the new cache state inline at
368
- // the same path. Either commit (records the boundary, suppresses
369
- // rollback) or fall through to the deferred rollback.
370
- if (shouldCommitTurn) {
371
- await session.commitTurn(turn, {
372
- kind: 'static',
373
- messageCount: history.length + 1,
374
- toolBlockCached: resolveToolBlockCached(payload, result.stats)
375
- });
376
- }
294
+ // Custom-key path: the file holds whatever the addon kept, cancelled and
295
+ // stopped turns included.
296
+ await saveCacheFile(model, turn.cachePath, requestLogger);
297
+ await session.commitTurn(turn, { kind: 'static' });
377
298
  return result;
378
299
  }
379
300
  // Auto-cache path.
380
- //
301
+ if (prefill && !signal.aborted && !result.stoppedAtContextBoundary) {
302
+ await saveCacheFile(model, turn.cachePath, requestLogger);
303
+ const cacheInfo = await getCurrentCacheInfo(modelId, configHash, history.map(toCacheMessage).map(normalizeCacheMessage), toolsActive ? toolsToRender : undefined);
304
+ await session.commitTurn(turn,
305
+ // Renaming to the held path would reacquire its non-reentrant write lock.
306
+ cacheInfo.cachePath === turn.cachePath
307
+ ? { kind: 'static' }
308
+ : { kind: 'autoRename', targetCachePath: cacheInfo.cachePath });
309
+ return result;
310
+ }
381
311
  // Tool-call turns: the auto-cache key is derived from
382
312
  // `result.responseText`, which here is raw tool-call markup rather
383
313
  // than a clean assistant message. There's no safe post-response key
384
- // to rename to, so we let the deferred rollback drop the file. Once
314
+ // to rename to, so the deferred rollback drops the turn, unwritten. Once
385
315
  // we support auto-cache for structured assistant/tool turns,
386
316
  // this becomes a normal commit path.
387
317
  if (result.toolCalls.length > 0) {
388
318
  logger.warn(`[kv-cache] Auto cache tool-call turn; rolling back to avoid disk leak. path=${turn.cachePath}`);
319
+ rollbackOnUnwind = true;
389
320
  return result;
390
321
  }
391
- if (!shouldCommitTurn) {
392
- // Cancelled, zero-token, or budget-exhausted turns do not establish
393
- // a trustworthy message boundary.
322
+ const shouldRename = shouldCommitCachedTurn({
323
+ aborted: signal.aborted,
324
+ producedTokens: result.producedTokens,
325
+ generatedTokens: result.stats?.generatedTokens,
326
+ predict: mergedGenerationParams?.predict ?? modelConfig.predict,
327
+ stoppedAtContextBoundary: result.stoppedAtContextBoundary
328
+ });
329
+ if (!shouldRename) {
330
+ // A cancelled, empty or cut-off reply is not one the caller will send
331
+ // back as-is, so there is no post-response key to move to. The file stays
332
+ // under the history it was found by, holding what the addon kept. A first
333
+ // turn found no file, and one it wrote now would only be rolled back.
334
+ if (!turn.createdCache)
335
+ await saveCacheFile(model, turn.cachePath, requestLogger);
394
336
  return result;
395
337
  }
396
- const savedHistory = buildAutoCacheSaveHistory(history.map((msg) => ({
397
- role: msg.role,
398
- content: msg.content,
399
- attachments: msg.attachments ?? undefined
400
- })), result.responseText);
401
- const postResponseCacheInfo = await getCurrentCacheInfo(modelId, configHash, savedHistory);
338
+ await saveCacheFile(model, turn.cachePath, requestLogger);
339
+ const savedHistory = buildAutoCacheSaveHistory(history.map(toCacheMessage), result.responseText);
340
+ const postResponseCacheInfo = await getCurrentCacheInfo(modelId, configHash, savedHistory, toolsActive ? toolsToRender : undefined);
402
341
  await session.commitTurn(turn, {
403
342
  kind: 'autoRename',
404
- targetCachePath: postResponseCacheInfo.cachePath,
405
- messageCount: savedHistory.length,
406
- toolBlockCached: resolveToolBlockCached(payload, result.stats)
343
+ targetCachePath: postResponseCacheInfo.cachePath
407
344
  });
408
345
  return result;
409
346
  }