@qvac/inference 0.0.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/LICENSE +179 -0
  2. package/NOTICE +986 -0
  3. package/README.md +138 -0
  4. package/dist/api/audio-gen.d.ts +18 -0
  5. package/dist/api/audio-gen.js +148 -0
  6. package/dist/api/batch-completion.d.ts +14 -0
  7. package/dist/api/batch-completion.js +332 -0
  8. package/dist/api/bci-transcribe.d.ts +56 -0
  9. package/dist/api/bci-transcribe.js +172 -0
  10. package/dist/api/cancel.d.ts +34 -0
  11. package/dist/api/cancel.js +82 -0
  12. package/dist/api/classify.d.ts +22 -0
  13. package/dist/api/classify.js +46 -0
  14. package/dist/api/completion-stream.d.ts +106 -0
  15. package/dist/api/completion-stream.js +351 -0
  16. package/dist/api/delete-cache.d.ts +29 -0
  17. package/dist/api/delete-cache.js +49 -0
  18. package/dist/api/diffusion.d.ts +92 -0
  19. package/dist/api/diffusion.js +176 -0
  20. package/dist/api/download-asset.d.ts +49 -0
  21. package/dist/api/download-asset.js +87 -0
  22. package/dist/api/embed.d.ts +41 -0
  23. package/dist/api/embed.js +29 -0
  24. package/dist/api/finetune.d.ts +89 -0
  25. package/dist/api/finetune.js +138 -0
  26. package/dist/api/get-loaded-model-info.d.ts +24 -0
  27. package/dist/api/get-loaded-model-info.js +34 -0
  28. package/dist/api/get-model-info.d.ts +43 -0
  29. package/dist/api/get-model-info.js +21 -0
  30. package/dist/api/get-system-resources.d.ts +2 -0
  31. package/dist/api/get-system-resources.js +16 -0
  32. package/dist/api/heartbeat.d.ts +29 -0
  33. package/dist/api/heartbeat.js +37 -0
  34. package/dist/api/index.d.ts +35 -0
  35. package/dist/api/index.js +35 -0
  36. package/dist/api/invoke-plugin.d.ts +30 -0
  37. package/dist/api/invoke-plugin.js +54 -0
  38. package/dist/api/load-model.d.ts +169 -0
  39. package/dist/api/load-model.js +129 -0
  40. package/dist/api/logging-stream-registry.d.ts +4 -0
  41. package/dist/api/logging-stream-registry.js +61 -0
  42. package/dist/api/logging-stream.d.ts +23 -0
  43. package/dist/api/logging-stream.js +36 -0
  44. package/dist/api/ocr.d.ts +30 -0
  45. package/dist/api/ocr.js +88 -0
  46. package/dist/api/parse-input.d.ts +2 -0
  47. package/dist/api/parse-input.js +14 -0
  48. package/dist/api/provide.d.ts +23 -0
  49. package/dist/api/provide.js +32 -0
  50. package/dist/api/rag.d.ts +251 -0
  51. package/dist/api/rag.js +485 -0
  52. package/dist/api/registry.d.ts +39 -0
  53. package/dist/api/registry.js +63 -0
  54. package/dist/api/resume.d.ts +29 -0
  55. package/dist/api/resume.js +36 -0
  56. package/dist/api/state.d.ts +22 -0
  57. package/dist/api/state.js +29 -0
  58. package/dist/api/stop-provider.d.ts +20 -0
  59. package/dist/api/stop-provider.js +30 -0
  60. package/dist/api/subscribe-logs.d.ts +27 -0
  61. package/dist/api/subscribe-logs.js +43 -0
  62. package/dist/api/suspend.d.ts +30 -0
  63. package/dist/api/suspend.js +37 -0
  64. package/dist/api/text-to-speech.d.ts +64 -0
  65. package/dist/api/text-to-speech.js +434 -0
  66. package/dist/api/transcribe.d.ts +75 -0
  67. package/dist/api/transcribe.js +256 -0
  68. package/dist/api/translate.d.ts +47 -0
  69. package/dist/api/translate.js +115 -0
  70. package/dist/api/unload-model.d.ts +17 -0
  71. package/dist/api/unload-model.js +42 -0
  72. package/dist/api/upscale.d.ts +44 -0
  73. package/dist/api/upscale.js +99 -0
  74. package/dist/api/video.d.ts +86 -0
  75. package/dist/api/video.js +176 -0
  76. package/dist/api/vla-helpers.d.ts +43 -0
  77. package/dist/api/vla-helpers.js +124 -0
  78. package/dist/api/vla.d.ts +68 -0
  79. package/dist/api/vla.js +113 -0
  80. package/dist/config/config-utils.d.ts +4 -0
  81. package/dist/config/config-utils.js +18 -0
  82. package/dist/config/resolve-config.d.ts +8 -0
  83. package/dist/config/resolve-config.js +74 -0
  84. package/dist/constants/audio.d.ts +1 -0
  85. package/dist/constants/audio.js +1 -0
  86. package/dist/constants/index.d.ts +2 -0
  87. package/dist/constants/index.js +2 -0
  88. package/dist/constants/registry.d.ts +1 -0
  89. package/dist/constants/registry.js +1 -0
  90. package/dist/dispatch.d.ts +18 -0
  91. package/dist/dispatch.js +271 -0
  92. package/dist/engine.d.ts +5 -0
  93. package/dist/engine.js +27 -0
  94. package/dist/errors/index.d.ts +416 -0
  95. package/dist/errors/index.js +681 -0
  96. package/dist/errors/options.d.ts +2 -0
  97. package/dist/errors/options.js +13 -0
  98. package/dist/handlers/cancelHandler.d.ts +29 -0
  99. package/dist/handlers/cancelHandler.js +67 -0
  100. package/dist/handlers/completion-orchestrate.d.ts +41 -0
  101. package/dist/handlers/completion-orchestrate.js +190 -0
  102. package/dist/handlers/delete-cache.d.ts +11 -0
  103. package/dist/handlers/delete-cache.js +37 -0
  104. package/dist/handlers/download-asset.d.ts +2 -0
  105. package/dist/handlers/download-asset.js +146 -0
  106. package/dist/handlers/finetune.d.ts +2 -0
  107. package/dist/handlers/finetune.js +18 -0
  108. package/dist/handlers/get-loaded-model-info.d.ts +2 -0
  109. package/dist/handlers/get-loaded-model-info.js +45 -0
  110. package/dist/handlers/get-model-info.d.ts +2 -0
  111. package/dist/handlers/get-model-info.js +173 -0
  112. package/dist/handlers/get-system-resources.d.ts +2 -0
  113. package/dist/handlers/get-system-resources.js +53 -0
  114. package/dist/handlers/heartbeat.d.ts +2 -0
  115. package/dist/handlers/heartbeat.js +3 -0
  116. package/dist/handlers/load-model/download-manager.d.ts +95 -0
  117. package/dist/handlers/load-model/download-manager.js +306 -0
  118. package/dist/handlers/load-model/download-stats.d.ts +7 -0
  119. package/dist/handlers/load-model/download-stats.js +126 -0
  120. package/dist/handlers/load-model/handler.d.ts +2 -0
  121. package/dist/handlers/load-model/handler.js +287 -0
  122. package/dist/handlers/load-model/http.d.ts +3 -0
  123. package/dist/handlers/load-model/http.js +857 -0
  124. package/dist/handlers/load-model/hyperdrive.d.ts +4 -0
  125. package/dist/handlers/load-model/hyperdrive.js +560 -0
  126. package/dist/handlers/load-model/index.d.ts +2 -0
  127. package/dist/handlers/load-model/index.js +2 -0
  128. package/dist/handlers/load-model/registry-client-options.d.ts +27 -0
  129. package/dist/handlers/load-model/registry-client-options.js +18 -0
  130. package/dist/handlers/load-model/registry-companion-set.d.ts +22 -0
  131. package/dist/handlers/load-model/registry-companion-set.js +206 -0
  132. package/dist/handlers/load-model/registry-download-utils.d.ts +23 -0
  133. package/dist/handlers/load-model/registry-download-utils.js +112 -0
  134. package/dist/handlers/load-model/registry-metadata.d.ts +10 -0
  135. package/dist/handlers/load-model/registry-metadata.js +23 -0
  136. package/dist/handlers/load-model/registry.d.ts +4 -0
  137. package/dist/handlers/load-model/registry.js +298 -0
  138. package/dist/handlers/load-model/resolve-session.d.ts +33 -0
  139. package/dist/handlers/load-model/resolve-session.js +62 -0
  140. package/dist/handlers/load-model/resolve.d.ts +5 -0
  141. package/dist/handlers/load-model/resolve.js +158 -0
  142. package/dist/handlers/load-model/types.d.ts +59 -0
  143. package/dist/handlers/load-model/types.js +28 -0
  144. package/dist/handlers/logging-stream.d.ts +2 -0
  145. package/dist/handlers/logging-stream.js +39 -0
  146. package/dist/handlers/plugin-dispatch.d.ts +3 -0
  147. package/dist/handlers/plugin-dispatch.js +63 -0
  148. package/dist/handlers/plugin-invoke.d.ts +3 -0
  149. package/dist/handlers/plugin-invoke.js +81 -0
  150. package/dist/handlers/registry.d.ts +4 -0
  151. package/dist/handlers/registry.js +156 -0
  152. package/dist/handlers/resume.d.ts +2 -0
  153. package/dist/handlers/resume.js +11 -0
  154. package/dist/handlers/state.d.ts +2 -0
  155. package/dist/handlers/state.js +7 -0
  156. package/dist/handlers/suspend.d.ts +2 -0
  157. package/dist/handlers/suspend.js +11 -0
  158. package/dist/handlers/types.d.ts +13 -0
  159. package/dist/handlers/types.js +1 -0
  160. package/dist/handlers/unload-model.d.ts +2 -0
  161. package/dist/handlers/unload-model.js +29 -0
  162. package/dist/index.d.ts +4 -0
  163. package/dist/index.js +12 -0
  164. package/dist/logging/addon.d.ts +5 -0
  165. package/dist/logging/addon.js +75 -0
  166. package/dist/logging/index.d.ts +5 -0
  167. package/dist/logging/index.js +4 -0
  168. package/dist/logging/logger.d.ts +41 -0
  169. package/dist/logging/logger.js +209 -0
  170. package/dist/logging/namespaces.d.ts +6 -0
  171. package/dist/logging/namespaces.js +7 -0
  172. package/dist/logging/types.d.ts +37 -0
  173. package/dist/logging/types.js +1 -0
  174. package/dist/logging/utils.d.ts +5 -0
  175. package/dist/logging/utils.js +113 -0
  176. package/dist/models/registry/index.d.ts +18 -0
  177. package/dist/models/registry/index.js +21 -0
  178. package/dist/models/registry/models.d.ts +22680 -0
  179. package/dist/models/registry/models.js +23908 -0
  180. package/dist/p2p/cancel-delegated.d.ts +3 -0
  181. package/dist/p2p/cancel-delegated.js +64 -0
  182. package/dist/p2p/completion-stream-delegated.d.ts +4 -0
  183. package/dist/p2p/completion-stream-delegated.js +51 -0
  184. package/dist/p2p/delegate-client.d.ts +9 -0
  185. package/dist/p2p/delegate-client.js +219 -0
  186. package/dist/p2p/delegate-connect-diagnostics.d.ts +8 -0
  187. package/dist/p2p/delegate-connect-diagnostics.js +38 -0
  188. package/dist/p2p/delegate-transport.d.ts +18 -0
  189. package/dist/p2p/delegate-transport.js +237 -0
  190. package/dist/p2p/error-handlers.d.ts +4 -0
  191. package/dist/p2p/error-handlers.js +31 -0
  192. package/dist/p2p/handle-request.d.ts +2 -0
  193. package/dist/p2p/handle-request.js +171 -0
  194. package/dist/p2p/handler-utils.d.ts +19 -0
  195. package/dist/p2p/handler-utils.js +180 -0
  196. package/dist/p2p/heartbeat-delegated.d.ts +3 -0
  197. package/dist/p2p/heartbeat-delegated.js +32 -0
  198. package/dist/p2p/load-model-delegated.d.ts +6 -0
  199. package/dist/p2p/load-model-delegated.js +132 -0
  200. package/dist/p2p/progress-throttle.d.ts +7 -0
  201. package/dist/p2p/progress-throttle.js +47 -0
  202. package/dist/p2p/provideHandler/connection.d.ts +2 -0
  203. package/dist/p2p/provideHandler/connection.js +38 -0
  204. package/dist/p2p/provideHandler/handler.d.ts +2 -0
  205. package/dist/p2p/provideHandler/handler.js +66 -0
  206. package/dist/p2p/provideHandler/proxy.d.ts +2 -0
  207. package/dist/p2p/provideHandler/proxy.js +24 -0
  208. package/dist/p2p/provideHandler.d.ts +1 -0
  209. package/dist/p2p/provideHandler.js +2 -0
  210. package/dist/p2p/rpc-utils.d.ts +4 -0
  211. package/dist/p2p/rpc-utils.js +8 -0
  212. package/dist/p2p/stopProvideHandler.d.ts +2 -0
  213. package/dist/p2p/stopProvideHandler.js +29 -0
  214. package/dist/p2p/swarm.d.ts +9 -0
  215. package/dist/p2p/swarm.js +103 -0
  216. package/dist/p2p/transport-selector.d.ts +2 -0
  217. package/dist/p2p/transport-selector.js +18 -0
  218. package/dist/p2p/unload-model-delegated.d.ts +2 -0
  219. package/dist/p2p/unload-model-delegated.js +31 -0
  220. package/dist/plugins/builtin/asr-ggml/config.d.ts +31 -0
  221. package/dist/plugins/builtin/asr-ggml/config.js +46 -0
  222. package/dist/plugins/builtin/asr-ggml/logging.d.ts +10 -0
  223. package/dist/plugins/builtin/asr-ggml/logging.js +12 -0
  224. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +17 -0
  225. package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
  226. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +2 -0
  227. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +163 -0
  228. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +212 -0
  229. package/dist/plugins/builtin/audiogen-ggml/plugin.js +58 -0
  230. package/dist/plugins/builtin/bci-whispercpp-transcription/args.d.ts +2 -0
  231. package/dist/plugins/builtin/bci-whispercpp-transcription/args.js +13 -0
  232. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +173 -0
  233. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +111 -0
  234. package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.d.ts +74 -0
  235. package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.js +11 -0
  236. package/dist/plugins/builtin/ggml-classification/ops/classify.d.ts +5 -0
  237. package/dist/plugins/builtin/ggml-classification/ops/classify.js +31 -0
  238. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +31 -0
  239. package/dist/plugins/builtin/ggml-classification/plugin.js +55 -0
  240. package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.d.ts +7 -0
  241. package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.js +119 -0
  242. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +102 -0
  243. package/dist/plugins/builtin/ggml-ocr/plugin.js +110 -0
  244. package/dist/plugins/builtin/ggml-ocr/resolve-config.d.ts +5 -0
  245. package/dist/plugins/builtin/ggml-ocr/resolve-config.js +72 -0
  246. package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.d.ts +2 -0
  247. package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.js +12 -0
  248. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -0
  249. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +78 -0
  250. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +65 -0
  251. package/dist/plugins/builtin/ggml-vla/plugin.js +57 -0
  252. package/dist/plugins/builtin/index.d.ts +11 -0
  253. package/dist/plugins/builtin/index.js +11 -0
  254. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +22 -0
  255. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +30 -0
  256. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.d.ts +31 -0
  257. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +99 -0
  258. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +6 -0
  259. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +36 -0
  260. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +20 -0
  261. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +48 -0
  262. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +45 -0
  263. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +374 -0
  264. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +28 -0
  265. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +65 -0
  266. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +10 -0
  267. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +209 -0
  268. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +163 -0
  269. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +581 -0
  270. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +56 -0
  271. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +58 -0
  272. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +671 -0
  273. package/dist/plugins/builtin/llamacpp-completion/plugin.js +481 -0
  274. package/dist/plugins/builtin/llamacpp-completion/transform.d.ts +9 -0
  275. package/dist/plugins/builtin/llamacpp-completion/transform.js +34 -0
  276. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +76 -0
  277. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +101 -0
  278. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +457 -0
  279. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +143 -0
  280. package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.d.ts +31 -0
  281. package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.js +114 -0
  282. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +133 -0
  283. package/dist/plugins/builtin/parakeet-transcription/plugin.js +130 -0
  284. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +2 -0
  285. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +64 -0
  286. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +2 -0
  287. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +41 -0
  288. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -0
  289. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +239 -0
  290. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +757 -0
  291. package/dist/plugins/builtin/sdcpp-generation/plugin.js +310 -0
  292. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +6 -0
  293. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +30 -0
  294. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +7 -0
  295. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +106 -0
  296. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +6 -0
  297. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +91 -0
  298. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +196 -0
  299. package/dist/plugins/builtin/tts-ggml/plugin.js +305 -0
  300. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +346 -0
  301. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +169 -0
  302. package/dist/plugins/index.d.ts +9 -0
  303. package/dist/plugins/index.js +12 -0
  304. package/dist/plugins/ops/bci-transcribe.d.ts +13 -0
  305. package/dist/plugins/ops/bci-transcribe.js +248 -0
  306. package/dist/plugins/ops/cancel.d.ts +25 -0
  307. package/dist/plugins/ops/cancel.js +43 -0
  308. package/dist/plugins/ops/embed.d.ts +15 -0
  309. package/dist/plugins/ops/embed.js +193 -0
  310. package/dist/plugins/ops/kv-cache-retention.d.ts +11 -0
  311. package/dist/plugins/ops/kv-cache-retention.js +121 -0
  312. package/dist/plugins/ops/kv-cache-utils.d.ts +21 -0
  313. package/dist/plugins/ops/kv-cache-utils.js +120 -0
  314. package/dist/plugins/ops/load-model.d.ts +8 -0
  315. package/dist/plugins/ops/load-model.js +82 -0
  316. package/dist/plugins/ops/transcribe.d.ts +20 -0
  317. package/dist/plugins/ops/transcribe.js +343 -0
  318. package/dist/plugins/ops/translate.d.ts +7 -0
  319. package/dist/plugins/ops/translate.js +256 -0
  320. package/dist/plugins/ops/unload-model.d.ts +2 -0
  321. package/dist/plugins/ops/unload-model.js +45 -0
  322. package/dist/plugins/registry.d.ts +9 -0
  323. package/dist/plugins/registry.js +107 -0
  324. package/dist/profiling/aggregator.d.ts +14 -0
  325. package/dist/profiling/aggregator.js +100 -0
  326. package/dist/profiling/backend-diagnostics.d.ts +8 -0
  327. package/dist/profiling/backend-diagnostics.js +36 -0
  328. package/dist/profiling/clock.d.ts +8 -0
  329. package/dist/profiling/clock.js +23 -0
  330. package/dist/profiling/controller.d.ts +34 -0
  331. package/dist/profiling/controller.js +130 -0
  332. package/dist/profiling/delegation-profiler.d.ts +31 -0
  333. package/dist/profiling/delegation-profiler.js +155 -0
  334. package/dist/profiling/envelope.d.ts +10 -0
  335. package/dist/profiling/envelope.js +45 -0
  336. package/dist/profiling/events.d.ts +19 -0
  337. package/dist/profiling/events.js +35 -0
  338. package/dist/profiling/exporters.d.ts +9 -0
  339. package/dist/profiling/exporters.js +242 -0
  340. package/dist/profiling/index.d.ts +91 -0
  341. package/dist/profiling/index.js +87 -0
  342. package/dist/profiling/model-execution.d.ts +15 -0
  343. package/dist/profiling/model-execution.js +34 -0
  344. package/dist/profiling/operation-metrics.d.ts +24 -0
  345. package/dist/profiling/operation-metrics.js +370 -0
  346. package/dist/profiling/operation-wrappers.d.ts +17 -0
  347. package/dist/profiling/operation-wrappers.js +196 -0
  348. package/dist/profiling/profiler.d.ts +27 -0
  349. package/dist/profiling/profiler.js +127 -0
  350. package/dist/profiling/ring-buffer.d.ts +18 -0
  351. package/dist/profiling/ring-buffer.js +83 -0
  352. package/dist/profiling/types.d.ts +52 -0
  353. package/dist/profiling/types.js +1 -0
  354. package/dist/rag/handler.d.ts +2 -0
  355. package/dist/rag/handler.js +272 -0
  356. package/dist/rag/handlers/chunk.d.ts +2 -0
  357. package/dist/rag/handlers/chunk.js +7 -0
  358. package/dist/rag/handlers/close-workspace.d.ts +2 -0
  359. package/dist/rag/handlers/close-workspace.js +9 -0
  360. package/dist/rag/handlers/delete-embeddings.d.ts +2 -0
  361. package/dist/rag/handlers/delete-embeddings.js +11 -0
  362. package/dist/rag/handlers/delete-workspace.d.ts +2 -0
  363. package/dist/rag/handlers/delete-workspace.js +14 -0
  364. package/dist/rag/handlers/ingest.d.ts +9 -0
  365. package/dist/rag/handlers/ingest.js +19 -0
  366. package/dist/rag/handlers/list-workspaces.d.ts +2 -0
  367. package/dist/rag/handlers/list-workspaces.js +4 -0
  368. package/dist/rag/handlers/reindex.d.ts +9 -0
  369. package/dist/rag/handlers/reindex.js +16 -0
  370. package/dist/rag/handlers/save-embeddings.d.ts +9 -0
  371. package/dist/rag/handlers/save-embeddings.js +15 -0
  372. package/dist/rag/handlers/search.d.ts +2 -0
  373. package/dist/rag/handlers/search.js +16 -0
  374. package/dist/rag/index.d.ts +11 -0
  375. package/dist/rag/index.js +11 -0
  376. package/dist/rag/rag-operation-manager.d.ts +28 -0
  377. package/dist/rag/rag-operation-manager.js +76 -0
  378. package/dist/rag/rag-workspace-manager.d.ts +14 -0
  379. package/dist/rag/rag-workspace-manager.js +143 -0
  380. package/dist/registry.d.ts +2 -0
  381. package/dist/registry.js +166 -0
  382. package/dist/resources/collector.d.ts +643 -0
  383. package/dist/resources/collector.js +226 -0
  384. package/dist/resources/instance.d.ts +4 -0
  385. package/dist/resources/instance.js +15 -0
  386. package/dist/resources/native.d.ts +2 -0
  387. package/dist/resources/native.js +35 -0
  388. package/dist/resources/normalize.d.ts +12 -0
  389. package/dist/resources/normalize.js +73 -0
  390. package/dist/resources/types.d.ts +65 -0
  391. package/dist/resources/types.js +1 -0
  392. package/dist/runtime/cache-lock.d.ts +2 -0
  393. package/dist/runtime/cache-lock.js +67 -0
  394. package/dist/runtime/disposable-scope.d.ts +35 -0
  395. package/dist/runtime/disposable-scope.js +69 -0
  396. package/dist/runtime/env.d.ts +16 -0
  397. package/dist/runtime/env.js +53 -0
  398. package/dist/runtime/index.d.ts +5 -0
  399. package/dist/runtime/index.js +2 -0
  400. package/dist/runtime/lifecycle.d.ts +20 -0
  401. package/dist/runtime/lifecycle.js +102 -0
  402. package/dist/runtime/logging-stream-registry.d.ts +27 -0
  403. package/dist/runtime/logging-stream-registry.js +133 -0
  404. package/dist/runtime/model-config-utils.d.ts +14 -0
  405. package/dist/runtime/model-config-utils.js +128 -0
  406. package/dist/runtime/model-registry.d.ts +74 -0
  407. package/dist/runtime/model-registry.js +130 -0
  408. package/dist/runtime/registry-client.d.ts +4 -0
  409. package/dist/runtime/registry-client.js +118 -0
  410. package/dist/runtime/request-context.d.ts +72 -0
  411. package/dist/runtime/request-context.js +114 -0
  412. package/dist/runtime/request-id.d.ts +17 -0
  413. package/dist/runtime/request-id.js +29 -0
  414. package/dist/runtime/request-registry.d.ts +194 -0
  415. package/dist/runtime/request-registry.js +546 -0
  416. package/dist/runtime/runtime-lifecycle.d.ts +37 -0
  417. package/dist/runtime/runtime-lifecycle.js +163 -0
  418. package/dist/runtime/state.d.ts +26 -0
  419. package/dist/runtime/state.js +138 -0
  420. package/dist/schemas/audio-gen.d.ts +252 -0
  421. package/dist/schemas/audio-gen.js +69 -0
  422. package/dist/schemas/batch-completion-stream.d.ts +467 -0
  423. package/dist/schemas/batch-completion-stream.js +123 -0
  424. package/dist/schemas/bci-config.d.ts +74 -0
  425. package/dist/schemas/bci-config.js +58 -0
  426. package/dist/schemas/bci.d.ts +178 -0
  427. package/dist/schemas/bci.js +89 -0
  428. package/dist/schemas/cancel.d.ts +203 -0
  429. package/dist/schemas/cancel.js +135 -0
  430. package/dist/schemas/classification.d.ts +52 -0
  431. package/dist/schemas/classification.js +33 -0
  432. package/dist/schemas/common.d.ts +6420 -0
  433. package/dist/schemas/common.js +133 -0
  434. package/dist/schemas/completion-event.d.ts +248 -0
  435. package/dist/schemas/completion-event.js +94 -0
  436. package/dist/schemas/completion-stream.d.ts +475 -0
  437. package/dist/schemas/completion-stream.js +236 -0
  438. package/dist/schemas/config.d.ts +5022 -0
  439. package/dist/schemas/config.js +169 -0
  440. package/dist/schemas/delegate.d.ts +29 -0
  441. package/dist/schemas/delegate.js +40 -0
  442. package/dist/schemas/download-asset.d.ts +225 -0
  443. package/dist/schemas/download-asset.js +71 -0
  444. package/dist/schemas/embed.d.ts +41 -0
  445. package/dist/schemas/embed.js +35 -0
  446. package/dist/schemas/engine-addon-map.d.ts +20 -0
  447. package/dist/schemas/engine-addon-map.js +78 -0
  448. package/dist/schemas/error.d.ts +40 -0
  449. package/dist/schemas/error.js +65 -0
  450. package/dist/schemas/errors.d.ts +119 -0
  451. package/dist/schemas/errors.js +615 -0
  452. package/dist/schemas/finetune.d.ts +621 -0
  453. package/dist/schemas/finetune.js +163 -0
  454. package/dist/schemas/index.d.ts +45 -0
  455. package/dist/schemas/index.js +45 -0
  456. package/dist/schemas/json-schema.d.ts +15 -0
  457. package/dist/schemas/json-schema.js +2 -0
  458. package/dist/schemas/lifecycle.d.ts +38 -0
  459. package/dist/schemas/lifecycle.js +21 -0
  460. package/dist/schemas/llamacpp-config.d.ts +403 -0
  461. package/dist/schemas/llamacpp-config.js +133 -0
  462. package/dist/schemas/load-model.d.ts +25635 -0
  463. package/dist/schemas/load-model.js +629 -0
  464. package/dist/schemas/logging-stream.d.ts +33 -0
  465. package/dist/schemas/logging-stream.js +17 -0
  466. package/dist/schemas/mcp-adapter.d.ts +46 -0
  467. package/dist/schemas/mcp-adapter.js +1 -0
  468. package/dist/schemas/model-file.d.ts +27 -0
  469. package/dist/schemas/model-file.js +29 -0
  470. package/dist/schemas/model-info.d.ts +197 -0
  471. package/dist/schemas/model-info.js +144 -0
  472. package/dist/schemas/model-ops.d.ts +36 -0
  473. package/dist/schemas/model-ops.js +40 -0
  474. package/dist/schemas/model-src-utils.d.ts +192 -0
  475. package/dist/schemas/model-src-utils.js +61 -0
  476. package/dist/schemas/model-types.d.ts +275 -0
  477. package/dist/schemas/model-types.js +223 -0
  478. package/dist/schemas/ocr.d.ts +138 -0
  479. package/dist/schemas/ocr.js +59 -0
  480. package/dist/schemas/path-validation.d.ts +9 -0
  481. package/dist/schemas/path-validation.js +22 -0
  482. package/dist/schemas/plugin.d.ts +341 -0
  483. package/dist/schemas/plugin.js +241 -0
  484. package/dist/schemas/profiling.d.ts +695 -0
  485. package/dist/schemas/profiling.js +100 -0
  486. package/dist/schemas/provide.d.ts +50 -0
  487. package/dist/schemas/provide.js +32 -0
  488. package/dist/schemas/rag.d.ts +301 -0
  489. package/dist/schemas/rag.js +211 -0
  490. package/dist/schemas/registry.d.ts +453 -0
  491. package/dist/schemas/registry.js +104 -0
  492. package/dist/schemas/reload-config.d.ts +585 -0
  493. package/dist/schemas/reload-config.js +42 -0
  494. package/dist/schemas/runtime-context.d.ts +18 -0
  495. package/dist/schemas/runtime-context.js +7 -0
  496. package/dist/schemas/sdcpp-config.d.ts +1126 -0
  497. package/dist/schemas/sdcpp-config.js +748 -0
  498. package/dist/schemas/system-resources.d.ts +3966 -0
  499. package/dist/schemas/system-resources.js +149 -0
  500. package/dist/schemas/text-to-speech.d.ts +1352 -0
  501. package/dist/schemas/text-to-speech.js +364 -0
  502. package/dist/schemas/tools.d.ts +80 -0
  503. package/dist/schemas/tools.js +52 -0
  504. package/dist/schemas/transcription-config.d.ts +161 -0
  505. package/dist/schemas/transcription-config.js +156 -0
  506. package/dist/schemas/transcription.d.ts +261 -0
  507. package/dist/schemas/transcription.js +145 -0
  508. package/dist/schemas/translate.d.ts +138 -0
  509. package/dist/schemas/translate.js +103 -0
  510. package/dist/schemas/translation-config.d.ts +1031 -0
  511. package/dist/schemas/translation-config.js +170 -0
  512. package/dist/schemas/vla.d.ts +110 -0
  513. package/dist/schemas/vla.js +151 -0
  514. package/dist/selection.d.ts +8 -0
  515. package/dist/selection.js +14 -0
  516. package/dist/surface.d.ts +18 -0
  517. package/dist/surface.js +30 -0
  518. package/dist/utils/addon-responses.d.ts +58 -0
  519. package/dist/utils/addon-responses.js +1 -0
  520. package/dist/utils/aggregate-events.d.ts +25 -0
  521. package/dist/utils/aggregate-events.js +73 -0
  522. package/dist/utils/archive.d.ts +4 -0
  523. package/dist/utils/archive.js +180 -0
  524. package/dist/utils/asr-events.d.ts +14 -0
  525. package/dist/utils/asr-events.js +30 -0
  526. package/dist/utils/audio/decoder.d.ts +4 -0
  527. package/dist/utils/audio/decoder.js +80 -0
  528. package/dist/utils/audio-input.d.ts +9 -0
  529. package/dist/utils/audio-input.js +34 -0
  530. package/dist/utils/cache/auto-history.d.ts +3 -0
  531. package/dist/utils/cache/auto-history.js +26 -0
  532. package/dist/utils/cache/paths.d.ts +42 -0
  533. package/dist/utils/cache/paths.js +133 -0
  534. package/dist/utils/cache/types.d.ts +7 -0
  535. package/dist/utils/cache/types.js +1 -0
  536. package/dist/utils/cache-normalize.d.ts +9 -0
  537. package/dist/utils/cache-normalize.js +20 -0
  538. package/dist/utils/checksum.d.ts +3 -0
  539. package/dist/utils/checksum.js +29 -0
  540. package/dist/utils/clear-storage.d.ts +4 -0
  541. package/dist/utils/clear-storage.js +14 -0
  542. package/dist/utils/completion-normalizer.d.ts +17 -0
  543. package/dist/utils/completion-normalizer.js +453 -0
  544. package/dist/utils/config-transform.d.ts +2 -0
  545. package/dist/utils/config-transform.js +21 -0
  546. package/dist/utils/decorate-promise.d.ts +33 -0
  547. package/dist/utils/decorate-promise.js +35 -0
  548. package/dist/utils/encoding.d.ts +16 -0
  549. package/dist/utils/encoding.js +27 -0
  550. package/dist/utils/expand-gguf-shards.d.ts +2 -0
  551. package/dist/utils/expand-gguf-shards.js +22 -0
  552. package/dist/utils/formatting.d.ts +18 -0
  553. package/dist/utils/formatting.js +56 -0
  554. package/dist/utils/gguf-tensor-extractor.d.ts +14 -0
  555. package/dist/utils/gguf-tensor-extractor.js +311 -0
  556. package/dist/utils/global-singleton.d.ts +1 -0
  557. package/dist/utils/global-singleton.js +10 -0
  558. package/dist/utils/index.d.ts +11 -0
  559. package/dist/utils/index.js +11 -0
  560. package/dist/utils/load-model-validation.d.ts +6 -0
  561. package/dist/utils/load-model-validation.js +18 -0
  562. package/dist/utils/mcp-adapter.d.ts +11 -0
  563. package/dist/utils/mcp-adapter.js +96 -0
  564. package/dist/utils/multi-gpu-mobile.d.ts +4 -0
  565. package/dist/utils/multi-gpu-mobile.js +10 -0
  566. package/dist/utils/object.d.ts +1 -0
  567. package/dist/utils/object.js +3 -0
  568. package/dist/utils/path-security.d.ts +22 -0
  569. package/dist/utils/path-security.js +69 -0
  570. package/dist/utils/qvac-paths.d.ts +1 -0
  571. package/dist/utils/qvac-paths.js +5 -0
  572. package/dist/utils/response-format.d.ts +2 -0
  573. package/dist/utils/response-format.js +20 -0
  574. package/dist/utils/shard-utils.d.ts +47 -0
  575. package/dist/utils/shard-utils.js +123 -0
  576. package/dist/utils/tool-helpers.d.ts +24 -0
  577. package/dist/utils/tool-helpers.js +111 -0
  578. package/dist/utils/tool-integration.d.ts +23 -0
  579. package/dist/utils/tool-integration.js +29 -0
  580. package/dist/utils/tools/dialect.d.ts +7 -0
  581. package/dist/utils/tools/dialect.js +20 -0
  582. package/dist/utils/tools/index.d.ts +2 -0
  583. package/dist/utils/tools/index.js +2 -0
  584. package/dist/utils/tools/parser.d.ts +5 -0
  585. package/dist/utils/tools/parser.js +69 -0
  586. package/dist/utils/tools/parsers/dsml.d.ts +3 -0
  587. package/dist/utils/tools/parsers/dsml.js +177 -0
  588. package/dist/utils/tools/parsers/gemma4native.d.ts +3 -0
  589. package/dist/utils/tools/parsers/gemma4native.js +65 -0
  590. package/dist/utils/tools/parsers/harmony.d.ts +3 -0
  591. package/dist/utils/tools/parsers/harmony.js +57 -0
  592. package/dist/utils/tools/parsers/hermes.d.ts +3 -0
  593. package/dist/utils/tools/parsers/hermes.js +122 -0
  594. package/dist/utils/tools/parsers/json.d.ts +5 -0
  595. package/dist/utils/tools/parsers/json.js +115 -0
  596. package/dist/utils/tools/parsers/pythonic.d.ts +3 -0
  597. package/dist/utils/tools/parsers/pythonic.js +267 -0
  598. package/dist/utils/tools/parsers/qwen35.d.ts +3 -0
  599. package/dist/utils/tools/parsers/qwen35.js +125 -0
  600. package/dist/utils/tools/shared.d.ts +17 -0
  601. package/dist/utils/tools/shared.js +52 -0
  602. package/dist/utils/transcribe-metadata.d.ts +20 -0
  603. package/dist/utils/transcribe-metadata.js +28 -0
  604. package/dist/utils/tts-stats.d.ts +25 -0
  605. package/dist/utils/tts-stats.js +16 -0
  606. package/dist/utils/withTimeout.d.ts +17 -0
  607. package/dist/utils/withTimeout.js +47 -0
  608. package/dist/utils/zod-error.d.ts +2 -0
  609. package/dist/utils/zod-error.js +4 -0
  610. package/package.json +266 -1
@@ -0,0 +1,45 @@
1
+ import type { AbortSignal } from 'bare-abort-controller';
2
+ import type { CompletionParams, CompletionStats, GenerationParams, ResponseFormat, Tool, ToolCall, ToolDialect } from '../../../../schemas/index.js';
3
+ import type { DisposableScope } from '../../../../runtime/disposable-scope.js';
4
+ import type { Logger } from '../../../../logging/types.js';
5
+ interface CompletionResult {
6
+ modelExecutionMs: number;
7
+ stats?: CompletionStats;
8
+ toolCalls: ToolCall[];
9
+ }
10
+ interface ChatHistory {
11
+ role?: string;
12
+ content?: string;
13
+ type?: string;
14
+ name?: string;
15
+ description?: string;
16
+ parameters?: unknown;
17
+ }
18
+ export type CompletionGenerationParams = GenerationParams & {
19
+ json_schema?: string;
20
+ };
21
+ export declare function transformMessages(messages: Array<{
22
+ role: string;
23
+ content: string;
24
+ attachments?: {
25
+ path: string;
26
+ }[] | undefined;
27
+ } | Tool>): ChatHistory[];
28
+ export declare function completion(params: CompletionParams & {
29
+ tools?: Tool[];
30
+ generationParams?: GenerationParams;
31
+ toolDialect?: ToolDialect;
32
+ responseFormat?: ResponseFormat;
33
+ }, opts: {
34
+ signal: AbortSignal;
35
+ scope: DisposableScope;
36
+ /**
37
+ * Request-scoped logger forwarded to `createKvCacheSession` so
38
+ * kv-cache lines share the request's lifecycle prefix. Falls
39
+ * back to the module-level server logger when omitted.
40
+ */
41
+ logger?: Logger;
42
+ }): AsyncGenerator<{
43
+ token: string;
44
+ }, CompletionResult, unknown>;
45
+ export {};
@@ -0,0 +1,374 @@
1
+ import { TOOLS_MODE } from '../../../../schemas/tools.js';
2
+ import { logCacheDisabled, logCacheInit, logCacheSave, logMessagesToAddon } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
3
+ import { extractSystemPrompt, getCurrentCacheInfo } from '../../../../plugins/ops/kv-cache-utils.js';
4
+ import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
5
+ import { decideCachedHistorySlice, shouldCommitCachedTurn } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-state.js';
6
+ import { createKvCacheSession, generateConfigHash } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
7
+ import { appendToolsToHistory, detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
8
+ import { parseToolCalls } from '../../../../utils/tools/index.js';
9
+ import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
10
+ import { buildAutoCacheSaveHistory } from '../../../../utils/index.js';
11
+ import { getEngineLogger } from '../../../../logging/index.js';
12
+ import { AttachmentNotFoundError } from '../../../../errors/index.js';
13
+ import { nowMs } from '../../../../profiling/index.js';
14
+ import { buildStreamResult } from '../../../../profiling/model-execution.js';
15
+ import { normalizeCompletionStats, withEmittedTokens } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
16
+ import fs from 'bare-fs';
17
+ const logger = getEngineLogger();
18
+ function transformMessage(message) {
19
+ const transformed = [];
20
+ // Check if it's a tool definition (has type: "function")
21
+ if ('type' in message && message.type === 'function') {
22
+ transformed.push({
23
+ type: 'function',
24
+ name: message.name,
25
+ description: message.description,
26
+ parameters: message.parameters
27
+ });
28
+ return transformed;
29
+ }
30
+ const msg = message;
31
+ if (msg.attachments && msg.attachments.length > 0) {
32
+ for (const attachment of msg.attachments) {
33
+ if (!fs.existsSync(attachment.path)) {
34
+ throw new AttachmentNotFoundError(attachment.path);
35
+ }
36
+ transformed.push({
37
+ role: msg.role,
38
+ content: attachment.path,
39
+ type: 'media'
40
+ });
41
+ }
42
+ }
43
+ transformed.push({
44
+ role: msg.role,
45
+ content: msg.content
46
+ });
47
+ return transformed;
48
+ }
49
+ function runModel(model, prompt, opts) {
50
+ return model.run(prompt, opts);
51
+ }
52
+ export function transformMessages(messages) {
53
+ const transformed = [];
54
+ for (const message of messages) {
55
+ transformed.push(...transformMessage(message));
56
+ }
57
+ return transformed;
58
+ }
59
+ async function initSystemPromptCache(model, cachePathToUse, systemPromptToUse, cacheKey, tools) {
60
+ const primeMessages = [{ role: 'system', content: systemPromptToUse }];
61
+ let toolCount = 0;
62
+ if (tools && tools.length > 0) {
63
+ const transformedTools = transformMessages(tools);
64
+ primeMessages.push(...transformedTools);
65
+ toolCount = tools.length;
66
+ }
67
+ logCacheInit(cacheKey, systemPromptToUse, toolCount);
68
+ logMessagesToAddon(primeMessages, 'CACHE_INIT');
69
+ const primeResponse = await runModel(model, primeMessages, {
70
+ cacheKey: cachePathToUse,
71
+ saveCacheToDisk: true,
72
+ prefill: true
73
+ });
74
+ await primeResponse.await();
75
+ }
76
+ /**
77
+ * Pick the messages that need to reach the model for the next turn.
78
+ *
79
+ * Static mode (no `tools` argument):
80
+ * - Cache miss: send the whole history minus the system message (which
81
+ * was primed during cache init).
82
+ * - Cache hit with a recorded `savedCount`: send only the unsaved tail
83
+ * (`history.slice(savedCount)`), so a multi-message turn (e.g. a
84
+ * consumer pushing both an assistant transcript and a follow-up user
85
+ * message between completions) all reaches the model.
86
+ * - Cache hit with a stale/missing `savedCount`: fall back to the full
87
+ * non-system history. The session is told (`dropStaleSavedCount`) so
88
+ * the bad boundary doesn't propagate into the next turn.
89
+ *
90
+ * Dynamic mode (`tools` argument set):
91
+ * - The addon anchors the tool block after the last user message and
92
+ * trims tools + the assistant's tool-call output from the cache once
93
+ * the chain resolves. After that trim, the cache only holds messages
94
+ * up to the last user turn, so we ship the right slice
95
+ * plus the (possibly new) tool set:
96
+ * * tool-chain continuation (last role is "tool"): send the trailing
97
+ * consecutive tool messages, no tool block — tools are still
98
+ * anchored in the cache from the previous round.
99
+ * * new user turn after a chain (prev role is "assistant"): send
100
+ * [assistant, user] so the model sees its own final reply before
101
+ * the new prompt, then re-anchor the tool block.
102
+ * * otherwise: send just the last message + tool block.
103
+ */
104
+ function prepareMessagesForCache(session, turn, cacheExists, history, tools) {
105
+ const addTools = tools?.length ? transformMessages(tools) : [];
106
+ const dynamic = addTools.length > 0;
107
+ if (!(cacheExists && history.length > 0)) {
108
+ const historyWithoutSystem = history.filter((msg) => msg.role !== 'system');
109
+ return [...transformMessages(historyWithoutSystem), ...addTools];
110
+ }
111
+ if (!dynamic) {
112
+ // Static path — slice from the turn's `savedCount` so callers can
113
+ // stage multiple messages between completions. `decideCachedHistorySlice`
114
+ // also guards against the QVAC-17780 stale-count regression: if the
115
+ // saved boundary would slice the history down to an empty payload
116
+ // (e.g. after a cancelled mid-decode), it falls back to the full
117
+ // non-system history and signals the caller to drop the bad entry.
118
+ // The session owns the entry; `dropStaleSavedCount` clears it
119
+ // without touching the on-disk file (the file is still trustworthy
120
+ // — only the boundary count is wrong).
121
+ const { messages, clearStaleCount } = decideCachedHistorySlice(turn.savedCount, cacheExists, history);
122
+ if (clearStaleCount) {
123
+ session.dropStaleSavedCount(turn);
124
+ }
125
+ return transformMessages(messages);
126
+ }
127
+ // Dynamic path. The addon trimmed tools after the previous round, so the
128
+ // cache no longer holds the saved-count we'd rely on for slicing — pick
129
+ // the right fragment based on the role of the last history message.
130
+ const lastMsg = history[history.length - 1];
131
+ if (lastMsg.role === 'tool') {
132
+ const trailingTools = [];
133
+ for (let i = history.length - 1; i >= 0; i--) {
134
+ const msg = history[i];
135
+ if (msg.role !== 'tool')
136
+ break;
137
+ trailingTools.unshift(msg);
138
+ }
139
+ return transformMessages(trailingTools);
140
+ }
141
+ if (lastMsg.role === 'user') {
142
+ const prevMsg = history[history.length - 2];
143
+ const tail = prevMsg?.role === 'assistant' ? [prevMsg, lastMsg] : [lastMsg];
144
+ return [...transformMessages(tail), ...addTools];
145
+ }
146
+ return [...transformMessages([lastMsg]), ...addTools];
147
+ }
148
+ async function* processModelResponse(model, messagesToSend, tools, generationParams, cacheOptions, dialect) {
149
+ const runOptions = {
150
+ ...(generationParams && { generationParams }),
151
+ ...(cacheOptions?.cacheKey !== undefined && {
152
+ cacheKey: cacheOptions.cacheKey
153
+ }),
154
+ ...(cacheOptions?.saveCacheToDisk !== undefined && {
155
+ saveCacheToDisk: cacheOptions.saveCacheToDisk
156
+ })
157
+ };
158
+ const hasRunOptions = Object.keys(runOptions).length > 0;
159
+ const modelStart = nowMs();
160
+ const response = await runModel(model, messagesToSend, hasRunOptions ? runOptions : undefined);
161
+ let accumulatedText = '';
162
+ let producedTokens = false;
163
+ let emittedPieces = 0;
164
+ let toolCallsResult = [];
165
+ for await (const token of response.iterate()) {
166
+ const tokenStr = token;
167
+ if (tokenStr.length > 0) {
168
+ producedTokens = true;
169
+ emittedPieces++;
170
+ }
171
+ accumulatedText += tokenStr;
172
+ yield { token: tokenStr };
173
+ }
174
+ const modelExecutionMs = nowMs() - modelStart;
175
+ if (cacheOptions?.saveCacheToDisk && cacheOptions.cacheKey) {
176
+ logCacheSave(cacheOptions.cacheKey);
177
+ }
178
+ if (tools && tools.length > 0) {
179
+ const { toolCalls } = parseToolCalls(accumulatedText, tools, dialect);
180
+ toolCallsResult = toolCalls;
181
+ }
182
+ const responseWithStats = response;
183
+ const stats = withEmittedTokens(normalizeCompletionStats(responseWithStats.stats), emittedPieces);
184
+ return {
185
+ ...buildStreamResult(modelExecutionMs, stats),
186
+ toolCalls: toolCallsResult,
187
+ responseText: accumulatedText,
188
+ producedTokens
189
+ };
190
+ }
191
+ export async function* completion(params, opts) {
192
+ const { history, modelId, kvCache, tools, generationParams, responseFormat } = params;
193
+ const { signal, scope } = opts;
194
+ const requestLogger = opts.logger ?? logger;
195
+ const modelConfig = getModelConfig(modelId);
196
+ const toolsEnabled = modelConfig.tools === true;
197
+ const toolsMode = modelConfig.toolsMode;
198
+ const dynamicTools = !!tools?.length && toolsEnabled && toolsMode === TOOLS_MODE.dynamic;
199
+ const staticTools = !!tools?.length && toolsEnabled && !dynamicTools;
200
+ const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
201
+ // `responseFormat` is forwarded to the addon as a per-request
202
+ // `generationParams.json_schema`, which the addon converts to GBNF and
203
+ // applies for the duration of the request only. This avoids mutating
204
+ // the shared `modelConfig` and is therefore safe under concurrent
205
+ // completions on the same model. `tools` still constrain output through
206
+ // their parameter schema and the dialect-specific parser chain (mutually
207
+ // exclusive with a non-text `responseFormat` at the schema layer).
208
+ let mergedGenerationParams = generationParams;
209
+ if (responseFormat && !(tools && tools.length > 0)) {
210
+ const jsonSchema = getResponseFormatJsonSchema(responseFormat);
211
+ if (jsonSchema !== undefined) {
212
+ mergedGenerationParams = {
213
+ ...(generationParams ?? {}),
214
+ json_schema: jsonSchema
215
+ };
216
+ }
217
+ }
218
+ const model = getModel(modelId);
219
+ // Hard-cancel wiring: when the registry aborts the request's signal,
220
+ // forward to the addon so the C++ work stops as soon as it can. The
221
+ // we still treat `signal.aborted` as the truth for cancel detection
222
+ // (post-completion bookkeeping below) — this listener only shortens
223
+ // the latency between "user clicked stop" and "addon stops decoding".
224
+ //
225
+ // Fire-and-forget by construction (event listeners can't `await`), but
226
+ // `addon.cancel()` returns a Promise — if it ever rejects the bare
227
+ // `void` would leak it as an unhandledRejection. Attach `.catch(...)`
228
+ // so a rejection is logged and the process stays clean; the iterator
229
+ // below still sees EOF/empty tokens via the addon's normal cancel path
230
+ // so callers aren't affected.
231
+ const onAbort = () => {
232
+ const addon = model.addon;
233
+ if (addon?.cancel) {
234
+ addon.cancel.call(addon).catch((err) => {
235
+ requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
236
+ });
237
+ }
238
+ };
239
+ signal.addEventListener('abort', onAbort, { once: true });
240
+ // `addEventListener("abort", ..., { once: true })` does *not* fire if
241
+ // the signal is already aborted at register time — but the registry
242
+ // synchronously aborts a fresh controller when `parentSignal` was
243
+ // already aborted at `begin(...)`. Without this fall-through, the
244
+ // addon would keep decoding until the post-loop check notices.
245
+ // Re-using `onAbort` here keeps the listener body as the single
246
+ // source of truth for "what cancel does."
247
+ if (signal.aborted)
248
+ onAbort();
249
+ // Detach the abort listener on every exit path (happy, throw, generator
250
+ // `return()` from upstream). `{ once: true }` already removes the
251
+ // listener if the signal fires, so the `removeEventListener` here is
252
+ // the cleanup hook for the signal-never-fired path.
253
+ scope.defer(() => {
254
+ signal.removeEventListener('abort', onAbort);
255
+ });
256
+ if (!kvCache) {
257
+ // KV-cache disabled — straight passthrough, no session involvement.
258
+ let historyWithTools = history;
259
+ if (staticTools && tools) {
260
+ historyWithTools = prependToolsToHistory(history, tools);
261
+ }
262
+ else if (dynamicTools && tools) {
263
+ historyWithTools = appendToolsToHistory(history, tools);
264
+ }
265
+ const transformedHistory = transformMessages(historyWithTools);
266
+ logCacheDisabled();
267
+ logMessagesToAddon(transformedHistory, 'NO_CACHE');
268
+ return yield* processModelResponse(model, transformedHistory, tools, mergedGenerationParams, undefined, dialect);
269
+ }
270
+ // ---- KV-cache path. The session owns all three bookkeeping layers
271
+ // (on-disk `.bin`, `initializedCaches`, `cachedMessageCounts`). The
272
+ // handler asks for a turn, registers rollback on the scope, and on
273
+ // the happy path calls `commitTurn` which short-circuits the deferred
274
+ // rollback. Cancellations / zero-token replies / rename failures all
275
+ // unwind through the same `scope.defer` hook. ----
276
+ const session = createKvCacheSession(modelId, { logger: requestLogger });
277
+ const systemPromptFromHistory = extractSystemPrompt(history);
278
+ // Dynamic mode lets each turn carry its own tool set, so the cache
279
+ // hash must not depend on the tool list — otherwise a tool change
280
+ // would force a fresh cache file and defeat the whole optimisation.
281
+ const configHash = generateConfigHash(systemPromptFromHistory, dynamicTools ? undefined : tools);
282
+ const systemPromptToUse = systemPromptFromHistory ||
283
+ modelConfig.system_prompt ||
284
+ 'You are a helpful assistant.';
285
+ const primeIfMissing = async (cachePath) => {
286
+ await initSystemPromptCache(model, cachePath, systemPromptToUse, typeof kvCache === 'string' ? kvCache : 'auto',
287
+ // Static-mode tools are baked into the system-prompt cache so
288
+ // they're shared across the session. Dynamic-mode tools belong
289
+ // to a per-turn anchor and must not enter the system cache.
290
+ staticTools ? tools : undefined);
291
+ };
292
+ let turn;
293
+ if (typeof kvCache === 'string') {
294
+ turn = await session.beginTurn({
295
+ kind: 'custom',
296
+ customKey: kvCache,
297
+ configHash,
298
+ primeIfMissing
299
+ });
300
+ }
301
+ else {
302
+ const cacheMessages = history.map((msg) => ({
303
+ role: msg.role,
304
+ content: msg.content,
305
+ attachments: msg.attachments ?? undefined
306
+ }));
307
+ turn = await session.beginTurn({
308
+ kind: 'auto',
309
+ configHash,
310
+ history: cacheMessages,
311
+ primeIfMissing
312
+ });
313
+ }
314
+ // Single cleanup hook for every non-success exit path. `commitTurn`
315
+ // flips the turn's internal `committed` flag so this becomes a no-op
316
+ // on the happy path. Scope unwinding is LIFO — registered after the
317
+ // `removeEventListener` defer above so rollback runs before the
318
+ // listener detach.
319
+ scope.defer(() => session.rollback(turn));
320
+ // `cacheExists` is implied by `beginTurn` — the session either found
321
+ // an existing cache or just primed one. Pass `true` to the message
322
+ // selector so the slicing branches engage.
323
+ const messagesToSend = prepareMessagesForCache(session, turn,
324
+ /* cacheExists */ true, history, dynamicTools ? tools : undefined);
325
+ logMessagesToAddon(messagesToSend, 'PROMPT_SEND');
326
+ const result = yield* processModelResponse(model, messagesToSend, tools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect);
327
+ const shouldCommitTurn = shouldCommitCachedTurn({
328
+ aborted: signal.aborted,
329
+ producedTokens: result.producedTokens,
330
+ generatedTokens: result.stats?.generatedTokens,
331
+ predict: mergedGenerationParams?.predict ?? modelConfig.predict
332
+ });
333
+ if (typeof kvCache === 'string') {
334
+ // Custom-key path: the addon wrote the new cache state inline at
335
+ // the same path. Either commit (records the boundary, suppresses
336
+ // rollback) or fall through to the deferred rollback.
337
+ if (shouldCommitTurn) {
338
+ await session.commitTurn(turn, {
339
+ kind: 'static',
340
+ messageCount: history.length + 1
341
+ });
342
+ }
343
+ return result;
344
+ }
345
+ // Auto-cache path.
346
+ //
347
+ // Tool-call turns: the auto-cache key is derived from
348
+ // `result.responseText`, which here is raw tool-call markup rather
349
+ // than a clean assistant message. There's no safe post-response key
350
+ // to rename to, so we let the deferred rollback drop the file. Once
351
+ // we support auto-cache for structured assistant/tool turns,
352
+ // this becomes a normal commit path.
353
+ if (result.toolCalls.length > 0) {
354
+ logger.warn(`[kv-cache] Auto cache tool-call turn; rolling back to avoid disk leak. path=${turn.cachePath}`);
355
+ return result;
356
+ }
357
+ if (!shouldCommitTurn) {
358
+ // Cancelled, zero-token, or budget-exhausted turns do not establish
359
+ // a trustworthy message boundary.
360
+ return result;
361
+ }
362
+ const savedHistory = buildAutoCacheSaveHistory(history.map((msg) => ({
363
+ role: msg.role,
364
+ content: msg.content,
365
+ attachments: msg.attachments ?? undefined
366
+ })), result.responseText);
367
+ const postResponseCacheInfo = await getCurrentCacheInfo(modelId, configHash, savedHistory);
368
+ await session.commitTurn(turn, {
369
+ kind: 'autoRename',
370
+ targetCachePath: postResponseCacheInfo.cachePath,
371
+ messageCount: savedHistory.length
372
+ });
373
+ return result;
374
+ }
@@ -0,0 +1,28 @@
1
+ /**
2
+ * Helpers for detecting and parsing the llama.cpp addon's
3
+ * `ContextOverflow` status error (`LlmErrors.hpp::ContextOverflow = 14`).
4
+ *
5
+ * The addon's JS-facing throw goes through `js_throw_error(env, code,
6
+ * msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
7
+ * `qvac_errors::StatusError::codeString()` and formats as
8
+ * `"[ <addonId> :: ContextOverflow ]"`. The message carries the
9
+ * C++-formatted detail (which may include the prompt/ctx sizes).
10
+ *
11
+ * These helpers let the plugin handler convert that addon error into a
12
+ * typed `ContextOverflowError` and let unit tests assert the detection
13
+ * + extraction logic without a real model load.
14
+ */
15
+ export declare function isAddonContextOverflowError(err: unknown): boolean;
16
+ /**
17
+ * Best-effort extraction of `promptTokens` / `ctxSize` from the addon
18
+ * error message. The C++ paths in `TextLlmContext.cpp` and
19
+ * `MtmdLlmContext.cpp` format both numbers into the message; the
20
+ * `LlamaModel::processPromptImpl` fallback path emits a bare
21
+ * `"<func>: context overflow\n"` with neither number. Returning
22
+ * `undefined` for either field is fine — `ContextOverflowError` holds
23
+ * them as optional and the message factory degrades gracefully.
24
+ */
25
+ export declare function parseContextOverflowMessage(message: string): {
26
+ promptTokens?: number;
27
+ ctxSize?: number;
28
+ };
@@ -0,0 +1,65 @@
1
+ /**
2
+ * Helpers for detecting and parsing the llama.cpp addon's
3
+ * `ContextOverflow` status error (`LlmErrors.hpp::ContextOverflow = 14`).
4
+ *
5
+ * The addon's JS-facing throw goes through `js_throw_error(env, code,
6
+ * msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
7
+ * `qvac_errors::StatusError::codeString()` and formats as
8
+ * `"[ <addonId> :: ContextOverflow ]"`. The message carries the
9
+ * C++-formatted detail (which may include the prompt/ctx sizes).
10
+ *
11
+ * These helpers let the plugin handler convert that addon error into a
12
+ * typed `ContextOverflowError` and let unit tests assert the detection
13
+ * + extraction logic without a real model load.
14
+ */
15
+ export function isAddonContextOverflowError(err) {
16
+ if (typeof err !== 'object' || err === null)
17
+ return false;
18
+ const code = err.code;
19
+ // Anchor to the codeString tail (`[ <addonId> :: ContextOverflow ]`)
20
+ // so we don't false-positive on a hypothetical sibling like
21
+ // `ContextOverflowRecovered` or `PostContextOverflow` after an
22
+ // addon-side rename.
23
+ if (typeof code === 'string' && /::\s*ContextOverflow\s*\]/.test(code)) {
24
+ return true;
25
+ }
26
+ // Message-substring fallback for the bare `LlamaModel::processPromptImpl`
27
+ // path, which doesn't go through `StatusError::codeString()`. Match the
28
+ // two known C++-emitted formats only — broader substring would catch
29
+ // wrapper / cause-chain text that mentions overflow without being one.
30
+ const message = err.message;
31
+ return (typeof message === 'string' &&
32
+ /(?:context overflow at prefill step|processPromptImpl: context overflow)/i.test(message));
33
+ }
34
+ /**
35
+ * Best-effort extraction of `promptTokens` / `ctxSize` from the addon
36
+ * error message. The C++ paths in `TextLlmContext.cpp` and
37
+ * `MtmdLlmContext.cpp` format both numbers into the message; the
38
+ * `LlamaModel::processPromptImpl` fallback path emits a bare
39
+ * `"<func>: context overflow\n"` with neither number. Returning
40
+ * `undefined` for either field is fine — `ContextOverflowError` holds
41
+ * them as optional and the message factory degrades gracefully.
42
+ */
43
+ export function parseContextOverflowMessage(message) {
44
+ // "[TextLlm] context overflow at prefill step: prompt tokens N, max context tokens M\n"
45
+ // Same-clause separator (no `[^]*?` cross-newline walk) so a future
46
+ // wrapper that pastes overflow text alongside unrelated numbers can't
47
+ // produce a mismatched pair.
48
+ const longForm = message.match(/prompt tokens (\d+)[,\s]+max context tokens (\d+)/i);
49
+ if (longForm?.[1] && longForm[2]) {
50
+ return {
51
+ promptTokens: Number(longForm[1]),
52
+ ctxSize: Number(longForm[2])
53
+ };
54
+ }
55
+ // "[TextLlm] context overflow at prefill step (N tokens, max M)\n"
56
+ // "[MtmdLlm] context overflow at prefill step (N tokens, max M)\n"
57
+ const shortForm = message.match(/\((\d+)\s+tokens,\s*max\s+(\d+)\)/i);
58
+ if (shortForm?.[1] && shortForm[2]) {
59
+ return {
60
+ promptTokens: Number(shortForm[1]),
61
+ ctxSize: Number(shortForm[2])
62
+ };
63
+ }
64
+ return {};
65
+ }
@@ -0,0 +1,10 @@
1
+ import type { FinetuneRunParams, FinetuneRunRequest, FinetuneProgress, FinetuneRequest, FinetuneResult, FinetuneStatus, FinetuneGetStateRequest } from '../../../../schemas/index.js';
2
+ type FinetuneOptions = FinetuneRunParams['options'];
3
+ export declare function clearFinetuneRuntimeState(modelId: string): void;
4
+ export declare function getFinetuneStateFromCheckpoints(options: FinetuneOptions): FinetuneStatus;
5
+ export declare function startFinetune(request: FinetuneRunRequest, onProgress?: (progress: FinetuneProgress) => void): Promise<FinetuneResult>;
6
+ export declare function pauseFinetune(modelId: string): Promise<FinetuneResult>;
7
+ export declare function cancelFinetune(modelId: string): Promise<FinetuneResult>;
8
+ export declare function getFinetuneState(params: FinetuneGetStateRequest): FinetuneResult;
9
+ export declare function finetune(request: FinetuneRequest, onProgress?: (progress: FinetuneProgress) => void): Promise<FinetuneResult>;
10
+ export {};