@qvac/inference 0.0.0 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/LICENSE +179 -0
  2. package/NOTICE +928 -0
  3. package/README.md +138 -0
  4. package/dist/api/audio-gen.d.ts +18 -0
  5. package/dist/api/audio-gen.js +148 -0
  6. package/dist/api/batch-completion.d.ts +14 -0
  7. package/dist/api/batch-completion.js +332 -0
  8. package/dist/api/bci-transcribe.d.ts +56 -0
  9. package/dist/api/bci-transcribe.js +172 -0
  10. package/dist/api/cancel.d.ts +34 -0
  11. package/dist/api/cancel.js +82 -0
  12. package/dist/api/classify.d.ts +22 -0
  13. package/dist/api/classify.js +46 -0
  14. package/dist/api/completion-stream.d.ts +106 -0
  15. package/dist/api/completion-stream.js +351 -0
  16. package/dist/api/delete-cache.d.ts +29 -0
  17. package/dist/api/delete-cache.js +49 -0
  18. package/dist/api/diffusion.d.ts +92 -0
  19. package/dist/api/diffusion.js +176 -0
  20. package/dist/api/download-asset.d.ts +49 -0
  21. package/dist/api/download-asset.js +87 -0
  22. package/dist/api/embed.d.ts +41 -0
  23. package/dist/api/embed.js +29 -0
  24. package/dist/api/finetune.d.ts +89 -0
  25. package/dist/api/finetune.js +138 -0
  26. package/dist/api/get-loaded-model-info.d.ts +24 -0
  27. package/dist/api/get-loaded-model-info.js +34 -0
  28. package/dist/api/get-model-info.d.ts +43 -0
  29. package/dist/api/get-model-info.js +21 -0
  30. package/dist/api/get-system-resources.d.ts +2 -0
  31. package/dist/api/get-system-resources.js +16 -0
  32. package/dist/api/heartbeat.d.ts +29 -0
  33. package/dist/api/heartbeat.js +37 -0
  34. package/dist/api/index.d.ts +35 -0
  35. package/dist/api/index.js +35 -0
  36. package/dist/api/invoke-plugin.d.ts +30 -0
  37. package/dist/api/invoke-plugin.js +54 -0
  38. package/dist/api/load-model.d.ts +169 -0
  39. package/dist/api/load-model.js +129 -0
  40. package/dist/api/logging-stream-registry.d.ts +4 -0
  41. package/dist/api/logging-stream-registry.js +61 -0
  42. package/dist/api/logging-stream.d.ts +23 -0
  43. package/dist/api/logging-stream.js +36 -0
  44. package/dist/api/ocr.d.ts +30 -0
  45. package/dist/api/ocr.js +88 -0
  46. package/dist/api/parse-input.d.ts +2 -0
  47. package/dist/api/parse-input.js +14 -0
  48. package/dist/api/provide.d.ts +23 -0
  49. package/dist/api/provide.js +32 -0
  50. package/dist/api/rag.d.ts +251 -0
  51. package/dist/api/rag.js +485 -0
  52. package/dist/api/registry.d.ts +39 -0
  53. package/dist/api/registry.js +63 -0
  54. package/dist/api/resume.d.ts +29 -0
  55. package/dist/api/resume.js +36 -0
  56. package/dist/api/state.d.ts +22 -0
  57. package/dist/api/state.js +29 -0
  58. package/dist/api/stop-provider.d.ts +20 -0
  59. package/dist/api/stop-provider.js +30 -0
  60. package/dist/api/subscribe-logs.d.ts +27 -0
  61. package/dist/api/subscribe-logs.js +43 -0
  62. package/dist/api/suspend.d.ts +30 -0
  63. package/dist/api/suspend.js +37 -0
  64. package/dist/api/text-to-speech.d.ts +64 -0
  65. package/dist/api/text-to-speech.js +434 -0
  66. package/dist/api/transcribe.d.ts +75 -0
  67. package/dist/api/transcribe.js +256 -0
  68. package/dist/api/translate.d.ts +47 -0
  69. package/dist/api/translate.js +115 -0
  70. package/dist/api/unload-model.d.ts +17 -0
  71. package/dist/api/unload-model.js +42 -0
  72. package/dist/api/upscale.d.ts +44 -0
  73. package/dist/api/upscale.js +99 -0
  74. package/dist/api/video.d.ts +86 -0
  75. package/dist/api/video.js +176 -0
  76. package/dist/api/vla-helpers.d.ts +43 -0
  77. package/dist/api/vla-helpers.js +124 -0
  78. package/dist/api/vla.d.ts +68 -0
  79. package/dist/api/vla.js +113 -0
  80. package/dist/config/config-utils.d.ts +4 -0
  81. package/dist/config/config-utils.js +18 -0
  82. package/dist/config/resolve-config.d.ts +8 -0
  83. package/dist/config/resolve-config.js +74 -0
  84. package/dist/constants/audio.d.ts +1 -0
  85. package/dist/constants/audio.js +1 -0
  86. package/dist/constants/index.d.ts +2 -0
  87. package/dist/constants/index.js +2 -0
  88. package/dist/constants/registry.d.ts +1 -0
  89. package/dist/constants/registry.js +1 -0
  90. package/dist/dispatch.d.ts +18 -0
  91. package/dist/dispatch.js +271 -0
  92. package/dist/engine.d.ts +5 -0
  93. package/dist/engine.js +27 -0
  94. package/dist/errors/index.d.ts +416 -0
  95. package/dist/errors/index.js +681 -0
  96. package/dist/errors/options.d.ts +2 -0
  97. package/dist/errors/options.js +13 -0
  98. package/dist/handlers/cancelHandler.d.ts +29 -0
  99. package/dist/handlers/cancelHandler.js +67 -0
  100. package/dist/handlers/completion-orchestrate.d.ts +41 -0
  101. package/dist/handlers/completion-orchestrate.js +190 -0
  102. package/dist/handlers/delete-cache.d.ts +11 -0
  103. package/dist/handlers/delete-cache.js +37 -0
  104. package/dist/handlers/download-asset.d.ts +2 -0
  105. package/dist/handlers/download-asset.js +146 -0
  106. package/dist/handlers/finetune.d.ts +2 -0
  107. package/dist/handlers/finetune.js +18 -0
  108. package/dist/handlers/get-loaded-model-info.d.ts +2 -0
  109. package/dist/handlers/get-loaded-model-info.js +45 -0
  110. package/dist/handlers/get-model-info.d.ts +2 -0
  111. package/dist/handlers/get-model-info.js +173 -0
  112. package/dist/handlers/get-system-resources.d.ts +2 -0
  113. package/dist/handlers/get-system-resources.js +53 -0
  114. package/dist/handlers/heartbeat.d.ts +2 -0
  115. package/dist/handlers/heartbeat.js +3 -0
  116. package/dist/handlers/load-model/download-manager.d.ts +95 -0
  117. package/dist/handlers/load-model/download-manager.js +306 -0
  118. package/dist/handlers/load-model/download-stats.d.ts +7 -0
  119. package/dist/handlers/load-model/download-stats.js +126 -0
  120. package/dist/handlers/load-model/handler.d.ts +2 -0
  121. package/dist/handlers/load-model/handler.js +287 -0
  122. package/dist/handlers/load-model/http.d.ts +3 -0
  123. package/dist/handlers/load-model/http.js +857 -0
  124. package/dist/handlers/load-model/hyperdrive.d.ts +4 -0
  125. package/dist/handlers/load-model/hyperdrive.js +560 -0
  126. package/dist/handlers/load-model/index.d.ts +2 -0
  127. package/dist/handlers/load-model/index.js +2 -0
  128. package/dist/handlers/load-model/registry-client-options.d.ts +27 -0
  129. package/dist/handlers/load-model/registry-client-options.js +18 -0
  130. package/dist/handlers/load-model/registry-companion-set.d.ts +22 -0
  131. package/dist/handlers/load-model/registry-companion-set.js +206 -0
  132. package/dist/handlers/load-model/registry-download-utils.d.ts +23 -0
  133. package/dist/handlers/load-model/registry-download-utils.js +112 -0
  134. package/dist/handlers/load-model/registry-metadata.d.ts +10 -0
  135. package/dist/handlers/load-model/registry-metadata.js +23 -0
  136. package/dist/handlers/load-model/registry.d.ts +4 -0
  137. package/dist/handlers/load-model/registry.js +298 -0
  138. package/dist/handlers/load-model/resolve-session.d.ts +33 -0
  139. package/dist/handlers/load-model/resolve-session.js +62 -0
  140. package/dist/handlers/load-model/resolve.d.ts +5 -0
  141. package/dist/handlers/load-model/resolve.js +158 -0
  142. package/dist/handlers/load-model/types.d.ts +59 -0
  143. package/dist/handlers/load-model/types.js +28 -0
  144. package/dist/handlers/logging-stream.d.ts +2 -0
  145. package/dist/handlers/logging-stream.js +39 -0
  146. package/dist/handlers/plugin-dispatch.d.ts +3 -0
  147. package/dist/handlers/plugin-dispatch.js +63 -0
  148. package/dist/handlers/plugin-invoke.d.ts +3 -0
  149. package/dist/handlers/plugin-invoke.js +81 -0
  150. package/dist/handlers/registry.d.ts +4 -0
  151. package/dist/handlers/registry.js +156 -0
  152. package/dist/handlers/resume.d.ts +2 -0
  153. package/dist/handlers/resume.js +11 -0
  154. package/dist/handlers/state.d.ts +2 -0
  155. package/dist/handlers/state.js +7 -0
  156. package/dist/handlers/suspend.d.ts +2 -0
  157. package/dist/handlers/suspend.js +11 -0
  158. package/dist/handlers/types.d.ts +13 -0
  159. package/dist/handlers/types.js +1 -0
  160. package/dist/handlers/unload-model.d.ts +2 -0
  161. package/dist/handlers/unload-model.js +29 -0
  162. package/dist/index.d.ts +4 -0
  163. package/dist/index.js +12 -0
  164. package/dist/logging/addon.d.ts +5 -0
  165. package/dist/logging/addon.js +75 -0
  166. package/dist/logging/index.d.ts +5 -0
  167. package/dist/logging/index.js +4 -0
  168. package/dist/logging/logger.d.ts +41 -0
  169. package/dist/logging/logger.js +209 -0
  170. package/dist/logging/namespaces.d.ts +6 -0
  171. package/dist/logging/namespaces.js +7 -0
  172. package/dist/logging/types.d.ts +37 -0
  173. package/dist/logging/types.js +1 -0
  174. package/dist/logging/utils.d.ts +5 -0
  175. package/dist/logging/utils.js +113 -0
  176. package/dist/models/registry/index.d.ts +18 -0
  177. package/dist/models/registry/index.js +21 -0
  178. package/dist/models/registry/models.d.ts +22680 -0
  179. package/dist/models/registry/models.js +23908 -0
  180. package/dist/p2p/cancel-delegated.d.ts +3 -0
  181. package/dist/p2p/cancel-delegated.js +64 -0
  182. package/dist/p2p/completion-stream-delegated.d.ts +4 -0
  183. package/dist/p2p/completion-stream-delegated.js +51 -0
  184. package/dist/p2p/delegate-client.d.ts +9 -0
  185. package/dist/p2p/delegate-client.js +219 -0
  186. package/dist/p2p/delegate-connect-diagnostics.d.ts +8 -0
  187. package/dist/p2p/delegate-connect-diagnostics.js +38 -0
  188. package/dist/p2p/delegate-transport.d.ts +18 -0
  189. package/dist/p2p/delegate-transport.js +237 -0
  190. package/dist/p2p/error-handlers.d.ts +4 -0
  191. package/dist/p2p/error-handlers.js +31 -0
  192. package/dist/p2p/handle-request.d.ts +2 -0
  193. package/dist/p2p/handle-request.js +171 -0
  194. package/dist/p2p/handler-utils.d.ts +19 -0
  195. package/dist/p2p/handler-utils.js +180 -0
  196. package/dist/p2p/heartbeat-delegated.d.ts +3 -0
  197. package/dist/p2p/heartbeat-delegated.js +32 -0
  198. package/dist/p2p/load-model-delegated.d.ts +6 -0
  199. package/dist/p2p/load-model-delegated.js +132 -0
  200. package/dist/p2p/progress-throttle.d.ts +7 -0
  201. package/dist/p2p/progress-throttle.js +47 -0
  202. package/dist/p2p/provideHandler/connection.d.ts +2 -0
  203. package/dist/p2p/provideHandler/connection.js +38 -0
  204. package/dist/p2p/provideHandler/handler.d.ts +2 -0
  205. package/dist/p2p/provideHandler/handler.js +66 -0
  206. package/dist/p2p/provideHandler/proxy.d.ts +2 -0
  207. package/dist/p2p/provideHandler/proxy.js +24 -0
  208. package/dist/p2p/provideHandler.d.ts +1 -0
  209. package/dist/p2p/provideHandler.js +2 -0
  210. package/dist/p2p/rpc-utils.d.ts +4 -0
  211. package/dist/p2p/rpc-utils.js +8 -0
  212. package/dist/p2p/stopProvideHandler.d.ts +2 -0
  213. package/dist/p2p/stopProvideHandler.js +29 -0
  214. package/dist/p2p/swarm.d.ts +9 -0
  215. package/dist/p2p/swarm.js +103 -0
  216. package/dist/p2p/transport-selector.d.ts +2 -0
  217. package/dist/p2p/transport-selector.js +18 -0
  218. package/dist/p2p/unload-model-delegated.d.ts +2 -0
  219. package/dist/p2p/unload-model-delegated.js +31 -0
  220. package/dist/plugins/builtin/asr-ggml/config.d.ts +31 -0
  221. package/dist/plugins/builtin/asr-ggml/config.js +46 -0
  222. package/dist/plugins/builtin/asr-ggml/logging.d.ts +10 -0
  223. package/dist/plugins/builtin/asr-ggml/logging.js +12 -0
  224. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +17 -0
  225. package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
  226. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +2 -0
  227. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +163 -0
  228. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +212 -0
  229. package/dist/plugins/builtin/audiogen-ggml/plugin.js +58 -0
  230. package/dist/plugins/builtin/bci-whispercpp-transcription/args.d.ts +2 -0
  231. package/dist/plugins/builtin/bci-whispercpp-transcription/args.js +13 -0
  232. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +173 -0
  233. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +111 -0
  234. package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.d.ts +74 -0
  235. package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.js +11 -0
  236. package/dist/plugins/builtin/ggml-classification/ops/classify.d.ts +5 -0
  237. package/dist/plugins/builtin/ggml-classification/ops/classify.js +31 -0
  238. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +31 -0
  239. package/dist/plugins/builtin/ggml-classification/plugin.js +55 -0
  240. package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.d.ts +7 -0
  241. package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.js +119 -0
  242. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +102 -0
  243. package/dist/plugins/builtin/ggml-ocr/plugin.js +110 -0
  244. package/dist/plugins/builtin/ggml-ocr/resolve-config.d.ts +5 -0
  245. package/dist/plugins/builtin/ggml-ocr/resolve-config.js +72 -0
  246. package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.d.ts +2 -0
  247. package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.js +12 -0
  248. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -0
  249. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +78 -0
  250. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +65 -0
  251. package/dist/plugins/builtin/ggml-vla/plugin.js +57 -0
  252. package/dist/plugins/builtin/index.d.ts +11 -0
  253. package/dist/plugins/builtin/index.js +11 -0
  254. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +22 -0
  255. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +30 -0
  256. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.d.ts +31 -0
  257. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +99 -0
  258. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +6 -0
  259. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +36 -0
  260. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +20 -0
  261. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +48 -0
  262. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +45 -0
  263. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +374 -0
  264. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +28 -0
  265. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +65 -0
  266. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +10 -0
  267. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +209 -0
  268. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +163 -0
  269. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +581 -0
  270. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +56 -0
  271. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +58 -0
  272. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +671 -0
  273. package/dist/plugins/builtin/llamacpp-completion/plugin.js +481 -0
  274. package/dist/plugins/builtin/llamacpp-completion/transform.d.ts +9 -0
  275. package/dist/plugins/builtin/llamacpp-completion/transform.js +34 -0
  276. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +76 -0
  277. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +101 -0
  278. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +457 -0
  279. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +143 -0
  280. package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.d.ts +31 -0
  281. package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.js +114 -0
  282. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +133 -0
  283. package/dist/plugins/builtin/parakeet-transcription/plugin.js +130 -0
  284. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +2 -0
  285. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +64 -0
  286. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +2 -0
  287. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +41 -0
  288. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -0
  289. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +239 -0
  290. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +757 -0
  291. package/dist/plugins/builtin/sdcpp-generation/plugin.js +310 -0
  292. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +6 -0
  293. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +30 -0
  294. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +7 -0
  295. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +106 -0
  296. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +6 -0
  297. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +91 -0
  298. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +196 -0
  299. package/dist/plugins/builtin/tts-ggml/plugin.js +305 -0
  300. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +346 -0
  301. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +169 -0
  302. package/dist/plugins/index.d.ts +9 -0
  303. package/dist/plugins/index.js +12 -0
  304. package/dist/plugins/ops/bci-transcribe.d.ts +13 -0
  305. package/dist/plugins/ops/bci-transcribe.js +248 -0
  306. package/dist/plugins/ops/cancel.d.ts +25 -0
  307. package/dist/plugins/ops/cancel.js +43 -0
  308. package/dist/plugins/ops/embed.d.ts +15 -0
  309. package/dist/plugins/ops/embed.js +193 -0
  310. package/dist/plugins/ops/kv-cache-retention.d.ts +11 -0
  311. package/dist/plugins/ops/kv-cache-retention.js +121 -0
  312. package/dist/plugins/ops/kv-cache-utils.d.ts +21 -0
  313. package/dist/plugins/ops/kv-cache-utils.js +120 -0
  314. package/dist/plugins/ops/load-model.d.ts +8 -0
  315. package/dist/plugins/ops/load-model.js +82 -0
  316. package/dist/plugins/ops/transcribe.d.ts +20 -0
  317. package/dist/plugins/ops/transcribe.js +343 -0
  318. package/dist/plugins/ops/translate.d.ts +7 -0
  319. package/dist/plugins/ops/translate.js +256 -0
  320. package/dist/plugins/ops/unload-model.d.ts +2 -0
  321. package/dist/plugins/ops/unload-model.js +45 -0
  322. package/dist/plugins/registry.d.ts +9 -0
  323. package/dist/plugins/registry.js +107 -0
  324. package/dist/profiling/aggregator.d.ts +14 -0
  325. package/dist/profiling/aggregator.js +100 -0
  326. package/dist/profiling/backend-diagnostics.d.ts +8 -0
  327. package/dist/profiling/backend-diagnostics.js +36 -0
  328. package/dist/profiling/clock.d.ts +8 -0
  329. package/dist/profiling/clock.js +23 -0
  330. package/dist/profiling/controller.d.ts +34 -0
  331. package/dist/profiling/controller.js +130 -0
  332. package/dist/profiling/delegation-profiler.d.ts +31 -0
  333. package/dist/profiling/delegation-profiler.js +155 -0
  334. package/dist/profiling/envelope.d.ts +10 -0
  335. package/dist/profiling/envelope.js +45 -0
  336. package/dist/profiling/events.d.ts +19 -0
  337. package/dist/profiling/events.js +35 -0
  338. package/dist/profiling/exporters.d.ts +9 -0
  339. package/dist/profiling/exporters.js +242 -0
  340. package/dist/profiling/index.d.ts +91 -0
  341. package/dist/profiling/index.js +87 -0
  342. package/dist/profiling/model-execution.d.ts +15 -0
  343. package/dist/profiling/model-execution.js +34 -0
  344. package/dist/profiling/operation-metrics.d.ts +24 -0
  345. package/dist/profiling/operation-metrics.js +370 -0
  346. package/dist/profiling/operation-wrappers.d.ts +17 -0
  347. package/dist/profiling/operation-wrappers.js +196 -0
  348. package/dist/profiling/profiler.d.ts +27 -0
  349. package/dist/profiling/profiler.js +127 -0
  350. package/dist/profiling/ring-buffer.d.ts +18 -0
  351. package/dist/profiling/ring-buffer.js +83 -0
  352. package/dist/profiling/types.d.ts +52 -0
  353. package/dist/profiling/types.js +1 -0
  354. package/dist/rag/handler.d.ts +2 -0
  355. package/dist/rag/handler.js +272 -0
  356. package/dist/rag/handlers/chunk.d.ts +2 -0
  357. package/dist/rag/handlers/chunk.js +7 -0
  358. package/dist/rag/handlers/close-workspace.d.ts +2 -0
  359. package/dist/rag/handlers/close-workspace.js +9 -0
  360. package/dist/rag/handlers/delete-embeddings.d.ts +2 -0
  361. package/dist/rag/handlers/delete-embeddings.js +11 -0
  362. package/dist/rag/handlers/delete-workspace.d.ts +2 -0
  363. package/dist/rag/handlers/delete-workspace.js +14 -0
  364. package/dist/rag/handlers/ingest.d.ts +9 -0
  365. package/dist/rag/handlers/ingest.js +19 -0
  366. package/dist/rag/handlers/list-workspaces.d.ts +2 -0
  367. package/dist/rag/handlers/list-workspaces.js +4 -0
  368. package/dist/rag/handlers/reindex.d.ts +9 -0
  369. package/dist/rag/handlers/reindex.js +16 -0
  370. package/dist/rag/handlers/save-embeddings.d.ts +9 -0
  371. package/dist/rag/handlers/save-embeddings.js +15 -0
  372. package/dist/rag/handlers/search.d.ts +2 -0
  373. package/dist/rag/handlers/search.js +16 -0
  374. package/dist/rag/index.d.ts +11 -0
  375. package/dist/rag/index.js +11 -0
  376. package/dist/rag/rag-operation-manager.d.ts +28 -0
  377. package/dist/rag/rag-operation-manager.js +76 -0
  378. package/dist/rag/rag-workspace-manager.d.ts +14 -0
  379. package/dist/rag/rag-workspace-manager.js +143 -0
  380. package/dist/registry.d.ts +2 -0
  381. package/dist/registry.js +166 -0
  382. package/dist/resources/collector.d.ts +643 -0
  383. package/dist/resources/collector.js +226 -0
  384. package/dist/resources/instance.d.ts +4 -0
  385. package/dist/resources/instance.js +15 -0
  386. package/dist/resources/native.d.ts +2 -0
  387. package/dist/resources/native.js +35 -0
  388. package/dist/resources/normalize.d.ts +12 -0
  389. package/dist/resources/normalize.js +73 -0
  390. package/dist/resources/types.d.ts +65 -0
  391. package/dist/resources/types.js +1 -0
  392. package/dist/runtime/cache-lock.d.ts +2 -0
  393. package/dist/runtime/cache-lock.js +67 -0
  394. package/dist/runtime/disposable-scope.d.ts +35 -0
  395. package/dist/runtime/disposable-scope.js +69 -0
  396. package/dist/runtime/env.d.ts +16 -0
  397. package/dist/runtime/env.js +53 -0
  398. package/dist/runtime/index.d.ts +5 -0
  399. package/dist/runtime/index.js +2 -0
  400. package/dist/runtime/lifecycle.d.ts +20 -0
  401. package/dist/runtime/lifecycle.js +102 -0
  402. package/dist/runtime/logging-stream-registry.d.ts +27 -0
  403. package/dist/runtime/logging-stream-registry.js +133 -0
  404. package/dist/runtime/model-config-utils.d.ts +14 -0
  405. package/dist/runtime/model-config-utils.js +128 -0
  406. package/dist/runtime/model-registry.d.ts +74 -0
  407. package/dist/runtime/model-registry.js +130 -0
  408. package/dist/runtime/registry-client.d.ts +4 -0
  409. package/dist/runtime/registry-client.js +118 -0
  410. package/dist/runtime/request-context.d.ts +72 -0
  411. package/dist/runtime/request-context.js +114 -0
  412. package/dist/runtime/request-id.d.ts +17 -0
  413. package/dist/runtime/request-id.js +29 -0
  414. package/dist/runtime/request-registry.d.ts +194 -0
  415. package/dist/runtime/request-registry.js +546 -0
  416. package/dist/runtime/runtime-lifecycle.d.ts +37 -0
  417. package/dist/runtime/runtime-lifecycle.js +163 -0
  418. package/dist/runtime/state.d.ts +26 -0
  419. package/dist/runtime/state.js +138 -0
  420. package/dist/schemas/audio-gen.d.ts +252 -0
  421. package/dist/schemas/audio-gen.js +69 -0
  422. package/dist/schemas/batch-completion-stream.d.ts +467 -0
  423. package/dist/schemas/batch-completion-stream.js +123 -0
  424. package/dist/schemas/bci-config.d.ts +74 -0
  425. package/dist/schemas/bci-config.js +58 -0
  426. package/dist/schemas/bci.d.ts +178 -0
  427. package/dist/schemas/bci.js +89 -0
  428. package/dist/schemas/cancel.d.ts +203 -0
  429. package/dist/schemas/cancel.js +135 -0
  430. package/dist/schemas/classification.d.ts +52 -0
  431. package/dist/schemas/classification.js +33 -0
  432. package/dist/schemas/common.d.ts +6420 -0
  433. package/dist/schemas/common.js +133 -0
  434. package/dist/schemas/completion-event.d.ts +248 -0
  435. package/dist/schemas/completion-event.js +94 -0
  436. package/dist/schemas/completion-stream.d.ts +475 -0
  437. package/dist/schemas/completion-stream.js +236 -0
  438. package/dist/schemas/config.d.ts +5022 -0
  439. package/dist/schemas/config.js +169 -0
  440. package/dist/schemas/delegate.d.ts +29 -0
  441. package/dist/schemas/delegate.js +40 -0
  442. package/dist/schemas/download-asset.d.ts +225 -0
  443. package/dist/schemas/download-asset.js +71 -0
  444. package/dist/schemas/embed.d.ts +41 -0
  445. package/dist/schemas/embed.js +35 -0
  446. package/dist/schemas/engine-addon-map.d.ts +20 -0
  447. package/dist/schemas/engine-addon-map.js +78 -0
  448. package/dist/schemas/error.d.ts +40 -0
  449. package/dist/schemas/error.js +65 -0
  450. package/dist/schemas/errors.d.ts +119 -0
  451. package/dist/schemas/errors.js +615 -0
  452. package/dist/schemas/finetune.d.ts +621 -0
  453. package/dist/schemas/finetune.js +163 -0
  454. package/dist/schemas/index.d.ts +45 -0
  455. package/dist/schemas/index.js +45 -0
  456. package/dist/schemas/json-schema.d.ts +15 -0
  457. package/dist/schemas/json-schema.js +2 -0
  458. package/dist/schemas/lifecycle.d.ts +38 -0
  459. package/dist/schemas/lifecycle.js +21 -0
  460. package/dist/schemas/llamacpp-config.d.ts +403 -0
  461. package/dist/schemas/llamacpp-config.js +133 -0
  462. package/dist/schemas/load-model.d.ts +25635 -0
  463. package/dist/schemas/load-model.js +629 -0
  464. package/dist/schemas/logging-stream.d.ts +33 -0
  465. package/dist/schemas/logging-stream.js +17 -0
  466. package/dist/schemas/mcp-adapter.d.ts +46 -0
  467. package/dist/schemas/mcp-adapter.js +1 -0
  468. package/dist/schemas/model-file.d.ts +27 -0
  469. package/dist/schemas/model-file.js +29 -0
  470. package/dist/schemas/model-info.d.ts +197 -0
  471. package/dist/schemas/model-info.js +144 -0
  472. package/dist/schemas/model-ops.d.ts +36 -0
  473. package/dist/schemas/model-ops.js +40 -0
  474. package/dist/schemas/model-src-utils.d.ts +192 -0
  475. package/dist/schemas/model-src-utils.js +61 -0
  476. package/dist/schemas/model-types.d.ts +275 -0
  477. package/dist/schemas/model-types.js +223 -0
  478. package/dist/schemas/ocr.d.ts +138 -0
  479. package/dist/schemas/ocr.js +59 -0
  480. package/dist/schemas/path-validation.d.ts +9 -0
  481. package/dist/schemas/path-validation.js +22 -0
  482. package/dist/schemas/plugin.d.ts +341 -0
  483. package/dist/schemas/plugin.js +241 -0
  484. package/dist/schemas/profiling.d.ts +695 -0
  485. package/dist/schemas/profiling.js +100 -0
  486. package/dist/schemas/provide.d.ts +50 -0
  487. package/dist/schemas/provide.js +32 -0
  488. package/dist/schemas/rag.d.ts +301 -0
  489. package/dist/schemas/rag.js +211 -0
  490. package/dist/schemas/registry.d.ts +453 -0
  491. package/dist/schemas/registry.js +104 -0
  492. package/dist/schemas/reload-config.d.ts +585 -0
  493. package/dist/schemas/reload-config.js +42 -0
  494. package/dist/schemas/runtime-context.d.ts +18 -0
  495. package/dist/schemas/runtime-context.js +7 -0
  496. package/dist/schemas/sdcpp-config.d.ts +1126 -0
  497. package/dist/schemas/sdcpp-config.js +748 -0
  498. package/dist/schemas/system-resources.d.ts +3966 -0
  499. package/dist/schemas/system-resources.js +149 -0
  500. package/dist/schemas/text-to-speech.d.ts +1352 -0
  501. package/dist/schemas/text-to-speech.js +364 -0
  502. package/dist/schemas/tools.d.ts +80 -0
  503. package/dist/schemas/tools.js +52 -0
  504. package/dist/schemas/transcription-config.d.ts +161 -0
  505. package/dist/schemas/transcription-config.js +156 -0
  506. package/dist/schemas/transcription.d.ts +261 -0
  507. package/dist/schemas/transcription.js +145 -0
  508. package/dist/schemas/translate.d.ts +138 -0
  509. package/dist/schemas/translate.js +103 -0
  510. package/dist/schemas/translation-config.d.ts +1031 -0
  511. package/dist/schemas/translation-config.js +170 -0
  512. package/dist/schemas/vla.d.ts +110 -0
  513. package/dist/schemas/vla.js +151 -0
  514. package/dist/selection.d.ts +8 -0
  515. package/dist/selection.js +14 -0
  516. package/dist/surface.d.ts +18 -0
  517. package/dist/surface.js +30 -0
  518. package/dist/utils/addon-responses.d.ts +58 -0
  519. package/dist/utils/addon-responses.js +1 -0
  520. package/dist/utils/aggregate-events.d.ts +25 -0
  521. package/dist/utils/aggregate-events.js +73 -0
  522. package/dist/utils/archive.d.ts +4 -0
  523. package/dist/utils/archive.js +180 -0
  524. package/dist/utils/asr-events.d.ts +14 -0
  525. package/dist/utils/asr-events.js +30 -0
  526. package/dist/utils/audio/decoder.d.ts +4 -0
  527. package/dist/utils/audio/decoder.js +80 -0
  528. package/dist/utils/audio-input.d.ts +9 -0
  529. package/dist/utils/audio-input.js +34 -0
  530. package/dist/utils/cache/auto-history.d.ts +3 -0
  531. package/dist/utils/cache/auto-history.js +26 -0
  532. package/dist/utils/cache/paths.d.ts +42 -0
  533. package/dist/utils/cache/paths.js +133 -0
  534. package/dist/utils/cache/types.d.ts +7 -0
  535. package/dist/utils/cache/types.js +1 -0
  536. package/dist/utils/cache-normalize.d.ts +9 -0
  537. package/dist/utils/cache-normalize.js +20 -0
  538. package/dist/utils/checksum.d.ts +3 -0
  539. package/dist/utils/checksum.js +29 -0
  540. package/dist/utils/clear-storage.d.ts +4 -0
  541. package/dist/utils/clear-storage.js +14 -0
  542. package/dist/utils/completion-normalizer.d.ts +17 -0
  543. package/dist/utils/completion-normalizer.js +453 -0
  544. package/dist/utils/config-transform.d.ts +2 -0
  545. package/dist/utils/config-transform.js +21 -0
  546. package/dist/utils/decorate-promise.d.ts +33 -0
  547. package/dist/utils/decorate-promise.js +35 -0
  548. package/dist/utils/encoding.d.ts +16 -0
  549. package/dist/utils/encoding.js +27 -0
  550. package/dist/utils/expand-gguf-shards.d.ts +2 -0
  551. package/dist/utils/expand-gguf-shards.js +22 -0
  552. package/dist/utils/formatting.d.ts +18 -0
  553. package/dist/utils/formatting.js +56 -0
  554. package/dist/utils/gguf-tensor-extractor.d.ts +14 -0
  555. package/dist/utils/gguf-tensor-extractor.js +311 -0
  556. package/dist/utils/global-singleton.d.ts +1 -0
  557. package/dist/utils/global-singleton.js +10 -0
  558. package/dist/utils/index.d.ts +11 -0
  559. package/dist/utils/index.js +11 -0
  560. package/dist/utils/load-model-validation.d.ts +6 -0
  561. package/dist/utils/load-model-validation.js +18 -0
  562. package/dist/utils/mcp-adapter.d.ts +11 -0
  563. package/dist/utils/mcp-adapter.js +96 -0
  564. package/dist/utils/multi-gpu-mobile.d.ts +4 -0
  565. package/dist/utils/multi-gpu-mobile.js +10 -0
  566. package/dist/utils/object.d.ts +1 -0
  567. package/dist/utils/object.js +3 -0
  568. package/dist/utils/path-security.d.ts +22 -0
  569. package/dist/utils/path-security.js +69 -0
  570. package/dist/utils/qvac-paths.d.ts +1 -0
  571. package/dist/utils/qvac-paths.js +5 -0
  572. package/dist/utils/response-format.d.ts +2 -0
  573. package/dist/utils/response-format.js +20 -0
  574. package/dist/utils/shard-utils.d.ts +47 -0
  575. package/dist/utils/shard-utils.js +123 -0
  576. package/dist/utils/tool-helpers.d.ts +24 -0
  577. package/dist/utils/tool-helpers.js +111 -0
  578. package/dist/utils/tool-integration.d.ts +23 -0
  579. package/dist/utils/tool-integration.js +29 -0
  580. package/dist/utils/tools/dialect.d.ts +7 -0
  581. package/dist/utils/tools/dialect.js +20 -0
  582. package/dist/utils/tools/index.d.ts +2 -0
  583. package/dist/utils/tools/index.js +2 -0
  584. package/dist/utils/tools/parser.d.ts +5 -0
  585. package/dist/utils/tools/parser.js +69 -0
  586. package/dist/utils/tools/parsers/dsml.d.ts +3 -0
  587. package/dist/utils/tools/parsers/dsml.js +177 -0
  588. package/dist/utils/tools/parsers/gemma4native.d.ts +3 -0
  589. package/dist/utils/tools/parsers/gemma4native.js +65 -0
  590. package/dist/utils/tools/parsers/harmony.d.ts +3 -0
  591. package/dist/utils/tools/parsers/harmony.js +57 -0
  592. package/dist/utils/tools/parsers/hermes.d.ts +3 -0
  593. package/dist/utils/tools/parsers/hermes.js +122 -0
  594. package/dist/utils/tools/parsers/json.d.ts +5 -0
  595. package/dist/utils/tools/parsers/json.js +115 -0
  596. package/dist/utils/tools/parsers/pythonic.d.ts +3 -0
  597. package/dist/utils/tools/parsers/pythonic.js +267 -0
  598. package/dist/utils/tools/parsers/qwen35.d.ts +3 -0
  599. package/dist/utils/tools/parsers/qwen35.js +125 -0
  600. package/dist/utils/tools/shared.d.ts +17 -0
  601. package/dist/utils/tools/shared.js +52 -0
  602. package/dist/utils/transcribe-metadata.d.ts +20 -0
  603. package/dist/utils/transcribe-metadata.js +28 -0
  604. package/dist/utils/tts-stats.d.ts +25 -0
  605. package/dist/utils/tts-stats.js +16 -0
  606. package/dist/utils/withTimeout.d.ts +17 -0
  607. package/dist/utils/withTimeout.js +47 -0
  608. package/dist/utils/zod-error.d.ts +2 -0
  609. package/dist/utils/zod-error.js +4 -0
  610. package/package.json +266 -1
@@ -0,0 +1,351 @@
1
+ import { stream as streamRpc } from '../dispatch.js';
2
+ import { getAppLogger } from '../logging/index.js';
3
+ import { completionStreamResponseSchema } from '../schemas/index.js';
4
+ import { CompletionFailedError, InferenceCancelledError } from '../errors/index.js';
5
+ import { getMcpToolsWithHandlers } from '../utils/mcp-adapter.js';
6
+ import { validateTools } from '../utils/tool-helpers.js';
7
+ import { buildFinalFromEvents } from '../utils/aggregate-events.js';
8
+ import { generateRequestId } from '../runtime/request-id.js';
9
+ const logger = getAppLogger();
10
+ /**
11
+ * Generates completion from a language model based on conversation history.
12
+ *
13
+ * Returns a `CompletionRun` whose canonical surfaces are:
14
+ *
15
+ * - `events` — `AsyncIterable<CompletionEvent>` of ordered, typed events.
16
+ * - `final` — `Promise<CompletionFinal>` with aggregated results once the
17
+ * stream ends (content, thinking, tool calls, stats, raw text).
18
+ *
19
+ * Legacy convenience fields (`tokenStream`, `text`, `toolCallStream`,
20
+ * `toolCalls`, `stats`) are still available but deprecated — they derive
21
+ * from `events` / `final` internally.
22
+ *
23
+ * @param params - The completion parameters
24
+ * @param params.modelId - The identifier of the model to use for completion
25
+ * @param params.history - Array of conversation messages with role, content, and optional attachments
26
+ * @param params.stream - Whether to stream tokens (true) or return complete response (false). Defaults to true
27
+ * @param params.tools - Optional array of tools (can be simple ToolInput with Zod schemas or full Tool objects)
28
+ * @param params.mcp - Optional array of MCP client inputs for tool integration
29
+ * @param params.captureThinking - Best-effort parsing of `<think>` blocks into `thinkingDelta` events; `final.raw.fullText` always preserves the original output
30
+ * @param params.emitRawDeltas - When true, every raw model token is also emitted as a `rawDelta` event
31
+ * @param params.toolDialect - Override name-based dialect detection. Supported values: `"hermes"`, `"pythonic"`, `"json"`, `"harmony"`, `"qwen35"` (Qwen3.5/3.6), `"gemma4"`. Use when the auto-router doesn't recognise your model name. Drives both streaming frame detection and finalization parsing.
32
+ * Common override case: Llama 3.x tool-calling fine-tunes that emit the native pythonic header (`<|start_header_id|>tool_call<|end_header_id|>...<|eot_id|>`).
33
+ * @param params.responseFormat - Optional structured-output constraint applied to the model's output:
34
+ * - `{ type: "text" }` — no constraint (default behavior)
35
+ * - `{ type: "json_object" }` — output must be a JSON object
36
+ * - `{ type: "json_schema", json_schema: { name, schema, description?, strict? } }` — output must validate against `schema`
37
+ *
38
+ * The schema is converted to GBNF natively by llama.cpp and applied for the
39
+ * duration of the request only. `json_schema.name` and `json_schema.description`
40
+ * are accepted for OpenAI compatibility but only used at the API boundary —
41
+ * they do not affect generation. **`json_schema.strict` is currently accepted
42
+ * for compatibility but does NOT trigger OpenAI's auto-tightening semantics**
43
+ * (implicit `additionalProperties: false`, all properties required). The
44
+ * schema is forwarded to the addon as-is, so callers who want strict
45
+ * validation must encode it explicitly in `schema`.
46
+ *
47
+ * Cannot be combined with `tools` (tools already constrain output via their parameter schema).
48
+ * @param params.kvCache - Optional KV cache configuration. Cache files are organized hierarchically:
49
+ * - Structure: `{kvCacheKey}/{modelId}/{configHash}.bin`
50
+ * - The configHash includes model config + system prompt to ensure cache isolation
51
+ * - `true`: Auto-generate cache key based on conversation history
52
+ * - `"custom-key"`: Use provided string as cache key for manual session management
53
+ * - `false` or `undefined`: No caching
54
+ * - ⚡ Performance: When cache exists, only the last message is sent to the model (includes multimodal attachments)
55
+ * - 🗑️ Cleanup: Use `deleteCache({ kvCacheKey })` to remove cached sessions
56
+ *
57
+ * **Auto-cache (`kvCache: true`) — assistant turn contract.** When
58
+ * pushing the assistant turn back into `history` for the next call,
59
+ * use `(await run.final).cacheableAssistantContent`. That's the exact
60
+ * string we persisted to the cache key on this turn, so re-using
61
+ * it verbatim guarantees the next-turn lookup hits.
62
+ * - Any post-processing of the assistant text (rewriting, summarizing,
63
+ * stripping model stop tokens like `<|im_end|>`) before pushing it
64
+ * back will miss the cache. Push the canonical string unchanged.
65
+ * - `cacheableAssistantContent` is omitted on tool-call turns - those
66
+ * can't be auto-cached today.
67
+ * @returns A CompletionRun — consume via `events` / `final`.
68
+ * @example
69
+ * ```typescript
70
+ * import { z } from "zod";
71
+ *
72
+ * const run = completion({
73
+ * modelId: "llama-2",
74
+ * history: [
75
+ * { role: "user", content: "What's the weather in Tokyo?" }
76
+ * ],
77
+ * stream: true,
78
+ * captureThinking: true,
79
+ * tools: [{
80
+ * name: "get_weather",
81
+ * description: "Get current weather",
82
+ * parameters: z.object({
83
+ * city: z.string().describe("City name"),
84
+ * }),
85
+ * handler: async (args) => {
86
+ * return { temperature: 22, condition: "sunny" };
87
+ * }
88
+ * }]
89
+ * });
90
+ *
91
+ * for await (const event of run.events) {
92
+ * if (event.type === "contentDelta") process.stdout.write(event.text);
93
+ * if (event.type === "toolCall") console.log(event.call.name, event.call.arguments);
94
+ * }
95
+ *
96
+ * const result = await run.final;
97
+ * for (const toolCall of await result.toolCalls) {
98
+ * if (toolCall.invoke) {
99
+ * const toolResult = await toolCall.invoke();
100
+ * console.log(toolResult);
101
+ * }
102
+ * }
103
+ * ```
104
+ */
105
+ export function completion(params) {
106
+ // Stable identity for this run, generated by the caller so it's
107
+ // available synchronously the moment we return — before any async work
108
+ // begins and therefore before the user could possibly
109
+ // have a "stop" handler. Surfaced on the returned `CompletionRun`
110
+ // (`run.requestId`) so callers can `cancel({ requestId })` at any
111
+ // point during the stream.
112
+ const requestId = generateRequestId();
113
+ let statsResolver = () => { };
114
+ let statsRejecter = () => { };
115
+ const statsPromise = new Promise((resolve, reject) => {
116
+ statsResolver = resolve;
117
+ statsRejecter = reject;
118
+ });
119
+ statsPromise.catch(() => { });
120
+ let toolCallsResolver = () => { };
121
+ let toolCallsRejecter = () => { };
122
+ const toolCallsPromise = new Promise((resolve, reject) => {
123
+ toolCallsResolver = resolve;
124
+ toolCallsRejecter = reject;
125
+ });
126
+ toolCallsPromise.catch(() => { });
127
+ let finalResolver = () => { };
128
+ let finalRejecter = () => { };
129
+ const finalPromise = new Promise((resolve, reject) => {
130
+ finalResolver = resolve;
131
+ finalRejecter = reject;
132
+ });
133
+ finalPromise.catch(() => { });
134
+ const tokenQueue = [];
135
+ const toolEventQueue = [];
136
+ const eventQueue = [];
137
+ let done = false;
138
+ let tokenResolve = null;
139
+ let toolResolve = null;
140
+ let eventResolve = null;
141
+ let streamError = null;
142
+ const allEvents = [];
143
+ function notifyWaiters() {
144
+ if (tokenResolve) {
145
+ tokenResolve();
146
+ tokenResolve = null;
147
+ }
148
+ if (toolResolve) {
149
+ toolResolve();
150
+ toolResolve = null;
151
+ }
152
+ if (eventResolve) {
153
+ eventResolve();
154
+ eventResolve = null;
155
+ }
156
+ }
157
+ const processResponses = async () => {
158
+ try {
159
+ let allTools = [];
160
+ const allHandlers = new Map();
161
+ if (params.tools) {
162
+ const { tools, handlers } = validateTools(params.tools);
163
+ allTools = tools;
164
+ for (const [name, handler] of handlers) {
165
+ if (allHandlers.has(name)) {
166
+ logger.warn(`Duplicate tool handler for "${name}", overwriting`);
167
+ }
168
+ allHandlers.set(name, handler);
169
+ }
170
+ }
171
+ if (params.mcp && params.mcp.length > 0) {
172
+ const { tools: mcpTools, handlers: mcpHandlers } = await getMcpToolsWithHandlers(params.mcp);
173
+ allTools = [...mcpTools, ...allTools];
174
+ for (const [name, handler] of mcpHandlers) {
175
+ if (allHandlers.has(name)) {
176
+ logger.warn(`Duplicate tool handler for "${name}", overwriting`);
177
+ }
178
+ allHandlers.set(name, handler);
179
+ }
180
+ }
181
+ const request = {
182
+ type: 'completionStream',
183
+ modelId: params.modelId,
184
+ history: params.history,
185
+ kvCache: params.kvCache,
186
+ tools: allTools.length > 0 ? allTools : undefined,
187
+ stream: params.stream ?? true,
188
+ generationParams: params.generationParams,
189
+ captureThinking: params.captureThinking,
190
+ emitRawDeltas: params.emitRawDeltas,
191
+ toolDialect: params.toolDialect,
192
+ responseFormat: params.responseFormat,
193
+ requestId
194
+ };
195
+ const responses = streamRpc(request, params.rpcOptions);
196
+ for await (const response of responses) {
197
+ if (response &&
198
+ typeof response === 'object' &&
199
+ 'type' in response &&
200
+ response.type === 'completionStream') {
201
+ const streamResponse = completionStreamResponseSchema.parse(response);
202
+ for (const event of streamResponse.events) {
203
+ allEvents.push(event);
204
+ eventQueue.push(event);
205
+ if (event.type === 'contentDelta') {
206
+ tokenQueue.push(event.text);
207
+ }
208
+ else if (event.type === 'toolCall') {
209
+ toolEventQueue.push(event);
210
+ }
211
+ }
212
+ notifyWaiters();
213
+ if (streamResponse.done) {
214
+ const { final, error, cancelled } = buildFinalFromEvents(allEvents, allHandlers);
215
+ if (error) {
216
+ const err = new CompletionFailedError(error.message, error);
217
+ finalRejecter(err);
218
+ statsRejecter(err);
219
+ toolCallsRejecter(err);
220
+ }
221
+ else if (cancelled) {
222
+ // The response stream ended with `stopReason: "cancelled"` — the
223
+ // run was aborted mid-flight. Cancellation contract: `events`
224
+ // ends normally (consumers iterating `run.events` see the
225
+ // cancelled `completionDone` and exit naturally), and the
226
+ // promise-aggregates reject with `InferenceCancelledError`
227
+ // carrying whatever the aggregator accumulated up to the
228
+ // cancel point. Consumers do `instanceof
229
+ // InferenceCancelledError` and read `.partial.text` /
230
+ // `.partial.toolCalls` / `.partial.stats` if they want the
231
+ // partial output.
232
+ const err = new InferenceCancelledError(requestId, {
233
+ text: final.contentText,
234
+ toolCalls: final.toolCalls,
235
+ ...(final.stats && { stats: final.stats })
236
+ });
237
+ finalRejecter(err);
238
+ statsRejecter(err);
239
+ toolCallsRejecter(err);
240
+ }
241
+ else {
242
+ finalResolver(final);
243
+ statsResolver(final.stats);
244
+ toolCallsResolver(final.toolCalls);
245
+ }
246
+ done = true;
247
+ notifyWaiters();
248
+ }
249
+ }
250
+ }
251
+ }
252
+ catch (error) {
253
+ streamError = error instanceof Error ? error : new Error(String(error));
254
+ statsRejecter(error);
255
+ toolCallsRejecter(error);
256
+ finalRejecter(error);
257
+ done = true;
258
+ notifyWaiters();
259
+ }
260
+ };
261
+ void processResponses();
262
+ const textPromise = (async () => {
263
+ const final = await finalPromise;
264
+ return final.contentText;
265
+ })();
266
+ textPromise.catch(() => { });
267
+ const eventStream = (async function* () {
268
+ while (true) {
269
+ if (eventQueue.length > 0) {
270
+ yield eventQueue.shift();
271
+ }
272
+ else if (done) {
273
+ if (streamError !== null) {
274
+ throw streamError;
275
+ }
276
+ break;
277
+ }
278
+ else {
279
+ await new Promise((resolve) => {
280
+ eventResolve = resolve;
281
+ });
282
+ }
283
+ }
284
+ })();
285
+ if (params.stream) {
286
+ const tokenStream = (async function* () {
287
+ while (true) {
288
+ if (tokenQueue.length > 0) {
289
+ yield tokenQueue.shift();
290
+ }
291
+ else if (done) {
292
+ if (streamError !== null) {
293
+ throw streamError;
294
+ }
295
+ break;
296
+ }
297
+ else {
298
+ await new Promise((resolve) => {
299
+ tokenResolve = resolve;
300
+ });
301
+ }
302
+ }
303
+ })();
304
+ const toolCallStream = (async function* () {
305
+ while (true) {
306
+ if (toolEventQueue.length > 0) {
307
+ yield toolEventQueue.shift();
308
+ }
309
+ else if (done) {
310
+ if (streamError !== null) {
311
+ throw streamError;
312
+ }
313
+ break;
314
+ }
315
+ else {
316
+ await new Promise((resolve) => {
317
+ toolResolve = resolve;
318
+ });
319
+ }
320
+ }
321
+ })();
322
+ return {
323
+ requestId,
324
+ events: eventStream,
325
+ final: finalPromise,
326
+ tokenStream,
327
+ toolCallStream,
328
+ text: textPromise,
329
+ stats: statsPromise,
330
+ toolCalls: toolCallsPromise
331
+ };
332
+ }
333
+ else {
334
+ const tokenStream = (async function* () {
335
+ //Empty generator for non-streaming mode
336
+ })();
337
+ const toolCallStream = (async function* () {
338
+ //Empty generator for non-streaming mode
339
+ })();
340
+ return {
341
+ requestId,
342
+ events: eventStream,
343
+ final: finalPromise,
344
+ tokenStream,
345
+ toolCallStream,
346
+ text: textPromise,
347
+ stats: statsPromise,
348
+ toolCalls: toolCallsPromise
349
+ };
350
+ }
351
+ }
@@ -0,0 +1,29 @@
1
+ /**
2
+ * Deletes KV cache files.
3
+ *
4
+ * @param params - The delete cache parameters
5
+ * @param params.all - If true, deletes all cache files
6
+ * @param params.kvCacheKey - The cache key to delete
7
+ * @param params.modelId - Optional: specific model ID to delete within the cache key. If not provided, deletes entire cache key.
8
+ * @returns Promise resolving to success status
9
+ * @throws {QvacErrorBase} When the cache parameters are invalid (`InvalidDeleteCacheParamsError`) or the engine reports a delete failure (`DeleteCacheFailedError`).
10
+ * @example
11
+ * ```typescript
12
+ * // Delete all caches
13
+ * await deleteCache({ all: true });
14
+ *
15
+ * // Delete entire cache key (all models)
16
+ * await deleteCache({ kvCacheKey: "my-session" });
17
+ *
18
+ * // Delete only specific model within cache key
19
+ * await deleteCache({ kvCacheKey: "my-session", modelId: "model-abc123" });
20
+ * ```
21
+ */
22
+ export declare function deleteCache(params: {
23
+ all: true;
24
+ } | {
25
+ kvCacheKey: string;
26
+ modelId?: string;
27
+ }): Promise<{
28
+ success: boolean;
29
+ }>;
@@ -0,0 +1,49 @@
1
+ import { send } from '../dispatch.js';
2
+ import { InvalidDeleteCacheParamsError, DeleteCacheFailedError } from '../errors/index.js';
3
+ /**
4
+ * Deletes KV cache files.
5
+ *
6
+ * @param params - The delete cache parameters
7
+ * @param params.all - If true, deletes all cache files
8
+ * @param params.kvCacheKey - The cache key to delete
9
+ * @param params.modelId - Optional: specific model ID to delete within the cache key. If not provided, deletes entire cache key.
10
+ * @returns Promise resolving to success status
11
+ * @throws {QvacErrorBase} When the cache parameters are invalid (`InvalidDeleteCacheParamsError`) or the engine reports a delete failure (`DeleteCacheFailedError`).
12
+ * @example
13
+ * ```typescript
14
+ * // Delete all caches
15
+ * await deleteCache({ all: true });
16
+ *
17
+ * // Delete entire cache key (all models)
18
+ * await deleteCache({ kvCacheKey: "my-session" });
19
+ *
20
+ * // Delete only specific model within cache key
21
+ * await deleteCache({ kvCacheKey: "my-session", modelId: "model-abc123" });
22
+ * ```
23
+ */
24
+ export async function deleteCache(params) {
25
+ let req;
26
+ if ('all' in params && params.all) {
27
+ req = {
28
+ type: 'deleteCache',
29
+ all: true
30
+ };
31
+ }
32
+ else if ('kvCacheKey' in params) {
33
+ req = {
34
+ type: 'deleteCache',
35
+ kvCacheKey: params.kvCacheKey,
36
+ modelId: params.modelId
37
+ };
38
+ }
39
+ else {
40
+ throw new InvalidDeleteCacheParamsError();
41
+ }
42
+ const response = (await send(req));
43
+ if (!response.success && response.error) {
44
+ throw new DeleteCacheFailedError(response.error);
45
+ }
46
+ return {
47
+ success: response.success
48
+ };
49
+ }
@@ -0,0 +1,92 @@
1
+ import { type DiffusionClientParams, type DiffusionStats } from '../schemas/index.js';
2
+ export interface DiffusionProgressTick {
3
+ step: number;
4
+ totalSteps: number;
5
+ elapsedMs: number;
6
+ }
7
+ interface DiffusionResult {
8
+ progressStream: AsyncGenerator<DiffusionProgressTick>;
9
+ outputs: Promise<Uint8Array[]>;
10
+ stats: Promise<DiffusionStats | undefined>;
11
+ }
12
+ /**
13
+ * Generates images using a loaded diffusion model.
14
+ *
15
+ * @param params - Diffusion request parameters (model, prompt, dimensions, sampler, seed, etc.).
16
+ * @returns A result object exposing `progressStream` (async iterator of `{ step, totalSteps, elapsedMs }`), `outputs` (promise of the generated image buffers), and `stats` (promise of generation statistics).
17
+ *
18
+ * Supports both txt2img (no `init_image`) and img2img (with `init_image`).
19
+ *
20
+ * @example
21
+ * ```typescript
22
+ * // txt2img
23
+ * const { outputs, stats } = diffusion({ modelId, prompt: "a cat" });
24
+ * const buffers = await outputs;
25
+ * fs.writeFileSync("output.png", buffers[0]);
26
+ *
27
+ * // img2img (SD/SDXL — SDEdit)
28
+ * const initImage = fs.readFileSync("input.png");
29
+ * const { outputs } = diffusion({ modelId, prompt: "oil painting style", init_image: initImage, strength: 0.7 });
30
+ *
31
+ * // img2img (FLUX.2 — in-context conditioning)
32
+ * // IMPORTANT: FLUX img2img requires `prediction: "flux2_flow"` to be set on the
33
+ * // model config at loadModel time (e.g. `loadModel(src, { modelType: "diffusion",
34
+ * // modelConfig: { prediction: "flux2_flow" } })`).
35
+ * const { outputs } = diffusion({ modelId, prompt: "turn into watercolor", init_image: initImage });
36
+ *
37
+ * // FLUX.2 multi-reference fusion
38
+ * // IMPORTANT: requires the model loaded with `modelConfig: { prediction: "flux2_flow" }`
39
+ * // and a Qwen3 text encoder via `llmModelSrc` (same loadModel requirements as the
40
+ * // FLUX.2 img2img example above). `init_image` and `init_images` are mutually
41
+ * // exclusive — pass one or the other, not both.
42
+ * const refA = fs.readFileSync("scientist-a.jpg");
43
+ * const refB = fs.readFileSync("scientist-b.jpg");
44
+ * const { outputs } = diffusion({
45
+ * modelId,
46
+ * prompt: "a portrait using most visual traits from @image1 and the eyes from @image2",
47
+ * init_images: [refA, refB],
48
+ * width: 768,
49
+ * height: 768,
50
+ * });
51
+ *
52
+ * // LoRA adapter for this generation (absolute path required).
53
+ * // Persistence across subsequent diffusion() calls is controlled at
54
+ * // loadModel time via `modelConfig.lora_apply_mode`.
55
+ * const { outputs } = diffusion({
56
+ * modelId,
57
+ * prompt: "a watercolor cat",
58
+ * lora: "/home/user/loras/watercolor.safetensors",
59
+ * });
60
+ *
61
+ * // ESRGAN upscale; requires the model to be loaded with `modelConfig.upscaler.model_src` set.
62
+ * const { outputs: singleOutputs } = diffusion({
63
+ * modelId,
64
+ * prompt: "a fox portrait",
65
+ * width: 128,
66
+ * height: 128,
67
+ * upscale: true, // one native-scale pass
68
+ * });
69
+ * const single = await singleOutputs;
70
+ * fs.writeFileSync("upscaled.png", single[0]);
71
+ *
72
+ * // Repeat upscale passes when needed.
73
+ * const { outputs: hiresOutputs } = diffusion({
74
+ * modelId,
75
+ * prompt: "a fox portrait",
76
+ * width: 128,
77
+ * height: 128,
78
+ * upscale: { repeats: 2 },
79
+ * });
80
+ * const hires = await hiresOutputs;
81
+ * fs.writeFileSync("hires.png", hires[0]);
82
+ *
83
+ * // With progress tracking
84
+ * const { progressStream, outputs } = diffusion({ modelId, prompt: "a cat" });
85
+ * for await (const { step, totalSteps } of progressStream) {
86
+ * console.log(`${step}/${totalSteps}`);
87
+ * }
88
+ * const buffers = await outputs;
89
+ * ```
90
+ */
91
+ export declare function diffusion(params: DiffusionClientParams): DiffusionResult;
92
+ export {};