@qvac/inference 0.0.0 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/LICENSE +179 -0
  2. package/NOTICE +928 -0
  3. package/README.md +138 -0
  4. package/dist/api/audio-gen.d.ts +18 -0
  5. package/dist/api/audio-gen.js +148 -0
  6. package/dist/api/batch-completion.d.ts +14 -0
  7. package/dist/api/batch-completion.js +332 -0
  8. package/dist/api/bci-transcribe.d.ts +56 -0
  9. package/dist/api/bci-transcribe.js +172 -0
  10. package/dist/api/cancel.d.ts +34 -0
  11. package/dist/api/cancel.js +82 -0
  12. package/dist/api/classify.d.ts +22 -0
  13. package/dist/api/classify.js +46 -0
  14. package/dist/api/completion-stream.d.ts +106 -0
  15. package/dist/api/completion-stream.js +351 -0
  16. package/dist/api/delete-cache.d.ts +29 -0
  17. package/dist/api/delete-cache.js +49 -0
  18. package/dist/api/diffusion.d.ts +92 -0
  19. package/dist/api/diffusion.js +176 -0
  20. package/dist/api/download-asset.d.ts +49 -0
  21. package/dist/api/download-asset.js +87 -0
  22. package/dist/api/embed.d.ts +41 -0
  23. package/dist/api/embed.js +29 -0
  24. package/dist/api/finetune.d.ts +89 -0
  25. package/dist/api/finetune.js +138 -0
  26. package/dist/api/get-loaded-model-info.d.ts +24 -0
  27. package/dist/api/get-loaded-model-info.js +34 -0
  28. package/dist/api/get-model-info.d.ts +43 -0
  29. package/dist/api/get-model-info.js +21 -0
  30. package/dist/api/get-system-resources.d.ts +2 -0
  31. package/dist/api/get-system-resources.js +16 -0
  32. package/dist/api/heartbeat.d.ts +29 -0
  33. package/dist/api/heartbeat.js +37 -0
  34. package/dist/api/index.d.ts +35 -0
  35. package/dist/api/index.js +35 -0
  36. package/dist/api/invoke-plugin.d.ts +30 -0
  37. package/dist/api/invoke-plugin.js +54 -0
  38. package/dist/api/load-model.d.ts +169 -0
  39. package/dist/api/load-model.js +129 -0
  40. package/dist/api/logging-stream-registry.d.ts +4 -0
  41. package/dist/api/logging-stream-registry.js +61 -0
  42. package/dist/api/logging-stream.d.ts +23 -0
  43. package/dist/api/logging-stream.js +36 -0
  44. package/dist/api/ocr.d.ts +30 -0
  45. package/dist/api/ocr.js +88 -0
  46. package/dist/api/parse-input.d.ts +2 -0
  47. package/dist/api/parse-input.js +14 -0
  48. package/dist/api/provide.d.ts +23 -0
  49. package/dist/api/provide.js +32 -0
  50. package/dist/api/rag.d.ts +251 -0
  51. package/dist/api/rag.js +485 -0
  52. package/dist/api/registry.d.ts +39 -0
  53. package/dist/api/registry.js +63 -0
  54. package/dist/api/resume.d.ts +29 -0
  55. package/dist/api/resume.js +36 -0
  56. package/dist/api/state.d.ts +22 -0
  57. package/dist/api/state.js +29 -0
  58. package/dist/api/stop-provider.d.ts +20 -0
  59. package/dist/api/stop-provider.js +30 -0
  60. package/dist/api/subscribe-logs.d.ts +27 -0
  61. package/dist/api/subscribe-logs.js +43 -0
  62. package/dist/api/suspend.d.ts +30 -0
  63. package/dist/api/suspend.js +37 -0
  64. package/dist/api/text-to-speech.d.ts +64 -0
  65. package/dist/api/text-to-speech.js +434 -0
  66. package/dist/api/transcribe.d.ts +75 -0
  67. package/dist/api/transcribe.js +256 -0
  68. package/dist/api/translate.d.ts +47 -0
  69. package/dist/api/translate.js +115 -0
  70. package/dist/api/unload-model.d.ts +17 -0
  71. package/dist/api/unload-model.js +42 -0
  72. package/dist/api/upscale.d.ts +44 -0
  73. package/dist/api/upscale.js +99 -0
  74. package/dist/api/video.d.ts +86 -0
  75. package/dist/api/video.js +176 -0
  76. package/dist/api/vla-helpers.d.ts +43 -0
  77. package/dist/api/vla-helpers.js +124 -0
  78. package/dist/api/vla.d.ts +68 -0
  79. package/dist/api/vla.js +113 -0
  80. package/dist/config/config-utils.d.ts +4 -0
  81. package/dist/config/config-utils.js +18 -0
  82. package/dist/config/resolve-config.d.ts +8 -0
  83. package/dist/config/resolve-config.js +74 -0
  84. package/dist/constants/audio.d.ts +1 -0
  85. package/dist/constants/audio.js +1 -0
  86. package/dist/constants/index.d.ts +2 -0
  87. package/dist/constants/index.js +2 -0
  88. package/dist/constants/registry.d.ts +1 -0
  89. package/dist/constants/registry.js +1 -0
  90. package/dist/dispatch.d.ts +18 -0
  91. package/dist/dispatch.js +271 -0
  92. package/dist/engine.d.ts +5 -0
  93. package/dist/engine.js +27 -0
  94. package/dist/errors/index.d.ts +416 -0
  95. package/dist/errors/index.js +681 -0
  96. package/dist/errors/options.d.ts +2 -0
  97. package/dist/errors/options.js +13 -0
  98. package/dist/handlers/cancelHandler.d.ts +29 -0
  99. package/dist/handlers/cancelHandler.js +67 -0
  100. package/dist/handlers/completion-orchestrate.d.ts +41 -0
  101. package/dist/handlers/completion-orchestrate.js +190 -0
  102. package/dist/handlers/delete-cache.d.ts +11 -0
  103. package/dist/handlers/delete-cache.js +37 -0
  104. package/dist/handlers/download-asset.d.ts +2 -0
  105. package/dist/handlers/download-asset.js +146 -0
  106. package/dist/handlers/finetune.d.ts +2 -0
  107. package/dist/handlers/finetune.js +18 -0
  108. package/dist/handlers/get-loaded-model-info.d.ts +2 -0
  109. package/dist/handlers/get-loaded-model-info.js +45 -0
  110. package/dist/handlers/get-model-info.d.ts +2 -0
  111. package/dist/handlers/get-model-info.js +173 -0
  112. package/dist/handlers/get-system-resources.d.ts +2 -0
  113. package/dist/handlers/get-system-resources.js +53 -0
  114. package/dist/handlers/heartbeat.d.ts +2 -0
  115. package/dist/handlers/heartbeat.js +3 -0
  116. package/dist/handlers/load-model/download-manager.d.ts +95 -0
  117. package/dist/handlers/load-model/download-manager.js +306 -0
  118. package/dist/handlers/load-model/download-stats.d.ts +7 -0
  119. package/dist/handlers/load-model/download-stats.js +126 -0
  120. package/dist/handlers/load-model/handler.d.ts +2 -0
  121. package/dist/handlers/load-model/handler.js +287 -0
  122. package/dist/handlers/load-model/http.d.ts +3 -0
  123. package/dist/handlers/load-model/http.js +857 -0
  124. package/dist/handlers/load-model/hyperdrive.d.ts +4 -0
  125. package/dist/handlers/load-model/hyperdrive.js +560 -0
  126. package/dist/handlers/load-model/index.d.ts +2 -0
  127. package/dist/handlers/load-model/index.js +2 -0
  128. package/dist/handlers/load-model/registry-client-options.d.ts +27 -0
  129. package/dist/handlers/load-model/registry-client-options.js +18 -0
  130. package/dist/handlers/load-model/registry-companion-set.d.ts +22 -0
  131. package/dist/handlers/load-model/registry-companion-set.js +206 -0
  132. package/dist/handlers/load-model/registry-download-utils.d.ts +23 -0
  133. package/dist/handlers/load-model/registry-download-utils.js +112 -0
  134. package/dist/handlers/load-model/registry-metadata.d.ts +10 -0
  135. package/dist/handlers/load-model/registry-metadata.js +23 -0
  136. package/dist/handlers/load-model/registry.d.ts +4 -0
  137. package/dist/handlers/load-model/registry.js +298 -0
  138. package/dist/handlers/load-model/resolve-session.d.ts +33 -0
  139. package/dist/handlers/load-model/resolve-session.js +62 -0
  140. package/dist/handlers/load-model/resolve.d.ts +5 -0
  141. package/dist/handlers/load-model/resolve.js +158 -0
  142. package/dist/handlers/load-model/types.d.ts +59 -0
  143. package/dist/handlers/load-model/types.js +28 -0
  144. package/dist/handlers/logging-stream.d.ts +2 -0
  145. package/dist/handlers/logging-stream.js +39 -0
  146. package/dist/handlers/plugin-dispatch.d.ts +3 -0
  147. package/dist/handlers/plugin-dispatch.js +63 -0
  148. package/dist/handlers/plugin-invoke.d.ts +3 -0
  149. package/dist/handlers/plugin-invoke.js +81 -0
  150. package/dist/handlers/registry.d.ts +4 -0
  151. package/dist/handlers/registry.js +156 -0
  152. package/dist/handlers/resume.d.ts +2 -0
  153. package/dist/handlers/resume.js +11 -0
  154. package/dist/handlers/state.d.ts +2 -0
  155. package/dist/handlers/state.js +7 -0
  156. package/dist/handlers/suspend.d.ts +2 -0
  157. package/dist/handlers/suspend.js +11 -0
  158. package/dist/handlers/types.d.ts +13 -0
  159. package/dist/handlers/types.js +1 -0
  160. package/dist/handlers/unload-model.d.ts +2 -0
  161. package/dist/handlers/unload-model.js +29 -0
  162. package/dist/index.d.ts +4 -0
  163. package/dist/index.js +12 -0
  164. package/dist/logging/addon.d.ts +5 -0
  165. package/dist/logging/addon.js +75 -0
  166. package/dist/logging/index.d.ts +5 -0
  167. package/dist/logging/index.js +4 -0
  168. package/dist/logging/logger.d.ts +41 -0
  169. package/dist/logging/logger.js +209 -0
  170. package/dist/logging/namespaces.d.ts +6 -0
  171. package/dist/logging/namespaces.js +7 -0
  172. package/dist/logging/types.d.ts +37 -0
  173. package/dist/logging/types.js +1 -0
  174. package/dist/logging/utils.d.ts +5 -0
  175. package/dist/logging/utils.js +113 -0
  176. package/dist/models/registry/index.d.ts +18 -0
  177. package/dist/models/registry/index.js +21 -0
  178. package/dist/models/registry/models.d.ts +22680 -0
  179. package/dist/models/registry/models.js +23908 -0
  180. package/dist/p2p/cancel-delegated.d.ts +3 -0
  181. package/dist/p2p/cancel-delegated.js +64 -0
  182. package/dist/p2p/completion-stream-delegated.d.ts +4 -0
  183. package/dist/p2p/completion-stream-delegated.js +51 -0
  184. package/dist/p2p/delegate-client.d.ts +9 -0
  185. package/dist/p2p/delegate-client.js +219 -0
  186. package/dist/p2p/delegate-connect-diagnostics.d.ts +8 -0
  187. package/dist/p2p/delegate-connect-diagnostics.js +38 -0
  188. package/dist/p2p/delegate-transport.d.ts +18 -0
  189. package/dist/p2p/delegate-transport.js +237 -0
  190. package/dist/p2p/error-handlers.d.ts +4 -0
  191. package/dist/p2p/error-handlers.js +31 -0
  192. package/dist/p2p/handle-request.d.ts +2 -0
  193. package/dist/p2p/handle-request.js +171 -0
  194. package/dist/p2p/handler-utils.d.ts +19 -0
  195. package/dist/p2p/handler-utils.js +180 -0
  196. package/dist/p2p/heartbeat-delegated.d.ts +3 -0
  197. package/dist/p2p/heartbeat-delegated.js +32 -0
  198. package/dist/p2p/load-model-delegated.d.ts +6 -0
  199. package/dist/p2p/load-model-delegated.js +132 -0
  200. package/dist/p2p/progress-throttle.d.ts +7 -0
  201. package/dist/p2p/progress-throttle.js +47 -0
  202. package/dist/p2p/provideHandler/connection.d.ts +2 -0
  203. package/dist/p2p/provideHandler/connection.js +38 -0
  204. package/dist/p2p/provideHandler/handler.d.ts +2 -0
  205. package/dist/p2p/provideHandler/handler.js +66 -0
  206. package/dist/p2p/provideHandler/proxy.d.ts +2 -0
  207. package/dist/p2p/provideHandler/proxy.js +24 -0
  208. package/dist/p2p/provideHandler.d.ts +1 -0
  209. package/dist/p2p/provideHandler.js +2 -0
  210. package/dist/p2p/rpc-utils.d.ts +4 -0
  211. package/dist/p2p/rpc-utils.js +8 -0
  212. package/dist/p2p/stopProvideHandler.d.ts +2 -0
  213. package/dist/p2p/stopProvideHandler.js +29 -0
  214. package/dist/p2p/swarm.d.ts +9 -0
  215. package/dist/p2p/swarm.js +103 -0
  216. package/dist/p2p/transport-selector.d.ts +2 -0
  217. package/dist/p2p/transport-selector.js +18 -0
  218. package/dist/p2p/unload-model-delegated.d.ts +2 -0
  219. package/dist/p2p/unload-model-delegated.js +31 -0
  220. package/dist/plugins/builtin/asr-ggml/config.d.ts +31 -0
  221. package/dist/plugins/builtin/asr-ggml/config.js +46 -0
  222. package/dist/plugins/builtin/asr-ggml/logging.d.ts +10 -0
  223. package/dist/plugins/builtin/asr-ggml/logging.js +12 -0
  224. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +17 -0
  225. package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
  226. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +2 -0
  227. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +163 -0
  228. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +212 -0
  229. package/dist/plugins/builtin/audiogen-ggml/plugin.js +58 -0
  230. package/dist/plugins/builtin/bci-whispercpp-transcription/args.d.ts +2 -0
  231. package/dist/plugins/builtin/bci-whispercpp-transcription/args.js +13 -0
  232. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +173 -0
  233. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +111 -0
  234. package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.d.ts +74 -0
  235. package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.js +11 -0
  236. package/dist/plugins/builtin/ggml-classification/ops/classify.d.ts +5 -0
  237. package/dist/plugins/builtin/ggml-classification/ops/classify.js +31 -0
  238. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +31 -0
  239. package/dist/plugins/builtin/ggml-classification/plugin.js +55 -0
  240. package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.d.ts +7 -0
  241. package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.js +119 -0
  242. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +102 -0
  243. package/dist/plugins/builtin/ggml-ocr/plugin.js +110 -0
  244. package/dist/plugins/builtin/ggml-ocr/resolve-config.d.ts +5 -0
  245. package/dist/plugins/builtin/ggml-ocr/resolve-config.js +72 -0
  246. package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.d.ts +2 -0
  247. package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.js +12 -0
  248. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -0
  249. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +78 -0
  250. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +65 -0
  251. package/dist/plugins/builtin/ggml-vla/plugin.js +57 -0
  252. package/dist/plugins/builtin/index.d.ts +11 -0
  253. package/dist/plugins/builtin/index.js +11 -0
  254. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +22 -0
  255. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +30 -0
  256. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.d.ts +31 -0
  257. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +99 -0
  258. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +6 -0
  259. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +36 -0
  260. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +20 -0
  261. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +48 -0
  262. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +45 -0
  263. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +374 -0
  264. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +28 -0
  265. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +65 -0
  266. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +10 -0
  267. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +209 -0
  268. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +163 -0
  269. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +581 -0
  270. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +56 -0
  271. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +58 -0
  272. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +671 -0
  273. package/dist/plugins/builtin/llamacpp-completion/plugin.js +481 -0
  274. package/dist/plugins/builtin/llamacpp-completion/transform.d.ts +9 -0
  275. package/dist/plugins/builtin/llamacpp-completion/transform.js +34 -0
  276. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +76 -0
  277. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +101 -0
  278. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +457 -0
  279. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +143 -0
  280. package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.d.ts +31 -0
  281. package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.js +114 -0
  282. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +133 -0
  283. package/dist/plugins/builtin/parakeet-transcription/plugin.js +130 -0
  284. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +2 -0
  285. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +64 -0
  286. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +2 -0
  287. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +41 -0
  288. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -0
  289. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +239 -0
  290. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +757 -0
  291. package/dist/plugins/builtin/sdcpp-generation/plugin.js +310 -0
  292. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +6 -0
  293. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +30 -0
  294. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +7 -0
  295. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +106 -0
  296. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +6 -0
  297. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +91 -0
  298. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +196 -0
  299. package/dist/plugins/builtin/tts-ggml/plugin.js +305 -0
  300. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +346 -0
  301. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +169 -0
  302. package/dist/plugins/index.d.ts +9 -0
  303. package/dist/plugins/index.js +12 -0
  304. package/dist/plugins/ops/bci-transcribe.d.ts +13 -0
  305. package/dist/plugins/ops/bci-transcribe.js +248 -0
  306. package/dist/plugins/ops/cancel.d.ts +25 -0
  307. package/dist/plugins/ops/cancel.js +43 -0
  308. package/dist/plugins/ops/embed.d.ts +15 -0
  309. package/dist/plugins/ops/embed.js +193 -0
  310. package/dist/plugins/ops/kv-cache-retention.d.ts +11 -0
  311. package/dist/plugins/ops/kv-cache-retention.js +121 -0
  312. package/dist/plugins/ops/kv-cache-utils.d.ts +21 -0
  313. package/dist/plugins/ops/kv-cache-utils.js +120 -0
  314. package/dist/plugins/ops/load-model.d.ts +8 -0
  315. package/dist/plugins/ops/load-model.js +82 -0
  316. package/dist/plugins/ops/transcribe.d.ts +20 -0
  317. package/dist/plugins/ops/transcribe.js +343 -0
  318. package/dist/plugins/ops/translate.d.ts +7 -0
  319. package/dist/plugins/ops/translate.js +256 -0
  320. package/dist/plugins/ops/unload-model.d.ts +2 -0
  321. package/dist/plugins/ops/unload-model.js +45 -0
  322. package/dist/plugins/registry.d.ts +9 -0
  323. package/dist/plugins/registry.js +107 -0
  324. package/dist/profiling/aggregator.d.ts +14 -0
  325. package/dist/profiling/aggregator.js +100 -0
  326. package/dist/profiling/backend-diagnostics.d.ts +8 -0
  327. package/dist/profiling/backend-diagnostics.js +36 -0
  328. package/dist/profiling/clock.d.ts +8 -0
  329. package/dist/profiling/clock.js +23 -0
  330. package/dist/profiling/controller.d.ts +34 -0
  331. package/dist/profiling/controller.js +130 -0
  332. package/dist/profiling/delegation-profiler.d.ts +31 -0
  333. package/dist/profiling/delegation-profiler.js +155 -0
  334. package/dist/profiling/envelope.d.ts +10 -0
  335. package/dist/profiling/envelope.js +45 -0
  336. package/dist/profiling/events.d.ts +19 -0
  337. package/dist/profiling/events.js +35 -0
  338. package/dist/profiling/exporters.d.ts +9 -0
  339. package/dist/profiling/exporters.js +242 -0
  340. package/dist/profiling/index.d.ts +91 -0
  341. package/dist/profiling/index.js +87 -0
  342. package/dist/profiling/model-execution.d.ts +15 -0
  343. package/dist/profiling/model-execution.js +34 -0
  344. package/dist/profiling/operation-metrics.d.ts +24 -0
  345. package/dist/profiling/operation-metrics.js +370 -0
  346. package/dist/profiling/operation-wrappers.d.ts +17 -0
  347. package/dist/profiling/operation-wrappers.js +196 -0
  348. package/dist/profiling/profiler.d.ts +27 -0
  349. package/dist/profiling/profiler.js +127 -0
  350. package/dist/profiling/ring-buffer.d.ts +18 -0
  351. package/dist/profiling/ring-buffer.js +83 -0
  352. package/dist/profiling/types.d.ts +52 -0
  353. package/dist/profiling/types.js +1 -0
  354. package/dist/rag/handler.d.ts +2 -0
  355. package/dist/rag/handler.js +272 -0
  356. package/dist/rag/handlers/chunk.d.ts +2 -0
  357. package/dist/rag/handlers/chunk.js +7 -0
  358. package/dist/rag/handlers/close-workspace.d.ts +2 -0
  359. package/dist/rag/handlers/close-workspace.js +9 -0
  360. package/dist/rag/handlers/delete-embeddings.d.ts +2 -0
  361. package/dist/rag/handlers/delete-embeddings.js +11 -0
  362. package/dist/rag/handlers/delete-workspace.d.ts +2 -0
  363. package/dist/rag/handlers/delete-workspace.js +14 -0
  364. package/dist/rag/handlers/ingest.d.ts +9 -0
  365. package/dist/rag/handlers/ingest.js +19 -0
  366. package/dist/rag/handlers/list-workspaces.d.ts +2 -0
  367. package/dist/rag/handlers/list-workspaces.js +4 -0
  368. package/dist/rag/handlers/reindex.d.ts +9 -0
  369. package/dist/rag/handlers/reindex.js +16 -0
  370. package/dist/rag/handlers/save-embeddings.d.ts +9 -0
  371. package/dist/rag/handlers/save-embeddings.js +15 -0
  372. package/dist/rag/handlers/search.d.ts +2 -0
  373. package/dist/rag/handlers/search.js +16 -0
  374. package/dist/rag/index.d.ts +11 -0
  375. package/dist/rag/index.js +11 -0
  376. package/dist/rag/rag-operation-manager.d.ts +28 -0
  377. package/dist/rag/rag-operation-manager.js +76 -0
  378. package/dist/rag/rag-workspace-manager.d.ts +14 -0
  379. package/dist/rag/rag-workspace-manager.js +143 -0
  380. package/dist/registry.d.ts +2 -0
  381. package/dist/registry.js +166 -0
  382. package/dist/resources/collector.d.ts +643 -0
  383. package/dist/resources/collector.js +226 -0
  384. package/dist/resources/instance.d.ts +4 -0
  385. package/dist/resources/instance.js +15 -0
  386. package/dist/resources/native.d.ts +2 -0
  387. package/dist/resources/native.js +35 -0
  388. package/dist/resources/normalize.d.ts +12 -0
  389. package/dist/resources/normalize.js +73 -0
  390. package/dist/resources/types.d.ts +65 -0
  391. package/dist/resources/types.js +1 -0
  392. package/dist/runtime/cache-lock.d.ts +2 -0
  393. package/dist/runtime/cache-lock.js +67 -0
  394. package/dist/runtime/disposable-scope.d.ts +35 -0
  395. package/dist/runtime/disposable-scope.js +69 -0
  396. package/dist/runtime/env.d.ts +16 -0
  397. package/dist/runtime/env.js +53 -0
  398. package/dist/runtime/index.d.ts +5 -0
  399. package/dist/runtime/index.js +2 -0
  400. package/dist/runtime/lifecycle.d.ts +20 -0
  401. package/dist/runtime/lifecycle.js +102 -0
  402. package/dist/runtime/logging-stream-registry.d.ts +27 -0
  403. package/dist/runtime/logging-stream-registry.js +133 -0
  404. package/dist/runtime/model-config-utils.d.ts +14 -0
  405. package/dist/runtime/model-config-utils.js +128 -0
  406. package/dist/runtime/model-registry.d.ts +74 -0
  407. package/dist/runtime/model-registry.js +130 -0
  408. package/dist/runtime/registry-client.d.ts +4 -0
  409. package/dist/runtime/registry-client.js +118 -0
  410. package/dist/runtime/request-context.d.ts +72 -0
  411. package/dist/runtime/request-context.js +114 -0
  412. package/dist/runtime/request-id.d.ts +17 -0
  413. package/dist/runtime/request-id.js +29 -0
  414. package/dist/runtime/request-registry.d.ts +194 -0
  415. package/dist/runtime/request-registry.js +546 -0
  416. package/dist/runtime/runtime-lifecycle.d.ts +37 -0
  417. package/dist/runtime/runtime-lifecycle.js +163 -0
  418. package/dist/runtime/state.d.ts +26 -0
  419. package/dist/runtime/state.js +138 -0
  420. package/dist/schemas/audio-gen.d.ts +252 -0
  421. package/dist/schemas/audio-gen.js +69 -0
  422. package/dist/schemas/batch-completion-stream.d.ts +467 -0
  423. package/dist/schemas/batch-completion-stream.js +123 -0
  424. package/dist/schemas/bci-config.d.ts +74 -0
  425. package/dist/schemas/bci-config.js +58 -0
  426. package/dist/schemas/bci.d.ts +178 -0
  427. package/dist/schemas/bci.js +89 -0
  428. package/dist/schemas/cancel.d.ts +203 -0
  429. package/dist/schemas/cancel.js +135 -0
  430. package/dist/schemas/classification.d.ts +52 -0
  431. package/dist/schemas/classification.js +33 -0
  432. package/dist/schemas/common.d.ts +6420 -0
  433. package/dist/schemas/common.js +133 -0
  434. package/dist/schemas/completion-event.d.ts +248 -0
  435. package/dist/schemas/completion-event.js +94 -0
  436. package/dist/schemas/completion-stream.d.ts +475 -0
  437. package/dist/schemas/completion-stream.js +236 -0
  438. package/dist/schemas/config.d.ts +5022 -0
  439. package/dist/schemas/config.js +169 -0
  440. package/dist/schemas/delegate.d.ts +29 -0
  441. package/dist/schemas/delegate.js +40 -0
  442. package/dist/schemas/download-asset.d.ts +225 -0
  443. package/dist/schemas/download-asset.js +71 -0
  444. package/dist/schemas/embed.d.ts +41 -0
  445. package/dist/schemas/embed.js +35 -0
  446. package/dist/schemas/engine-addon-map.d.ts +20 -0
  447. package/dist/schemas/engine-addon-map.js +78 -0
  448. package/dist/schemas/error.d.ts +40 -0
  449. package/dist/schemas/error.js +65 -0
  450. package/dist/schemas/errors.d.ts +119 -0
  451. package/dist/schemas/errors.js +615 -0
  452. package/dist/schemas/finetune.d.ts +621 -0
  453. package/dist/schemas/finetune.js +163 -0
  454. package/dist/schemas/index.d.ts +45 -0
  455. package/dist/schemas/index.js +45 -0
  456. package/dist/schemas/json-schema.d.ts +15 -0
  457. package/dist/schemas/json-schema.js +2 -0
  458. package/dist/schemas/lifecycle.d.ts +38 -0
  459. package/dist/schemas/lifecycle.js +21 -0
  460. package/dist/schemas/llamacpp-config.d.ts +403 -0
  461. package/dist/schemas/llamacpp-config.js +133 -0
  462. package/dist/schemas/load-model.d.ts +25635 -0
  463. package/dist/schemas/load-model.js +629 -0
  464. package/dist/schemas/logging-stream.d.ts +33 -0
  465. package/dist/schemas/logging-stream.js +17 -0
  466. package/dist/schemas/mcp-adapter.d.ts +46 -0
  467. package/dist/schemas/mcp-adapter.js +1 -0
  468. package/dist/schemas/model-file.d.ts +27 -0
  469. package/dist/schemas/model-file.js +29 -0
  470. package/dist/schemas/model-info.d.ts +197 -0
  471. package/dist/schemas/model-info.js +144 -0
  472. package/dist/schemas/model-ops.d.ts +36 -0
  473. package/dist/schemas/model-ops.js +40 -0
  474. package/dist/schemas/model-src-utils.d.ts +192 -0
  475. package/dist/schemas/model-src-utils.js +61 -0
  476. package/dist/schemas/model-types.d.ts +275 -0
  477. package/dist/schemas/model-types.js +223 -0
  478. package/dist/schemas/ocr.d.ts +138 -0
  479. package/dist/schemas/ocr.js +59 -0
  480. package/dist/schemas/path-validation.d.ts +9 -0
  481. package/dist/schemas/path-validation.js +22 -0
  482. package/dist/schemas/plugin.d.ts +341 -0
  483. package/dist/schemas/plugin.js +241 -0
  484. package/dist/schemas/profiling.d.ts +695 -0
  485. package/dist/schemas/profiling.js +100 -0
  486. package/dist/schemas/provide.d.ts +50 -0
  487. package/dist/schemas/provide.js +32 -0
  488. package/dist/schemas/rag.d.ts +301 -0
  489. package/dist/schemas/rag.js +211 -0
  490. package/dist/schemas/registry.d.ts +453 -0
  491. package/dist/schemas/registry.js +104 -0
  492. package/dist/schemas/reload-config.d.ts +585 -0
  493. package/dist/schemas/reload-config.js +42 -0
  494. package/dist/schemas/runtime-context.d.ts +18 -0
  495. package/dist/schemas/runtime-context.js +7 -0
  496. package/dist/schemas/sdcpp-config.d.ts +1126 -0
  497. package/dist/schemas/sdcpp-config.js +748 -0
  498. package/dist/schemas/system-resources.d.ts +3966 -0
  499. package/dist/schemas/system-resources.js +149 -0
  500. package/dist/schemas/text-to-speech.d.ts +1352 -0
  501. package/dist/schemas/text-to-speech.js +364 -0
  502. package/dist/schemas/tools.d.ts +80 -0
  503. package/dist/schemas/tools.js +52 -0
  504. package/dist/schemas/transcription-config.d.ts +161 -0
  505. package/dist/schemas/transcription-config.js +156 -0
  506. package/dist/schemas/transcription.d.ts +261 -0
  507. package/dist/schemas/transcription.js +145 -0
  508. package/dist/schemas/translate.d.ts +138 -0
  509. package/dist/schemas/translate.js +103 -0
  510. package/dist/schemas/translation-config.d.ts +1031 -0
  511. package/dist/schemas/translation-config.js +170 -0
  512. package/dist/schemas/vla.d.ts +110 -0
  513. package/dist/schemas/vla.js +151 -0
  514. package/dist/selection.d.ts +8 -0
  515. package/dist/selection.js +14 -0
  516. package/dist/surface.d.ts +18 -0
  517. package/dist/surface.js +30 -0
  518. package/dist/utils/addon-responses.d.ts +58 -0
  519. package/dist/utils/addon-responses.js +1 -0
  520. package/dist/utils/aggregate-events.d.ts +25 -0
  521. package/dist/utils/aggregate-events.js +73 -0
  522. package/dist/utils/archive.d.ts +4 -0
  523. package/dist/utils/archive.js +180 -0
  524. package/dist/utils/asr-events.d.ts +14 -0
  525. package/dist/utils/asr-events.js +30 -0
  526. package/dist/utils/audio/decoder.d.ts +4 -0
  527. package/dist/utils/audio/decoder.js +80 -0
  528. package/dist/utils/audio-input.d.ts +9 -0
  529. package/dist/utils/audio-input.js +34 -0
  530. package/dist/utils/cache/auto-history.d.ts +3 -0
  531. package/dist/utils/cache/auto-history.js +26 -0
  532. package/dist/utils/cache/paths.d.ts +42 -0
  533. package/dist/utils/cache/paths.js +133 -0
  534. package/dist/utils/cache/types.d.ts +7 -0
  535. package/dist/utils/cache/types.js +1 -0
  536. package/dist/utils/cache-normalize.d.ts +9 -0
  537. package/dist/utils/cache-normalize.js +20 -0
  538. package/dist/utils/checksum.d.ts +3 -0
  539. package/dist/utils/checksum.js +29 -0
  540. package/dist/utils/clear-storage.d.ts +4 -0
  541. package/dist/utils/clear-storage.js +14 -0
  542. package/dist/utils/completion-normalizer.d.ts +17 -0
  543. package/dist/utils/completion-normalizer.js +453 -0
  544. package/dist/utils/config-transform.d.ts +2 -0
  545. package/dist/utils/config-transform.js +21 -0
  546. package/dist/utils/decorate-promise.d.ts +33 -0
  547. package/dist/utils/decorate-promise.js +35 -0
  548. package/dist/utils/encoding.d.ts +16 -0
  549. package/dist/utils/encoding.js +27 -0
  550. package/dist/utils/expand-gguf-shards.d.ts +2 -0
  551. package/dist/utils/expand-gguf-shards.js +22 -0
  552. package/dist/utils/formatting.d.ts +18 -0
  553. package/dist/utils/formatting.js +56 -0
  554. package/dist/utils/gguf-tensor-extractor.d.ts +14 -0
  555. package/dist/utils/gguf-tensor-extractor.js +311 -0
  556. package/dist/utils/global-singleton.d.ts +1 -0
  557. package/dist/utils/global-singleton.js +10 -0
  558. package/dist/utils/index.d.ts +11 -0
  559. package/dist/utils/index.js +11 -0
  560. package/dist/utils/load-model-validation.d.ts +6 -0
  561. package/dist/utils/load-model-validation.js +18 -0
  562. package/dist/utils/mcp-adapter.d.ts +11 -0
  563. package/dist/utils/mcp-adapter.js +96 -0
  564. package/dist/utils/multi-gpu-mobile.d.ts +4 -0
  565. package/dist/utils/multi-gpu-mobile.js +10 -0
  566. package/dist/utils/object.d.ts +1 -0
  567. package/dist/utils/object.js +3 -0
  568. package/dist/utils/path-security.d.ts +22 -0
  569. package/dist/utils/path-security.js +69 -0
  570. package/dist/utils/qvac-paths.d.ts +1 -0
  571. package/dist/utils/qvac-paths.js +5 -0
  572. package/dist/utils/response-format.d.ts +2 -0
  573. package/dist/utils/response-format.js +20 -0
  574. package/dist/utils/shard-utils.d.ts +47 -0
  575. package/dist/utils/shard-utils.js +123 -0
  576. package/dist/utils/tool-helpers.d.ts +24 -0
  577. package/dist/utils/tool-helpers.js +111 -0
  578. package/dist/utils/tool-integration.d.ts +23 -0
  579. package/dist/utils/tool-integration.js +29 -0
  580. package/dist/utils/tools/dialect.d.ts +7 -0
  581. package/dist/utils/tools/dialect.js +20 -0
  582. package/dist/utils/tools/index.d.ts +2 -0
  583. package/dist/utils/tools/index.js +2 -0
  584. package/dist/utils/tools/parser.d.ts +5 -0
  585. package/dist/utils/tools/parser.js +69 -0
  586. package/dist/utils/tools/parsers/dsml.d.ts +3 -0
  587. package/dist/utils/tools/parsers/dsml.js +177 -0
  588. package/dist/utils/tools/parsers/gemma4native.d.ts +3 -0
  589. package/dist/utils/tools/parsers/gemma4native.js +65 -0
  590. package/dist/utils/tools/parsers/harmony.d.ts +3 -0
  591. package/dist/utils/tools/parsers/harmony.js +57 -0
  592. package/dist/utils/tools/parsers/hermes.d.ts +3 -0
  593. package/dist/utils/tools/parsers/hermes.js +122 -0
  594. package/dist/utils/tools/parsers/json.d.ts +5 -0
  595. package/dist/utils/tools/parsers/json.js +115 -0
  596. package/dist/utils/tools/parsers/pythonic.d.ts +3 -0
  597. package/dist/utils/tools/parsers/pythonic.js +267 -0
  598. package/dist/utils/tools/parsers/qwen35.d.ts +3 -0
  599. package/dist/utils/tools/parsers/qwen35.js +125 -0
  600. package/dist/utils/tools/shared.d.ts +17 -0
  601. package/dist/utils/tools/shared.js +52 -0
  602. package/dist/utils/transcribe-metadata.d.ts +20 -0
  603. package/dist/utils/transcribe-metadata.js +28 -0
  604. package/dist/utils/tts-stats.d.ts +25 -0
  605. package/dist/utils/tts-stats.js +16 -0
  606. package/dist/utils/withTimeout.d.ts +17 -0
  607. package/dist/utils/withTimeout.js +47 -0
  608. package/dist/utils/zod-error.d.ts +2 -0
  609. package/dist/utils/zod-error.js +4 -0
  610. package/package.json +266 -1
@@ -0,0 +1,29 @@
1
+ import type { CancelRequest, CancelResponse } from '../schemas/cancel.js';
2
+ /**
3
+ * Cancel handler entry point. Every long-running handler registers
4
+ * itself on the process-singleton `RequestRegistry`, so the cancel
5
+ * surface is two paths that route through the same registry primitive:
6
+ *
7
+ * - `{ operation: "request", requestId, clearCache? }` — targeted
8
+ * cancel by caller-generated id. Looks up the registry entry,
9
+ * fires its abort signal, optionally marks the underlying download
10
+ * transfer for cache clear. The "stop-button race" case (a cancel
11
+ * beats its own begin) is handled inside the
12
+ * registry via the cancel-before-begin tripwire.
13
+ *
14
+ * - `{ operation: "broad", modelId, kind? }` — abort every in-flight
15
+ * request on a model (optionally narrowed by `kind`). Used for
16
+ * model unload, app shutdown, and admin sweeps where the caller
17
+ * has no `requestId`. Delegates to the `cancel` bare op so the
18
+ * `ModelNotLoadedError` validation is shared with internal
19
+ * engine-side broad cancels.
20
+ *
21
+ * Always returns `success: true` plus a `cancelled` count (the number
22
+ * of contexts this call flipped to `cancelling` — already-cancelled
23
+ * contexts are not counted). A targeted cancel with no in-flight
24
+ * match still returns `success: true, cancelled: 0`; the
25
+ * cancel-before-begin tripwire ensures the cancel is applied
26
+ * retroactively if a matching begin arrives within the registry's
27
+ * race window.
28
+ */
29
+ export declare function cancelHandler(request: CancelRequest): CancelResponse;
@@ -0,0 +1,67 @@
1
+ import { cancel as cancelByModelId } from '../plugins/ops/cancel.js';
2
+ import { getRequestRegistry } from '../runtime/index.js';
3
+ import { markClearCacheForRequest } from '../handlers/load-model/download-manager.js';
4
+ import { getEngineLogger } from '../logging/index.js';
5
+ const logger = getEngineLogger();
6
+ /**
7
+ * Cancel handler entry point. Every long-running handler registers
8
+ * itself on the process-singleton `RequestRegistry`, so the cancel
9
+ * surface is two paths that route through the same registry primitive:
10
+ *
11
+ * - `{ operation: "request", requestId, clearCache? }` — targeted
12
+ * cancel by caller-generated id. Looks up the registry entry,
13
+ * fires its abort signal, optionally marks the underlying download
14
+ * transfer for cache clear. The "stop-button race" case (a cancel
15
+ * beats its own begin) is handled inside the
16
+ * registry via the cancel-before-begin tripwire.
17
+ *
18
+ * - `{ operation: "broad", modelId, kind? }` — abort every in-flight
19
+ * request on a model (optionally narrowed by `kind`). Used for
20
+ * model unload, app shutdown, and admin sweeps where the caller
21
+ * has no `requestId`. Delegates to the `cancel` bare op so the
22
+ * `ModelNotLoadedError` validation is shared with internal
23
+ * engine-side broad cancels.
24
+ *
25
+ * Always returns `success: true` plus a `cancelled` count (the number
26
+ * of contexts this call flipped to `cancelling` — already-cancelled
27
+ * contexts are not counted). A targeted cancel with no in-flight
28
+ * match still returns `success: true, cancelled: 0`; the
29
+ * cancel-before-begin tripwire ensures the cancel is applied
30
+ * retroactively if a matching begin arrives within the registry's
31
+ * race window.
32
+ */
33
+ export function cancelHandler(request) {
34
+ try {
35
+ if (request.operation === 'request') {
36
+ if (request.clearCache) {
37
+ markClearCacheForRequest(request.requestId);
38
+ }
39
+ const cancelled = getRequestRegistry().cancel({
40
+ requestId: request.requestId
41
+ });
42
+ if (cancelled === 0) {
43
+ // info-level (not debug) because the decorated-promise pattern
44
+ // makes "no in-flight match" a common and user-visible case:
45
+ // a Stop button fired after the request settled but before
46
+ // the UI cleared lands here. The cancel-before-begin tripwire
47
+ // inside the registry already captured the cancel for any
48
+ // matching begin in flight; this log just helps operators
49
+ // debugging "my Stop button isn't working" without lowering
50
+ // the log level.
51
+ logger.info(`[cancel] no in-flight request matched requestId=${request.requestId}`);
52
+ }
53
+ return { type: 'cancel', success: true, cancelled };
54
+ }
55
+ // operation === "broad"
56
+ const cancelled = cancelByModelId({ modelId: request.modelId }, request.kind ? { kind: request.kind } : undefined);
57
+ return { type: 'cancel', success: true, cancelled };
58
+ }
59
+ catch (error) {
60
+ logger.error('Error during cancellation:', error);
61
+ return {
62
+ type: 'cancel',
63
+ success: false,
64
+ error: error instanceof Error ? error.message : 'Unknown error'
65
+ };
66
+ }
67
+ }
@@ -0,0 +1,41 @@
1
+ import type { CompletionOrchestrateRequest, CompletionOrchestrateResponse, CompletionStreamRequest, CompletionStreamResponse } from '../schemas/index.js';
2
+ import { type ToolCallbackResult } from '../schemas/completion-stream.js';
3
+ /**
4
+ * Reads newline-delimited JSON tool results off the duplex request stream
5
+ * and resolves per-callId waiters. Chunk boundaries don't align with lines,
6
+ * so a partial tail is buffered until its newline arrives.
7
+ *
8
+ * The pump is torn down by `close()` (called from a `finally` when
9
+ * orchestration ends, however it ends) so the upstream iterator is returned
10
+ * and the reader stops holding the request stream open.
11
+ */
12
+ export declare class ToolResultReader {
13
+ private buffer;
14
+ private readonly results;
15
+ private readonly waiters;
16
+ private failed;
17
+ private closed;
18
+ private readonly iterator;
19
+ constructor(inputStream: AsyncIterable<Buffer>);
20
+ private pump;
21
+ private failWaiters;
22
+ private accept;
23
+ waitFor(callId: string, timeoutMs?: number): Promise<ToolCallbackResult>;
24
+ /**
25
+ * Stop the pump and release the upstream iterator. Idempotent. Pending
26
+ * waiters are failed so a caller blocked in `waitFor` can't hang after the
27
+ * stream is torn down.
28
+ */
29
+ close(): void;
30
+ }
31
+ type RunTurn = (request: CompletionStreamRequest) => AsyncGenerator<CompletionStreamResponse, void, unknown>;
32
+ /**
33
+ * The orchestration core, pure over its collaborators so it unit-tests
34
+ * without the plugin registry: runs completion turns, forwards their events
35
+ * downstream, and when a turn requests tool calls, emits `toolCallback`
36
+ * frames and blocks on the client's results before starting the next turn
37
+ * with the extended history.
38
+ */
39
+ export declare function orchestrateCompletion(request: CompletionOrchestrateRequest, runTurn: RunTurn, reader: Pick<ToolResultReader, 'waitFor'>): AsyncGenerator<CompletionOrchestrateResponse>;
40
+ export declare function handleCompletionOrchestrate(request: CompletionOrchestrateRequest, inputStream: AsyncIterable<Buffer>): AsyncGenerator<CompletionOrchestrateResponse>;
41
+ export {};
@@ -0,0 +1,190 @@
1
+ import { toolCallbackResultSchema } from '../schemas/completion-stream.js';
2
+ import { aggregateEvents } from '../utils/aggregate-events.js';
3
+ import { CompletionFailedError } from '../errors/index.js';
4
+ import { dispatchPluginStream } from '../handlers/plugin-dispatch.js';
5
+ const DEFAULT_MAX_TOOL_TURNS = 8;
6
+ // Generous ceiling on one client-side tool execution: long enough for a slow
7
+ // MCP round trip, short enough that a client that stopped reading can't pin a
8
+ // model slot forever.
9
+ const TOOL_RESULT_TIMEOUT_MS = 5 * 60 * 1000;
10
+ // A single tool result is small JSON; anything past this without a newline is a
11
+ // malformed/hostile client trying to grow the buffer without bound. Fail loudly
12
+ // instead of accumulating forever.
13
+ const MAX_TOOL_RESULT_LINE_BYTES = 16 * 1024 * 1024;
14
+ /**
15
+ * Reads newline-delimited JSON tool results off the duplex request stream
16
+ * and resolves per-callId waiters. Chunk boundaries don't align with lines,
17
+ * so a partial tail is buffered until its newline arrives.
18
+ *
19
+ * The pump is torn down by `close()` (called from a `finally` when
20
+ * orchestration ends, however it ends) so the upstream iterator is returned
21
+ * and the reader stops holding the request stream open.
22
+ */
23
+ export class ToolResultReader {
24
+ buffer = '';
25
+ results = new Map();
26
+ waiters = new Map();
27
+ failed;
28
+ closed = false;
29
+ iterator;
30
+ constructor(inputStream) {
31
+ this.iterator = inputStream[Symbol.asyncIterator]();
32
+ void this.pump();
33
+ }
34
+ async pump() {
35
+ try {
36
+ while (!this.closed) {
37
+ const next = await this.iterator.next();
38
+ if (next.done)
39
+ break;
40
+ this.buffer += next.value.toString();
41
+ let nl;
42
+ while ((nl = this.buffer.indexOf('\n')) !== -1) {
43
+ const line = this.buffer.slice(0, nl);
44
+ this.buffer = this.buffer.slice(nl + 1);
45
+ if (line.trim())
46
+ this.accept(line);
47
+ }
48
+ if (this.buffer.length > MAX_TOOL_RESULT_LINE_BYTES) {
49
+ throw new CompletionFailedError('tool result frame exceeds the maximum size');
50
+ }
51
+ }
52
+ if (!this.closed && this.buffer.trim())
53
+ this.accept(this.buffer);
54
+ }
55
+ catch (error) {
56
+ this.failWaiters(error instanceof Error ? error : new Error(String(error)));
57
+ }
58
+ }
59
+ failWaiters(error) {
60
+ this.failed = error;
61
+ // Wake every waiter; each rethrows via waitFor's failure check.
62
+ for (const [callId, resolve] of this.waiters) {
63
+ this.waiters.delete(callId);
64
+ resolve({ callId, error: error.message });
65
+ }
66
+ }
67
+ accept(line) {
68
+ const parsed = toolCallbackResultSchema.parse(JSON.parse(line));
69
+ const waiter = this.waiters.get(parsed.callId);
70
+ if (waiter) {
71
+ this.waiters.delete(parsed.callId);
72
+ waiter(parsed);
73
+ }
74
+ else {
75
+ this.results.set(parsed.callId, parsed);
76
+ }
77
+ }
78
+ async waitFor(callId, timeoutMs = TOOL_RESULT_TIMEOUT_MS) {
79
+ const buffered = this.results.get(callId);
80
+ if (buffered) {
81
+ this.results.delete(callId);
82
+ return buffered;
83
+ }
84
+ if (this.failed)
85
+ throw this.failed;
86
+ return await new Promise((resolve, reject) => {
87
+ const timer = setTimeout(() => {
88
+ this.waiters.delete(callId);
89
+ reject(new CompletionFailedError(`tool callback ${callId} timed out`));
90
+ }, timeoutMs);
91
+ this.waiters.set(callId, (result) => {
92
+ clearTimeout(timer);
93
+ resolve(result);
94
+ });
95
+ });
96
+ }
97
+ /**
98
+ * Stop the pump and release the upstream iterator. Idempotent. Pending
99
+ * waiters are failed so a caller blocked in `waitFor` can't hang after the
100
+ * stream is torn down.
101
+ */
102
+ close() {
103
+ if (this.closed)
104
+ return;
105
+ this.closed = true;
106
+ this.buffer = '';
107
+ this.results.clear();
108
+ if (!this.failed)
109
+ this.failWaiters(new CompletionFailedError('tool result stream closed'));
110
+ void this.iterator.return?.();
111
+ }
112
+ }
113
+ /**
114
+ * The orchestration core, pure over its collaborators so it unit-tests
115
+ * without the plugin registry: runs completion turns, forwards their events
116
+ * downstream, and when a turn requests tool calls, emits `toolCallback`
117
+ * frames and blocks on the client's results before starting the next turn
118
+ * with the extended history.
119
+ */
120
+ export async function* orchestrateCompletion(request, runTurn, reader) {
121
+ // Pull the loop bound out of the per-turn base; everything else (including
122
+ // `requestId`) flows into each inner completionStream turn. Threading the
123
+ // orchestrate `requestId` down is what makes `cancel({ requestId })` reach
124
+ // the turn that's currently generating — the expensive part a Stop button
125
+ // targets. Turns run strictly sequentially: turn N's request-registry entry
126
+ // is disposed (on the `for await` break below) before turn N+1 calls
127
+ // `begin(...)`, so reusing the id can't collide in `entries`. A cancel that
128
+ // lands between turns (during a tool-result wait) is caught by the
129
+ // registry's cancel-before-begin tripwire and aborts the next turn's
130
+ // `begin(...)`. Each turn's `history` and `type` are set below.
131
+ const { maxToolTurns, ...turnBase } = request;
132
+ const maxTurns = maxToolTurns ?? DEFAULT_MAX_TOOL_TURNS;
133
+ let history = [...request.history];
134
+ for (let turn = 0; turn < maxTurns; turn++) {
135
+ const turnEvents = [];
136
+ const innerRequest = {
137
+ ...turnBase,
138
+ history,
139
+ type: 'completionStream'
140
+ };
141
+ for await (const response of runTurn(innerRequest)) {
142
+ turnEvents.push(...response.events);
143
+ yield { type: 'completionOrchestrate', turn, events: response.events };
144
+ if (response.done)
145
+ break;
146
+ }
147
+ const { toolCalls, rawFullText, contentText, error, cancelled } = aggregateEvents(turnEvents);
148
+ // Terminal turn: the model answered (or failed/was cancelled) without
149
+ // requesting tools — the client folds the forwarded events itself.
150
+ if (error || cancelled || toolCalls.length === 0) {
151
+ yield { type: 'completionOrchestrate', done: true };
152
+ return;
153
+ }
154
+ // The assistant's tool-call turn goes back verbatim so the model sees
155
+ // its own call syntax; each result follows as a `tool` message.
156
+ history = [...history, { role: 'assistant', content: rawFullText ?? contentText }];
157
+ for (const call of toolCalls) {
158
+ yield {
159
+ type: 'completionOrchestrate',
160
+ turn,
161
+ toolCallback: { callId: call.id, name: call.name, arguments: call.arguments }
162
+ };
163
+ const result = await reader.waitFor(call.id);
164
+ const content = result.error !== undefined
165
+ ? JSON.stringify({ error: result.error })
166
+ : JSON.stringify(result.result ?? null);
167
+ history = [...history, { role: 'tool', content }];
168
+ }
169
+ }
170
+ // Turn cap reached without a natural end. Emit a terminal done frame with a
171
+ // stopReason so the client can tell this was truncated (the model still
172
+ // wanted a tool) rather than a natural finish; the last turn's events carry
173
+ // whatever the model produced.
174
+ yield { type: 'completionOrchestrate', done: true, stopReason: 'maxToolTurns' };
175
+ }
176
+ export async function* handleCompletionOrchestrate(request, inputStream) {
177
+ const reader = new ToolResultReader(inputStream);
178
+ const runTurn = async function* (innerRequest) {
179
+ yield* dispatchPluginStream(innerRequest.modelId, 'completionStream', innerRequest);
180
+ };
181
+ try {
182
+ yield* orchestrateCompletion(request, runTurn, reader);
183
+ }
184
+ finally {
185
+ // Tear the reader down however orchestration ends -- natural finish, turn
186
+ // cap, error, or the client abandoning the stream -- so the pump stops and
187
+ // the request stream isn't held open.
188
+ reader.close();
189
+ }
190
+ }
@@ -0,0 +1,11 @@
1
+ import type { DeleteCacheRequest, DeleteCacheResponse } from '../schemas/index.js';
2
+ /**
3
+ * RPC handler for `deleteCache(...)`. The three KV-cache bookkeeping
4
+ * layers (on-disk `.bin`, `initializedCaches`, `cachedMessageCounts`)
5
+ * are private to `kv-cache-session.ts`; this handler delegates to that
6
+ * module's single administrative entry point (`deleteKvCacheState`).
7
+ * The handler must have **zero** direct references to
8
+ * `fsPromises.unlink`, the `initializedCaches` set, or the
9
+ * `cachedMessageCounts` map.
10
+ */
11
+ export declare function handleDeleteCache(request: DeleteCacheRequest): Promise<DeleteCacheResponse>;
@@ -0,0 +1,37 @@
1
+ import { deleteKvCacheState } from '../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
2
+ import { getEngineLogger } from '../logging/index.js';
3
+ const logger = getEngineLogger();
4
+ /**
5
+ * RPC handler for `deleteCache(...)`. The three KV-cache bookkeeping
6
+ * layers (on-disk `.bin`, `initializedCaches`, `cachedMessageCounts`)
7
+ * are private to `kv-cache-session.ts`; this handler delegates to that
8
+ * module's single administrative entry point (`deleteKvCacheState`).
9
+ * The handler must have **zero** direct references to
10
+ * `fsPromises.unlink`, the `initializedCaches` set, or the
11
+ * `cachedMessageCounts` map.
12
+ */
13
+ export async function handleDeleteCache(request) {
14
+ try {
15
+ if ('all' in request && request.all) {
16
+ await deleteKvCacheState({ all: true });
17
+ }
18
+ else if ('kvCacheKey' in request) {
19
+ await deleteKvCacheState({
20
+ kvCacheKey: request.kvCacheKey,
21
+ ...(request.modelId !== undefined && { modelId: request.modelId })
22
+ });
23
+ }
24
+ return {
25
+ type: 'deleteCache',
26
+ success: true
27
+ };
28
+ }
29
+ catch (error) {
30
+ logger.error('Error deleting cache:', error);
31
+ return {
32
+ type: 'deleteCache',
33
+ success: false,
34
+ error: error instanceof Error ? error.message : 'Unknown error'
35
+ };
36
+ }
37
+ }
@@ -0,0 +1,2 @@
1
+ import type { DownloadAssetRequest, DownloadAssetResponse, ModelProgressUpdate } from '../schemas/index.js';
2
+ export declare function handleDownloadAsset(request: DownloadAssetRequest, progressCallback?: (update: ModelProgressUpdate) => void): Promise<DownloadAssetResponse>;
@@ -0,0 +1,146 @@
1
+ var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
+ if (value !== null && value !== void 0) {
3
+ if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
+ var dispose, inner;
5
+ if (async) {
6
+ if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
+ dispose = value[Symbol.asyncDispose];
8
+ }
9
+ if (dispose === void 0) {
10
+ if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
+ dispose = value[Symbol.dispose];
12
+ if (async) inner = dispose;
13
+ }
14
+ if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
+ if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
+ env.stack.push({ value: value, dispose: dispose, async: async });
17
+ }
18
+ else if (async) {
19
+ env.stack.push({ async: true });
20
+ }
21
+ return value;
22
+ };
23
+ var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
+ return function (env) {
25
+ function fail(e) {
26
+ env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
+ env.hasError = true;
28
+ }
29
+ var r, s = 0;
30
+ function next() {
31
+ while (r = env.stack.pop()) {
32
+ try {
33
+ if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
+ if (r.dispose) {
35
+ var result = r.dispose.call(r.value);
36
+ if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
+ }
38
+ else s |= 1;
39
+ }
40
+ catch (e) {
41
+ fail(e);
42
+ }
43
+ }
44
+ if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
+ if (env.hasError) throw env.error;
46
+ }
47
+ return next();
48
+ };
49
+ })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
+ var e = new Error(message);
51
+ return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
+ });
53
+ import { PROFILING_KEY, OPERATION_EVENT_KEY } from '../schemas/index.js';
54
+ import { resolveModelPath, resolveModelPathWithStats } from '../handlers/load-model/resolve.js';
55
+ import { buildDownloadProfilingFields } from '../handlers/load-model/types.js';
56
+ import { nowMs, generateProfileId } from '../profiling/clock.js';
57
+ import { getEngineLogger } from '../logging/index.js';
58
+ import { getRequestRegistry, withRequestContext } from '../runtime/index.js';
59
+ import { generateRandomRequestId } from '../runtime/request-id.js';
60
+ import { InferenceCancelledError } from '../errors/index.js';
61
+ const logger = getEngineLogger();
62
+ export async function handleDownloadAsset(request, progressCallback) {
63
+ const env_1 = { stack: [], error: void 0, hasError: false };
64
+ try {
65
+ const { assetSrc, seed } = request;
66
+ const profilingMeta = request[PROFILING_KEY];
67
+ const profilingEnabled = profilingMeta?.enabled !== false && !!profilingMeta;
68
+ const requestId = request.requestId ?? generateRandomRequestId();
69
+ // `downloadAsset` is artifact-shaped, not model-shaped — there is no
70
+ // `modelId` to register on the registry entry. Cancel by `requestId`
71
+ // is the primary path; `cancel({ modelId })` is intentionally a
72
+ // non-match for this kind.
73
+ const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
74
+ requestId,
75
+ kind: 'downloadAsset'
76
+ }), true);
77
+ const log = withRequestContext(getEngineLogger(), ctx);
78
+ log.debug(`downloadAsset start assetSrc=${assetSrc}`);
79
+ const hooks = {
80
+ requestBinding: {
81
+ signal: ctx.signal,
82
+ scope: ctx.scope,
83
+ requestId
84
+ }
85
+ };
86
+ try {
87
+ const totalDownloadStart = profilingEnabled ? nowMs() : 0;
88
+ let sourceType;
89
+ let downloadStats;
90
+ if (profilingEnabled) {
91
+ const result = await resolveModelPathWithStats(assetSrc, progressCallback, seed, ctx.signal, hooks);
92
+ sourceType = result.sourceType;
93
+ downloadStats = result.downloadStats;
94
+ }
95
+ else {
96
+ await resolveModelPath(assetSrc, progressCallback, seed, ctx.signal, hooks);
97
+ }
98
+ const response = {
99
+ type: 'downloadAsset',
100
+ success: true,
101
+ assetId: assetSrc
102
+ };
103
+ if (profilingEnabled) {
104
+ const totalDownloadTimeMs = nowMs() - totalDownloadStart;
105
+ const profileId = profilingMeta?.id ?? generateProfileId();
106
+ const { gauges, tags } = buildDownloadProfilingFields(downloadStats, sourceType);
107
+ gauges['totalDownloadTime'] = totalDownloadTimeMs;
108
+ const operationEvent = {
109
+ op: 'downloadAsset',
110
+ kind: 'handler',
111
+ ms: totalDownloadTimeMs,
112
+ profileId,
113
+ gauges: Object.keys(gauges).length > 0 ? gauges : undefined,
114
+ tags: Object.keys(tags).length > 0 ? tags : undefined
115
+ };
116
+ response[OPERATION_EVENT_KEY] = operationEvent;
117
+ }
118
+ return response;
119
+ }
120
+ catch (error) {
121
+ // Mirror the load-model handler's cancel contract: a typed cancel
122
+ // bubbles up as the rejection so client-side callers can branch on
123
+ // `instanceof InferenceCancelledError`. Every other error keeps
124
+ // the legacy `success: false` envelope.
125
+ if (error instanceof InferenceCancelledError) {
126
+ log.info(`downloadAsset cancelled requestId=${requestId}`);
127
+ throw error;
128
+ }
129
+ logger.error('Error downloading asset:', error);
130
+ return {
131
+ type: 'downloadAsset',
132
+ success: false,
133
+ error: error instanceof Error ? error.message : 'Unknown error'
134
+ };
135
+ }
136
+ }
137
+ catch (e_1) {
138
+ env_1.error = e_1;
139
+ env_1.hasError = true;
140
+ }
141
+ finally {
142
+ const result_1 = __disposeResources(env_1);
143
+ if (result_1)
144
+ await result_1;
145
+ }
146
+ }
@@ -0,0 +1,2 @@
1
+ import type { FinetuneProgressResponse, FinetuneRequest, FinetuneResult } from '../schemas/index.js';
2
+ export declare function handleFinetune(request: FinetuneRequest, progressCallback?: (update: FinetuneProgressResponse) => void): Promise<FinetuneResult>;
@@ -0,0 +1,18 @@
1
+ import { dispatchPluginReply } from '../handlers/plugin-dispatch.js';
2
+ import { startFinetune } from '../plugins/builtin/llamacpp-completion/ops/finetune.js';
3
+ export async function handleFinetune(request, progressCallback) {
4
+ if (progressCallback &&
5
+ (request.operation === 'start' ||
6
+ request.operation === 'resume' ||
7
+ request.operation === undefined)) {
8
+ return startFinetune(request, (progress) => {
9
+ const update = {
10
+ type: 'finetune:progress',
11
+ modelId: request.modelId,
12
+ ...progress
13
+ };
14
+ progressCallback(update);
15
+ });
16
+ }
17
+ return dispatchPluginReply(request.modelId, 'finetune', request);
18
+ }
@@ -0,0 +1,2 @@
1
+ import type { GetLoadedModelInfoRequest, GetLoadedModelInfoResponse } from '../schemas/index.js';
2
+ export declare function handleGetLoadedModelInfo(request: GetLoadedModelInfoRequest): GetLoadedModelInfoResponse;
@@ -0,0 +1,45 @@
1
+ import { getModelEntry } from '../runtime/model-registry.js';
2
+ import { getPlugin } from '../plugins/registry.js';
3
+ import { detectToolDialectFromName } from '../utils/tools/index.js';
4
+ import { ModelNotFoundError } from '../errors/index.js';
5
+ import { getEngineLogger } from '../logging/index.js';
6
+ const logger = getEngineLogger();
7
+ export function handleGetLoadedModelInfo(request) {
8
+ const { modelId } = request;
9
+ const entry = getModelEntry(modelId);
10
+ if (!entry) {
11
+ throw new ModelNotFoundError(modelId);
12
+ }
13
+ if (entry.isDelegated) {
14
+ const info = {
15
+ modelId: entry.id,
16
+ isDelegated: true,
17
+ handlers: [],
18
+ providerInfo: {
19
+ providerPublicKey: entry.delegated.providerPublicKey
20
+ }
21
+ };
22
+ return { type: 'getLoadedModelInfo', info };
23
+ }
24
+ const plugin = getPlugin(entry.local.modelType);
25
+ if (!plugin) {
26
+ logger.warn(`getLoadedModelInfo: no plugin registered for modelType "${entry.local.modelType}" on loaded model "${modelId}"`);
27
+ }
28
+ const handlers = plugin ? Object.keys(plugin.handlers) : [];
29
+ const info = {
30
+ modelId: entry.id,
31
+ isDelegated: false,
32
+ modelType: entry.local.modelType,
33
+ handlers,
34
+ loadedAt: entry.local.loadedAt,
35
+ ...(plugin && { displayName: plugin.displayName }),
36
+ ...(plugin && { addonPackage: plugin.addonPackage }),
37
+ ...(entry.local.name && { name: entry.local.name }),
38
+ ...(entry.local.path && { path: entry.local.path }),
39
+ // Same detection the completion normalizer uses, so callers see the dialect it parses.
40
+ ...(handlers.includes('completionStream') && {
41
+ toolDialect: detectToolDialectFromName(entry.local.name, entry.local.path)
42
+ })
43
+ };
44
+ return { type: 'getLoadedModelInfo', info };
45
+ }
@@ -0,0 +1,2 @@
1
+ import type { GetModelInfoRequest, GetModelInfoResponse } from '../schemas/index.js';
2
+ export declare function handleGetModelInfo(request: GetModelInfoRequest): Promise<GetModelInfoResponse>;