@qvac/inference 0.0.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/LICENSE +179 -0
  2. package/NOTICE +986 -0
  3. package/README.md +138 -0
  4. package/dist/api/audio-gen.d.ts +18 -0
  5. package/dist/api/audio-gen.js +148 -0
  6. package/dist/api/batch-completion.d.ts +14 -0
  7. package/dist/api/batch-completion.js +332 -0
  8. package/dist/api/bci-transcribe.d.ts +56 -0
  9. package/dist/api/bci-transcribe.js +172 -0
  10. package/dist/api/cancel.d.ts +34 -0
  11. package/dist/api/cancel.js +82 -0
  12. package/dist/api/classify.d.ts +22 -0
  13. package/dist/api/classify.js +46 -0
  14. package/dist/api/completion-stream.d.ts +106 -0
  15. package/dist/api/completion-stream.js +351 -0
  16. package/dist/api/delete-cache.d.ts +29 -0
  17. package/dist/api/delete-cache.js +49 -0
  18. package/dist/api/diffusion.d.ts +92 -0
  19. package/dist/api/diffusion.js +176 -0
  20. package/dist/api/download-asset.d.ts +49 -0
  21. package/dist/api/download-asset.js +87 -0
  22. package/dist/api/embed.d.ts +41 -0
  23. package/dist/api/embed.js +29 -0
  24. package/dist/api/finetune.d.ts +89 -0
  25. package/dist/api/finetune.js +138 -0
  26. package/dist/api/get-loaded-model-info.d.ts +24 -0
  27. package/dist/api/get-loaded-model-info.js +34 -0
  28. package/dist/api/get-model-info.d.ts +43 -0
  29. package/dist/api/get-model-info.js +21 -0
  30. package/dist/api/get-system-resources.d.ts +2 -0
  31. package/dist/api/get-system-resources.js +16 -0
  32. package/dist/api/heartbeat.d.ts +29 -0
  33. package/dist/api/heartbeat.js +37 -0
  34. package/dist/api/index.d.ts +35 -0
  35. package/dist/api/index.js +35 -0
  36. package/dist/api/invoke-plugin.d.ts +30 -0
  37. package/dist/api/invoke-plugin.js +54 -0
  38. package/dist/api/load-model.d.ts +169 -0
  39. package/dist/api/load-model.js +129 -0
  40. package/dist/api/logging-stream-registry.d.ts +4 -0
  41. package/dist/api/logging-stream-registry.js +61 -0
  42. package/dist/api/logging-stream.d.ts +23 -0
  43. package/dist/api/logging-stream.js +36 -0
  44. package/dist/api/ocr.d.ts +30 -0
  45. package/dist/api/ocr.js +88 -0
  46. package/dist/api/parse-input.d.ts +2 -0
  47. package/dist/api/parse-input.js +14 -0
  48. package/dist/api/provide.d.ts +23 -0
  49. package/dist/api/provide.js +32 -0
  50. package/dist/api/rag.d.ts +251 -0
  51. package/dist/api/rag.js +485 -0
  52. package/dist/api/registry.d.ts +39 -0
  53. package/dist/api/registry.js +63 -0
  54. package/dist/api/resume.d.ts +29 -0
  55. package/dist/api/resume.js +36 -0
  56. package/dist/api/state.d.ts +22 -0
  57. package/dist/api/state.js +29 -0
  58. package/dist/api/stop-provider.d.ts +20 -0
  59. package/dist/api/stop-provider.js +30 -0
  60. package/dist/api/subscribe-logs.d.ts +27 -0
  61. package/dist/api/subscribe-logs.js +43 -0
  62. package/dist/api/suspend.d.ts +30 -0
  63. package/dist/api/suspend.js +37 -0
  64. package/dist/api/text-to-speech.d.ts +64 -0
  65. package/dist/api/text-to-speech.js +434 -0
  66. package/dist/api/transcribe.d.ts +75 -0
  67. package/dist/api/transcribe.js +256 -0
  68. package/dist/api/translate.d.ts +47 -0
  69. package/dist/api/translate.js +115 -0
  70. package/dist/api/unload-model.d.ts +17 -0
  71. package/dist/api/unload-model.js +42 -0
  72. package/dist/api/upscale.d.ts +44 -0
  73. package/dist/api/upscale.js +99 -0
  74. package/dist/api/video.d.ts +86 -0
  75. package/dist/api/video.js +176 -0
  76. package/dist/api/vla-helpers.d.ts +43 -0
  77. package/dist/api/vla-helpers.js +124 -0
  78. package/dist/api/vla.d.ts +68 -0
  79. package/dist/api/vla.js +113 -0
  80. package/dist/config/config-utils.d.ts +4 -0
  81. package/dist/config/config-utils.js +18 -0
  82. package/dist/config/resolve-config.d.ts +8 -0
  83. package/dist/config/resolve-config.js +74 -0
  84. package/dist/constants/audio.d.ts +1 -0
  85. package/dist/constants/audio.js +1 -0
  86. package/dist/constants/index.d.ts +2 -0
  87. package/dist/constants/index.js +2 -0
  88. package/dist/constants/registry.d.ts +1 -0
  89. package/dist/constants/registry.js +1 -0
  90. package/dist/dispatch.d.ts +18 -0
  91. package/dist/dispatch.js +271 -0
  92. package/dist/engine.d.ts +5 -0
  93. package/dist/engine.js +27 -0
  94. package/dist/errors/index.d.ts +416 -0
  95. package/dist/errors/index.js +681 -0
  96. package/dist/errors/options.d.ts +2 -0
  97. package/dist/errors/options.js +13 -0
  98. package/dist/handlers/cancelHandler.d.ts +29 -0
  99. package/dist/handlers/cancelHandler.js +67 -0
  100. package/dist/handlers/completion-orchestrate.d.ts +41 -0
  101. package/dist/handlers/completion-orchestrate.js +190 -0
  102. package/dist/handlers/delete-cache.d.ts +11 -0
  103. package/dist/handlers/delete-cache.js +37 -0
  104. package/dist/handlers/download-asset.d.ts +2 -0
  105. package/dist/handlers/download-asset.js +146 -0
  106. package/dist/handlers/finetune.d.ts +2 -0
  107. package/dist/handlers/finetune.js +18 -0
  108. package/dist/handlers/get-loaded-model-info.d.ts +2 -0
  109. package/dist/handlers/get-loaded-model-info.js +45 -0
  110. package/dist/handlers/get-model-info.d.ts +2 -0
  111. package/dist/handlers/get-model-info.js +173 -0
  112. package/dist/handlers/get-system-resources.d.ts +2 -0
  113. package/dist/handlers/get-system-resources.js +53 -0
  114. package/dist/handlers/heartbeat.d.ts +2 -0
  115. package/dist/handlers/heartbeat.js +3 -0
  116. package/dist/handlers/load-model/download-manager.d.ts +95 -0
  117. package/dist/handlers/load-model/download-manager.js +306 -0
  118. package/dist/handlers/load-model/download-stats.d.ts +7 -0
  119. package/dist/handlers/load-model/download-stats.js +126 -0
  120. package/dist/handlers/load-model/handler.d.ts +2 -0
  121. package/dist/handlers/load-model/handler.js +287 -0
  122. package/dist/handlers/load-model/http.d.ts +3 -0
  123. package/dist/handlers/load-model/http.js +857 -0
  124. package/dist/handlers/load-model/hyperdrive.d.ts +4 -0
  125. package/dist/handlers/load-model/hyperdrive.js +560 -0
  126. package/dist/handlers/load-model/index.d.ts +2 -0
  127. package/dist/handlers/load-model/index.js +2 -0
  128. package/dist/handlers/load-model/registry-client-options.d.ts +27 -0
  129. package/dist/handlers/load-model/registry-client-options.js +18 -0
  130. package/dist/handlers/load-model/registry-companion-set.d.ts +22 -0
  131. package/dist/handlers/load-model/registry-companion-set.js +206 -0
  132. package/dist/handlers/load-model/registry-download-utils.d.ts +23 -0
  133. package/dist/handlers/load-model/registry-download-utils.js +112 -0
  134. package/dist/handlers/load-model/registry-metadata.d.ts +10 -0
  135. package/dist/handlers/load-model/registry-metadata.js +23 -0
  136. package/dist/handlers/load-model/registry.d.ts +4 -0
  137. package/dist/handlers/load-model/registry.js +298 -0
  138. package/dist/handlers/load-model/resolve-session.d.ts +33 -0
  139. package/dist/handlers/load-model/resolve-session.js +62 -0
  140. package/dist/handlers/load-model/resolve.d.ts +5 -0
  141. package/dist/handlers/load-model/resolve.js +158 -0
  142. package/dist/handlers/load-model/types.d.ts +59 -0
  143. package/dist/handlers/load-model/types.js +28 -0
  144. package/dist/handlers/logging-stream.d.ts +2 -0
  145. package/dist/handlers/logging-stream.js +39 -0
  146. package/dist/handlers/plugin-dispatch.d.ts +3 -0
  147. package/dist/handlers/plugin-dispatch.js +63 -0
  148. package/dist/handlers/plugin-invoke.d.ts +3 -0
  149. package/dist/handlers/plugin-invoke.js +81 -0
  150. package/dist/handlers/registry.d.ts +4 -0
  151. package/dist/handlers/registry.js +156 -0
  152. package/dist/handlers/resume.d.ts +2 -0
  153. package/dist/handlers/resume.js +11 -0
  154. package/dist/handlers/state.d.ts +2 -0
  155. package/dist/handlers/state.js +7 -0
  156. package/dist/handlers/suspend.d.ts +2 -0
  157. package/dist/handlers/suspend.js +11 -0
  158. package/dist/handlers/types.d.ts +13 -0
  159. package/dist/handlers/types.js +1 -0
  160. package/dist/handlers/unload-model.d.ts +2 -0
  161. package/dist/handlers/unload-model.js +29 -0
  162. package/dist/index.d.ts +4 -0
  163. package/dist/index.js +12 -0
  164. package/dist/logging/addon.d.ts +5 -0
  165. package/dist/logging/addon.js +75 -0
  166. package/dist/logging/index.d.ts +5 -0
  167. package/dist/logging/index.js +4 -0
  168. package/dist/logging/logger.d.ts +41 -0
  169. package/dist/logging/logger.js +209 -0
  170. package/dist/logging/namespaces.d.ts +6 -0
  171. package/dist/logging/namespaces.js +7 -0
  172. package/dist/logging/types.d.ts +37 -0
  173. package/dist/logging/types.js +1 -0
  174. package/dist/logging/utils.d.ts +5 -0
  175. package/dist/logging/utils.js +113 -0
  176. package/dist/models/registry/index.d.ts +18 -0
  177. package/dist/models/registry/index.js +21 -0
  178. package/dist/models/registry/models.d.ts +22680 -0
  179. package/dist/models/registry/models.js +23908 -0
  180. package/dist/p2p/cancel-delegated.d.ts +3 -0
  181. package/dist/p2p/cancel-delegated.js +64 -0
  182. package/dist/p2p/completion-stream-delegated.d.ts +4 -0
  183. package/dist/p2p/completion-stream-delegated.js +51 -0
  184. package/dist/p2p/delegate-client.d.ts +9 -0
  185. package/dist/p2p/delegate-client.js +219 -0
  186. package/dist/p2p/delegate-connect-diagnostics.d.ts +8 -0
  187. package/dist/p2p/delegate-connect-diagnostics.js +38 -0
  188. package/dist/p2p/delegate-transport.d.ts +18 -0
  189. package/dist/p2p/delegate-transport.js +237 -0
  190. package/dist/p2p/error-handlers.d.ts +4 -0
  191. package/dist/p2p/error-handlers.js +31 -0
  192. package/dist/p2p/handle-request.d.ts +2 -0
  193. package/dist/p2p/handle-request.js +171 -0
  194. package/dist/p2p/handler-utils.d.ts +19 -0
  195. package/dist/p2p/handler-utils.js +180 -0
  196. package/dist/p2p/heartbeat-delegated.d.ts +3 -0
  197. package/dist/p2p/heartbeat-delegated.js +32 -0
  198. package/dist/p2p/load-model-delegated.d.ts +6 -0
  199. package/dist/p2p/load-model-delegated.js +132 -0
  200. package/dist/p2p/progress-throttle.d.ts +7 -0
  201. package/dist/p2p/progress-throttle.js +47 -0
  202. package/dist/p2p/provideHandler/connection.d.ts +2 -0
  203. package/dist/p2p/provideHandler/connection.js +38 -0
  204. package/dist/p2p/provideHandler/handler.d.ts +2 -0
  205. package/dist/p2p/provideHandler/handler.js +66 -0
  206. package/dist/p2p/provideHandler/proxy.d.ts +2 -0
  207. package/dist/p2p/provideHandler/proxy.js +24 -0
  208. package/dist/p2p/provideHandler.d.ts +1 -0
  209. package/dist/p2p/provideHandler.js +2 -0
  210. package/dist/p2p/rpc-utils.d.ts +4 -0
  211. package/dist/p2p/rpc-utils.js +8 -0
  212. package/dist/p2p/stopProvideHandler.d.ts +2 -0
  213. package/dist/p2p/stopProvideHandler.js +29 -0
  214. package/dist/p2p/swarm.d.ts +9 -0
  215. package/dist/p2p/swarm.js +103 -0
  216. package/dist/p2p/transport-selector.d.ts +2 -0
  217. package/dist/p2p/transport-selector.js +18 -0
  218. package/dist/p2p/unload-model-delegated.d.ts +2 -0
  219. package/dist/p2p/unload-model-delegated.js +31 -0
  220. package/dist/plugins/builtin/asr-ggml/config.d.ts +31 -0
  221. package/dist/plugins/builtin/asr-ggml/config.js +46 -0
  222. package/dist/plugins/builtin/asr-ggml/logging.d.ts +10 -0
  223. package/dist/plugins/builtin/asr-ggml/logging.js +12 -0
  224. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +17 -0
  225. package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
  226. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +2 -0
  227. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +163 -0
  228. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +212 -0
  229. package/dist/plugins/builtin/audiogen-ggml/plugin.js +58 -0
  230. package/dist/plugins/builtin/bci-whispercpp-transcription/args.d.ts +2 -0
  231. package/dist/plugins/builtin/bci-whispercpp-transcription/args.js +13 -0
  232. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +173 -0
  233. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +111 -0
  234. package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.d.ts +74 -0
  235. package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.js +11 -0
  236. package/dist/plugins/builtin/ggml-classification/ops/classify.d.ts +5 -0
  237. package/dist/plugins/builtin/ggml-classification/ops/classify.js +31 -0
  238. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +31 -0
  239. package/dist/plugins/builtin/ggml-classification/plugin.js +55 -0
  240. package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.d.ts +7 -0
  241. package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.js +119 -0
  242. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +102 -0
  243. package/dist/plugins/builtin/ggml-ocr/plugin.js +110 -0
  244. package/dist/plugins/builtin/ggml-ocr/resolve-config.d.ts +5 -0
  245. package/dist/plugins/builtin/ggml-ocr/resolve-config.js +72 -0
  246. package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.d.ts +2 -0
  247. package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.js +12 -0
  248. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -0
  249. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +78 -0
  250. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +65 -0
  251. package/dist/plugins/builtin/ggml-vla/plugin.js +57 -0
  252. package/dist/plugins/builtin/index.d.ts +11 -0
  253. package/dist/plugins/builtin/index.js +11 -0
  254. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +22 -0
  255. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +30 -0
  256. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.d.ts +31 -0
  257. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +99 -0
  258. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +6 -0
  259. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +36 -0
  260. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +20 -0
  261. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +48 -0
  262. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +45 -0
  263. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +374 -0
  264. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +28 -0
  265. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +65 -0
  266. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +10 -0
  267. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +209 -0
  268. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +163 -0
  269. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +581 -0
  270. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +56 -0
  271. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +58 -0
  272. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +671 -0
  273. package/dist/plugins/builtin/llamacpp-completion/plugin.js +481 -0
  274. package/dist/plugins/builtin/llamacpp-completion/transform.d.ts +9 -0
  275. package/dist/plugins/builtin/llamacpp-completion/transform.js +34 -0
  276. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +76 -0
  277. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +101 -0
  278. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +457 -0
  279. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +143 -0
  280. package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.d.ts +31 -0
  281. package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.js +114 -0
  282. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +133 -0
  283. package/dist/plugins/builtin/parakeet-transcription/plugin.js +130 -0
  284. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +2 -0
  285. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +64 -0
  286. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +2 -0
  287. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +41 -0
  288. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -0
  289. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +239 -0
  290. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +757 -0
  291. package/dist/plugins/builtin/sdcpp-generation/plugin.js +310 -0
  292. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +6 -0
  293. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +30 -0
  294. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +7 -0
  295. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +106 -0
  296. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +6 -0
  297. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +91 -0
  298. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +196 -0
  299. package/dist/plugins/builtin/tts-ggml/plugin.js +305 -0
  300. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +346 -0
  301. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +169 -0
  302. package/dist/plugins/index.d.ts +9 -0
  303. package/dist/plugins/index.js +12 -0
  304. package/dist/plugins/ops/bci-transcribe.d.ts +13 -0
  305. package/dist/plugins/ops/bci-transcribe.js +248 -0
  306. package/dist/plugins/ops/cancel.d.ts +25 -0
  307. package/dist/plugins/ops/cancel.js +43 -0
  308. package/dist/plugins/ops/embed.d.ts +15 -0
  309. package/dist/plugins/ops/embed.js +193 -0
  310. package/dist/plugins/ops/kv-cache-retention.d.ts +11 -0
  311. package/dist/plugins/ops/kv-cache-retention.js +121 -0
  312. package/dist/plugins/ops/kv-cache-utils.d.ts +21 -0
  313. package/dist/plugins/ops/kv-cache-utils.js +120 -0
  314. package/dist/plugins/ops/load-model.d.ts +8 -0
  315. package/dist/plugins/ops/load-model.js +82 -0
  316. package/dist/plugins/ops/transcribe.d.ts +20 -0
  317. package/dist/plugins/ops/transcribe.js +343 -0
  318. package/dist/plugins/ops/translate.d.ts +7 -0
  319. package/dist/plugins/ops/translate.js +256 -0
  320. package/dist/plugins/ops/unload-model.d.ts +2 -0
  321. package/dist/plugins/ops/unload-model.js +45 -0
  322. package/dist/plugins/registry.d.ts +9 -0
  323. package/dist/plugins/registry.js +107 -0
  324. package/dist/profiling/aggregator.d.ts +14 -0
  325. package/dist/profiling/aggregator.js +100 -0
  326. package/dist/profiling/backend-diagnostics.d.ts +8 -0
  327. package/dist/profiling/backend-diagnostics.js +36 -0
  328. package/dist/profiling/clock.d.ts +8 -0
  329. package/dist/profiling/clock.js +23 -0
  330. package/dist/profiling/controller.d.ts +34 -0
  331. package/dist/profiling/controller.js +130 -0
  332. package/dist/profiling/delegation-profiler.d.ts +31 -0
  333. package/dist/profiling/delegation-profiler.js +155 -0
  334. package/dist/profiling/envelope.d.ts +10 -0
  335. package/dist/profiling/envelope.js +45 -0
  336. package/dist/profiling/events.d.ts +19 -0
  337. package/dist/profiling/events.js +35 -0
  338. package/dist/profiling/exporters.d.ts +9 -0
  339. package/dist/profiling/exporters.js +242 -0
  340. package/dist/profiling/index.d.ts +91 -0
  341. package/dist/profiling/index.js +87 -0
  342. package/dist/profiling/model-execution.d.ts +15 -0
  343. package/dist/profiling/model-execution.js +34 -0
  344. package/dist/profiling/operation-metrics.d.ts +24 -0
  345. package/dist/profiling/operation-metrics.js +370 -0
  346. package/dist/profiling/operation-wrappers.d.ts +17 -0
  347. package/dist/profiling/operation-wrappers.js +196 -0
  348. package/dist/profiling/profiler.d.ts +27 -0
  349. package/dist/profiling/profiler.js +127 -0
  350. package/dist/profiling/ring-buffer.d.ts +18 -0
  351. package/dist/profiling/ring-buffer.js +83 -0
  352. package/dist/profiling/types.d.ts +52 -0
  353. package/dist/profiling/types.js +1 -0
  354. package/dist/rag/handler.d.ts +2 -0
  355. package/dist/rag/handler.js +272 -0
  356. package/dist/rag/handlers/chunk.d.ts +2 -0
  357. package/dist/rag/handlers/chunk.js +7 -0
  358. package/dist/rag/handlers/close-workspace.d.ts +2 -0
  359. package/dist/rag/handlers/close-workspace.js +9 -0
  360. package/dist/rag/handlers/delete-embeddings.d.ts +2 -0
  361. package/dist/rag/handlers/delete-embeddings.js +11 -0
  362. package/dist/rag/handlers/delete-workspace.d.ts +2 -0
  363. package/dist/rag/handlers/delete-workspace.js +14 -0
  364. package/dist/rag/handlers/ingest.d.ts +9 -0
  365. package/dist/rag/handlers/ingest.js +19 -0
  366. package/dist/rag/handlers/list-workspaces.d.ts +2 -0
  367. package/dist/rag/handlers/list-workspaces.js +4 -0
  368. package/dist/rag/handlers/reindex.d.ts +9 -0
  369. package/dist/rag/handlers/reindex.js +16 -0
  370. package/dist/rag/handlers/save-embeddings.d.ts +9 -0
  371. package/dist/rag/handlers/save-embeddings.js +15 -0
  372. package/dist/rag/handlers/search.d.ts +2 -0
  373. package/dist/rag/handlers/search.js +16 -0
  374. package/dist/rag/index.d.ts +11 -0
  375. package/dist/rag/index.js +11 -0
  376. package/dist/rag/rag-operation-manager.d.ts +28 -0
  377. package/dist/rag/rag-operation-manager.js +76 -0
  378. package/dist/rag/rag-workspace-manager.d.ts +14 -0
  379. package/dist/rag/rag-workspace-manager.js +143 -0
  380. package/dist/registry.d.ts +2 -0
  381. package/dist/registry.js +166 -0
  382. package/dist/resources/collector.d.ts +643 -0
  383. package/dist/resources/collector.js +226 -0
  384. package/dist/resources/instance.d.ts +4 -0
  385. package/dist/resources/instance.js +15 -0
  386. package/dist/resources/native.d.ts +2 -0
  387. package/dist/resources/native.js +35 -0
  388. package/dist/resources/normalize.d.ts +12 -0
  389. package/dist/resources/normalize.js +73 -0
  390. package/dist/resources/types.d.ts +65 -0
  391. package/dist/resources/types.js +1 -0
  392. package/dist/runtime/cache-lock.d.ts +2 -0
  393. package/dist/runtime/cache-lock.js +67 -0
  394. package/dist/runtime/disposable-scope.d.ts +35 -0
  395. package/dist/runtime/disposable-scope.js +69 -0
  396. package/dist/runtime/env.d.ts +16 -0
  397. package/dist/runtime/env.js +53 -0
  398. package/dist/runtime/index.d.ts +5 -0
  399. package/dist/runtime/index.js +2 -0
  400. package/dist/runtime/lifecycle.d.ts +20 -0
  401. package/dist/runtime/lifecycle.js +102 -0
  402. package/dist/runtime/logging-stream-registry.d.ts +27 -0
  403. package/dist/runtime/logging-stream-registry.js +133 -0
  404. package/dist/runtime/model-config-utils.d.ts +14 -0
  405. package/dist/runtime/model-config-utils.js +128 -0
  406. package/dist/runtime/model-registry.d.ts +74 -0
  407. package/dist/runtime/model-registry.js +130 -0
  408. package/dist/runtime/registry-client.d.ts +4 -0
  409. package/dist/runtime/registry-client.js +118 -0
  410. package/dist/runtime/request-context.d.ts +72 -0
  411. package/dist/runtime/request-context.js +114 -0
  412. package/dist/runtime/request-id.d.ts +17 -0
  413. package/dist/runtime/request-id.js +29 -0
  414. package/dist/runtime/request-registry.d.ts +194 -0
  415. package/dist/runtime/request-registry.js +546 -0
  416. package/dist/runtime/runtime-lifecycle.d.ts +37 -0
  417. package/dist/runtime/runtime-lifecycle.js +163 -0
  418. package/dist/runtime/state.d.ts +26 -0
  419. package/dist/runtime/state.js +138 -0
  420. package/dist/schemas/audio-gen.d.ts +252 -0
  421. package/dist/schemas/audio-gen.js +69 -0
  422. package/dist/schemas/batch-completion-stream.d.ts +467 -0
  423. package/dist/schemas/batch-completion-stream.js +123 -0
  424. package/dist/schemas/bci-config.d.ts +74 -0
  425. package/dist/schemas/bci-config.js +58 -0
  426. package/dist/schemas/bci.d.ts +178 -0
  427. package/dist/schemas/bci.js +89 -0
  428. package/dist/schemas/cancel.d.ts +203 -0
  429. package/dist/schemas/cancel.js +135 -0
  430. package/dist/schemas/classification.d.ts +52 -0
  431. package/dist/schemas/classification.js +33 -0
  432. package/dist/schemas/common.d.ts +6420 -0
  433. package/dist/schemas/common.js +133 -0
  434. package/dist/schemas/completion-event.d.ts +248 -0
  435. package/dist/schemas/completion-event.js +94 -0
  436. package/dist/schemas/completion-stream.d.ts +475 -0
  437. package/dist/schemas/completion-stream.js +236 -0
  438. package/dist/schemas/config.d.ts +5022 -0
  439. package/dist/schemas/config.js +169 -0
  440. package/dist/schemas/delegate.d.ts +29 -0
  441. package/dist/schemas/delegate.js +40 -0
  442. package/dist/schemas/download-asset.d.ts +225 -0
  443. package/dist/schemas/download-asset.js +71 -0
  444. package/dist/schemas/embed.d.ts +41 -0
  445. package/dist/schemas/embed.js +35 -0
  446. package/dist/schemas/engine-addon-map.d.ts +20 -0
  447. package/dist/schemas/engine-addon-map.js +78 -0
  448. package/dist/schemas/error.d.ts +40 -0
  449. package/dist/schemas/error.js +65 -0
  450. package/dist/schemas/errors.d.ts +119 -0
  451. package/dist/schemas/errors.js +615 -0
  452. package/dist/schemas/finetune.d.ts +621 -0
  453. package/dist/schemas/finetune.js +163 -0
  454. package/dist/schemas/index.d.ts +45 -0
  455. package/dist/schemas/index.js +45 -0
  456. package/dist/schemas/json-schema.d.ts +15 -0
  457. package/dist/schemas/json-schema.js +2 -0
  458. package/dist/schemas/lifecycle.d.ts +38 -0
  459. package/dist/schemas/lifecycle.js +21 -0
  460. package/dist/schemas/llamacpp-config.d.ts +403 -0
  461. package/dist/schemas/llamacpp-config.js +133 -0
  462. package/dist/schemas/load-model.d.ts +25635 -0
  463. package/dist/schemas/load-model.js +629 -0
  464. package/dist/schemas/logging-stream.d.ts +33 -0
  465. package/dist/schemas/logging-stream.js +17 -0
  466. package/dist/schemas/mcp-adapter.d.ts +46 -0
  467. package/dist/schemas/mcp-adapter.js +1 -0
  468. package/dist/schemas/model-file.d.ts +27 -0
  469. package/dist/schemas/model-file.js +29 -0
  470. package/dist/schemas/model-info.d.ts +197 -0
  471. package/dist/schemas/model-info.js +144 -0
  472. package/dist/schemas/model-ops.d.ts +36 -0
  473. package/dist/schemas/model-ops.js +40 -0
  474. package/dist/schemas/model-src-utils.d.ts +192 -0
  475. package/dist/schemas/model-src-utils.js +61 -0
  476. package/dist/schemas/model-types.d.ts +275 -0
  477. package/dist/schemas/model-types.js +223 -0
  478. package/dist/schemas/ocr.d.ts +138 -0
  479. package/dist/schemas/ocr.js +59 -0
  480. package/dist/schemas/path-validation.d.ts +9 -0
  481. package/dist/schemas/path-validation.js +22 -0
  482. package/dist/schemas/plugin.d.ts +341 -0
  483. package/dist/schemas/plugin.js +241 -0
  484. package/dist/schemas/profiling.d.ts +695 -0
  485. package/dist/schemas/profiling.js +100 -0
  486. package/dist/schemas/provide.d.ts +50 -0
  487. package/dist/schemas/provide.js +32 -0
  488. package/dist/schemas/rag.d.ts +301 -0
  489. package/dist/schemas/rag.js +211 -0
  490. package/dist/schemas/registry.d.ts +453 -0
  491. package/dist/schemas/registry.js +104 -0
  492. package/dist/schemas/reload-config.d.ts +585 -0
  493. package/dist/schemas/reload-config.js +42 -0
  494. package/dist/schemas/runtime-context.d.ts +18 -0
  495. package/dist/schemas/runtime-context.js +7 -0
  496. package/dist/schemas/sdcpp-config.d.ts +1126 -0
  497. package/dist/schemas/sdcpp-config.js +748 -0
  498. package/dist/schemas/system-resources.d.ts +3966 -0
  499. package/dist/schemas/system-resources.js +149 -0
  500. package/dist/schemas/text-to-speech.d.ts +1352 -0
  501. package/dist/schemas/text-to-speech.js +364 -0
  502. package/dist/schemas/tools.d.ts +80 -0
  503. package/dist/schemas/tools.js +52 -0
  504. package/dist/schemas/transcription-config.d.ts +161 -0
  505. package/dist/schemas/transcription-config.js +156 -0
  506. package/dist/schemas/transcription.d.ts +261 -0
  507. package/dist/schemas/transcription.js +145 -0
  508. package/dist/schemas/translate.d.ts +138 -0
  509. package/dist/schemas/translate.js +103 -0
  510. package/dist/schemas/translation-config.d.ts +1031 -0
  511. package/dist/schemas/translation-config.js +170 -0
  512. package/dist/schemas/vla.d.ts +110 -0
  513. package/dist/schemas/vla.js +151 -0
  514. package/dist/selection.d.ts +8 -0
  515. package/dist/selection.js +14 -0
  516. package/dist/surface.d.ts +18 -0
  517. package/dist/surface.js +30 -0
  518. package/dist/utils/addon-responses.d.ts +58 -0
  519. package/dist/utils/addon-responses.js +1 -0
  520. package/dist/utils/aggregate-events.d.ts +25 -0
  521. package/dist/utils/aggregate-events.js +73 -0
  522. package/dist/utils/archive.d.ts +4 -0
  523. package/dist/utils/archive.js +180 -0
  524. package/dist/utils/asr-events.d.ts +14 -0
  525. package/dist/utils/asr-events.js +30 -0
  526. package/dist/utils/audio/decoder.d.ts +4 -0
  527. package/dist/utils/audio/decoder.js +80 -0
  528. package/dist/utils/audio-input.d.ts +9 -0
  529. package/dist/utils/audio-input.js +34 -0
  530. package/dist/utils/cache/auto-history.d.ts +3 -0
  531. package/dist/utils/cache/auto-history.js +26 -0
  532. package/dist/utils/cache/paths.d.ts +42 -0
  533. package/dist/utils/cache/paths.js +133 -0
  534. package/dist/utils/cache/types.d.ts +7 -0
  535. package/dist/utils/cache/types.js +1 -0
  536. package/dist/utils/cache-normalize.d.ts +9 -0
  537. package/dist/utils/cache-normalize.js +20 -0
  538. package/dist/utils/checksum.d.ts +3 -0
  539. package/dist/utils/checksum.js +29 -0
  540. package/dist/utils/clear-storage.d.ts +4 -0
  541. package/dist/utils/clear-storage.js +14 -0
  542. package/dist/utils/completion-normalizer.d.ts +17 -0
  543. package/dist/utils/completion-normalizer.js +453 -0
  544. package/dist/utils/config-transform.d.ts +2 -0
  545. package/dist/utils/config-transform.js +21 -0
  546. package/dist/utils/decorate-promise.d.ts +33 -0
  547. package/dist/utils/decorate-promise.js +35 -0
  548. package/dist/utils/encoding.d.ts +16 -0
  549. package/dist/utils/encoding.js +27 -0
  550. package/dist/utils/expand-gguf-shards.d.ts +2 -0
  551. package/dist/utils/expand-gguf-shards.js +22 -0
  552. package/dist/utils/formatting.d.ts +18 -0
  553. package/dist/utils/formatting.js +56 -0
  554. package/dist/utils/gguf-tensor-extractor.d.ts +14 -0
  555. package/dist/utils/gguf-tensor-extractor.js +311 -0
  556. package/dist/utils/global-singleton.d.ts +1 -0
  557. package/dist/utils/global-singleton.js +10 -0
  558. package/dist/utils/index.d.ts +11 -0
  559. package/dist/utils/index.js +11 -0
  560. package/dist/utils/load-model-validation.d.ts +6 -0
  561. package/dist/utils/load-model-validation.js +18 -0
  562. package/dist/utils/mcp-adapter.d.ts +11 -0
  563. package/dist/utils/mcp-adapter.js +96 -0
  564. package/dist/utils/multi-gpu-mobile.d.ts +4 -0
  565. package/dist/utils/multi-gpu-mobile.js +10 -0
  566. package/dist/utils/object.d.ts +1 -0
  567. package/dist/utils/object.js +3 -0
  568. package/dist/utils/path-security.d.ts +22 -0
  569. package/dist/utils/path-security.js +69 -0
  570. package/dist/utils/qvac-paths.d.ts +1 -0
  571. package/dist/utils/qvac-paths.js +5 -0
  572. package/dist/utils/response-format.d.ts +2 -0
  573. package/dist/utils/response-format.js +20 -0
  574. package/dist/utils/shard-utils.d.ts +47 -0
  575. package/dist/utils/shard-utils.js +123 -0
  576. package/dist/utils/tool-helpers.d.ts +24 -0
  577. package/dist/utils/tool-helpers.js +111 -0
  578. package/dist/utils/tool-integration.d.ts +23 -0
  579. package/dist/utils/tool-integration.js +29 -0
  580. package/dist/utils/tools/dialect.d.ts +7 -0
  581. package/dist/utils/tools/dialect.js +20 -0
  582. package/dist/utils/tools/index.d.ts +2 -0
  583. package/dist/utils/tools/index.js +2 -0
  584. package/dist/utils/tools/parser.d.ts +5 -0
  585. package/dist/utils/tools/parser.js +69 -0
  586. package/dist/utils/tools/parsers/dsml.d.ts +3 -0
  587. package/dist/utils/tools/parsers/dsml.js +177 -0
  588. package/dist/utils/tools/parsers/gemma4native.d.ts +3 -0
  589. package/dist/utils/tools/parsers/gemma4native.js +65 -0
  590. package/dist/utils/tools/parsers/harmony.d.ts +3 -0
  591. package/dist/utils/tools/parsers/harmony.js +57 -0
  592. package/dist/utils/tools/parsers/hermes.d.ts +3 -0
  593. package/dist/utils/tools/parsers/hermes.js +122 -0
  594. package/dist/utils/tools/parsers/json.d.ts +5 -0
  595. package/dist/utils/tools/parsers/json.js +115 -0
  596. package/dist/utils/tools/parsers/pythonic.d.ts +3 -0
  597. package/dist/utils/tools/parsers/pythonic.js +267 -0
  598. package/dist/utils/tools/parsers/qwen35.d.ts +3 -0
  599. package/dist/utils/tools/parsers/qwen35.js +125 -0
  600. package/dist/utils/tools/shared.d.ts +17 -0
  601. package/dist/utils/tools/shared.js +52 -0
  602. package/dist/utils/transcribe-metadata.d.ts +20 -0
  603. package/dist/utils/transcribe-metadata.js +28 -0
  604. package/dist/utils/tts-stats.d.ts +25 -0
  605. package/dist/utils/tts-stats.js +16 -0
  606. package/dist/utils/withTimeout.d.ts +17 -0
  607. package/dist/utils/withTimeout.js +47 -0
  608. package/dist/utils/zod-error.d.ts +2 -0
  609. package/dist/utils/zod-error.js +4 -0
  610. package/package.json +266 -1
@@ -0,0 +1,256 @@
1
+ var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
+ if (value !== null && value !== void 0) {
3
+ if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
+ var dispose, inner;
5
+ if (async) {
6
+ if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
+ dispose = value[Symbol.asyncDispose];
8
+ }
9
+ if (dispose === void 0) {
10
+ if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
+ dispose = value[Symbol.dispose];
12
+ if (async) inner = dispose;
13
+ }
14
+ if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
+ if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
+ env.stack.push({ value: value, dispose: dispose, async: async });
17
+ }
18
+ else if (async) {
19
+ env.stack.push({ async: true });
20
+ }
21
+ return value;
22
+ };
23
+ var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
+ return function (env) {
25
+ function fail(e) {
26
+ env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
+ env.hasError = true;
28
+ }
29
+ var r, s = 0;
30
+ function next() {
31
+ while (r = env.stack.pop()) {
32
+ try {
33
+ if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
+ if (r.dispose) {
35
+ var result = r.dispose.call(r.value);
36
+ if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
+ }
38
+ else s |= 1;
39
+ }
40
+ catch (e) {
41
+ fail(e);
42
+ }
43
+ }
44
+ if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
+ if (env.hasError) throw env.error;
46
+ }
47
+ return next();
48
+ };
49
+ })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
+ var e = new Error(message);
51
+ return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
+ });
53
+ import { getModelEntry } from '../../runtime/model-registry.js';
54
+ import { translateServerParamsSchema, normalizeModelType, ModelType, AFRICAN_LANGUAGES_MAP } from '../../schemas/index.js';
55
+ import { getLangName, detectOne } from '@qvac/langdetect-text';
56
+ import { nowMs } from '../../profiling/index.js';
57
+ import { buildStreamResult } from '../../profiling/model-execution.js';
58
+ import { ModelIsDelegatedError, ModelNotFoundError, ModelTypeMismatchError, TranslationFailedError } from '../../errors/index.js';
59
+ import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
60
+ import { generateRandomRequestId } from '../../runtime/request-id.js';
61
+ import { getEngineLogger } from '../../logging/index.js';
62
+ export function getLanguage(code) {
63
+ if (!code)
64
+ return '';
65
+ if (AFRICAN_LANGUAGES_MAP.has(code))
66
+ return AFRICAN_LANGUAGES_MAP.get(code);
67
+ const fullName = getLangName(code);
68
+ return fullName ?? code.toUpperCase();
69
+ }
70
+ export function isAfrican(code) {
71
+ return !!code && AFRICAN_LANGUAGES_MAP.has(code);
72
+ }
73
+ const LLM_TRANSLATE_GENERATION_PARAMS = {
74
+ temp: 0,
75
+ top_k: 1,
76
+ top_p: 1,
77
+ repeat_penalty: 1.3,
78
+ seed: 42,
79
+ predict: 256
80
+ };
81
+ function shouldSkipPerCallSampling(modelName) {
82
+ return !!modelName && modelName.startsWith('AFRICAN_');
83
+ }
84
+ export async function* translate(params, requestId) {
85
+ const env_1 = { stack: [], error: void 0, hasError: false };
86
+ try {
87
+ const { modelId, text, modelType: inputModelType } = params;
88
+ const entry = getModelEntry(modelId);
89
+ if (!entry) {
90
+ throw new ModelNotFoundError(modelId);
91
+ }
92
+ if (entry.isDelegated) {
93
+ throw new ModelIsDelegatedError(modelId);
94
+ }
95
+ const canonicalModelType = entry.local.modelType;
96
+ const model = entry.local.model;
97
+ if (inputModelType !== undefined) {
98
+ const requestedCanonical = normalizeModelType(inputModelType);
99
+ if (requestedCanonical !== canonicalModelType) {
100
+ throw new ModelTypeMismatchError(canonicalModelType, requestedCanonical);
101
+ }
102
+ }
103
+ const isLlm = canonicalModelType === ModelType.llamacppCompletion;
104
+ let from = isLlm ? params.from : undefined;
105
+ const to = isLlm ? params.to : undefined;
106
+ const context = isLlm ? params.context : undefined;
107
+ translateServerParamsSchema.parse(params);
108
+ // Auto-detect the source language when the caller didn't pass `from` (LLM
109
+ // only). This used to run in each client; moving it here gives every
110
+ // language binding one detector instead of each shipping its own (lingua in
111
+ // Python vs langdetect-text in JS drifted on the same input). Store the
112
+ // detected code — not the language name — so the explicit-`from` and
113
+ // detected paths feed getLanguage() identically.
114
+ if (isLlm && !from) {
115
+ const detected = detectOne(text);
116
+ if (detected.code === 'und' || detected.language === 'Undetermined') {
117
+ throw new TranslationFailedError("Could not detect the source language. Please specify the 'from' parameter explicitly.");
118
+ }
119
+ from = detected.code;
120
+ }
121
+ const afriquePrompt = isLlm && (isAfrican(from) || isAfrican(to));
122
+ const fromLanguage = getLanguage(from);
123
+ const toLanguage = getLanguage(to);
124
+ // Open a request-scoped lifecycle for both engine branches. LLM-
125
+ // translate inherits llamacpp-completion's `{ scope: "model",
126
+ // hard: true }` cancel surface; NMT-translate inherits nmtcpp's
127
+ // `{ scope: "none" }` — so the addon-cancel wiring below only
128
+ // engages on the LLM path. NMT cancel is purely soft: the loop
129
+ // exits on `signal.aborted`, scope unwinds, and the addon may run
130
+ // to completion in the background — acceptable because the result
131
+ // is dropped either way.
132
+ const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
133
+ requestId: requestId ?? generateRandomRequestId(),
134
+ kind: 'translate',
135
+ modelId
136
+ }), true);
137
+ const requestLogger = withRequestContext(getEngineLogger(), ctx);
138
+ if (isLlm) {
139
+ const onAbort = () => {
140
+ const addon = model.addon;
141
+ if (addon?.cancel) {
142
+ addon.cancel.call(addon).catch((err) => {
143
+ requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
144
+ });
145
+ }
146
+ };
147
+ ctx.signal.addEventListener('abort', onAbort, { once: true });
148
+ if (ctx.signal.aborted)
149
+ onAbort();
150
+ ctx.scope.defer(() => {
151
+ ctx.signal.removeEventListener('abort', onAbort);
152
+ });
153
+ }
154
+ // Check if input is an array and model type is NMT
155
+ if (Array.isArray(text) && canonicalModelType === ModelType.nmtcppTranslation) {
156
+ // Use runBatch for batch processing
157
+ const modelStart = nowMs();
158
+ const translations = await model.runBatch(text);
159
+ const modelExecutionMs = nowMs() - modelStart;
160
+ // Soft-cancel boundary: if `cancel({ requestId })` landed while
161
+ // the addon was running the batch, bail before yielding anything.
162
+ // The addon may have completed its work; the result is dropped.
163
+ if (ctx.signal.aborted) {
164
+ return { modelExecutionMs };
165
+ }
166
+ // Yield each translation with a newline separator
167
+ for (let i = 0; i < translations.length; i++) {
168
+ if (ctx.signal.aborted)
169
+ break;
170
+ const translation = translations[i];
171
+ yield translation;
172
+ if (i < translations.length - 1) {
173
+ yield '\n';
174
+ }
175
+ }
176
+ return { modelExecutionMs };
177
+ }
178
+ // Single text processing (for NMT or LLM)
179
+ const singleText = Array.isArray(text) ? text[0] : text;
180
+ // Prepare input based on model type
181
+ const input = canonicalModelType === ModelType.nmtcppTranslation
182
+ ? singleText
183
+ : [
184
+ {
185
+ role: afriquePrompt ? 'user' : 'system',
186
+ content: afriquePrompt
187
+ ? `Translate ${fromLanguage} to ${toLanguage}.\n${fromLanguage}: ${singleText}\n${toLanguage}:`
188
+ : `${context ? `${context}. ` : ''}Translate the following text from ${fromLanguage} into ${toLanguage}. Only output the translation, nothing else.\n\n${fromLanguage}: ${singleText}\n${toLanguage}:`
189
+ }
190
+ ];
191
+ const modelStart = nowMs();
192
+ let response;
193
+ if (canonicalModelType === ModelType.llamacppCompletion &&
194
+ !shouldSkipPerCallSampling(entry.local.name)) {
195
+ response = await model.run(input, {
196
+ generationParams: LLM_TRANSLATE_GENERATION_PARAMS
197
+ });
198
+ }
199
+ else {
200
+ response = await model.run(input);
201
+ }
202
+ // Check if the response has an iterate method (like LLM models)
203
+ if (canonicalModelType === ModelType.llamacppCompletion &&
204
+ typeof response.iterate === 'function') {
205
+ const llmResponse = response;
206
+ for await (const token of llmResponse.iterate()) {
207
+ if (ctx.signal.aborted)
208
+ break;
209
+ yield token;
210
+ }
211
+ const modelExecutionMs = nowMs() - modelStart;
212
+ const stats = {
213
+ ...(llmResponse.stats?.TPS !== undefined && { tokensPerSecond: llmResponse.stats.TPS }),
214
+ ...(llmResponse.stats?.TTFT !== undefined && { timeToFirstToken: llmResponse.stats.TTFT }),
215
+ ...(llmResponse.stats?.CacheTokens !== undefined && {
216
+ cacheTokens: llmResponse.stats.CacheTokens
217
+ }),
218
+ ...(llmResponse.stats?.generatedTokens !== undefined && {
219
+ totalTokens: llmResponse.stats.generatedTokens
220
+ })
221
+ };
222
+ return buildStreamResult(modelExecutionMs, stats);
223
+ }
224
+ const nmtResponse = response;
225
+ for await (const token of nmtResponse.iterate()) {
226
+ if (ctx.signal.aborted)
227
+ break;
228
+ yield token;
229
+ }
230
+ const modelExecutionMs = nowMs() - modelStart;
231
+ const stats = {
232
+ ...(nmtResponse.stats?.totalTime !== undefined && { totalTime: nmtResponse.stats.totalTime }),
233
+ ...(nmtResponse.stats?.totalTokens !== undefined && {
234
+ totalTokens: nmtResponse.stats.totalTokens
235
+ }),
236
+ ...(nmtResponse.stats?.decodeTime !== undefined && {
237
+ decodeTime: nmtResponse.stats.decodeTime
238
+ }),
239
+ ...(nmtResponse.stats?.encodeTime !== undefined && {
240
+ encodeTime: nmtResponse.stats.encodeTime
241
+ }),
242
+ ...(nmtResponse.stats?.TPS !== undefined && { tokensPerSecond: nmtResponse.stats.TPS }),
243
+ ...(nmtResponse.stats?.TTFT !== undefined && { timeToFirstToken: nmtResponse.stats.TTFT })
244
+ };
245
+ return buildStreamResult(modelExecutionMs, stats);
246
+ }
247
+ catch (e_1) {
248
+ env_1.error = e_1;
249
+ env_1.hasError = true;
250
+ }
251
+ finally {
252
+ const result_1 = __disposeResources(env_1);
253
+ if (result_1)
254
+ await result_1;
255
+ }
256
+ }
@@ -0,0 +1,2 @@
1
+ import { type UnloadModelParams } from '../../schemas/index.js';
2
+ export declare function unloadModel(params: UnloadModelParams): Promise<void>;
@@ -0,0 +1,45 @@
1
+ import { promises as fsPromises } from 'bare-fs';
2
+ import path from 'bare-path';
3
+ import { unregisterModel } from '../../runtime/model-registry.js';
4
+ import { unregisterAllLoggingStreams } from '../../runtime/logging-stream-registry.js';
5
+ import { clearFinetuneRuntimeState } from '../../plugins/builtin/llamacpp-completion/ops/finetune.js';
6
+ import { unregisterAddonLogger, getEngineLogger } from '../../logging/index.js';
7
+ import { unloadModelParamsSchema } from '../../schemas/index.js';
8
+ import { ModelNotLoadedError } from '../../errors/index.js';
9
+ import { detectShardedModel } from '../../utils/index.js';
10
+ import { getClearStorageTarget } from '../../utils/cache/paths.js';
11
+ const logger = getEngineLogger();
12
+ export async function unloadModel(params) {
13
+ const { modelId, clearStorage } = unloadModelParamsSchema.parse(params);
14
+ const entry = unregisterModel(modelId);
15
+ if (!entry) {
16
+ throw new ModelNotLoadedError(modelId);
17
+ }
18
+ clearFinetuneRuntimeState(modelId);
19
+ if (!entry.isDelegated) {
20
+ if (entry.local.model.unload) {
21
+ await entry.local.model.unload();
22
+ }
23
+ if (clearStorage && entry.local.path) {
24
+ const modelPath = entry.local.path;
25
+ const modelFileName = path.basename(modelPath);
26
+ const shardInfo = detectShardedModel(modelFileName);
27
+ if (shardInfo.isSharded) {
28
+ const shardDir = path.dirname(modelPath);
29
+ await fsPromises.rm(shardDir, { recursive: true, force: true });
30
+ logger.info(`Sharded model storage cleared: ${shardDir}`);
31
+ }
32
+ else {
33
+ const target = getClearStorageTarget(modelPath);
34
+ await fsPromises.rm(target.path, {
35
+ recursive: target.kind === 'directory',
36
+ force: true
37
+ });
38
+ logger.info(`Model storage cleared (${target.kind}): ${target.path}`);
39
+ }
40
+ }
41
+ }
42
+ unregisterAddonLogger(modelId);
43
+ unregisterAllLoggingStreams(modelId);
44
+ logger.info(`Model ${modelId} unloaded`);
45
+ }
@@ -0,0 +1,9 @@
1
+ import { type QvacPlugin, type PluginHandlerDefinition } from '../schemas/plugin.js';
2
+ export declare function registerPlugin(plugin: QvacPlugin): void;
3
+ export declare function registerPlugins(pluginList: readonly QvacPlugin[]): void;
4
+ export declare function getPlugin(modelType: string): QvacPlugin | undefined;
5
+ export declare function getPluginHandler(modelType: string, handlerName: string): PluginHandlerDefinition | undefined;
6
+ export declare function hasPlugin(modelType: string): boolean;
7
+ export declare function unregisterPlugin(modelType: string): boolean;
8
+ export declare function getAllPlugins(): QvacPlugin[];
9
+ export declare function clearPlugins(): void;
@@ -0,0 +1,107 @@
1
+ import { pluginDefinitionRuntimeSchema } from '../schemas/plugin.js';
2
+ import { isModelTypeAlias } from '../schemas/index.js';
3
+ import { PluginAlreadyRegisteredError, PluginDefinitionInvalidError, PluginLoggingInvalidError, PluginModelTypeReservedError } from '../errors/index.js';
4
+ import { createAddonLoggerCallback } from '../logging/addon.js';
5
+ import { getEngineLogger } from '../logging/index.js';
6
+ import { formatZodError } from '../utils/zod-error.js';
7
+ const plugins = new Map();
8
+ function getLoggingModule(plugin) {
9
+ return plugin.logging?.module;
10
+ }
11
+ function findPluginUsingLoggingModule(loggingModule) {
12
+ return Array.from(plugins.values()).find((plugin) => plugin.logging?.module === loggingModule);
13
+ }
14
+ function getModelTypeForError(plugin) {
15
+ if (!plugin || typeof plugin !== 'object')
16
+ return '(unknown)';
17
+ if (!('modelType' in plugin))
18
+ return '(unknown)';
19
+ const modelType = plugin.modelType;
20
+ return typeof modelType === 'string' && modelType.length > 0 ? modelType : '(unknown)';
21
+ }
22
+ function validatePluginDefinition(plugin) {
23
+ const result = pluginDefinitionRuntimeSchema.safeParse(plugin);
24
+ if (result.success)
25
+ return;
26
+ throw new PluginDefinitionInvalidError(getModelTypeForError(plugin), formatZodError(result.error));
27
+ }
28
+ export function registerPlugin(plugin) {
29
+ validatePluginDefinition(plugin);
30
+ if (isModelTypeAlias(plugin.modelType)) {
31
+ throw new PluginModelTypeReservedError(plugin.modelType);
32
+ }
33
+ if (plugins.has(plugin.modelType)) {
34
+ throw new PluginAlreadyRegisteredError(plugin.modelType);
35
+ }
36
+ // Validate logging module shape if provided
37
+ if (plugin.logging?.module) {
38
+ const loggingModule = plugin.logging.module;
39
+ if (typeof loggingModule['setLogger'] !== 'function') {
40
+ throw new PluginLoggingInvalidError(plugin.modelType, 'logging.module must have a setLogger(callback) function');
41
+ }
42
+ }
43
+ const loggingModule = getLoggingModule(plugin);
44
+ const pluginUsingLoggingModule = loggingModule
45
+ ? findPluginUsingLoggingModule(loggingModule)
46
+ : undefined;
47
+ if (pluginUsingLoggingModule &&
48
+ pluginUsingLoggingModule.logging?.namespace !== plugin.logging?.namespace) {
49
+ throw new PluginLoggingInvalidError(plugin.modelType, 'plugins sharing logging.module must use the same namespace');
50
+ }
51
+ plugins.set(plugin.modelType, plugin);
52
+ if (loggingModule && plugin.logging?.namespace && !pluginUsingLoggingModule) {
53
+ loggingModule.setLogger(createAddonLoggerCallback(plugin.logging.namespace));
54
+ }
55
+ }
56
+ export function registerPlugins(pluginList) {
57
+ for (const plugin of pluginList) {
58
+ registerPlugin(plugin);
59
+ }
60
+ }
61
+ export function getPlugin(modelType) {
62
+ return plugins.get(modelType);
63
+ }
64
+ export function getPluginHandler(modelType, handlerName) {
65
+ const plugin = plugins.get(modelType);
66
+ if (!plugin)
67
+ return undefined;
68
+ return plugin.handlers[handlerName];
69
+ }
70
+ export function hasPlugin(modelType) {
71
+ return plugins.has(modelType);
72
+ }
73
+ export function unregisterPlugin(modelType) {
74
+ const plugin = plugins.get(modelType);
75
+ if (!plugin)
76
+ return false;
77
+ const loggingModule = getLoggingModule(plugin);
78
+ plugins.delete(modelType);
79
+ if (loggingModule && !findPluginUsingLoggingModule(loggingModule)) {
80
+ loggingModule.releaseLogger?.();
81
+ }
82
+ return true;
83
+ }
84
+ export function getAllPlugins() {
85
+ return Array.from(plugins.values());
86
+ }
87
+ export function clearPlugins() {
88
+ const loggingModules = new Map();
89
+ for (const plugin of plugins.values()) {
90
+ const loggingModule = getLoggingModule(plugin);
91
+ if (loggingModule && !loggingModules.has(loggingModule)) {
92
+ loggingModules.set(loggingModule, plugin.modelType);
93
+ }
94
+ }
95
+ plugins.clear();
96
+ for (const [loggingModule, modelType] of loggingModules) {
97
+ try {
98
+ loggingModule.releaseLogger?.();
99
+ }
100
+ catch (error) {
101
+ // A plugin's logger teardown must not abort the sweep or leave the
102
+ // registry half-cleared for the next caller — but surface it, so a leaked
103
+ // reference or async handle is not masked as a clean teardown.
104
+ getEngineLogger().warn(`[${modelType}] releaseLogger failed during clearPlugins: ${error instanceof Error ? error.message : String(error)}`);
105
+ }
106
+ }
107
+ }
@@ -0,0 +1,14 @@
1
+ /**
2
+ * Accumulates profiling statistics and maintains a bounded buffer of recent events.
3
+ * Uses a ring buffer for O(1) event insertion.
4
+ */
5
+ import type { ProfilingEvent, AggregatedStats } from '../profiling/types.js';
6
+ export declare function createAggregator(maxRecentEvents?: number): void;
7
+ export declare function recordEvent(event: ProfilingEvent, storeInBuffer?: boolean): void;
8
+ export declare function getAggregates(): Record<string, AggregatedStats>;
9
+ /** Returns events in chronological order (oldest first). */
10
+ export declare function getRecentEvents(): ProfilingEvent[];
11
+ export declare function getEventCount(): number;
12
+ export declare function getDroppedCount(): number;
13
+ export declare function clearAggregator(): void;
14
+ export declare function setMaxRecentEvents(max: number): void;
@@ -0,0 +1,100 @@
1
+ /**
2
+ * Accumulates profiling statistics and maintains a bounded buffer of recent events.
3
+ * Uses a ring buffer for O(1) event insertion.
4
+ */
5
+ import { getGlobalSingleton } from '../utils/global-singleton.js';
6
+ import { createRingBuffer, ringBufferPush, ringBufferToArray, ringBufferClear, ringBufferResize, ringBufferDroppedCount } from '../profiling/ring-buffer.js';
7
+ const DEFAULT_MAX_RECENT_EVENTS = 1000;
8
+ const AGGREGATOR_STATE_KEY = Symbol.for('@qvac/inference:profiler-aggregator-state');
9
+ function getAggregatorState() {
10
+ return getGlobalSingleton(AGGREGATOR_STATE_KEY, () => {
11
+ return {
12
+ stats: new Map(),
13
+ eventBuffer: createRingBuffer(DEFAULT_MAX_RECENT_EVENTS)
14
+ };
15
+ });
16
+ }
17
+ export function createAggregator(maxRecentEvents = DEFAULT_MAX_RECENT_EVENTS) {
18
+ const state = getAggregatorState();
19
+ state.stats = new Map();
20
+ state.eventBuffer = createRingBuffer(maxRecentEvents);
21
+ }
22
+ function recordStats(key, value) {
23
+ const aggregator = getAggregatorState();
24
+ let statsState = aggregator.stats.get(key);
25
+ if (!statsState) {
26
+ statsState = {
27
+ count: 0,
28
+ min: Infinity,
29
+ max: -Infinity,
30
+ sum: 0,
31
+ last: 0
32
+ };
33
+ aggregator.stats.set(key, statsState);
34
+ }
35
+ statsState.count++;
36
+ statsState.sum += value;
37
+ statsState.last = value;
38
+ if (value < statsState.min)
39
+ statsState.min = value;
40
+ if (value > statsState.max)
41
+ statsState.max = value;
42
+ }
43
+ function getStats(state) {
44
+ return {
45
+ count: state.count,
46
+ min: state.count > 0 ? state.min : 0,
47
+ max: state.count > 0 ? state.max : 0,
48
+ avg: state.count > 0 ? state.sum / state.count : 0,
49
+ sum: state.sum,
50
+ last: state.last
51
+ };
52
+ }
53
+ export function recordEvent(event, storeInBuffer = true) {
54
+ const state = getAggregatorState();
55
+ if (storeInBuffer) {
56
+ ringBufferPush(state.eventBuffer, event);
57
+ }
58
+ if (event.ms !== undefined) {
59
+ const key = event.phase ? `${event.op}.${event.phase}` : event.op;
60
+ recordStats(key, event.ms);
61
+ }
62
+ if (event.gauges) {
63
+ const baseKey = event.phase ? `${event.op}.${event.phase}` : event.op;
64
+ for (const [gaugeName, value] of Object.entries(event.gauges)) {
65
+ recordStats(`${baseKey}.${gaugeName}`, value);
66
+ }
67
+ }
68
+ }
69
+ export function getAggregates() {
70
+ const state = getAggregatorState();
71
+ const result = {};
72
+ for (const [key, statsState] of state.stats) {
73
+ result[key] = getStats(statsState);
74
+ }
75
+ return result;
76
+ }
77
+ /** Returns events in chronological order (oldest first). */
78
+ export function getRecentEvents() {
79
+ return ringBufferToArray(getAggregatorState().eventBuffer);
80
+ }
81
+ export function getEventCount() {
82
+ const state = getAggregatorState();
83
+ let total = 0;
84
+ for (const statsState of state.stats.values()) {
85
+ total += statsState.count;
86
+ }
87
+ return total;
88
+ }
89
+ export function getDroppedCount() {
90
+ return ringBufferDroppedCount(getAggregatorState().eventBuffer);
91
+ }
92
+ export function clearAggregator() {
93
+ const state = getAggregatorState();
94
+ state.stats.clear();
95
+ ringBufferClear(state.eventBuffer);
96
+ }
97
+ export function setMaxRecentEvents(max) {
98
+ const state = getAggregatorState();
99
+ state.eventBuffer = ringBufferResize(state.eventBuffer, max);
100
+ }
@@ -0,0 +1,8 @@
1
+ import { type InferenceBackendDiagnostics } from '../schemas/index.js';
2
+ /**
3
+ * Attaches diagnostics to a unary addon result, a plugin model-factory result,
4
+ * or the terminal chunk of a streaming addon result. Earlier stream chunks are not inspected.
5
+ */
6
+ export declare function attachBackendDiagnostics<T extends object>(target: T, diagnostics: InferenceBackendDiagnostics): T;
7
+ export declare function forwardBackendDiagnostics<T extends object>(target: T, source: unknown): T;
8
+ export declare function readBackendDiagnostics(target: unknown): InferenceBackendDiagnostics | undefined;
@@ -0,0 +1,36 @@
1
+ import { getEngineLogger } from '../logging/index.js';
2
+ import { BACKEND_DIAGNOSTICS_KEY, inferenceBackendDiagnosticsSchema } from '../schemas/index.js';
3
+ const logger = getEngineLogger();
4
+ /**
5
+ * Attaches diagnostics to a unary addon result, a plugin model-factory result,
6
+ * or the terminal chunk of a streaming addon result. Earlier stream chunks are not inspected.
7
+ */
8
+ export function attachBackendDiagnostics(target, diagnostics) {
9
+ const parsed = inferenceBackendDiagnosticsSchema.parse(diagnostics);
10
+ target[BACKEND_DIAGNOSTICS_KEY] =
11
+ parsed;
12
+ return target;
13
+ }
14
+ export function forwardBackendDiagnostics(target, source) {
15
+ if (!source || typeof source !== 'object')
16
+ return target;
17
+ const diagnostics = source[BACKEND_DIAGNOSTICS_KEY];
18
+ if (diagnostics !== undefined) {
19
+ ;
20
+ target[BACKEND_DIAGNOSTICS_KEY] = diagnostics;
21
+ }
22
+ return target;
23
+ }
24
+ export function readBackendDiagnostics(target) {
25
+ if (!target || typeof target !== 'object')
26
+ return undefined;
27
+ const diagnostics = target[BACKEND_DIAGNOSTICS_KEY];
28
+ if (diagnostics === undefined)
29
+ return undefined;
30
+ const result = inferenceBackendDiagnosticsSchema.safeParse(diagnostics);
31
+ if (!result.success) {
32
+ logger.debug('Ignoring invalid backend diagnostics', result.error);
33
+ return undefined;
34
+ }
35
+ return result.data;
36
+ }
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Millisecond clock for profiling durations. Bare has no monotonic clock
3
+ * global, so this uses `Date.now` — ample resolution for inference timing.
4
+ */
5
+ export declare function nowMs(): number;
6
+ export declare function measureAsync<T>(fn: () => Promise<T>): Promise<[T, number]>;
7
+ export declare function measureSync<T>(fn: () => T): [T, number];
8
+ export declare function generateProfileId(): string;
@@ -0,0 +1,23 @@
1
+ /**
2
+ * Millisecond clock for profiling durations. Bare has no monotonic clock
3
+ * global, so this uses `Date.now` — ample resolution for inference timing.
4
+ */
5
+ const startMs = Date.now();
6
+ export function nowMs() {
7
+ return Date.now() - startMs;
8
+ }
9
+ export async function measureAsync(fn) {
10
+ const start = nowMs();
11
+ const result = await fn();
12
+ return [result, nowMs() - start];
13
+ }
14
+ export function measureSync(fn) {
15
+ const start = nowMs();
16
+ const result = fn();
17
+ return [result, nowMs() - start];
18
+ }
19
+ export function generateProfileId() {
20
+ const timestamp = Date.now().toString(36);
21
+ const random = Math.random().toString(36).substring(2, 8);
22
+ return `${timestamp}-${random}`;
23
+ }
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Central state management for profiling enablement, configuration, and recording.
3
+ *
4
+ * Precedence (highest to lowest):
5
+ * 1. Per-call override
6
+ * 2. Runtime API (enable/disable)
7
+ * 3. Disabled default
8
+ */
9
+ import type { ProfilerMode, PerCallProfiling } from '../schemas/index.js';
10
+ import type { ProfilerRuntimeOptions, ProfilingEvent, AggregatedStats } from '../profiling/types.js';
11
+ export interface ResolvedProfilerConfig {
12
+ enabled: boolean;
13
+ mode: ProfilerMode;
14
+ includeServerBreakdown: boolean;
15
+ includeResourceGauges: boolean;
16
+ operationFilters: string[];
17
+ maxRecentEvents: number;
18
+ }
19
+ type RecordCallback = (event: ProfilingEvent) => void;
20
+ export declare function enable(options?: ProfilerRuntimeOptions): void;
21
+ export declare function disable(): void;
22
+ export declare function isEnabled(): boolean;
23
+ export declare function getEffectiveConfig(): ResolvedProfilerConfig;
24
+ export declare function shouldProfile(operation: string, perCallOptions?: PerCallProfiling): boolean;
25
+ export declare function shouldIncludeServerBreakdown(perCallOptions?: PerCallProfiling): boolean;
26
+ export declare function shouldIncludeResourceGauges(perCallOptions?: PerCallProfiling): boolean;
27
+ export declare function generateId(): string;
28
+ export declare function record(event: ProfilingEvent): void;
29
+ /** Returns unsubscribe function. */
30
+ export declare function onRecord(callback: RecordCallback): () => void;
31
+ export declare function getAggregates(): Record<string, AggregatedStats>;
32
+ export declare function getRecentEvents(): ProfilingEvent[];
33
+ export declare function clear(): void;
34
+ export {};