@qvac/inference 0.0.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (610) hide show
  1. package/LICENSE +179 -0
  2. package/NOTICE +986 -0
  3. package/README.md +138 -0
  4. package/dist/api/audio-gen.d.ts +18 -0
  5. package/dist/api/audio-gen.js +148 -0
  6. package/dist/api/batch-completion.d.ts +14 -0
  7. package/dist/api/batch-completion.js +332 -0
  8. package/dist/api/bci-transcribe.d.ts +56 -0
  9. package/dist/api/bci-transcribe.js +172 -0
  10. package/dist/api/cancel.d.ts +34 -0
  11. package/dist/api/cancel.js +82 -0
  12. package/dist/api/classify.d.ts +22 -0
  13. package/dist/api/classify.js +46 -0
  14. package/dist/api/completion-stream.d.ts +106 -0
  15. package/dist/api/completion-stream.js +351 -0
  16. package/dist/api/delete-cache.d.ts +29 -0
  17. package/dist/api/delete-cache.js +49 -0
  18. package/dist/api/diffusion.d.ts +92 -0
  19. package/dist/api/diffusion.js +176 -0
  20. package/dist/api/download-asset.d.ts +49 -0
  21. package/dist/api/download-asset.js +87 -0
  22. package/dist/api/embed.d.ts +41 -0
  23. package/dist/api/embed.js +29 -0
  24. package/dist/api/finetune.d.ts +89 -0
  25. package/dist/api/finetune.js +138 -0
  26. package/dist/api/get-loaded-model-info.d.ts +24 -0
  27. package/dist/api/get-loaded-model-info.js +34 -0
  28. package/dist/api/get-model-info.d.ts +43 -0
  29. package/dist/api/get-model-info.js +21 -0
  30. package/dist/api/get-system-resources.d.ts +2 -0
  31. package/dist/api/get-system-resources.js +16 -0
  32. package/dist/api/heartbeat.d.ts +29 -0
  33. package/dist/api/heartbeat.js +37 -0
  34. package/dist/api/index.d.ts +35 -0
  35. package/dist/api/index.js +35 -0
  36. package/dist/api/invoke-plugin.d.ts +30 -0
  37. package/dist/api/invoke-plugin.js +54 -0
  38. package/dist/api/load-model.d.ts +169 -0
  39. package/dist/api/load-model.js +129 -0
  40. package/dist/api/logging-stream-registry.d.ts +4 -0
  41. package/dist/api/logging-stream-registry.js +61 -0
  42. package/dist/api/logging-stream.d.ts +23 -0
  43. package/dist/api/logging-stream.js +36 -0
  44. package/dist/api/ocr.d.ts +30 -0
  45. package/dist/api/ocr.js +88 -0
  46. package/dist/api/parse-input.d.ts +2 -0
  47. package/dist/api/parse-input.js +14 -0
  48. package/dist/api/provide.d.ts +23 -0
  49. package/dist/api/provide.js +32 -0
  50. package/dist/api/rag.d.ts +251 -0
  51. package/dist/api/rag.js +485 -0
  52. package/dist/api/registry.d.ts +39 -0
  53. package/dist/api/registry.js +63 -0
  54. package/dist/api/resume.d.ts +29 -0
  55. package/dist/api/resume.js +36 -0
  56. package/dist/api/state.d.ts +22 -0
  57. package/dist/api/state.js +29 -0
  58. package/dist/api/stop-provider.d.ts +20 -0
  59. package/dist/api/stop-provider.js +30 -0
  60. package/dist/api/subscribe-logs.d.ts +27 -0
  61. package/dist/api/subscribe-logs.js +43 -0
  62. package/dist/api/suspend.d.ts +30 -0
  63. package/dist/api/suspend.js +37 -0
  64. package/dist/api/text-to-speech.d.ts +64 -0
  65. package/dist/api/text-to-speech.js +434 -0
  66. package/dist/api/transcribe.d.ts +75 -0
  67. package/dist/api/transcribe.js +256 -0
  68. package/dist/api/translate.d.ts +47 -0
  69. package/dist/api/translate.js +115 -0
  70. package/dist/api/unload-model.d.ts +17 -0
  71. package/dist/api/unload-model.js +42 -0
  72. package/dist/api/upscale.d.ts +44 -0
  73. package/dist/api/upscale.js +99 -0
  74. package/dist/api/video.d.ts +86 -0
  75. package/dist/api/video.js +176 -0
  76. package/dist/api/vla-helpers.d.ts +43 -0
  77. package/dist/api/vla-helpers.js +124 -0
  78. package/dist/api/vla.d.ts +68 -0
  79. package/dist/api/vla.js +113 -0
  80. package/dist/config/config-utils.d.ts +4 -0
  81. package/dist/config/config-utils.js +18 -0
  82. package/dist/config/resolve-config.d.ts +8 -0
  83. package/dist/config/resolve-config.js +74 -0
  84. package/dist/constants/audio.d.ts +1 -0
  85. package/dist/constants/audio.js +1 -0
  86. package/dist/constants/index.d.ts +2 -0
  87. package/dist/constants/index.js +2 -0
  88. package/dist/constants/registry.d.ts +1 -0
  89. package/dist/constants/registry.js +1 -0
  90. package/dist/dispatch.d.ts +18 -0
  91. package/dist/dispatch.js +271 -0
  92. package/dist/engine.d.ts +5 -0
  93. package/dist/engine.js +27 -0
  94. package/dist/errors/index.d.ts +416 -0
  95. package/dist/errors/index.js +681 -0
  96. package/dist/errors/options.d.ts +2 -0
  97. package/dist/errors/options.js +13 -0
  98. package/dist/handlers/cancelHandler.d.ts +29 -0
  99. package/dist/handlers/cancelHandler.js +67 -0
  100. package/dist/handlers/completion-orchestrate.d.ts +41 -0
  101. package/dist/handlers/completion-orchestrate.js +190 -0
  102. package/dist/handlers/delete-cache.d.ts +11 -0
  103. package/dist/handlers/delete-cache.js +37 -0
  104. package/dist/handlers/download-asset.d.ts +2 -0
  105. package/dist/handlers/download-asset.js +146 -0
  106. package/dist/handlers/finetune.d.ts +2 -0
  107. package/dist/handlers/finetune.js +18 -0
  108. package/dist/handlers/get-loaded-model-info.d.ts +2 -0
  109. package/dist/handlers/get-loaded-model-info.js +45 -0
  110. package/dist/handlers/get-model-info.d.ts +2 -0
  111. package/dist/handlers/get-model-info.js +173 -0
  112. package/dist/handlers/get-system-resources.d.ts +2 -0
  113. package/dist/handlers/get-system-resources.js +53 -0
  114. package/dist/handlers/heartbeat.d.ts +2 -0
  115. package/dist/handlers/heartbeat.js +3 -0
  116. package/dist/handlers/load-model/download-manager.d.ts +95 -0
  117. package/dist/handlers/load-model/download-manager.js +306 -0
  118. package/dist/handlers/load-model/download-stats.d.ts +7 -0
  119. package/dist/handlers/load-model/download-stats.js +126 -0
  120. package/dist/handlers/load-model/handler.d.ts +2 -0
  121. package/dist/handlers/load-model/handler.js +287 -0
  122. package/dist/handlers/load-model/http.d.ts +3 -0
  123. package/dist/handlers/load-model/http.js +857 -0
  124. package/dist/handlers/load-model/hyperdrive.d.ts +4 -0
  125. package/dist/handlers/load-model/hyperdrive.js +560 -0
  126. package/dist/handlers/load-model/index.d.ts +2 -0
  127. package/dist/handlers/load-model/index.js +2 -0
  128. package/dist/handlers/load-model/registry-client-options.d.ts +27 -0
  129. package/dist/handlers/load-model/registry-client-options.js +18 -0
  130. package/dist/handlers/load-model/registry-companion-set.d.ts +22 -0
  131. package/dist/handlers/load-model/registry-companion-set.js +206 -0
  132. package/dist/handlers/load-model/registry-download-utils.d.ts +23 -0
  133. package/dist/handlers/load-model/registry-download-utils.js +112 -0
  134. package/dist/handlers/load-model/registry-metadata.d.ts +10 -0
  135. package/dist/handlers/load-model/registry-metadata.js +23 -0
  136. package/dist/handlers/load-model/registry.d.ts +4 -0
  137. package/dist/handlers/load-model/registry.js +298 -0
  138. package/dist/handlers/load-model/resolve-session.d.ts +33 -0
  139. package/dist/handlers/load-model/resolve-session.js +62 -0
  140. package/dist/handlers/load-model/resolve.d.ts +5 -0
  141. package/dist/handlers/load-model/resolve.js +158 -0
  142. package/dist/handlers/load-model/types.d.ts +59 -0
  143. package/dist/handlers/load-model/types.js +28 -0
  144. package/dist/handlers/logging-stream.d.ts +2 -0
  145. package/dist/handlers/logging-stream.js +39 -0
  146. package/dist/handlers/plugin-dispatch.d.ts +3 -0
  147. package/dist/handlers/plugin-dispatch.js +63 -0
  148. package/dist/handlers/plugin-invoke.d.ts +3 -0
  149. package/dist/handlers/plugin-invoke.js +81 -0
  150. package/dist/handlers/registry.d.ts +4 -0
  151. package/dist/handlers/registry.js +156 -0
  152. package/dist/handlers/resume.d.ts +2 -0
  153. package/dist/handlers/resume.js +11 -0
  154. package/dist/handlers/state.d.ts +2 -0
  155. package/dist/handlers/state.js +7 -0
  156. package/dist/handlers/suspend.d.ts +2 -0
  157. package/dist/handlers/suspend.js +11 -0
  158. package/dist/handlers/types.d.ts +13 -0
  159. package/dist/handlers/types.js +1 -0
  160. package/dist/handlers/unload-model.d.ts +2 -0
  161. package/dist/handlers/unload-model.js +29 -0
  162. package/dist/index.d.ts +4 -0
  163. package/dist/index.js +12 -0
  164. package/dist/logging/addon.d.ts +5 -0
  165. package/dist/logging/addon.js +75 -0
  166. package/dist/logging/index.d.ts +5 -0
  167. package/dist/logging/index.js +4 -0
  168. package/dist/logging/logger.d.ts +41 -0
  169. package/dist/logging/logger.js +209 -0
  170. package/dist/logging/namespaces.d.ts +6 -0
  171. package/dist/logging/namespaces.js +7 -0
  172. package/dist/logging/types.d.ts +37 -0
  173. package/dist/logging/types.js +1 -0
  174. package/dist/logging/utils.d.ts +5 -0
  175. package/dist/logging/utils.js +113 -0
  176. package/dist/models/registry/index.d.ts +18 -0
  177. package/dist/models/registry/index.js +21 -0
  178. package/dist/models/registry/models.d.ts +22680 -0
  179. package/dist/models/registry/models.js +23908 -0
  180. package/dist/p2p/cancel-delegated.d.ts +3 -0
  181. package/dist/p2p/cancel-delegated.js +64 -0
  182. package/dist/p2p/completion-stream-delegated.d.ts +4 -0
  183. package/dist/p2p/completion-stream-delegated.js +51 -0
  184. package/dist/p2p/delegate-client.d.ts +9 -0
  185. package/dist/p2p/delegate-client.js +219 -0
  186. package/dist/p2p/delegate-connect-diagnostics.d.ts +8 -0
  187. package/dist/p2p/delegate-connect-diagnostics.js +38 -0
  188. package/dist/p2p/delegate-transport.d.ts +18 -0
  189. package/dist/p2p/delegate-transport.js +237 -0
  190. package/dist/p2p/error-handlers.d.ts +4 -0
  191. package/dist/p2p/error-handlers.js +31 -0
  192. package/dist/p2p/handle-request.d.ts +2 -0
  193. package/dist/p2p/handle-request.js +171 -0
  194. package/dist/p2p/handler-utils.d.ts +19 -0
  195. package/dist/p2p/handler-utils.js +180 -0
  196. package/dist/p2p/heartbeat-delegated.d.ts +3 -0
  197. package/dist/p2p/heartbeat-delegated.js +32 -0
  198. package/dist/p2p/load-model-delegated.d.ts +6 -0
  199. package/dist/p2p/load-model-delegated.js +132 -0
  200. package/dist/p2p/progress-throttle.d.ts +7 -0
  201. package/dist/p2p/progress-throttle.js +47 -0
  202. package/dist/p2p/provideHandler/connection.d.ts +2 -0
  203. package/dist/p2p/provideHandler/connection.js +38 -0
  204. package/dist/p2p/provideHandler/handler.d.ts +2 -0
  205. package/dist/p2p/provideHandler/handler.js +66 -0
  206. package/dist/p2p/provideHandler/proxy.d.ts +2 -0
  207. package/dist/p2p/provideHandler/proxy.js +24 -0
  208. package/dist/p2p/provideHandler.d.ts +1 -0
  209. package/dist/p2p/provideHandler.js +2 -0
  210. package/dist/p2p/rpc-utils.d.ts +4 -0
  211. package/dist/p2p/rpc-utils.js +8 -0
  212. package/dist/p2p/stopProvideHandler.d.ts +2 -0
  213. package/dist/p2p/stopProvideHandler.js +29 -0
  214. package/dist/p2p/swarm.d.ts +9 -0
  215. package/dist/p2p/swarm.js +103 -0
  216. package/dist/p2p/transport-selector.d.ts +2 -0
  217. package/dist/p2p/transport-selector.js +18 -0
  218. package/dist/p2p/unload-model-delegated.d.ts +2 -0
  219. package/dist/p2p/unload-model-delegated.js +31 -0
  220. package/dist/plugins/builtin/asr-ggml/config.d.ts +31 -0
  221. package/dist/plugins/builtin/asr-ggml/config.js +46 -0
  222. package/dist/plugins/builtin/asr-ggml/logging.d.ts +10 -0
  223. package/dist/plugins/builtin/asr-ggml/logging.js +12 -0
  224. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +17 -0
  225. package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
  226. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +2 -0
  227. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +163 -0
  228. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +212 -0
  229. package/dist/plugins/builtin/audiogen-ggml/plugin.js +58 -0
  230. package/dist/plugins/builtin/bci-whispercpp-transcription/args.d.ts +2 -0
  231. package/dist/plugins/builtin/bci-whispercpp-transcription/args.js +13 -0
  232. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +173 -0
  233. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +111 -0
  234. package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.d.ts +74 -0
  235. package/dist/plugins/builtin/bci-whispercpp-transcription/resolve-config.js +11 -0
  236. package/dist/plugins/builtin/ggml-classification/ops/classify.d.ts +5 -0
  237. package/dist/plugins/builtin/ggml-classification/ops/classify.js +31 -0
  238. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +31 -0
  239. package/dist/plugins/builtin/ggml-classification/plugin.js +55 -0
  240. package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.d.ts +7 -0
  241. package/dist/plugins/builtin/ggml-ocr/ops/ocr-stream.js +119 -0
  242. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +102 -0
  243. package/dist/plugins/builtin/ggml-ocr/plugin.js +110 -0
  244. package/dist/plugins/builtin/ggml-ocr/resolve-config.d.ts +5 -0
  245. package/dist/plugins/builtin/ggml-ocr/resolve-config.js +72 -0
  246. package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.d.ts +2 -0
  247. package/dist/plugins/builtin/ggml-vla/ops/vla-hparams.js +12 -0
  248. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -0
  249. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +78 -0
  250. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +65 -0
  251. package/dist/plugins/builtin/ggml-vla/plugin.js +57 -0
  252. package/dist/plugins/builtin/index.d.ts +11 -0
  253. package/dist/plugins/builtin/index.js +11 -0
  254. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +22 -0
  255. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +30 -0
  256. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.d.ts +31 -0
  257. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +99 -0
  258. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +6 -0
  259. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +36 -0
  260. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +20 -0
  261. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +48 -0
  262. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +45 -0
  263. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +374 -0
  264. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +28 -0
  265. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +65 -0
  266. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +10 -0
  267. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +209 -0
  268. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +163 -0
  269. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +581 -0
  270. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +56 -0
  271. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +58 -0
  272. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +671 -0
  273. package/dist/plugins/builtin/llamacpp-completion/plugin.js +481 -0
  274. package/dist/plugins/builtin/llamacpp-completion/transform.d.ts +9 -0
  275. package/dist/plugins/builtin/llamacpp-completion/transform.js +34 -0
  276. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +76 -0
  277. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +101 -0
  278. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +457 -0
  279. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +143 -0
  280. package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.d.ts +31 -0
  281. package/dist/plugins/builtin/nmtcpp-translation/resolve-vocab.js +114 -0
  282. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +133 -0
  283. package/dist/plugins/builtin/parakeet-transcription/plugin.js +130 -0
  284. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +2 -0
  285. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +64 -0
  286. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +2 -0
  287. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +41 -0
  288. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -0
  289. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +239 -0
  290. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +757 -0
  291. package/dist/plugins/builtin/sdcpp-generation/plugin.js +310 -0
  292. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +6 -0
  293. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +30 -0
  294. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +7 -0
  295. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +106 -0
  296. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +6 -0
  297. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +91 -0
  298. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +196 -0
  299. package/dist/plugins/builtin/tts-ggml/plugin.js +305 -0
  300. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +346 -0
  301. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +169 -0
  302. package/dist/plugins/index.d.ts +9 -0
  303. package/dist/plugins/index.js +12 -0
  304. package/dist/plugins/ops/bci-transcribe.d.ts +13 -0
  305. package/dist/plugins/ops/bci-transcribe.js +248 -0
  306. package/dist/plugins/ops/cancel.d.ts +25 -0
  307. package/dist/plugins/ops/cancel.js +43 -0
  308. package/dist/plugins/ops/embed.d.ts +15 -0
  309. package/dist/plugins/ops/embed.js +193 -0
  310. package/dist/plugins/ops/kv-cache-retention.d.ts +11 -0
  311. package/dist/plugins/ops/kv-cache-retention.js +121 -0
  312. package/dist/plugins/ops/kv-cache-utils.d.ts +21 -0
  313. package/dist/plugins/ops/kv-cache-utils.js +120 -0
  314. package/dist/plugins/ops/load-model.d.ts +8 -0
  315. package/dist/plugins/ops/load-model.js +82 -0
  316. package/dist/plugins/ops/transcribe.d.ts +20 -0
  317. package/dist/plugins/ops/transcribe.js +343 -0
  318. package/dist/plugins/ops/translate.d.ts +7 -0
  319. package/dist/plugins/ops/translate.js +256 -0
  320. package/dist/plugins/ops/unload-model.d.ts +2 -0
  321. package/dist/plugins/ops/unload-model.js +45 -0
  322. package/dist/plugins/registry.d.ts +9 -0
  323. package/dist/plugins/registry.js +107 -0
  324. package/dist/profiling/aggregator.d.ts +14 -0
  325. package/dist/profiling/aggregator.js +100 -0
  326. package/dist/profiling/backend-diagnostics.d.ts +8 -0
  327. package/dist/profiling/backend-diagnostics.js +36 -0
  328. package/dist/profiling/clock.d.ts +8 -0
  329. package/dist/profiling/clock.js +23 -0
  330. package/dist/profiling/controller.d.ts +34 -0
  331. package/dist/profiling/controller.js +130 -0
  332. package/dist/profiling/delegation-profiler.d.ts +31 -0
  333. package/dist/profiling/delegation-profiler.js +155 -0
  334. package/dist/profiling/envelope.d.ts +10 -0
  335. package/dist/profiling/envelope.js +45 -0
  336. package/dist/profiling/events.d.ts +19 -0
  337. package/dist/profiling/events.js +35 -0
  338. package/dist/profiling/exporters.d.ts +9 -0
  339. package/dist/profiling/exporters.js +242 -0
  340. package/dist/profiling/index.d.ts +91 -0
  341. package/dist/profiling/index.js +87 -0
  342. package/dist/profiling/model-execution.d.ts +15 -0
  343. package/dist/profiling/model-execution.js +34 -0
  344. package/dist/profiling/operation-metrics.d.ts +24 -0
  345. package/dist/profiling/operation-metrics.js +370 -0
  346. package/dist/profiling/operation-wrappers.d.ts +17 -0
  347. package/dist/profiling/operation-wrappers.js +196 -0
  348. package/dist/profiling/profiler.d.ts +27 -0
  349. package/dist/profiling/profiler.js +127 -0
  350. package/dist/profiling/ring-buffer.d.ts +18 -0
  351. package/dist/profiling/ring-buffer.js +83 -0
  352. package/dist/profiling/types.d.ts +52 -0
  353. package/dist/profiling/types.js +1 -0
  354. package/dist/rag/handler.d.ts +2 -0
  355. package/dist/rag/handler.js +272 -0
  356. package/dist/rag/handlers/chunk.d.ts +2 -0
  357. package/dist/rag/handlers/chunk.js +7 -0
  358. package/dist/rag/handlers/close-workspace.d.ts +2 -0
  359. package/dist/rag/handlers/close-workspace.js +9 -0
  360. package/dist/rag/handlers/delete-embeddings.d.ts +2 -0
  361. package/dist/rag/handlers/delete-embeddings.js +11 -0
  362. package/dist/rag/handlers/delete-workspace.d.ts +2 -0
  363. package/dist/rag/handlers/delete-workspace.js +14 -0
  364. package/dist/rag/handlers/ingest.d.ts +9 -0
  365. package/dist/rag/handlers/ingest.js +19 -0
  366. package/dist/rag/handlers/list-workspaces.d.ts +2 -0
  367. package/dist/rag/handlers/list-workspaces.js +4 -0
  368. package/dist/rag/handlers/reindex.d.ts +9 -0
  369. package/dist/rag/handlers/reindex.js +16 -0
  370. package/dist/rag/handlers/save-embeddings.d.ts +9 -0
  371. package/dist/rag/handlers/save-embeddings.js +15 -0
  372. package/dist/rag/handlers/search.d.ts +2 -0
  373. package/dist/rag/handlers/search.js +16 -0
  374. package/dist/rag/index.d.ts +11 -0
  375. package/dist/rag/index.js +11 -0
  376. package/dist/rag/rag-operation-manager.d.ts +28 -0
  377. package/dist/rag/rag-operation-manager.js +76 -0
  378. package/dist/rag/rag-workspace-manager.d.ts +14 -0
  379. package/dist/rag/rag-workspace-manager.js +143 -0
  380. package/dist/registry.d.ts +2 -0
  381. package/dist/registry.js +166 -0
  382. package/dist/resources/collector.d.ts +643 -0
  383. package/dist/resources/collector.js +226 -0
  384. package/dist/resources/instance.d.ts +4 -0
  385. package/dist/resources/instance.js +15 -0
  386. package/dist/resources/native.d.ts +2 -0
  387. package/dist/resources/native.js +35 -0
  388. package/dist/resources/normalize.d.ts +12 -0
  389. package/dist/resources/normalize.js +73 -0
  390. package/dist/resources/types.d.ts +65 -0
  391. package/dist/resources/types.js +1 -0
  392. package/dist/runtime/cache-lock.d.ts +2 -0
  393. package/dist/runtime/cache-lock.js +67 -0
  394. package/dist/runtime/disposable-scope.d.ts +35 -0
  395. package/dist/runtime/disposable-scope.js +69 -0
  396. package/dist/runtime/env.d.ts +16 -0
  397. package/dist/runtime/env.js +53 -0
  398. package/dist/runtime/index.d.ts +5 -0
  399. package/dist/runtime/index.js +2 -0
  400. package/dist/runtime/lifecycle.d.ts +20 -0
  401. package/dist/runtime/lifecycle.js +102 -0
  402. package/dist/runtime/logging-stream-registry.d.ts +27 -0
  403. package/dist/runtime/logging-stream-registry.js +133 -0
  404. package/dist/runtime/model-config-utils.d.ts +14 -0
  405. package/dist/runtime/model-config-utils.js +128 -0
  406. package/dist/runtime/model-registry.d.ts +74 -0
  407. package/dist/runtime/model-registry.js +130 -0
  408. package/dist/runtime/registry-client.d.ts +4 -0
  409. package/dist/runtime/registry-client.js +118 -0
  410. package/dist/runtime/request-context.d.ts +72 -0
  411. package/dist/runtime/request-context.js +114 -0
  412. package/dist/runtime/request-id.d.ts +17 -0
  413. package/dist/runtime/request-id.js +29 -0
  414. package/dist/runtime/request-registry.d.ts +194 -0
  415. package/dist/runtime/request-registry.js +546 -0
  416. package/dist/runtime/runtime-lifecycle.d.ts +37 -0
  417. package/dist/runtime/runtime-lifecycle.js +163 -0
  418. package/dist/runtime/state.d.ts +26 -0
  419. package/dist/runtime/state.js +138 -0
  420. package/dist/schemas/audio-gen.d.ts +252 -0
  421. package/dist/schemas/audio-gen.js +69 -0
  422. package/dist/schemas/batch-completion-stream.d.ts +467 -0
  423. package/dist/schemas/batch-completion-stream.js +123 -0
  424. package/dist/schemas/bci-config.d.ts +74 -0
  425. package/dist/schemas/bci-config.js +58 -0
  426. package/dist/schemas/bci.d.ts +178 -0
  427. package/dist/schemas/bci.js +89 -0
  428. package/dist/schemas/cancel.d.ts +203 -0
  429. package/dist/schemas/cancel.js +135 -0
  430. package/dist/schemas/classification.d.ts +52 -0
  431. package/dist/schemas/classification.js +33 -0
  432. package/dist/schemas/common.d.ts +6420 -0
  433. package/dist/schemas/common.js +133 -0
  434. package/dist/schemas/completion-event.d.ts +248 -0
  435. package/dist/schemas/completion-event.js +94 -0
  436. package/dist/schemas/completion-stream.d.ts +475 -0
  437. package/dist/schemas/completion-stream.js +236 -0
  438. package/dist/schemas/config.d.ts +5022 -0
  439. package/dist/schemas/config.js +169 -0
  440. package/dist/schemas/delegate.d.ts +29 -0
  441. package/dist/schemas/delegate.js +40 -0
  442. package/dist/schemas/download-asset.d.ts +225 -0
  443. package/dist/schemas/download-asset.js +71 -0
  444. package/dist/schemas/embed.d.ts +41 -0
  445. package/dist/schemas/embed.js +35 -0
  446. package/dist/schemas/engine-addon-map.d.ts +20 -0
  447. package/dist/schemas/engine-addon-map.js +78 -0
  448. package/dist/schemas/error.d.ts +40 -0
  449. package/dist/schemas/error.js +65 -0
  450. package/dist/schemas/errors.d.ts +119 -0
  451. package/dist/schemas/errors.js +615 -0
  452. package/dist/schemas/finetune.d.ts +621 -0
  453. package/dist/schemas/finetune.js +163 -0
  454. package/dist/schemas/index.d.ts +45 -0
  455. package/dist/schemas/index.js +45 -0
  456. package/dist/schemas/json-schema.d.ts +15 -0
  457. package/dist/schemas/json-schema.js +2 -0
  458. package/dist/schemas/lifecycle.d.ts +38 -0
  459. package/dist/schemas/lifecycle.js +21 -0
  460. package/dist/schemas/llamacpp-config.d.ts +403 -0
  461. package/dist/schemas/llamacpp-config.js +133 -0
  462. package/dist/schemas/load-model.d.ts +25635 -0
  463. package/dist/schemas/load-model.js +629 -0
  464. package/dist/schemas/logging-stream.d.ts +33 -0
  465. package/dist/schemas/logging-stream.js +17 -0
  466. package/dist/schemas/mcp-adapter.d.ts +46 -0
  467. package/dist/schemas/mcp-adapter.js +1 -0
  468. package/dist/schemas/model-file.d.ts +27 -0
  469. package/dist/schemas/model-file.js +29 -0
  470. package/dist/schemas/model-info.d.ts +197 -0
  471. package/dist/schemas/model-info.js +144 -0
  472. package/dist/schemas/model-ops.d.ts +36 -0
  473. package/dist/schemas/model-ops.js +40 -0
  474. package/dist/schemas/model-src-utils.d.ts +192 -0
  475. package/dist/schemas/model-src-utils.js +61 -0
  476. package/dist/schemas/model-types.d.ts +275 -0
  477. package/dist/schemas/model-types.js +223 -0
  478. package/dist/schemas/ocr.d.ts +138 -0
  479. package/dist/schemas/ocr.js +59 -0
  480. package/dist/schemas/path-validation.d.ts +9 -0
  481. package/dist/schemas/path-validation.js +22 -0
  482. package/dist/schemas/plugin.d.ts +341 -0
  483. package/dist/schemas/plugin.js +241 -0
  484. package/dist/schemas/profiling.d.ts +695 -0
  485. package/dist/schemas/profiling.js +100 -0
  486. package/dist/schemas/provide.d.ts +50 -0
  487. package/dist/schemas/provide.js +32 -0
  488. package/dist/schemas/rag.d.ts +301 -0
  489. package/dist/schemas/rag.js +211 -0
  490. package/dist/schemas/registry.d.ts +453 -0
  491. package/dist/schemas/registry.js +104 -0
  492. package/dist/schemas/reload-config.d.ts +585 -0
  493. package/dist/schemas/reload-config.js +42 -0
  494. package/dist/schemas/runtime-context.d.ts +18 -0
  495. package/dist/schemas/runtime-context.js +7 -0
  496. package/dist/schemas/sdcpp-config.d.ts +1126 -0
  497. package/dist/schemas/sdcpp-config.js +748 -0
  498. package/dist/schemas/system-resources.d.ts +3966 -0
  499. package/dist/schemas/system-resources.js +149 -0
  500. package/dist/schemas/text-to-speech.d.ts +1352 -0
  501. package/dist/schemas/text-to-speech.js +364 -0
  502. package/dist/schemas/tools.d.ts +80 -0
  503. package/dist/schemas/tools.js +52 -0
  504. package/dist/schemas/transcription-config.d.ts +161 -0
  505. package/dist/schemas/transcription-config.js +156 -0
  506. package/dist/schemas/transcription.d.ts +261 -0
  507. package/dist/schemas/transcription.js +145 -0
  508. package/dist/schemas/translate.d.ts +138 -0
  509. package/dist/schemas/translate.js +103 -0
  510. package/dist/schemas/translation-config.d.ts +1031 -0
  511. package/dist/schemas/translation-config.js +170 -0
  512. package/dist/schemas/vla.d.ts +110 -0
  513. package/dist/schemas/vla.js +151 -0
  514. package/dist/selection.d.ts +8 -0
  515. package/dist/selection.js +14 -0
  516. package/dist/surface.d.ts +18 -0
  517. package/dist/surface.js +30 -0
  518. package/dist/utils/addon-responses.d.ts +58 -0
  519. package/dist/utils/addon-responses.js +1 -0
  520. package/dist/utils/aggregate-events.d.ts +25 -0
  521. package/dist/utils/aggregate-events.js +73 -0
  522. package/dist/utils/archive.d.ts +4 -0
  523. package/dist/utils/archive.js +180 -0
  524. package/dist/utils/asr-events.d.ts +14 -0
  525. package/dist/utils/asr-events.js +30 -0
  526. package/dist/utils/audio/decoder.d.ts +4 -0
  527. package/dist/utils/audio/decoder.js +80 -0
  528. package/dist/utils/audio-input.d.ts +9 -0
  529. package/dist/utils/audio-input.js +34 -0
  530. package/dist/utils/cache/auto-history.d.ts +3 -0
  531. package/dist/utils/cache/auto-history.js +26 -0
  532. package/dist/utils/cache/paths.d.ts +42 -0
  533. package/dist/utils/cache/paths.js +133 -0
  534. package/dist/utils/cache/types.d.ts +7 -0
  535. package/dist/utils/cache/types.js +1 -0
  536. package/dist/utils/cache-normalize.d.ts +9 -0
  537. package/dist/utils/cache-normalize.js +20 -0
  538. package/dist/utils/checksum.d.ts +3 -0
  539. package/dist/utils/checksum.js +29 -0
  540. package/dist/utils/clear-storage.d.ts +4 -0
  541. package/dist/utils/clear-storage.js +14 -0
  542. package/dist/utils/completion-normalizer.d.ts +17 -0
  543. package/dist/utils/completion-normalizer.js +453 -0
  544. package/dist/utils/config-transform.d.ts +2 -0
  545. package/dist/utils/config-transform.js +21 -0
  546. package/dist/utils/decorate-promise.d.ts +33 -0
  547. package/dist/utils/decorate-promise.js +35 -0
  548. package/dist/utils/encoding.d.ts +16 -0
  549. package/dist/utils/encoding.js +27 -0
  550. package/dist/utils/expand-gguf-shards.d.ts +2 -0
  551. package/dist/utils/expand-gguf-shards.js +22 -0
  552. package/dist/utils/formatting.d.ts +18 -0
  553. package/dist/utils/formatting.js +56 -0
  554. package/dist/utils/gguf-tensor-extractor.d.ts +14 -0
  555. package/dist/utils/gguf-tensor-extractor.js +311 -0
  556. package/dist/utils/global-singleton.d.ts +1 -0
  557. package/dist/utils/global-singleton.js +10 -0
  558. package/dist/utils/index.d.ts +11 -0
  559. package/dist/utils/index.js +11 -0
  560. package/dist/utils/load-model-validation.d.ts +6 -0
  561. package/dist/utils/load-model-validation.js +18 -0
  562. package/dist/utils/mcp-adapter.d.ts +11 -0
  563. package/dist/utils/mcp-adapter.js +96 -0
  564. package/dist/utils/multi-gpu-mobile.d.ts +4 -0
  565. package/dist/utils/multi-gpu-mobile.js +10 -0
  566. package/dist/utils/object.d.ts +1 -0
  567. package/dist/utils/object.js +3 -0
  568. package/dist/utils/path-security.d.ts +22 -0
  569. package/dist/utils/path-security.js +69 -0
  570. package/dist/utils/qvac-paths.d.ts +1 -0
  571. package/dist/utils/qvac-paths.js +5 -0
  572. package/dist/utils/response-format.d.ts +2 -0
  573. package/dist/utils/response-format.js +20 -0
  574. package/dist/utils/shard-utils.d.ts +47 -0
  575. package/dist/utils/shard-utils.js +123 -0
  576. package/dist/utils/tool-helpers.d.ts +24 -0
  577. package/dist/utils/tool-helpers.js +111 -0
  578. package/dist/utils/tool-integration.d.ts +23 -0
  579. package/dist/utils/tool-integration.js +29 -0
  580. package/dist/utils/tools/dialect.d.ts +7 -0
  581. package/dist/utils/tools/dialect.js +20 -0
  582. package/dist/utils/tools/index.d.ts +2 -0
  583. package/dist/utils/tools/index.js +2 -0
  584. package/dist/utils/tools/parser.d.ts +5 -0
  585. package/dist/utils/tools/parser.js +69 -0
  586. package/dist/utils/tools/parsers/dsml.d.ts +3 -0
  587. package/dist/utils/tools/parsers/dsml.js +177 -0
  588. package/dist/utils/tools/parsers/gemma4native.d.ts +3 -0
  589. package/dist/utils/tools/parsers/gemma4native.js +65 -0
  590. package/dist/utils/tools/parsers/harmony.d.ts +3 -0
  591. package/dist/utils/tools/parsers/harmony.js +57 -0
  592. package/dist/utils/tools/parsers/hermes.d.ts +3 -0
  593. package/dist/utils/tools/parsers/hermes.js +122 -0
  594. package/dist/utils/tools/parsers/json.d.ts +5 -0
  595. package/dist/utils/tools/parsers/json.js +115 -0
  596. package/dist/utils/tools/parsers/pythonic.d.ts +3 -0
  597. package/dist/utils/tools/parsers/pythonic.js +267 -0
  598. package/dist/utils/tools/parsers/qwen35.d.ts +3 -0
  599. package/dist/utils/tools/parsers/qwen35.js +125 -0
  600. package/dist/utils/tools/shared.d.ts +17 -0
  601. package/dist/utils/tools/shared.js +52 -0
  602. package/dist/utils/transcribe-metadata.d.ts +20 -0
  603. package/dist/utils/transcribe-metadata.js +28 -0
  604. package/dist/utils/tts-stats.d.ts +25 -0
  605. package/dist/utils/tts-stats.js +16 -0
  606. package/dist/utils/withTimeout.d.ts +17 -0
  607. package/dist/utils/withTimeout.js +47 -0
  608. package/dist/utils/zod-error.d.ts +2 -0
  609. package/dist/utils/zod-error.js +4 -0
  610. package/package.json +266 -1
@@ -0,0 +1,21 @@
1
+ import { type CacheMessage } from '../../utils/index.js';
2
+ export declare function extractSystemPrompt(messages: CacheMessage[]): string | null;
3
+ export declare function generateConfigHash(systemPrompt: string | null, tools?: unknown): string;
4
+ export declare function generateCacheKey(messages: CacheMessage[]): string;
5
+ export declare function getCacheFilePath(modelId: string, configHash: string, cacheKey: string): Promise<string>;
6
+ export declare function findMatchingCache(modelId: string, configHash: string, currentHistory: CacheMessage[]): Promise<{
7
+ cacheKey: string;
8
+ cachePath: string;
9
+ } | null>;
10
+ export declare function getCurrentCacheInfo(modelId: string, configHash: string, currentHistory: CacheMessage[]): Promise<{
11
+ cacheKey: string;
12
+ cachePath: string;
13
+ }>;
14
+ export declare function renameCacheFile(oldPath: string, newPath: string): Promise<boolean>;
15
+ export declare function pruneEmptyCacheDirectories(cacheFilePath: string): Promise<void>;
16
+ export declare function deleteCache(options: {
17
+ all: true;
18
+ } | {
19
+ kvCacheKey: string;
20
+ modelId?: string;
21
+ }): Promise<string>;
@@ -0,0 +1,120 @@
1
+ import crypto from 'bare-crypto';
2
+ import { promises as fsPromises } from 'bare-fs';
3
+ import path from 'bare-path';
4
+ import { getAutoCacheLookupHistory, getKVCacheDir, validateAndJoinPath } from '../../utils/index.js';
5
+ import { getEngineLogger } from '../../logging/index.js';
6
+ import { Buffer } from 'bare-buffer';
7
+ import { markAutoCacheKey } from '../../plugins/ops/kv-cache-retention.js';
8
+ const logger = getEngineLogger();
9
+ // In-memory KV-cache state lives in `KvCacheSession` (the single
10
+ // mutation point for all three KV-cache bookkeeping layers). This
11
+ // module keeps only the pure path / hash utilities that don't touch
12
+ // in-memory state.
13
+ export function extractSystemPrompt(messages) {
14
+ const systemMessage = messages.find((msg) => msg.role === 'system');
15
+ return systemMessage ? systemMessage.content : null;
16
+ }
17
+ function getToolNamesForHash(tools) {
18
+ if (!Array.isArray(tools))
19
+ return [];
20
+ return tools
21
+ .map((t) => t.name)
22
+ .filter((n) => typeof n === 'string')
23
+ .sort();
24
+ }
25
+ // Cache hash based on system prompt + tool names
26
+ // Different tools = different cache (tools anchored in first message, protected from n_discarded)
27
+ export function generateConfigHash(systemPrompt, tools) {
28
+ const hash = crypto.createHash('sha-256');
29
+ const toolNames = getToolNamesForHash(tools);
30
+ hash.update(Buffer.from(JSON.stringify({ systemPrompt, toolNames }), 'utf8'));
31
+ return hash.digest('hex').substring(0, 16);
32
+ }
33
+ export function generateCacheKey(messages) {
34
+ const hash = crypto.createHash('sha-256');
35
+ const historyString = JSON.stringify(messages);
36
+ const historyBuffer = Buffer.from(historyString, 'utf8');
37
+ hash.update(historyBuffer);
38
+ const hashString = hash.digest('hex');
39
+ return hashString.substring(0, 16);
40
+ }
41
+ function resolveCacheFilePath(modelId, configHash, cacheKey) {
42
+ const cacheDir = getKVCacheDir();
43
+ const sessionCacheDir = validateAndJoinPath(cacheDir, cacheKey);
44
+ const modelCacheDir = validateAndJoinPath(sessionCacheDir, modelId);
45
+ return path.join(modelCacheDir, `${configHash}.bin`);
46
+ }
47
+ export async function getCacheFilePath(modelId, configHash, cacheKey) {
48
+ const cachePath = resolveCacheFilePath(modelId, configHash, cacheKey);
49
+ const modelCacheDir = path.dirname(cachePath);
50
+ try {
51
+ await fsPromises.mkdir(modelCacheDir, { recursive: true });
52
+ }
53
+ catch {
54
+ // Ignore if directories already exist
55
+ }
56
+ return cachePath;
57
+ }
58
+ // Used for auto-generated cache key
59
+ export async function findMatchingCache(modelId, configHash, currentHistory) {
60
+ if (currentHistory.length <= 1) {
61
+ return null;
62
+ }
63
+ const previousHistory = getAutoCacheLookupHistory(currentHistory);
64
+ const cacheKey = generateCacheKey(previousHistory);
65
+ const cachePath = resolveCacheFilePath(modelId, configHash, cacheKey);
66
+ try {
67
+ await fsPromises.access(cachePath);
68
+ await markAutoCacheKey(cacheKey);
69
+ return { cacheKey, cachePath };
70
+ }
71
+ catch {
72
+ return null;
73
+ }
74
+ }
75
+ export async function getCurrentCacheInfo(modelId, configHash, currentHistory) {
76
+ const cacheKey = generateCacheKey(currentHistory);
77
+ await markAutoCacheKey(cacheKey);
78
+ const cachePath = await getCacheFilePath(modelId, configHash, cacheKey);
79
+ return { cacheKey, cachePath };
80
+ }
81
+ export async function renameCacheFile(oldPath, newPath) {
82
+ try {
83
+ await fsPromises.rename(oldPath, newPath);
84
+ return true;
85
+ }
86
+ catch (error) {
87
+ logger.error('Error renaming cache file:', error instanceof Error ? error.message : String(error));
88
+ return false;
89
+ }
90
+ }
91
+ export async function pruneEmptyCacheDirectories(cacheFilePath) {
92
+ const cacheDir = getKVCacheDir();
93
+ const cacheDirPrefix = `${cacheDir}${path.sep}`;
94
+ let currentDirectory = path.dirname(cacheFilePath);
95
+ while (currentDirectory.startsWith(cacheDirPrefix)) {
96
+ try {
97
+ await fsPromises.rmdir(currentDirectory);
98
+ }
99
+ catch {
100
+ return;
101
+ }
102
+ currentDirectory = path.dirname(currentDirectory);
103
+ }
104
+ }
105
+ // Cache-existence probing (in-memory registry first, fall back to
106
+ // `fs.access`) lives in `KvCacheSession.beginTurn(...)`. Keeping the
107
+ // in-memory `initializedCaches` set private to the session module
108
+ // avoids drift between the two layers.
109
+ export async function deleteCache(options) {
110
+ const cacheDir = getKVCacheDir();
111
+ if ('all' in options) {
112
+ await fsPromises.rm(cacheDir, { recursive: true, force: true });
113
+ await fsPromises.mkdir(cacheDir, { recursive: true });
114
+ return cacheDir;
115
+ }
116
+ const kvCacheDir = validateAndJoinPath(cacheDir, options.kvCacheKey);
117
+ const targetDir = options.modelId !== undefined ? validateAndJoinPath(kvCacheDir, options.modelId) : kvCacheDir;
118
+ await fsPromises.rm(targetDir, { recursive: true, force: true });
119
+ return targetDir;
120
+ }
@@ -0,0 +1,8 @@
1
+ import { type LoadModelServerParams } from '../../schemas/index.js';
2
+ import type { LoadTimingStats } from '../../profiling/types.js';
3
+ export interface LoadModelResult {
4
+ timing?: LoadTimingStats;
5
+ }
6
+ export declare function loadModel(params: LoadModelServerParams, options?: {
7
+ collectTiming?: boolean;
8
+ }): Promise<LoadModelResult>;
@@ -0,0 +1,82 @@
1
+ import { loadModelServerParamsSchema, normalizeModelType } from '../../schemas/index.js';
2
+ import { nowMs } from '../../profiling/clock.js';
3
+ import { forwardBackendDiagnostics } from '../../profiling/backend-diagnostics.js';
4
+ import { isModelLoaded, registerModel } from '../../runtime/model-registry.js';
5
+ import { startLogBuffering, stopLogBufferingWithTimeout } from '../../runtime/logging-stream-registry.js';
6
+ import { detectShardedModel, generateShardFilenames, validateShardedModelCache } from '../../utils/index.js';
7
+ import { PluginNotFoundError, ModelFileNotFoundError, ModelFileNotFoundInDirError, ModelFileLocateFailedError } from '../../errors/index.js';
8
+ import { getPlugin } from '../../plugins/index.js';
9
+ import { promises as fsPromises } from 'bare-fs';
10
+ import path from 'bare-path';
11
+ import { getEngineLogger } from '../../logging/index.js';
12
+ const logger = getEngineLogger();
13
+ export async function loadModel(params, options) {
14
+ const { modelId, modelPath, options: modelOptions, artifacts, modelName } = loadModelServerParamsSchema.parse(params);
15
+ const { modelConfig, modelType: rawModelType } = modelOptions;
16
+ // Normalize modelType to canonical form (handles aliases and custom types)
17
+ const modelType = normalizeModelType(rawModelType);
18
+ // Check if model is already loaded
19
+ if (isModelLoaded(modelId)) {
20
+ logger.info(`${modelType} model ${modelId} is already loaded`);
21
+ return {};
22
+ }
23
+ // Detect if sharded model
24
+ const modelFileName = path.basename(modelPath);
25
+ const shardInfo = detectShardedModel(modelFileName);
26
+ const isShardedModel = shardInfo.isSharded;
27
+ const plugin = getPlugin(modelType);
28
+ if (!plugin) {
29
+ throw new PluginNotFoundError(modelType);
30
+ }
31
+ if (isShardedModel) {
32
+ // For sharded models, validate all shards and tensors.txt exist
33
+ const shardDir = path.dirname(modelPath);
34
+ const isValid = await validateShardedModelCache(shardDir, modelFileName);
35
+ if (!isValid) {
36
+ const numberedShards = generateShardFilenames(modelFileName);
37
+ throw new ModelFileNotFoundError(`Missing shards or ${shardInfo.baseFilename}.tensors.txt. Expected ${numberedShards.length} shard files + tensors.txt in ${shardDir}`);
38
+ }
39
+ }
40
+ else if (!plugin.skipPrimaryModelPathValidation) {
41
+ // For non-sharded models, validate single file exists
42
+ try {
43
+ const modelDir = path.dirname(modelPath);
44
+ const modelFile = path.basename(modelPath);
45
+ const files = (await fsPromises.readdir(modelDir));
46
+ if (!files.includes(modelFile)) {
47
+ throw new ModelFileNotFoundInDirError(modelFile, modelDir, modelType);
48
+ }
49
+ }
50
+ catch (error) {
51
+ logger.error(`Error reading ${modelType} model directory:`, error instanceof Error ? error.message : String(error));
52
+ throw new ModelFileLocateFailedError(modelType, modelPath, error);
53
+ }
54
+ }
55
+ logger.info(`${modelType}: Loading model ${modelId}...`);
56
+ startLogBuffering(modelId);
57
+ try {
58
+ const initStart = options?.collectTiming ? nowMs() : 0;
59
+ const result = plugin.createModel({
60
+ modelId,
61
+ modelPath,
62
+ modelConfig: modelConfig,
63
+ modelName,
64
+ artifacts
65
+ });
66
+ await result.model.load(false);
67
+ const modelInitializationTimeMs = options?.collectTiming ? nowMs() - initStart : undefined;
68
+ logger.info(`${modelType} model ${modelId} loaded`);
69
+ registerModel(modelId, {
70
+ model: result.model,
71
+ path: modelPath,
72
+ config: modelConfig,
73
+ modelType: modelType,
74
+ name: modelName
75
+ });
76
+ const loadResult = modelInitializationTimeMs !== undefined ? { timing: { modelInitializationTimeMs } } : {};
77
+ return forwardBackendDiagnostics(loadResult, result);
78
+ }
79
+ finally {
80
+ stopLogBufferingWithTimeout(modelId);
81
+ }
82
+ }
@@ -0,0 +1,20 @@
1
+ import { type TranscribeParams, type TranscribeSegment, type TranscribeStats, type TranscribeStreamEvent, type ParakeetStreamingRunConfig } from '../../schemas/index.js';
2
+ import type Buffer from 'bare-buffer';
3
+ import { assertMetadataSupported, toTranscribeSegment, type WhisperAddonSegment } from '../../utils/transcribe-metadata.js';
4
+ export { assertMetadataSupported, toTranscribeSegment, type WhisperAddonSegment };
5
+ type TranscribeReturn = {
6
+ modelExecutionMs: number;
7
+ stats?: TranscribeStats;
8
+ };
9
+ export declare function transcribe(params: TranscribeParams & {
10
+ metadata: true;
11
+ }, requestId?: string): AsyncGenerator<TranscribeSegment, TranscribeReturn, void>;
12
+ export declare function transcribe(params: TranscribeParams, requestId?: string): AsyncGenerator<string, TranscribeReturn, void>;
13
+ export interface TranscribeStreamOpts {
14
+ emitVadEvents?: boolean;
15
+ endOfTurnSilenceMs?: number;
16
+ vadRunIntervalMs?: number;
17
+ parakeetStreamingConfig?: ParakeetStreamingRunConfig;
18
+ }
19
+ export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent, void, void>;
20
+ export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<string | TranscribeStreamEvent, void, void>;
@@ -0,0 +1,343 @@
1
+ var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
2
+ if (value !== null && value !== void 0) {
3
+ if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
4
+ var dispose, inner;
5
+ if (async) {
6
+ if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
7
+ dispose = value[Symbol.asyncDispose];
8
+ }
9
+ if (dispose === void 0) {
10
+ if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
11
+ dispose = value[Symbol.dispose];
12
+ if (async) inner = dispose;
13
+ }
14
+ if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
15
+ if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
16
+ env.stack.push({ value: value, dispose: dispose, async: async });
17
+ }
18
+ else if (async) {
19
+ env.stack.push({ async: true });
20
+ }
21
+ return value;
22
+ };
23
+ var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
24
+ return function (env) {
25
+ function fail(e) {
26
+ env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
27
+ env.hasError = true;
28
+ }
29
+ var r, s = 0;
30
+ function next() {
31
+ while (r = env.stack.pop()) {
32
+ try {
33
+ if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
34
+ if (r.dispose) {
35
+ var result = r.dispose.call(r.value);
36
+ if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
37
+ }
38
+ else s |= 1;
39
+ }
40
+ catch (e) {
41
+ fail(e);
42
+ }
43
+ }
44
+ if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
45
+ if (env.hasError) throw env.error;
46
+ }
47
+ return next();
48
+ };
49
+ })(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
50
+ var e = new Error(message);
51
+ return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
+ });
53
+ import { getModel, getModelConfig, getModelEntry } from '../../runtime/model-registry.js';
54
+ import { ModelType } from '../../schemas/index.js';
55
+ import { createAudioStream } from '../../utils/audio-input.js';
56
+ import { getEngineLogger } from '../../logging/index.js';
57
+ import { TranscriptionFailedError } from '../../errors/index.js';
58
+ import { nowMs } from '../../profiling/index.js';
59
+ import { buildStreamResult } from '../../profiling/model-execution.js';
60
+ import { assertMetadataSupported, toTranscribeSegment } from '../../utils/transcribe-metadata.js';
61
+ import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
62
+ import { generateRandomRequestId } from '../../runtime/request-id.js';
63
+ import { isEndOfTurnEvent, isVadEvent, toEndOfTurnEvent, toVadStateEvent } from '../../utils/asr-events.js';
64
+ import { buildWhisperReloadConfig } from '../../plugins/builtin/asr-ggml/config.js';
65
+ export { assertMetadataSupported, toTranscribeSegment };
66
+ function hasRunStreaming(model) {
67
+ return 'runStreaming' in model && typeof model.runStreaming === 'function';
68
+ }
69
+ const SILENCE_MARKERS = {
70
+ [ModelType.whispercppTranscription]: '[BLANK_AUDIO]',
71
+ [ModelType.parakeetTranscription]: '[No speech detected]'
72
+ };
73
+ function getEngineModelType(modelId) {
74
+ const entry = getModelEntry(modelId);
75
+ if (!entry || entry.isDelegated)
76
+ return '';
77
+ return entry.local.modelType;
78
+ }
79
+ function getAudioFormat(modelId, engineType) {
80
+ if (engineType === ModelType.whispercppTranscription) {
81
+ const config = getModelConfig(modelId);
82
+ return config.audio_format || 's16le';
83
+ }
84
+ return 's16le';
85
+ }
86
+ async function applyPrompt(modelId, prompt, engineType) {
87
+ if (engineType !== ModelType.whispercppTranscription || !prompt) {
88
+ return null;
89
+ }
90
+ const model = getModel(modelId);
91
+ if (typeof model.reload !== 'function')
92
+ return null;
93
+ const originalConfig = getModelConfig(modelId);
94
+ await model.reload(buildWhisperReloadConfig({ ...originalConfig, initial_prompt: prompt }));
95
+ return originalConfig;
96
+ }
97
+ async function restorePrompt(modelId, originalConfig) {
98
+ const model = getModel(modelId);
99
+ if (typeof model.reload !== 'function')
100
+ return;
101
+ await model.reload(buildWhisperReloadConfig({ ...originalConfig, initial_prompt: '' }));
102
+ }
103
+ export async function* transcribe(params, requestId) {
104
+ const env_1 = { stack: [], error: void 0, hasError: false };
105
+ try {
106
+ const { modelId, metadata } = params;
107
+ // Open a request-scoped lifecycle. The registry routes
108
+ // `cancel({ requestId })` and `cancel({ modelId, kind: "transcribe" })`
109
+ // through this context. Falls back to a generated id if the
110
+ // caller didn't send one.
111
+ const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
112
+ requestId: requestId ?? generateRandomRequestId(),
113
+ kind: 'transcribe',
114
+ modelId
115
+ }), true);
116
+ const requestLogger = withRequestContext(getEngineLogger(), ctx);
117
+ const engineType = getEngineModelType(modelId);
118
+ assertMetadataSupported(modelId, engineType, metadata);
119
+ const silenceMarker = SILENCE_MARKERS[engineType] ?? '';
120
+ const audioFormat = getAudioFormat(modelId, engineType);
121
+ const originalConfig = await applyPrompt(modelId, params.prompt, engineType);
122
+ if (originalConfig) {
123
+ // `restorePrompt` runs on every exit path — happy, throw, cancel —
124
+ // via the scope. LIFO unwinding pairs with the addon-cancel detach
125
+ // below.
126
+ ctx.scope.defer(() => restorePrompt(modelId, originalConfig));
127
+ }
128
+ const model = getModel(modelId);
129
+ // Hard-cancel wiring: whisper.cpp / parakeet expose model-wide
130
+ // `addon.cancel()`. The listener forwards an abort so the
131
+ // currently-running transcription stops decoding ASAP. The
132
+ // `if (ctx.signal.aborted) break` guard inside the iterate loop is
133
+ // the soft-cancel safety net for the case where the abort fires
134
+ // between the addon flag flipping and the iterator's next pull.
135
+ const onAbort = () => {
136
+ const addon = model.addon;
137
+ if (addon?.cancel) {
138
+ addon.cancel.call(addon).catch((err) => {
139
+ requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
140
+ });
141
+ }
142
+ };
143
+ ctx.signal.addEventListener('abort', onAbort, { once: true });
144
+ if (ctx.signal.aborted)
145
+ onAbort();
146
+ ctx.scope.defer(() => {
147
+ ctx.signal.removeEventListener('abort', onAbort);
148
+ });
149
+ const audioStream = await createAudioStream(params.audioChunk, audioFormat);
150
+ const modelStart = nowMs();
151
+ const response = (await model.run(audioStream));
152
+ for await (const output of response.iterate()) {
153
+ if (ctx.signal.aborted)
154
+ break;
155
+ requestLogger.debug('Streaming Transcription Update:', output);
156
+ const chunks = (Array.isArray(output) ? output : [output]);
157
+ if (metadata) {
158
+ for (const chunk of chunks) {
159
+ if (!chunk.text)
160
+ continue;
161
+ if (silenceMarker && chunk.text.includes(silenceMarker))
162
+ continue;
163
+ yield toTranscribeSegment(chunk);
164
+ }
165
+ continue;
166
+ }
167
+ const text = chunks
168
+ .filter((chunk) => !silenceMarker || !chunk.text.includes(silenceMarker))
169
+ .map((chunk) => chunk.text)
170
+ .join('');
171
+ if (text.trim()) {
172
+ yield text;
173
+ }
174
+ }
175
+ const modelExecutionMs = nowMs() - modelStart;
176
+ const stats = {
177
+ ...(response.stats?.audioDurationMs !== undefined && {
178
+ audioDuration: response.stats.audioDurationMs
179
+ }),
180
+ ...(response.stats?.realTimeFactor !== undefined && {
181
+ realTimeFactor: response.stats.realTimeFactor
182
+ }),
183
+ ...(response.stats?.tokensPerSecond !== undefined && {
184
+ tokensPerSecond: response.stats.tokensPerSecond
185
+ }),
186
+ ...(response.stats?.totalTokens !== undefined && {
187
+ totalTokens: response.stats.totalTokens
188
+ }),
189
+ ...(response.stats?.totalSegments !== undefined && {
190
+ totalSegments: response.stats.totalSegments
191
+ }),
192
+ ...(response.stats?.whisperEncodeMs !== undefined && {
193
+ whisperEncodeTime: response.stats.whisperEncodeMs
194
+ }),
195
+ ...(response.stats?.whisperDecodeMs !== undefined && {
196
+ whisperDecodeTime: response.stats.whisperDecodeMs
197
+ }),
198
+ ...(response.stats?.encoderMs !== undefined && {
199
+ encoderTime: response.stats.encoderMs
200
+ }),
201
+ ...(response.stats?.decoderMs !== undefined && {
202
+ decoderTime: response.stats.decoderMs
203
+ }),
204
+ ...(response.stats?.melSpecMs !== undefined && {
205
+ melSpecTime: response.stats.melSpecMs
206
+ }),
207
+ ...(response.stats?.backendDevice !== undefined && {
208
+ backendDevice: response.stats.backendDevice
209
+ }),
210
+ ...(response.stats?.backendId !== undefined && {
211
+ backendId: response.stats.backendId
212
+ }),
213
+ ...(response.stats?.gpuUnsupported !== undefined && {
214
+ gpuUnsupported: response.stats.gpuUnsupported
215
+ }),
216
+ ...(response.stats?.gpuMemTotalMb !== undefined && {
217
+ gpuMemTotalMb: response.stats.gpuMemTotalMb
218
+ }),
219
+ ...(response.stats?.gpuMemFreeMb !== undefined && {
220
+ gpuMemFreeMb: response.stats.gpuMemFreeMb
221
+ })
222
+ };
223
+ return buildStreamResult(modelExecutionMs, stats);
224
+ }
225
+ catch (e_1) {
226
+ env_1.error = e_1;
227
+ env_1.hasError = true;
228
+ }
229
+ finally {
230
+ const result_1 = __disposeResources(env_1);
231
+ if (result_1)
232
+ await result_1;
233
+ }
234
+ }
235
+ function buildRunStreamingOpts(engineType, opts) {
236
+ if (engineType === ModelType.parakeetTranscription) {
237
+ return opts?.parakeetStreamingConfig;
238
+ }
239
+ const runOpts = {};
240
+ if (opts?.emitVadEvents)
241
+ runOpts.emitVadEvents = true;
242
+ if (opts?.endOfTurnSilenceMs !== undefined) {
243
+ runOpts.endOfTurnSilenceMs = opts.endOfTurnSilenceMs;
244
+ }
245
+ if (opts?.vadRunIntervalMs !== undefined) {
246
+ runOpts.vadRunIntervalMs = opts.vadRunIntervalMs;
247
+ }
248
+ return runOpts;
249
+ }
250
+ export async function* transcribeStream(modelId, audioInputStream, prompt, metadata, opts, requestId) {
251
+ const env_2 = { stack: [], error: void 0, hasError: false };
252
+ try {
253
+ // Same `kind: "transcribe"` as the unary variant — the registry
254
+ // doesn't distinguish streaming vs non-streaming variants of the same
255
+ // operation, so `cancel({ modelId, kind: "transcribe" })` cancels
256
+ // either shape.
257
+ const ctx = __addDisposableResource(env_2, await getRequestRegistry().begin({
258
+ requestId: requestId ?? generateRandomRequestId(),
259
+ kind: 'transcribe',
260
+ modelId
261
+ }), true);
262
+ const requestLogger = withRequestContext(getEngineLogger(), ctx);
263
+ const engineType = getEngineModelType(modelId);
264
+ assertMetadataSupported(modelId, engineType, metadata);
265
+ const silenceMarker = SILENCE_MARKERS[engineType] ?? '';
266
+ const originalConfig = await applyPrompt(modelId, prompt, engineType);
267
+ if (originalConfig) {
268
+ ctx.scope.defer(() => restorePrompt(modelId, originalConfig));
269
+ }
270
+ const model = getModel(modelId);
271
+ if (!hasRunStreaming(model)) {
272
+ throw new TranscriptionFailedError(`Model ${modelId} does not support streaming transcription`);
273
+ }
274
+ const onAbort = () => {
275
+ const addon = model.addon;
276
+ if (addon?.cancel) {
277
+ addon.cancel.call(addon).catch((err) => {
278
+ requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
279
+ });
280
+ }
281
+ };
282
+ ctx.signal.addEventListener('abort', onAbort, { once: true });
283
+ if (ctx.signal.aborted)
284
+ onAbort();
285
+ ctx.scope.defer(() => {
286
+ ctx.signal.removeEventListener('abort', onAbort);
287
+ });
288
+ const runOpts = buildRunStreamingOpts(engineType, opts);
289
+ const response = await model.runStreaming(audioInputStream, runOpts);
290
+ for await (const output of response.iterate()) {
291
+ if (ctx.signal.aborted)
292
+ break;
293
+ requestLogger.debug('Live Transcription Update:', output);
294
+ if (!Array.isArray(output)) {
295
+ if (isVadEvent(output)) {
296
+ yield {
297
+ type: 'vad',
298
+ ...toVadStateEvent(output)
299
+ };
300
+ continue;
301
+ }
302
+ if (isEndOfTurnEvent(output)) {
303
+ const event = toEndOfTurnEvent(output);
304
+ if (event) {
305
+ yield {
306
+ type: 'endOfTurn',
307
+ ...event
308
+ };
309
+ }
310
+ continue;
311
+ }
312
+ yield* emitSegment(output, metadata, silenceMarker);
313
+ continue;
314
+ }
315
+ for (const segment of output) {
316
+ yield* emitSegment(segment, metadata, silenceMarker);
317
+ }
318
+ }
319
+ }
320
+ catch (e_2) {
321
+ env_2.error = e_2;
322
+ env_2.hasError = true;
323
+ }
324
+ finally {
325
+ const result_2 = __disposeResources(env_2);
326
+ if (result_2)
327
+ await result_2;
328
+ }
329
+ }
330
+ function* emitSegment(segment, metadata, silenceMarker) {
331
+ if (!segment.text) {
332
+ return;
333
+ }
334
+ if (silenceMarker && segment.text.includes(silenceMarker)) {
335
+ return;
336
+ }
337
+ if (metadata) {
338
+ yield toTranscribeSegment(segment);
339
+ }
340
+ else if (segment.text.trim()) {
341
+ yield segment.text;
342
+ }
343
+ }
@@ -0,0 +1,7 @@
1
+ import { type TranslateParams, type TranslationStats } from '../../schemas/index.js';
2
+ export declare function getLanguage(code: string | undefined): string;
3
+ export declare function isAfrican(code: string | undefined): boolean;
4
+ export declare function translate(params: TranslateParams, requestId?: string): AsyncGenerator<string, {
5
+ modelExecutionMs: number;
6
+ stats?: TranslationStats;
7
+ }, unknown>;