@qvac/inference 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/NOTICE +60 -28
  2. package/README.md +3 -1
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +135 -44
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +1 -1
  13. package/dist/api/completion-stream.js +2 -1
  14. package/dist/api/delete-cache.d.ts +7 -1
  15. package/dist/api/delete-cache.js +11 -1
  16. package/dist/api/diffusion.d.ts +14 -1
  17. package/dist/api/diffusion.js +14 -1
  18. package/dist/api/embed.d.ts +82 -29
  19. package/dist/api/embed.js +64 -18
  20. package/dist/api/get-model-info.d.ts +1 -1
  21. package/dist/api/index.d.ts +2 -1
  22. package/dist/api/index.js +2 -1
  23. package/dist/api/rag.d.ts +1 -1
  24. package/dist/api/rag.js +1 -1
  25. package/dist/api/registry.d.ts +16 -6
  26. package/dist/api/registry.js +32 -9
  27. package/dist/api/text-to-speech.d.ts +14 -5
  28. package/dist/api/text-to-speech.js +83 -29
  29. package/dist/api/transcribe.d.ts +6 -2
  30. package/dist/api/transcribe.js +7 -1
  31. package/dist/api/vector-index.d.ts +25 -3
  32. package/dist/api/vector-index.js +17 -2
  33. package/dist/api/video-request.d.ts +15 -15
  34. package/dist/dispatch.d.ts +10 -4
  35. package/dist/dispatch.js +65 -15
  36. package/dist/engine.d.ts +1 -1
  37. package/dist/errors/index.d.ts +70 -0
  38. package/dist/errors/index.js +128 -0
  39. package/dist/handlers/assess-model-fit.d.ts +3 -21
  40. package/dist/handlers/assess-model-fit.js +4 -303
  41. package/dist/handlers/cancelHandler.d.ts +10 -9
  42. package/dist/handlers/cancelHandler.js +17 -13
  43. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  44. package/dist/handlers/completion-orchestrate.js +164 -22
  45. package/dist/handlers/delete-cache.d.ts +2 -1
  46. package/dist/handlers/delete-cache.js +6 -1
  47. package/dist/handlers/download-asset.js +11 -4
  48. package/dist/handlers/get-loaded-model-info.js +1 -0
  49. package/dist/handlers/get-model-info.js +71 -2
  50. package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
  51. package/dist/handlers/load-model/coreml-sidecars.js +16 -119
  52. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  53. package/dist/handlers/load-model/download-lock.js +213 -0
  54. package/dist/handlers/load-model/download-manager.d.ts +10 -2
  55. package/dist/handlers/load-model/download-manager.js +62 -11
  56. package/dist/handlers/load-model/handler.js +88 -35
  57. package/dist/handlers/load-model/http.js +327 -205
  58. package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
  59. package/dist/handlers/load-model/hyperdrive.js +123 -50
  60. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  61. package/dist/handlers/load-model/registry-client-options.js +4 -2
  62. package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
  63. package/dist/handlers/load-model/registry-companion-set.js +19 -1
  64. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  65. package/dist/handlers/load-model/registry-delete.js +269 -0
  66. package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
  67. package/dist/handlers/load-model/registry-download-utils.js +93 -22
  68. package/dist/handlers/load-model/registry.d.ts +20 -0
  69. package/dist/handlers/load-model/registry.js +308 -175
  70. package/dist/handlers/load-model/resolve.js +4 -1
  71. package/dist/handlers/load-model/types.d.ts +2 -0
  72. package/dist/handlers/logging-stream.js +17 -5
  73. package/dist/handlers/plugin-invoke.js +152 -39
  74. package/dist/handlers/registry.d.ts +2 -2
  75. package/dist/handlers/registry.js +105 -28
  76. package/dist/handlers/rpc-server.js +7 -7
  77. package/dist/handlers/session.d.ts +11 -0
  78. package/dist/handlers/session.js +40 -0
  79. package/dist/handlers/unload-model.js +8 -3
  80. package/dist/handlers/vector-index.js +118 -19
  81. package/dist/index.d.ts +1 -1
  82. package/dist/index.js +1 -1
  83. package/dist/logging/utils.js +10 -2
  84. package/dist/model-cache-layout.d.ts +1 -0
  85. package/dist/model-cache-layout.js +2 -0
  86. package/dist/model-fit/assess.d.ts +18 -67
  87. package/dist/model-fit/assess.js +96 -665
  88. package/dist/model-fit/describe.d.ts +12 -0
  89. package/dist/model-fit/describe.js +29 -0
  90. package/dist/model-fit/description/fetch.d.ts +1 -1
  91. package/dist/model-fit/description/fetch.js +95 -16
  92. package/dist/model-fit/description/resolve-load.d.ts +5 -3
  93. package/dist/model-fit/description/resolve-load.js +14 -14
  94. package/dist/model-fit/fit-load.d.ts +23 -0
  95. package/dist/model-fit/fit-load.js +29 -0
  96. package/dist/model-fit/load-check.d.ts +15 -33
  97. package/dist/model-fit/load-check.js +50 -195
  98. package/dist/model-fit/run/engine.d.ts +12 -72
  99. package/dist/model-fit/run/engine.js +4 -8
  100. package/dist/model-fit/run/run.d.ts +10 -7
  101. package/dist/model-fit/run/run.js +22 -18
  102. package/dist/model-fit/sum.d.ts +32 -0
  103. package/dist/model-fit/sum.js +60 -0
  104. package/dist/models/registry/models.d.ts +865 -128
  105. package/dist/models/registry/models.js +3737 -2963
  106. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  107. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  108. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  109. package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
  110. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  111. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  112. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  113. package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  114. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  115. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
  116. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  117. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  118. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  119. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  120. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  121. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
  122. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  123. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
  124. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
  125. package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
  126. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  127. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  128. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
  129. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
  130. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
  131. package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
  132. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  133. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  134. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  135. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  136. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  137. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  138. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  139. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
  140. package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
  141. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  142. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  143. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
  144. package/dist/plugins/builtin/index.d.ts +1 -0
  145. package/dist/plugins/builtin/index.js +1 -0
  146. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  147. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  148. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
  149. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  150. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  151. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  152. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  153. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
  154. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
  155. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  156. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  157. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
  158. package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
  159. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  160. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
  161. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
  162. package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
  163. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  164. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  165. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
  166. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
  167. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  168. package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
  169. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
  170. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
  171. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
  172. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
  173. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
  174. package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
  175. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  176. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  177. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
  178. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
  179. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
  180. package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
  181. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  182. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  183. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  184. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  185. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
  186. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
  187. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  188. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  189. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
  190. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
  191. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
  192. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
  193. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
  194. package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
  195. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  196. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  197. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  198. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  199. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  200. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
  201. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
  202. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  203. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
  204. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  205. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  206. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
  207. package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
  208. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
  209. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
  210. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  211. package/dist/plugins/ops/bci-transcribe.js +18 -4
  212. package/dist/plugins/ops/cancel.d.ts +1 -0
  213. package/dist/plugins/ops/cancel.js +11 -2
  214. package/dist/plugins/ops/embed.d.ts +5 -3
  215. package/dist/plugins/ops/embed.js +89 -34
  216. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
  217. package/dist/plugins/ops/kv-cache-utils.js +6 -2
  218. package/dist/plugins/ops/load-model.d.ts +2 -0
  219. package/dist/plugins/ops/load-model.js +33 -16
  220. package/dist/plugins/ops/transcribe.d.ts +5 -4
  221. package/dist/plugins/ops/transcribe.js +62 -27
  222. package/dist/plugins/ops/translate.d.ts +2 -1
  223. package/dist/plugins/ops/translate.js +4 -1
  224. package/dist/plugins/ops/unload-model.d.ts +17 -1
  225. package/dist/plugins/ops/unload-model.js +31 -2
  226. package/dist/plugins/turbovec-backend.d.ts +1 -1
  227. package/dist/plugins/turbovec-backend.js +16 -1
  228. package/dist/profiling/operation-metrics.js +22 -19
  229. package/dist/rag/embedding-scope.d.ts +16 -0
  230. package/dist/rag/embedding-scope.js +81 -0
  231. package/dist/rag/handler.d.ts +25 -0
  232. package/dist/rag/handler.js +50 -16
  233. package/dist/rag/handlers/ingest.js +2 -6
  234. package/dist/rag/handlers/search.d.ts +10 -1
  235. package/dist/rag/handlers/search.js +16 -7
  236. package/dist/rag/rag-operation-manager.d.ts +27 -3
  237. package/dist/rag/rag-operation-manager.js +10 -27
  238. package/dist/registry.js +5 -0
  239. package/dist/resources/collector.d.ts +70 -36
  240. package/dist/resources/collector.js +8 -1
  241. package/dist/resources/types.d.ts +2 -0
  242. package/dist/rpc/inventory.js +1 -12
  243. package/dist/runtime/env.d.ts +5 -1
  244. package/dist/runtime/env.js +6 -21
  245. package/dist/runtime/index.d.ts +4 -1
  246. package/dist/runtime/index.js +2 -0
  247. package/dist/runtime/lifecycle.d.ts +6 -4
  248. package/dist/runtime/lifecycle.js +9 -5
  249. package/dist/runtime/model-config-utils.js +5 -1
  250. package/dist/runtime/model-registry.d.ts +28 -4
  251. package/dist/runtime/model-registry.js +46 -1
  252. package/dist/runtime/model-work-lock.d.ts +13 -0
  253. package/dist/runtime/model-work-lock.js +48 -0
  254. package/dist/runtime/registry-client.d.ts +47 -1
  255. package/dist/runtime/registry-client.js +446 -88
  256. package/dist/runtime/request-context.d.ts +9 -1
  257. package/dist/runtime/request-context.js +25 -1
  258. package/dist/runtime/request-registry.d.ts +55 -7
  259. package/dist/runtime/request-registry.js +189 -71
  260. package/dist/runtime/runtime-lifecycle.js +10 -1
  261. package/dist/runtime/session-id.d.ts +18 -0
  262. package/dist/runtime/session-id.js +20 -0
  263. package/dist/runtime/session-registry.d.ts +39 -0
  264. package/dist/runtime/session-registry.js +114 -0
  265. package/dist/runtime/vector-index-backend.d.ts +4 -0
  266. package/dist/runtime/vector-index-registry.d.ts +21 -0
  267. package/dist/runtime/vector-index-registry.js +56 -18
  268. package/dist/schemas/abort-signal.d.ts +8 -0
  269. package/dist/schemas/abort-signal.js +1 -0
  270. package/dist/schemas/assess-model-fit.d.ts +132 -282
  271. package/dist/schemas/assess-model-fit.js +82 -216
  272. package/dist/schemas/audio-gen.d.ts +278 -24
  273. package/dist/schemas/audio-gen.js +127 -23
  274. package/dist/schemas/batch-completion-stream.d.ts +105 -13
  275. package/dist/schemas/batch-completion-stream.js +8 -4
  276. package/dist/schemas/bci-config.d.ts +2 -1
  277. package/dist/schemas/bci.d.ts +54 -3
  278. package/dist/schemas/bci.js +11 -16
  279. package/dist/schemas/calc-tokens.d.ts +141 -0
  280. package/dist/schemas/calc-tokens.js +53 -0
  281. package/dist/schemas/cancel.d.ts +26 -0
  282. package/dist/schemas/cancel.js +10 -2
  283. package/dist/schemas/classification.d.ts +3 -0
  284. package/dist/schemas/classification.js +6 -2
  285. package/dist/schemas/common.d.ts +1831 -348
  286. package/dist/schemas/common.js +18 -2
  287. package/dist/schemas/completion-event.d.ts +3 -0
  288. package/dist/schemas/completion-event.js +1 -1
  289. package/dist/schemas/completion-stream.d.ts +162 -17
  290. package/dist/schemas/completion-stream.js +83 -13
  291. package/dist/schemas/config.d.ts +1212 -169
  292. package/dist/schemas/config.js +9 -13
  293. package/dist/schemas/decision.d.ts +4 -2
  294. package/dist/schemas/decision.js +2 -1
  295. package/dist/schemas/download-asset.d.ts +10 -5
  296. package/dist/schemas/download-asset.js +3 -6
  297. package/dist/schemas/embed.d.ts +286 -3
  298. package/dist/schemas/embed.js +162 -16
  299. package/dist/schemas/engine-addon-map.d.ts +1 -0
  300. package/dist/schemas/engine-addon-map.js +1 -0
  301. package/dist/schemas/errors.d.ts +12 -0
  302. package/dist/schemas/errors.js +66 -1
  303. package/dist/schemas/finetune.d.ts +14 -4
  304. package/dist/schemas/finetune.js +5 -6
  305. package/dist/schemas/heartbeat.d.ts +1 -0
  306. package/dist/schemas/heartbeat.js +5 -2
  307. package/dist/schemas/image-embedding.d.ts +27 -0
  308. package/dist/schemas/image-embedding.js +107 -0
  309. package/dist/schemas/index.d.ts +7 -3
  310. package/dist/schemas/index.js +7 -3
  311. package/dist/schemas/lifecycle.d.ts +3 -0
  312. package/dist/schemas/lifecycle.js +13 -6
  313. package/dist/schemas/llamacpp-config.d.ts +269 -22
  314. package/dist/schemas/llamacpp-config.js +86 -7
  315. package/dist/schemas/load-model.d.ts +6418 -1960
  316. package/dist/schemas/load-model.js +39 -8
  317. package/dist/schemas/logging-stream.d.ts +1 -0
  318. package/dist/schemas/logging-stream.js +5 -2
  319. package/dist/schemas/model-config-schemas.d.ts +542 -116
  320. package/dist/schemas/model-config-schemas.js +2 -0
  321. package/dist/schemas/model-fit-policy.d.ts +2 -2
  322. package/dist/schemas/model-info.d.ts +54 -56
  323. package/dist/schemas/model-info.js +17 -6
  324. package/dist/schemas/model-ops.d.ts +12 -0
  325. package/dist/schemas/model-ops.js +33 -5
  326. package/dist/schemas/model-src-utils.d.ts +10 -6
  327. package/dist/schemas/model-types.d.ts +8 -4
  328. package/dist/schemas/model-types.js +1 -0
  329. package/dist/schemas/ocr.d.ts +9 -4
  330. package/dist/schemas/ocr.js +6 -2
  331. package/dist/schemas/plugin.d.ts +62 -6
  332. package/dist/schemas/plugin.js +26 -4
  333. package/dist/schemas/profiling.d.ts +32 -32
  334. package/dist/schemas/rag.d.ts +9 -0
  335. package/dist/schemas/rag.js +10 -16
  336. package/dist/schemas/registry.d.ts +30 -10
  337. package/dist/schemas/registry.js +14 -6
  338. package/dist/schemas/reload-config.d.ts +41 -6
  339. package/dist/schemas/reload-config.js +11 -4
  340. package/dist/schemas/request-base.d.ts +39 -0
  341. package/dist/schemas/request-base.js +64 -0
  342. package/dist/schemas/rerank.d.ts +8 -7
  343. package/dist/schemas/rerank.js +6 -4
  344. package/dist/schemas/rpc-server.d.ts +3 -0
  345. package/dist/schemas/rpc-server.js +15 -15
  346. package/dist/schemas/sdcpp-config.d.ts +90 -20
  347. package/dist/schemas/sdcpp-config.js +96 -19
  348. package/dist/schemas/session.d.ts +37 -0
  349. package/dist/schemas/session.js +53 -0
  350. package/dist/schemas/system-resources.d.ts +364 -179
  351. package/dist/schemas/system-resources.js +8 -3
  352. package/dist/schemas/text-to-speech.d.ts +1343 -271
  353. package/dist/schemas/text-to-speech.js +547 -33
  354. package/dist/schemas/tools.d.ts +7 -0
  355. package/dist/schemas/tools.js +29 -0
  356. package/dist/schemas/transcription-config.d.ts +262 -2
  357. package/dist/schemas/transcription-config.js +122 -6
  358. package/dist/schemas/transcription.d.ts +93 -6
  359. package/dist/schemas/transcription.js +98 -21
  360. package/dist/schemas/translate.d.ts +2 -0
  361. package/dist/schemas/translate.js +5 -9
  362. package/dist/schemas/translation-config.d.ts +30 -20
  363. package/dist/schemas/tts-pcm.d.ts +35 -0
  364. package/dist/schemas/tts-pcm.js +69 -0
  365. package/dist/schemas/vector-index.d.ts +47 -1
  366. package/dist/schemas/vector-index.js +37 -10
  367. package/dist/schemas/vla.d.ts +3 -3
  368. package/dist/surface.d.ts +3 -3
  369. package/dist/surface.js +3 -3
  370. package/dist/utils/addon-responses.d.ts +69 -2
  371. package/dist/utils/archive.d.ts +7 -0
  372. package/dist/utils/archive.js +88 -29
  373. package/dist/utils/asr-events.d.ts +5 -0
  374. package/dist/utils/asr-events.js +10 -1
  375. package/dist/utils/cache/auto-history.js +3 -1
  376. package/dist/utils/cache/layout.d.ts +48 -0
  377. package/dist/utils/cache/layout.js +190 -0
  378. package/dist/utils/cache/paths.js +2 -4
  379. package/dist/utils/cache/types.d.ts +2 -1
  380. package/dist/utils/checksum.d.ts +5 -0
  381. package/dist/utils/checksum.js +18 -0
  382. package/dist/utils/completion-normalizer.js +286 -87
  383. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  384. package/dist/utils/gguf-tensor-extractor.js +1 -50
  385. package/dist/utils/shard-utils.d.ts +0 -21
  386. package/dist/utils/shard-utils.js +12 -62
  387. package/dist/utils/tool-integration.d.ts +2 -2
  388. package/dist/utils/tools/defer.d.ts +39 -10
  389. package/dist/utils/tools/defer.js +72 -17
  390. package/dist/utils/tools/index.d.ts +1 -1
  391. package/dist/utils/tools/index.js +1 -1
  392. package/dist/utils/transcribe-metadata.d.ts +28 -2
  393. package/dist/utils/transcribe-metadata.js +16 -0
  394. package/dist/utils/tts-stats.d.ts +42 -0
  395. package/dist/utils/tts-stats.js +120 -22
  396. package/package.json +43 -39
  397. package/dist/model-fit/classify.d.ts +0 -6
  398. package/dist/model-fit/classify.js +0 -166
  399. package/dist/model-fit/description/project.d.ts +0 -27
  400. package/dist/model-fit/description/project.js +0 -33
  401. package/dist/model-fit/floor.d.ts +0 -31
  402. package/dist/model-fit/floor.js +0 -65
  403. package/dist/model-fit/kv-cache.d.ts +0 -23
  404. package/dist/model-fit/kv-cache.js +0 -85
  405. package/dist/model-fit/platform.d.ts +0 -14
  406. package/dist/model-fit/platform.js +0 -26
  407. package/dist/model-fit/request/create.d.ts +0 -31
  408. package/dist/model-fit/request/create.js +0 -61
  409. package/dist/model-fit/request/engines/asr.d.ts +0 -14
  410. package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
  411. package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
  412. package/dist/model-fit/request/engines/diffusion.js +0 -93
  413. package/dist/model-fit/request/engines/llama.d.ts +0 -12
  414. package/dist/model-fit/request/engines/llama.js +0 -44
  415. package/dist/model-fit/request/engines/nmt.d.ts +0 -8
  416. package/dist/model-fit/request/engines/nmt.js +0 -11
  417. package/dist/model-fit/request/engines/ocr.d.ts +0 -9
  418. package/dist/model-fit/request/engines/ocr.js +0 -13
  419. package/dist/model-fit/request/engines/tts.d.ts +0 -10
  420. package/dist/model-fit/request/engines/tts.js +0 -152
  421. package/dist/models/registry/resource-profiles.d.ts +0 -17
  422. package/dist/models/registry/resource-profiles.js +0 -5859
  423. package/dist/schemas/model-resource-profile.d.ts +0 -109
  424. package/dist/schemas/model-resource-profile.js +0 -80
  425. /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
  426. /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
@@ -8,7 +8,9 @@ import { diffusion } from '../../../plugins/builtin/sdcpp-generation/ops/diffusi
8
8
  import { markLtxVideoModel, markH3VideoModel, markMoeCapableVideoModel, video } from '../../../plugins/builtin/sdcpp-generation/ops/video.js';
9
9
  import { upscale } from '../../../plugins/builtin/sdcpp-generation/ops/upscale.js';
10
10
  import { createWorldSession, worldCreateScene, worldScenePath, worldStep } from '../../../plugins/builtin/sdcpp-generation/ops/world.js';
11
- import { flattenUpscalerKeys, toEsrganAddonConfig } from '../../../plugins/builtin/sdcpp-generation/upscaler-config.js';
11
+ import { sdcppFitRequest } from '../../../plugins/builtin/sdcpp-generation/fit.js';
12
+ import { assertUpscalerResolved, imageConfigFor, imageFilesFor, videoConfigFor, videoFilesFor } from '../../../plugins/builtin/sdcpp-generation/load-files.js';
13
+ import { toEsrganAddonConfig } from '../../../plugins/builtin/sdcpp-generation/upscaler-config.js';
12
14
  import { splitWorldConfig } from '../../../plugins/builtin/sdcpp-generation/world-config.js';
13
15
  // Wait for op cleanup before emitting completion.
14
16
  async function* withCompletionFrame(stream, complete) {
@@ -35,6 +37,7 @@ export const diffusionPlugin = definePlugin({
35
37
  displayName: 'Image Generation & Upscaling (stable-diffusion.cpp)',
36
38
  addonPackage: ADDON_DIFFUSION,
37
39
  loadConfigSchema: sdcppConfigSchema,
40
+ fitRequest: sdcppFitRequest,
38
41
  assessFit: diffusionAssessFit,
39
42
  async resolveConfig(cfg, ctx) {
40
43
  if (cfg.projectionModelSrc && cfg.mode !== 'diffusion') {
@@ -52,6 +55,14 @@ export const diffusionPlugin = definePlugin({
52
55
  throw new ModelLoadFailedError('modelConfig.uncondModelSrc selects the Ideogram 4 layout and requires ' +
53
56
  'modelConfig.llmModelSrc (Qwen3-VL) and modelConfig.vaeModelSrc.');
54
57
  }
58
+ if (cfg.tokenizerSrc && cfg.mode !== 'diffusion') {
59
+ throw new ModelLoadFailedError('modelConfig.tokenizerSrc is MOSS-Image diffusion only. ' +
60
+ "Use mode: 'diffusion' or remove tokenizerSrc.");
61
+ }
62
+ if (cfg.tokenizerSrc && (!cfg.llmModelSrc || !cfg.vaeModelSrc)) {
63
+ throw new ModelLoadFailedError('modelConfig.tokenizerSrc selects the MOSS-Image layout and requires ' +
64
+ 'modelConfig.llmModelSrc (Qwen3.5) and modelConfig.vaeModelSrc.');
65
+ }
55
66
  // A high-noise expert only has somewhere to go in the Wan 2.2 A14B video
56
67
  // layout: every other mode builds a model with no second-expert slot, and
57
68
  // embeddingsConnectorsModelSrc selects LTX-2 instead. Check it ahead of the
@@ -140,7 +151,7 @@ export const diffusionPlugin = definePlugin({
140
151
  if (cfg.mode === 'upscale') {
141
152
  return { config: cfg };
142
153
  }
143
- const { clipLModelSrc, clipGModelSrc, clipVisionModelSrc, t5XxlModelSrc, llmModelSrc, projectionModelSrc, vaeModelSrc, highNoiseDiffusionModelSrc, uncondModelSrc, audioVaeModelSrc, embeddingsConnectorsModelSrc, taehvModelSrc, sceneSrc, upscaler, ...rest } = cfg;
154
+ const { clipLModelSrc, clipGModelSrc, clipVisionModelSrc, t5XxlModelSrc, llmModelSrc, projectionModelSrc, vaeModelSrc, highNoiseDiffusionModelSrc, uncondModelSrc, tokenizerSrc, audioVaeModelSrc, embeddingsConnectorsModelSrc, taehvModelSrc, sceneSrc, upscaler, ...rest } = cfg;
144
155
  if (embeddingsConnectorsModelSrc && cfg.mode !== 'video') {
145
156
  throw new ModelLoadFailedError('modelConfig.embeddingsConnectorsModelSrc selects the LTX-2 video ' +
146
157
  "layout and is only valid with mode: 'video'.");
@@ -184,6 +195,7 @@ export const diffusionPlugin = definePlugin({
184
195
  vaeModelSrc,
185
196
  highNoiseDiffusionModelSrc,
186
197
  uncondModelSrc,
198
+ tokenizerSrc,
187
199
  audioVaeModelSrc,
188
200
  embeddingsConnectorsModelSrc,
189
201
  esrganModelSrc,
@@ -195,7 +207,7 @@ export const diffusionPlugin = definePlugin({
195
207
  return { config: runtimeConfig };
196
208
  }
197
209
  const resolve = ctx.resolveModelPath;
198
- const [clipLModelPath, clipGModelPath, clipVisionModelPath, t5XxlModelPath, llmModelPath, projectionModelPath, vaeModelPath, highNoiseDiffusionModelPath, uncondModelPath, audioVaeModelPath, embeddingsConnectorsModelPath, esrganModelPath, taehvModelPath, seedScenePath] = await Promise.all([
210
+ const [clipLModelPath, clipGModelPath, clipVisionModelPath, t5XxlModelPath, llmModelPath, projectionModelPath, vaeModelPath, highNoiseDiffusionModelPath, uncondModelPath, tokenizerPath, audioVaeModelPath, embeddingsConnectorsModelPath, esrganModelPath, taehvModelPath, seedScenePath] = await Promise.all([
199
211
  clipLModelSrc ? resolve(clipLModelSrc) : undefined,
200
212
  clipGModelSrc ? resolve(clipGModelSrc) : undefined,
201
213
  clipVisionModelSrc ? resolve(clipVisionModelSrc) : undefined,
@@ -205,6 +217,7 @@ export const diffusionPlugin = definePlugin({
205
217
  vaeModelSrc ? resolve(vaeModelSrc) : undefined,
206
218
  highNoiseDiffusionModelSrc ? resolve(highNoiseDiffusionModelSrc) : undefined,
207
219
  uncondModelSrc ? resolve(uncondModelSrc) : undefined,
220
+ tokenizerSrc ? resolve(tokenizerSrc) : undefined,
208
221
  audioVaeModelSrc ? resolve(audioVaeModelSrc) : undefined,
209
222
  embeddingsConnectorsModelSrc ? resolve(embeddingsConnectorsModelSrc) : undefined,
210
223
  esrganModelSrc ? resolve(esrganModelSrc) : undefined,
@@ -223,6 +236,7 @@ export const diffusionPlugin = definePlugin({
223
236
  ...(vaeModelPath && { vaeModelPath }),
224
237
  ...(highNoiseDiffusionModelPath && { highNoiseDiffusionModelPath }),
225
238
  ...(uncondModelPath && { uncondModelPath }),
239
+ ...(tokenizerPath && { tokenizerPath }),
226
240
  ...(audioVaeModelPath && { audioVaeModelPath }),
227
241
  ...(embeddingsConnectorsModelPath && { embeddingsConnectorsModelPath }),
228
242
  ...(esrganModelPath && { esrganModelPath }),
@@ -241,19 +255,10 @@ export const diffusionPlugin = definePlugin({
241
255
  getEngineLogger().warn(`[${ModelType.sdcppGeneration}:${modelId}] Multi-GPU parameters (${stripped.join(', ')}) are not supported on mobile (single-GPU device) — removing from config; model will load with single-GPU defaults`);
242
256
  }
243
257
  }
244
- // In diffusion mode the ESRGAN file (when post-generation upscale is
245
- // wanted) must come from upscaler.model_src — the primary modelPath is
246
- // the main diffusion checkpoint. Reject early with a clear error
247
- // instead of letting the native addon fail mid-load. Done before any
248
- // logger / native side-effects so callers can recover cleanly.
249
- if (config.mode === 'diffusion' &&
250
- config.upscaler !== undefined &&
251
- !artifacts?.['esrganModelPath']) {
252
- throw new ModelLoadFailedError('modelConfig.upscaler.model_src is required when modelConfig.upscaler ' +
253
- 'is set in diffusion mode. Provide the ESRGAN model, omit the ' +
254
- "upscaler block, or switch to modelConfig.mode = 'upscale' to load " +
255
- 'a standalone upscaler.');
256
- }
258
+ // Reject early with a clear error instead of letting the native addon fail
259
+ // mid-load. Done before any logger / native side-effects so callers can
260
+ // recover cleanly.
261
+ assertUpscalerResolved(config, artifacts);
257
262
  const logger = createStreamLogger(modelId, ModelType.sdcppGeneration);
258
263
  registerAddonLogger(modelId, ModelType.sdcppGeneration, logger);
259
264
  if (config.mode === 'upscale') {
@@ -292,86 +297,14 @@ export const diffusionPlugin = definePlugin({
292
297
  return { model };
293
298
  }
294
299
  if (config.mode === 'video') {
295
- // Companion layout selection, mirroring how the addon self-detects it:
296
- // the LTX-2 text-embedding connectors switch to the LTX-2 layout (Gemma
297
- // via llm + video VAE + connectors, optional audio VAE —
298
- // `SdModel::isLtxModel_ = !embeddingsConnectorsPath.empty()`); without
299
- // them, an audio VAE (or an llm with no Wan T5) means MiniMax-H3
300
- // (llm + video VAE, with an optional audio VAE); everything else is Wan
301
- // via t5Xxl.
302
- // Native code verifies the model's tensors.
303
- const embeddingsConnectorsModelPath = artifacts?.['embeddingsConnectorsModelPath'];
304
- if (!artifacts?.['vaeModelPath']) {
305
- throw new ModelLoadFailedError('modelConfig.vaeModelSrc is required in video mode. ' +
306
- 'Provide the Wan, LTX-2 or MiniMax-H3 video VAE before loading the video pipeline.');
307
- }
308
- const vaeModelPath = artifacts['vaeModelPath'];
309
- let files;
310
- if (embeddingsConnectorsModelPath) {
311
- if (!artifacts['llmModelPath']) {
312
- throw new ModelLoadFailedError('modelConfig.llmModelSrc is required for LTX-2 video. ' +
313
- 'Provide the Gemma text encoder model before loading the LTX-2 pipeline.');
314
- }
315
- files = {
316
- model: modelPath,
317
- vae: vaeModelPath,
318
- llm: artifacts['llmModelPath'],
319
- embeddingsConnectors: embeddingsConnectorsModelPath,
320
- ...(artifacts['audioVaeModelPath'] && {
321
- audioVae: artifacts['audioVaeModelPath']
322
- }),
323
- ...(artifacts['esrganModelPath'] && { esrgan: artifacts['esrganModelPath'] })
324
- };
325
- }
326
- else if (artifacts['audioVaeModelPath'] ||
327
- (artifacts['llmModelPath'] && !artifacts['t5XxlModelPath'])) {
328
- if (!artifacts['llmModelPath']) {
329
- throw new ModelLoadFailedError('MiniMax-H3 requires llmModelSrc and vaeModelSrc.');
330
- }
331
- if (artifacts['t5XxlModelPath'] ||
332
- artifacts['highNoiseDiffusionModelPath'] ||
333
- artifacts['clipVisionModelPath'] ||
334
- artifacts['clipLModelPath'] ||
335
- artifacts['clipGModelPath']) {
336
- throw new ModelLoadFailedError('MiniMax-H3 cannot use Wan T5, high-noise, CLIP-vision or CLIP-L/G companions.');
337
- }
338
- files = {
339
- model: modelPath,
340
- vae: vaeModelPath,
341
- llm: artifacts['llmModelPath'],
342
- ...(artifacts['audioVaeModelPath'] && {
343
- audioVae: artifacts['audioVaeModelPath']
344
- })
345
- };
346
- }
347
- else {
348
- if (!artifacts['t5XxlModelPath']) {
349
- throw new ModelLoadFailedError('modelConfig.t5XxlModelSrc is required in video mode. ' +
350
- 'Provide the Wan text encoder model before loading the video pipeline.');
351
- }
352
- files = {
353
- model: modelPath,
354
- vae: vaeModelPath,
355
- t5Xxl: artifacts['t5XxlModelPath'],
356
- ...(artifacts['highNoiseDiffusionModelPath'] && {
357
- highNoiseDiffusionModel: artifacts['highNoiseDiffusionModelPath']
358
- }),
359
- ...(artifacts['clipVisionModelPath'] && {
360
- clipVision: artifacts['clipVisionModelPath']
361
- }),
362
- ...(artifacts['esrganModelPath'] && { esrgan: artifacts['esrganModelPath'] })
363
- };
364
- }
365
- /* eslint-disable @typescript-eslint/no-unused-vars */
366
- const { clipLModelSrc, clipGModelSrc, clipVisionModelSrc, t5XxlModelSrc, llmModelSrc, projectionModelSrc, vaeModelSrc, highNoiseDiffusionModelSrc, uncondModelSrc, audioVaeModelSrc, embeddingsConnectorsModelSrc, upscaler, mode, ...rest } = config;
367
- /* eslint-enable @typescript-eslint/no-unused-vars */
300
+ const files = videoFilesFor(modelPath, artifacts);
368
301
  const model = new VideoStableDiffusion({
369
302
  files,
370
- config: rest,
303
+ config: videoConfigFor(config),
371
304
  logger,
372
305
  opts: { stats: true }
373
306
  });
374
- if (embeddingsConnectorsModelPath)
307
+ if (files.embeddingsConnectors)
375
308
  markLtxVideoModel(model);
376
309
  else if (files.llm)
377
310
  markH3VideoModel(model);
@@ -379,31 +312,9 @@ export const diffusionPlugin = definePlugin({
379
312
  markMoeCapableVideoModel(model);
380
313
  return { model };
381
314
  }
382
- const files = {
383
- model: modelPath,
384
- ...(artifacts?.['clipLModelPath'] && { clipL: artifacts['clipLModelPath'] }),
385
- ...(artifacts?.['clipGModelPath'] && { clipG: artifacts['clipGModelPath'] }),
386
- ...(artifacts?.['t5XxlModelPath'] && { t5Xxl: artifacts['t5XxlModelPath'] }),
387
- ...(artifacts?.['llmModelPath'] && { llm: artifacts['llmModelPath'] }),
388
- ...(artifacts?.['projectionModelPath'] && { llmVision: artifacts['projectionModelPath'] }),
389
- ...(artifacts?.['vaeModelPath'] && { vae: artifacts['vaeModelPath'] }),
390
- ...(artifacts?.['uncondModelPath'] && {
391
- uncondModel: artifacts['uncondModelPath']
392
- }),
393
- ...(artifacts?.['esrganModelPath'] && { esrgan: artifacts['esrganModelPath'] })
394
- };
395
- // `mode` is consumed by this plugin to select the model class above; the
396
- // stable-diffusion.cpp native config does not understand it. Strip it
397
- // before forwarding `rest` to the addon.
398
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
399
- const { upscaler, mode, ...rest } = config;
400
- const addonConfig = {
401
- ...rest,
402
- ...flattenUpscalerKeys(upscaler)
403
- };
404
315
  const model = new ImgStableDiffusion({
405
- files,
406
- config: addonConfig,
316
+ files: imageFilesFor(modelPath, artifacts),
317
+ config: imageConfigFor(config),
407
318
  logger,
408
319
  opts: { stats: true }
409
320
  });
@@ -417,8 +328,9 @@ export const diffusionPlugin = definePlugin({
417
328
  // sdcpp diffusion exposes a model-wide hard cancel — compute
418
329
  // is interrupted on the currently-running generation.
419
330
  cancel: { scope: 'model', hard: true },
420
- handler: (request) => withCompletionFrame(diffusion(request), ({ stats }) => ({
331
+ handler: (request, _inputStream, context) => withCompletionFrame(diffusion(request, context), ({ requestId, stats }) => ({
421
332
  type: 'diffusionStream',
333
+ requestId,
422
334
  done: true,
423
335
  stats
424
336
  }))
@@ -428,7 +340,7 @@ export const diffusionPlugin = definePlugin({
428
340
  responseSchema: videoStreamResponseSchema,
429
341
  streaming: true,
430
342
  cancel: { scope: 'model', hard: true },
431
- handler: (request) => withCompletionFrame(video(request), ({ stats }) => ({
343
+ handler: (request, _inputStream, context) => withCompletionFrame(video(request, context), ({ stats }) => ({
432
344
  type: 'videoStream',
433
345
  done: true,
434
346
  stats
@@ -0,0 +1,3 @@
1
+ import type { FitLoad, FitRequestPlan } from '../../../schemas/plugin.js';
2
+ /** The tts-ggml fit request for a load, built for the voice engine it names. */
3
+ export declare function ttsFitRequest(load: FitLoad): FitRequestPlan;
@@ -0,0 +1,162 @@
1
+ import fs from 'bare-fs';
2
+ import path from 'bare-path';
3
+ import { gpuLayersFromCount } from '../../../model-fit/gpu.js';
4
+ const COSYVOICE_COMPANIONS_MISSING = 'the cosyvoice companion set is not laid out beside the checkpoint';
5
+ /**
6
+ * CosyVoice's companion set co-locates the flow, HiFT and voice GGUFs beside
7
+ * the LLM checkpoint, and the loader finds them by name in that directory. The
8
+ * fitter takes explicit paths, so the same directory is read here.
9
+ */
10
+ function cosyvoiceRequest(modelPath, common) {
11
+ const dir = path.dirname(modelPath);
12
+ let entries;
13
+ try {
14
+ entries = fs.readdirSync(dir);
15
+ }
16
+ catch {
17
+ return { unknown: COSYVOICE_COMPANIONS_MISSING };
18
+ }
19
+ const companion = (matches) => {
20
+ const found = entries.find(matches);
21
+ return found === undefined ? undefined : path.join(dir, found);
22
+ };
23
+ const flowPath = companion((name) => name.includes('-flow-'));
24
+ const hiftPath = companion((name) => name.includes('-hift-'));
25
+ const voicePath = companion((name) => name === 'voice.gguf');
26
+ if (flowPath === undefined || hiftPath === undefined || voicePath === undefined) {
27
+ return { unknown: COSYVOICE_COMPANIONS_MISSING };
28
+ }
29
+ return {
30
+ request: {
31
+ engineType: 'cosyvoice3',
32
+ cosyvoiceLlmModelPath: modelPath,
33
+ cosyvoiceFlowModelPath: flowPath,
34
+ cosyvoiceHiftModelPath: hiftPath,
35
+ cosyvoiceVoiceModelPath: voicePath,
36
+ ...common
37
+ }
38
+ };
39
+ }
40
+ /** The tts-ggml fit request for a load, built for the voice engine it names. */
41
+ export function ttsFitRequest(load) {
42
+ const config = (load.modelConfig ?? {});
43
+ const artifacts = load.artifacts ?? {};
44
+ const workload = load.ttsWorkload;
45
+ // The enhancer and denoiser load next to the voice, and no fitter sizes them.
46
+ if (artifacts['lavasrEnhancerPath'] !== undefined ||
47
+ artifacts['lavasrDenoiserPath'] !== undefined) {
48
+ return { unknown: 'the LavaSR stages have no fitter' };
49
+ }
50
+ // The explicit backend selector resolves the same device the load would, so
51
+ // the projection lands on that device rather than on the `useGPU` policy's.
52
+ const common = {
53
+ nGpuLayers: gpuLayersFromCount(config.useGPU, config.nGpuLayers),
54
+ ...(config.backend !== undefined && { backend: config.backend }),
55
+ ...(config.backendsDir !== undefined && { backendsDir: config.backendsDir })
56
+ };
57
+ const request = (fit) => ({ request: fit });
58
+ switch (config.ttsEngine) {
59
+ case 'supertonic':
60
+ // Precision and the f16 weight override change the weight footprint the
61
+ // fitter sizes, so they travel with the request.
62
+ return request({
63
+ engineType: 'supertonic',
64
+ supertonicModelPath: load.modelPath,
65
+ ...common,
66
+ ...(config.precision !== undefined && { precision: config.precision }),
67
+ ...(config.f16Weights !== undefined && { f16Weights: config.f16Weights }),
68
+ ...(config.ttsNumInferenceSteps !== undefined && { steps: config.ttsNumInferenceSteps })
69
+ });
70
+ case 'parler':
71
+ // The decoder sidecar policy decides whether the decoder weights load on
72
+ // ggml at all; the fitter matches the load's choice.
73
+ return request({
74
+ engineType: 'parler',
75
+ parlerModelPath: load.modelPath,
76
+ ...common,
77
+ ...(config.coremlDecoder !== undefined && { coremlDecoder: config.coremlDecoder }),
78
+ ...(config.maxFrames !== undefined && { maxFrames: config.maxFrames })
79
+ });
80
+ case 'audio8': {
81
+ const codecDecoderPath = artifacts['audio8CodecDecoderPath'];
82
+ if (codecDecoderPath === undefined) {
83
+ return { unknown: 'the audio8 codec decoder is not resolved' };
84
+ }
85
+ const codecEncoderPath = artifacts['audio8CodecEncoderPath'];
86
+ return request({
87
+ engineType: 'audio8',
88
+ audio8LmPath: load.modelPath,
89
+ audio8CodecDecoderPath: codecDecoderPath,
90
+ ...(codecEncoderPath !== undefined && { audio8CodecEncoderPath: codecEncoderPath }),
91
+ ...common,
92
+ ...(config.maxFrames !== undefined && { maxFrames: config.maxFrames })
93
+ });
94
+ }
95
+ case 'cosyvoice3':
96
+ return cosyvoiceRequest(load.modelPath, common);
97
+ case 'moss': {
98
+ const decoder = artifacts['mossCodecDecoderPath'];
99
+ const encoder = artifacts['mossCodecEncoderPath'];
100
+ if (decoder === undefined)
101
+ return { unknown: 'the moss codec decoder is not resolved' };
102
+ if (workload === undefined || !('promptRows' in workload)) {
103
+ return {
104
+ unknown: 'a MOSS projection needs a ttsWorkload with promptRows, referenceSamples and streaming'
105
+ };
106
+ }
107
+ if (workload.referenceSamples > 0 && encoder === undefined) {
108
+ return { unknown: 'MOSS reference encoding needs the codec encoder' };
109
+ }
110
+ return request({
111
+ engineType: 'moss',
112
+ mossBackbonePath: load.modelPath,
113
+ mossCodecDecoderPath: decoder,
114
+ ...(encoder !== undefined && { mossCodecEncoderPath: encoder }),
115
+ ...common,
116
+ ...workload,
117
+ ...(config.threads !== undefined && { threads: config.threads }),
118
+ ...(config.streamChunkTokens !== undefined && {
119
+ streamChunkTokens: config.streamChunkTokens
120
+ }),
121
+ ...(config.durationTokens !== undefined && { durationTokens: config.durationTokens }),
122
+ // The context and generation budgets size the KV cache and the
123
+ // generation buffers the fitter projects.
124
+ ...(config.nCtx !== undefined && { nCtx: config.nCtx }),
125
+ ...(config.maxNewTokens !== undefined && { maxNewTokens: config.maxNewTokens })
126
+ });
127
+ }
128
+ case 'moss-sfx':
129
+ if (workload === undefined || !('prompt' in workload)) {
130
+ return {
131
+ unknown: 'a MOSS-SoundEffect projection needs a ttsWorkload with prompt and seconds'
132
+ };
133
+ }
134
+ return request({
135
+ engineType: 'moss-sfx',
136
+ mossSoundEffectPath: load.modelPath,
137
+ ...common,
138
+ ...workload,
139
+ ...(config.negativePrompt !== undefined && { negativePrompt: config.negativePrompt }),
140
+ ...(config.steps !== undefined && { steps: config.steps }),
141
+ ...(config.guidance !== undefined && { guidance: config.guidance }),
142
+ ...(config.shift !== undefined && { shift: config.shift }),
143
+ ...(config.threads !== undefined && { threads: config.threads })
144
+ });
145
+ case 'moss-speech':
146
+ return { unknown: 'MOSS-Speech has no fitter' };
147
+ default: {
148
+ const s3genPath = artifacts['s3genPath'];
149
+ if (s3genPath === undefined) {
150
+ return { unknown: 'the chatterbox s3gen checkpoint is not resolved' };
151
+ }
152
+ return request({
153
+ engineType: 'chatterbox',
154
+ t3ModelPath: load.modelPath,
155
+ s3genModelPath: s3genPath,
156
+ ...common,
157
+ ...(config.nCtx !== undefined && config.nCtx > 0 && { nCtx: config.nCtx }),
158
+ ...(config.kvCacheType !== undefined && { kvCacheType: config.kvCacheType })
159
+ });
160
+ }
161
+ }
162
+ }
@@ -1,3 +1,4 @@
1
+ import { type PluginHandlerContext } from '../../../../schemas/plugin.js';
1
2
  import type { AbortSignal } from 'bare-abort-controller';
2
3
  /**
3
4
  * The abort is persistent state, not a one-shot poke. The addon's `cancel()`
@@ -26,4 +27,4 @@ export declare function cancelIfAborted(model: unknown, signal: AbortSignal): Pr
26
27
  * never owned the model slot, so cancelling then would interrupt the run that
27
28
  * does.
28
29
  */
29
- export declare function bindTtsCancel(model: unknown, modelId: string, requestId?: string): Promise<import("../../../../runtime/index.js").ManagedRequestContext>;
30
+ export declare function bindTtsCancel(model: unknown, modelId: string, requestId?: string, sessionId?: string, context?: PluginHandlerContext): Promise<import("../../../../runtime/index.js").ManagedRequestContext>;
@@ -1,3 +1,4 @@
1
+ import { requestParent } from '../../../../schemas/plugin.js';
1
2
  import { getEngineLogger } from '../../../../logging/index.js';
2
3
  import { getRequestRegistry, withRequestContext } from '../../../../runtime/index.js';
3
4
  import { generateRequestId } from '../../../../runtime/request-id.js';
@@ -36,9 +37,11 @@ export async function cancelIfAborted(model, signal) {
36
37
  * never owned the model slot, so cancelling then would interrupt the run that
37
38
  * does.
38
39
  */
39
- export async function bindTtsCancel(model, modelId, requestId) {
40
+ export async function bindTtsCancel(model, modelId, requestId, sessionId, context) {
40
41
  const ctx = await getRequestRegistry().begin({
41
42
  requestId: requestId ?? generateRequestId(),
43
+ sessionId,
44
+ ...requestParent(context),
42
45
  kind: 'tts',
43
46
  modelId
44
47
  });
@@ -15,7 +15,7 @@ export declare function getMossJobOptions(model: unknown, request: TtsRequest):
15
15
  sampleRate?: number;
16
16
  audio?: Float32Array<ArrayBuffer>;
17
17
  text?: string;
18
- role: "assistant" | "system" | "user";
18
+ role: "system" | "assistant" | "user";
19
19
  }[];
20
20
  sampleRate?: number;
21
21
  audio?: Float32Array<ArrayBuffer>;
@@ -1,5 +1,6 @@
1
+ import type { PluginHandlerContext } from '../../../../schemas/plugin.js';
1
2
  import { type TextToSpeechStreamRequest } from '../../../../schemas/index.js';
2
3
  import Buffer from 'bare-buffer';
3
4
  import { type TtsOpYield } from '../../../../utils/tts-stats.js';
4
5
  import type { TtsOpResult } from '../../../../plugins/builtin/tts-ggml/ops/text-to-speech.js';
5
- export declare function textToSpeechStream(params: TextToSpeechStreamRequest, inputStream: AsyncIterable<Buffer>): AsyncGenerator<TtsOpYield, TtsOpResult, unknown>;
6
+ export declare function textToSpeechStream(params: TextToSpeechStreamRequest, inputStream: AsyncIterable<Buffer>, context?: PluginHandlerContext): AsyncGenerator<TtsOpYield, TtsOpResult, unknown>;
@@ -56,9 +56,10 @@ import Buffer from 'bare-buffer';
56
56
  import { nowMs } from '../../../../profiling/index.js';
57
57
  import { buildStreamResult, hasDefinedValues } from '../../../../profiling/model-execution.js';
58
58
  import { TextToSpeechStreamFailedError } from '../../../../errors/index.js';
59
- import { collectTtsStats, chunkMetadata } from '../../../../utils/tts-stats.js';
59
+ import { carriesSpeechTokenIds, collectTtsStats, pcmFrame, chunkMetadata } from '../../../../utils/tts-stats.js';
60
60
  import { assertParlerJobOptionsSupported, getParlerJobOptions, getTtsEngineType } from '../../../../plugins/builtin/tts-ggml/ops/parler-options.js';
61
61
  import { bindTtsCancel, cancelIfAborted } from '../../../../plugins/builtin/tts-ggml/ops/cancel-binding.js';
62
+ import { assertVoiceJobOptionsSupported, resolveVoiceJobOptions } from '../../../../plugins/builtin/tts-ggml/ops/voice-options.js';
62
63
  // Ends the text source the moment the run is cancelled. Without this an idle
63
64
  // duplex session — cancelled while the accumulator is waiting for the next
64
65
  // fragment, with no native job live — would sit in `runStreaming` holding the
@@ -149,7 +150,7 @@ async function* buffersToUtf8Fragments(inputStream) {
149
150
  }
150
151
  }
151
152
  }
152
- function buildRunStreamingOptions(request, parlerJobOptions) {
153
+ function buildRunStreamingOptions(request, parlerJobOptions, voiceJobOptions) {
153
154
  const o = {};
154
155
  if (request.accumulateSentences !== undefined) {
155
156
  o['accumulateSentences'] = request.accumulateSentences;
@@ -163,9 +164,9 @@ function buildRunStreamingOptions(request, parlerJobOptions) {
163
164
  if (request.flushAfterMs !== undefined) {
164
165
  o['flushAfterMs'] = request.flushAfterMs;
165
166
  }
166
- return { ...o, ...parlerJobOptions };
167
+ return { ...o, ...parlerJobOptions, ...voiceJobOptions };
167
168
  }
168
- export async function* textToSpeechStream(params, inputStream) {
169
+ export async function* textToSpeechStream(params, inputStream, context) {
169
170
  const env_1 = { stack: [], error: void 0, hasError: false };
170
171
  try {
171
172
  const request = textToSpeechStreamRequestSchema.parse(params);
@@ -175,15 +176,26 @@ export async function* textToSpeechStream(params, inputStream) {
175
176
  }
176
177
  const parlerJobOptions = getParlerJobOptions(request);
177
178
  assertParlerJobOptionsSupported(model, parlerJobOptions, 'textToSpeechStream');
178
- const ctx = __addDisposableResource(env_1, await bindTtsCancel(model, request.modelId, request.requestId), true);
179
+ assertVoiceJobOptionsSupported(model, request, 'textToSpeechStream');
180
+ const ctx = __addDisposableResource(env_1, await bindTtsCancel(model, request.modelId, request.requestId, request.sessionId, context), true);
179
181
  if (ctx.signal.aborted)
180
182
  return { ...buildStreamResult(0), cancelled: true };
183
+ // Resolved inside the request context: an uncached remote reference
184
+ // recording downloads here, and a cancel has to be able to stop that.
185
+ const voiceJobOptions = await resolveVoiceJobOptions(request, {
186
+ signal: ctx.signal,
187
+ scope: ctx.scope,
188
+ requestId: ctx.requestId,
189
+ sessionId: ctx.sessionId
190
+ });
191
+ if (voiceJobOptions === null)
192
+ return { ...buildStreamResult(0), cancelled: true };
181
193
  const modelStart = nowMs();
182
194
  if (!hasRunStreaming(model)) {
183
195
  throw new TextToSpeechStreamFailedError('textToSpeechStream requires a TTS model with runStreaming');
184
196
  }
185
197
  const textSource = untilAborted(buffersToUtf8Fragments(inputStream), ctx.signal);
186
- const streamOpts = buildRunStreamingOptions(request, parlerJobOptions);
198
+ const streamOpts = buildRunStreamingOptions(request, parlerJobOptions, voiceJobOptions);
187
199
  const response = await model.runStreaming(textSource, Object.keys(streamOpts).length > 0 ? streamOpts : undefined);
188
200
  try {
189
201
  for await (const data of response.iterate()) {
@@ -193,11 +205,12 @@ export async function* textToSpeechStream(params, inputStream) {
193
205
  if (data.outputArray == null) {
194
206
  continue;
195
207
  }
196
- const buf = Array.from(data.outputArray);
197
- if (buf.length === 0) {
208
+ // An empty chunk is a native end marker, unless it carries the sentence's
209
+ // speech tokens (CosyVoice3 native streaming), which the caller wants.
210
+ if (data.outputArray.length === 0 && !carriesSpeechTokenIds(data)) {
198
211
  continue;
199
212
  }
200
- yield { buffer: buf, ...chunkMetadata(data) };
213
+ yield { ...pcmFrame(data.outputArray), ...chunkMetadata(data) };
201
214
  }
202
215
  }
203
216
  catch (error) {
@@ -1,3 +1,4 @@
1
+ import type { PluginHandlerContext } from '../../../../schemas/plugin.js';
1
2
  import { type TtsRequest, type TtsStats } from '../../../../schemas/index.js';
2
3
  import { type TtsOpYield } from '../../../../utils/tts-stats.js';
3
4
  /** What the handler turns into the terminal frame. */
@@ -6,4 +7,4 @@ export type TtsOpResult = {
6
7
  stats?: TtsStats;
7
8
  cancelled?: boolean;
8
9
  };
9
- export declare function textToSpeech(params: TtsRequest): AsyncGenerator<TtsOpYield, TtsOpResult>;
10
+ export declare function textToSpeech(params: TtsRequest, context?: PluginHandlerContext): AsyncGenerator<TtsOpYield, TtsOpResult>;