@qvac/inference 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (569) hide show
  1. package/NOTICE +111 -60
  2. package/README.md +4 -2
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +136 -45
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +2 -1
  13. package/dist/api/completion-stream.js +4 -1
  14. package/dist/api/decide.js +1 -1
  15. package/dist/api/delete-cache.d.ts +7 -1
  16. package/dist/api/delete-cache.js +11 -1
  17. package/dist/api/diffusion.d.ts +32 -2
  18. package/dist/api/diffusion.js +32 -2
  19. package/dist/api/download-asset.js +1 -1
  20. package/dist/api/embed.d.ts +82 -29
  21. package/dist/api/embed.js +64 -18
  22. package/dist/api/finetune.js +1 -1
  23. package/dist/api/get-model-info.d.ts +1 -1
  24. package/dist/api/index.d.ts +40 -38
  25. package/dist/api/index.js +39 -37
  26. package/dist/api/load-model.js +2 -2
  27. package/dist/api/logging-stream-registry.js +11 -7
  28. package/dist/api/logging-stream.d.ts +11 -1
  29. package/dist/api/logging-stream.js +11 -2
  30. package/dist/api/rag.d.ts +1 -1
  31. package/dist/api/rag.js +1 -1
  32. package/dist/api/registry.d.ts +18 -7
  33. package/dist/api/registry.js +37 -13
  34. package/dist/api/rerank.d.ts +17 -0
  35. package/dist/api/rerank.js +36 -0
  36. package/dist/api/subscribe-logs.js +5 -3
  37. package/dist/api/text-to-speech.d.ts +14 -5
  38. package/dist/api/text-to-speech.js +83 -29
  39. package/dist/api/transcribe.d.ts +6 -2
  40. package/dist/api/transcribe.js +7 -1
  41. package/dist/api/unload-model.js +1 -1
  42. package/dist/api/vector-index.d.ts +25 -3
  43. package/dist/api/vector-index.js +17 -2
  44. package/dist/api/video-request.d.ts +15 -15
  45. package/dist/api/video.js +1 -1
  46. package/dist/api/vla.js +1 -1
  47. package/dist/api/world-result.js +1 -1
  48. package/dist/api/world.d.ts +1 -1
  49. package/dist/api/world.js +1 -1
  50. package/dist/config/resolve-config.d.ts +1 -1
  51. package/dist/config/resolve-config.js +1 -1
  52. package/dist/constants/index.d.ts +2 -2
  53. package/dist/constants/index.js +2 -2
  54. package/dist/dispatch.d.ts +11 -4
  55. package/dist/dispatch.js +80 -18
  56. package/dist/engine.d.ts +1 -1
  57. package/dist/errors/index.d.ts +76 -0
  58. package/dist/errors/index.js +141 -1
  59. package/dist/handlers/assess-model-fit.d.ts +3 -21
  60. package/dist/handlers/assess-model-fit.js +4 -303
  61. package/dist/handlers/cancelHandler.d.ts +10 -9
  62. package/dist/handlers/cancelHandler.js +18 -14
  63. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  64. package/dist/handlers/completion-orchestrate.js +166 -24
  65. package/dist/handlers/delete-cache.d.ts +2 -1
  66. package/dist/handlers/delete-cache.js +6 -1
  67. package/dist/handlers/download-asset.js +13 -6
  68. package/dist/handlers/finetune.js +1 -1
  69. package/dist/handlers/get-loaded-model-info.js +3 -3
  70. package/dist/handlers/get-model-info.js +72 -3
  71. package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
  72. package/dist/handlers/load-model/coreml-sidecars.js +51 -91
  73. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  74. package/dist/handlers/load-model/download-lock.js +213 -0
  75. package/dist/handlers/load-model/download-manager.d.ts +11 -3
  76. package/dist/handlers/load-model/download-manager.js +62 -11
  77. package/dist/handlers/load-model/download-stats.d.ts +2 -2
  78. package/dist/handlers/load-model/download-stats.js +3 -3
  79. package/dist/handlers/load-model/handler.js +97 -41
  80. package/dist/handlers/load-model/http-verify.d.ts +1 -1
  81. package/dist/handlers/load-model/http.d.ts +1 -1
  82. package/dist/handlers/load-model/http.js +330 -208
  83. package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
  84. package/dist/handlers/load-model/hyperdrive.js +125 -52
  85. package/dist/handlers/load-model/index.d.ts +2 -2
  86. package/dist/handlers/load-model/index.js +2 -2
  87. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  88. package/dist/handlers/load-model/registry-client-options.js +4 -2
  89. package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
  90. package/dist/handlers/load-model/registry-companion-set.js +20 -2
  91. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  92. package/dist/handlers/load-model/registry-delete.js +269 -0
  93. package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
  94. package/dist/handlers/load-model/registry-download-utils.js +95 -24
  95. package/dist/handlers/load-model/registry.d.ts +22 -2
  96. package/dist/handlers/load-model/registry.js +312 -179
  97. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  98. package/dist/handlers/load-model/resolve-session.js +11 -3
  99. package/dist/handlers/load-model/resolve.d.ts +1 -1
  100. package/dist/handlers/load-model/resolve.js +9 -6
  101. package/dist/handlers/load-model/types.d.ts +2 -0
  102. package/dist/handlers/logging-stream.d.ts +2 -1
  103. package/dist/handlers/logging-stream.js +35 -7
  104. package/dist/handlers/plugin-invoke.js +152 -39
  105. package/dist/handlers/registry.d.ts +2 -2
  106. package/dist/handlers/registry.js +108 -29
  107. package/dist/handlers/rpc-server.js +7 -7
  108. package/dist/handlers/session.d.ts +11 -0
  109. package/dist/handlers/session.js +40 -0
  110. package/dist/handlers/types.d.ts +7 -1
  111. package/dist/handlers/unload-model.js +8 -3
  112. package/dist/handlers/vector-index.js +118 -19
  113. package/dist/index.d.ts +1 -1
  114. package/dist/index.js +1 -1
  115. package/dist/logging/addon.d.ts +1 -1
  116. package/dist/logging/addon.js +1 -1
  117. package/dist/logging/index.d.ts +5 -5
  118. package/dist/logging/index.js +4 -4
  119. package/dist/logging/logger.d.ts +1 -1
  120. package/dist/logging/logger.js +3 -3
  121. package/dist/logging/utils.js +11 -3
  122. package/dist/model-cache-layout.d.ts +1 -0
  123. package/dist/model-cache-layout.js +2 -0
  124. package/dist/model-fit/assess.d.ts +23 -0
  125. package/dist/model-fit/assess.js +110 -0
  126. package/dist/model-fit/describe.d.ts +12 -0
  127. package/dist/model-fit/describe.js +29 -0
  128. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
  129. package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
  130. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
  131. package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
  132. package/dist/model-fit/fit-load.d.ts +23 -0
  133. package/dist/model-fit/fit-load.js +29 -0
  134. package/dist/model-fit/load-check.d.ts +34 -0
  135. package/dist/model-fit/load-check.js +77 -0
  136. package/dist/model-fit/run/engine.d.ts +12 -0
  137. package/dist/model-fit/run/engine.js +9 -0
  138. package/dist/model-fit/run/run.d.ts +46 -0
  139. package/dist/model-fit/run/run.js +208 -0
  140. package/dist/model-fit/sum.d.ts +32 -0
  141. package/dist/model-fit/sum.js +60 -0
  142. package/dist/models/registry/index.d.ts +1 -1
  143. package/dist/models/registry/index.js +2 -2
  144. package/dist/models/registry/models.d.ts +1377 -130
  145. package/dist/models/registry/models.js +3622 -2312
  146. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  147. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  148. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  149. package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
  150. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  151. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  152. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  153. package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  154. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  155. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
  156. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  157. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  158. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  159. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  160. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  161. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
  162. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  163. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
  164. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
  165. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
  166. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  167. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  168. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
  169. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
  170. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
  171. package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
  172. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  173. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  174. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  175. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  176. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  177. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  178. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  179. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
  180. package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
  181. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  182. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  183. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
  184. package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
  185. package/dist/plugins/builtin/index.d.ts +16 -14
  186. package/dist/plugins/builtin/index.js +16 -14
  187. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  188. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  189. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
  190. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  191. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  192. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  193. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
  194. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  195. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
  196. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
  197. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  198. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  199. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
  200. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
  201. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
  202. package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
  203. package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
  204. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
  205. package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
  206. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  207. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
  208. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
  209. package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
  210. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  211. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  212. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
  213. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
  214. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  215. package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
  216. package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
  217. package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
  218. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
  219. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
  220. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
  221. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
  222. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
  223. package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
  224. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  225. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  226. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
  227. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
  228. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
  229. package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
  230. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  231. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  232. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  233. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  234. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
  235. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
  236. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  237. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  238. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
  239. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
  240. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
  241. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
  242. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
  243. package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
  244. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
  245. package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
  246. package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
  247. package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
  248. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  249. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  250. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  251. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  252. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  253. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
  254. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
  255. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  256. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
  257. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  258. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  259. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
  260. package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
  261. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
  262. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
  263. package/dist/plugins/index.d.ts +1 -1
  264. package/dist/plugins/index.js +2 -2
  265. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  266. package/dist/plugins/ops/bci-transcribe.js +18 -4
  267. package/dist/plugins/ops/cancel.d.ts +1 -0
  268. package/dist/plugins/ops/cancel.js +11 -2
  269. package/dist/plugins/ops/embed.d.ts +5 -3
  270. package/dist/plugins/ops/embed.js +89 -34
  271. package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
  272. package/dist/plugins/ops/kv-cache-utils.js +21 -10
  273. package/dist/plugins/ops/load-model.d.ts +10 -2
  274. package/dist/plugins/ops/load-model.js +45 -19
  275. package/dist/plugins/ops/transcribe.d.ts +5 -4
  276. package/dist/plugins/ops/transcribe.js +63 -28
  277. package/dist/plugins/ops/translate.d.ts +2 -1
  278. package/dist/plugins/ops/translate.js +7 -4
  279. package/dist/plugins/ops/unload-model.d.ts +17 -1
  280. package/dist/plugins/ops/unload-model.js +32 -3
  281. package/dist/plugins/turbovec-backend.d.ts +1 -1
  282. package/dist/plugins/turbovec-backend.js +21 -6
  283. package/dist/profiling/aggregator.d.ts +1 -1
  284. package/dist/profiling/aggregator.js +1 -1
  285. package/dist/profiling/controller.d.ts +1 -1
  286. package/dist/profiling/controller.js +2 -2
  287. package/dist/profiling/events.d.ts +1 -1
  288. package/dist/profiling/events.js +2 -2
  289. package/dist/profiling/exporters.d.ts +1 -1
  290. package/dist/profiling/exporters.js +3 -3
  291. package/dist/profiling/index.d.ts +13 -13
  292. package/dist/profiling/index.js +12 -12
  293. package/dist/profiling/operation-metrics.d.ts +1 -1
  294. package/dist/profiling/operation-metrics.js +24 -21
  295. package/dist/profiling/operation-wrappers.js +3 -3
  296. package/dist/profiling/profiler.js +1 -1
  297. package/dist/rag/embedding-scope.d.ts +16 -0
  298. package/dist/rag/embedding-scope.js +81 -0
  299. package/dist/rag/handler.d.ts +25 -0
  300. package/dist/rag/handler.js +51 -17
  301. package/dist/rag/handlers/close-workspace.js +1 -1
  302. package/dist/rag/handlers/delete-embeddings.js +1 -1
  303. package/dist/rag/handlers/delete-workspace.js +1 -1
  304. package/dist/rag/handlers/ingest.js +3 -7
  305. package/dist/rag/handlers/list-workspaces.d.ts +1 -1
  306. package/dist/rag/handlers/list-workspaces.js +1 -1
  307. package/dist/rag/handlers/reindex.js +1 -1
  308. package/dist/rag/handlers/save-embeddings.js +1 -1
  309. package/dist/rag/handlers/search.d.ts +10 -1
  310. package/dist/rag/handlers/search.js +17 -8
  311. package/dist/rag/index.d.ts +11 -11
  312. package/dist/rag/index.js +11 -11
  313. package/dist/rag/rag-operation-manager.d.ts +27 -3
  314. package/dist/rag/rag-operation-manager.js +10 -27
  315. package/dist/rag/rag-workspace-manager.js +1 -1
  316. package/dist/registry.js +7 -1
  317. package/dist/resources/collector.d.ts +71 -37
  318. package/dist/resources/collector.js +9 -2
  319. package/dist/resources/instance.d.ts +1 -1
  320. package/dist/resources/instance.js +1 -1
  321. package/dist/resources/native.d.ts +1 -1
  322. package/dist/resources/normalize.d.ts +1 -1
  323. package/dist/resources/types.d.ts +2 -0
  324. package/dist/rpc/inventory.js +1 -12
  325. package/dist/runtime/env.d.ts +5 -1
  326. package/dist/runtime/env.js +6 -21
  327. package/dist/runtime/index.d.ts +8 -5
  328. package/dist/runtime/index.js +4 -2
  329. package/dist/runtime/lifecycle.d.ts +6 -4
  330. package/dist/runtime/lifecycle.js +16 -12
  331. package/dist/runtime/model-config-utils.js +8 -1
  332. package/dist/runtime/model-registry.d.ts +30 -4
  333. package/dist/runtime/model-registry.js +48 -2
  334. package/dist/runtime/model-work-lock.d.ts +13 -0
  335. package/dist/runtime/model-work-lock.js +48 -0
  336. package/dist/runtime/registry-client.d.ts +47 -1
  337. package/dist/runtime/registry-client.js +446 -88
  338. package/dist/runtime/request-context.d.ts +11 -3
  339. package/dist/runtime/request-context.js +39 -2
  340. package/dist/runtime/request-registry.d.ts +56 -8
  341. package/dist/runtime/request-registry.js +190 -72
  342. package/dist/runtime/runtime-lifecycle.js +10 -1
  343. package/dist/runtime/session-id.d.ts +18 -0
  344. package/dist/runtime/session-id.js +20 -0
  345. package/dist/runtime/session-registry.d.ts +39 -0
  346. package/dist/runtime/session-registry.js +114 -0
  347. package/dist/runtime/state.d.ts +1 -1
  348. package/dist/runtime/state.js +2 -2
  349. package/dist/runtime/vector-index-backend.d.ts +6 -2
  350. package/dist/runtime/vector-index-registry.d.ts +26 -5
  351. package/dist/runtime/vector-index-registry.js +88 -27
  352. package/dist/schemas/abort-signal.d.ts +8 -0
  353. package/dist/schemas/assess-model-fit.d.ts +138 -282
  354. package/dist/schemas/assess-model-fit.js +85 -219
  355. package/dist/schemas/audio-gen.d.ts +291 -25
  356. package/dist/schemas/audio-gen.js +130 -26
  357. package/dist/schemas/batch-completion-stream.d.ts +113 -15
  358. package/dist/schemas/batch-completion-stream.js +10 -6
  359. package/dist/schemas/bci-config.d.ts +3 -1
  360. package/dist/schemas/bci-config.js +2 -2
  361. package/dist/schemas/bci.d.ts +55 -4
  362. package/dist/schemas/bci.js +13 -18
  363. package/dist/schemas/calc-tokens.d.ts +141 -0
  364. package/dist/schemas/calc-tokens.js +53 -0
  365. package/dist/schemas/cancel.d.ts +31 -0
  366. package/dist/schemas/cancel.js +11 -2
  367. package/dist/schemas/classification.d.ts +3 -0
  368. package/dist/schemas/classification.js +6 -2
  369. package/dist/schemas/common.d.ts +2209 -348
  370. package/dist/schemas/common.js +52 -33
  371. package/dist/schemas/completion-event.d.ts +11 -2
  372. package/dist/schemas/completion-event.js +5 -2
  373. package/dist/schemas/completion-stream.d.ts +173 -19
  374. package/dist/schemas/completion-stream.js +117 -20
  375. package/dist/schemas/config.d.ts +2064 -178
  376. package/dist/schemas/config.js +21 -23
  377. package/dist/schemas/decision.d.ts +12 -2
  378. package/dist/schemas/decision.js +16 -1
  379. package/dist/schemas/download-asset.d.ts +13 -6
  380. package/dist/schemas/download-asset.js +4 -7
  381. package/dist/schemas/embed.d.ts +286 -3
  382. package/dist/schemas/embed.js +162 -16
  383. package/dist/schemas/engine-addon-map.d.ts +3 -1
  384. package/dist/schemas/engine-addon-map.js +5 -3
  385. package/dist/schemas/errors.d.ts +13 -0
  386. package/dist/schemas/errors.js +71 -1
  387. package/dist/schemas/finetune.d.ts +14 -4
  388. package/dist/schemas/finetune.js +5 -6
  389. package/dist/schemas/heartbeat.d.ts +1 -0
  390. package/dist/schemas/heartbeat.js +5 -2
  391. package/dist/schemas/image-embedding.d.ts +27 -0
  392. package/dist/schemas/image-embedding.js +107 -0
  393. package/dist/schemas/index.d.ts +55 -50
  394. package/dist/schemas/index.js +54 -49
  395. package/dist/schemas/lifecycle.d.ts +3 -0
  396. package/dist/schemas/lifecycle.js +13 -6
  397. package/dist/schemas/llamacpp-config.d.ts +443 -20
  398. package/dist/schemas/llamacpp-config.js +188 -14
  399. package/dist/schemas/load-model.d.ts +8940 -1952
  400. package/dist/schemas/load-model.js +73 -9
  401. package/dist/schemas/logging-stream.d.ts +1 -0
  402. package/dist/schemas/logging-stream.js +5 -2
  403. package/dist/schemas/mcp-adapter.d.ts +1 -1
  404. package/dist/schemas/model-config-schemas.d.ts +731 -111
  405. package/dist/schemas/model-config-schemas.js +16 -12
  406. package/dist/schemas/model-fit-policy.d.ts +2 -2
  407. package/dist/schemas/model-info.d.ts +54 -56
  408. package/dist/schemas/model-info.js +18 -7
  409. package/dist/schemas/model-ops.d.ts +12 -0
  410. package/dist/schemas/model-ops.js +33 -5
  411. package/dist/schemas/model-src-utils.d.ts +14 -6
  412. package/dist/schemas/model-src-utils.js +2 -2
  413. package/dist/schemas/model-types.d.ts +12 -4
  414. package/dist/schemas/model-types.js +2 -0
  415. package/dist/schemas/ocr.d.ts +11 -4
  416. package/dist/schemas/ocr.js +7 -3
  417. package/dist/schemas/plugin.d.ts +70 -8
  418. package/dist/schemas/plugin.js +32 -4
  419. package/dist/schemas/profiling.d.ts +32 -32
  420. package/dist/schemas/profiling.js +1 -1
  421. package/dist/schemas/rag.d.ts +9 -0
  422. package/dist/schemas/rag.js +11 -17
  423. package/dist/schemas/registry.d.ts +38 -10
  424. package/dist/schemas/registry.js +16 -7
  425. package/dist/schemas/reload-config.d.ts +45 -6
  426. package/dist/schemas/reload-config.js +13 -6
  427. package/dist/schemas/request-base.d.ts +39 -0
  428. package/dist/schemas/request-base.js +64 -0
  429. package/dist/schemas/rerank.d.ts +132 -0
  430. package/dist/schemas/rerank.js +104 -0
  431. package/dist/schemas/rpc-server.d.ts +3 -0
  432. package/dist/schemas/rpc-server.js +15 -15
  433. package/dist/schemas/sdcpp-config.d.ts +144 -18
  434. package/dist/schemas/sdcpp-config.js +123 -28
  435. package/dist/schemas/session.d.ts +37 -0
  436. package/dist/schemas/session.js +53 -0
  437. package/dist/schemas/system-resources.d.ts +364 -179
  438. package/dist/schemas/system-resources.js +8 -3
  439. package/dist/schemas/text-to-speech.d.ts +1408 -275
  440. package/dist/schemas/text-to-speech.js +547 -33
  441. package/dist/schemas/tools.d.ts +7 -0
  442. package/dist/schemas/tools.js +29 -0
  443. package/dist/schemas/transcription-config.d.ts +267 -2
  444. package/dist/schemas/transcription-config.js +138 -7
  445. package/dist/schemas/transcription.d.ts +94 -7
  446. package/dist/schemas/transcription.js +99 -22
  447. package/dist/schemas/translate.d.ts +4 -2
  448. package/dist/schemas/translate.js +6 -10
  449. package/dist/schemas/translation-config.d.ts +40 -20
  450. package/dist/schemas/translation-config.js +1 -1
  451. package/dist/schemas/tts-pcm.d.ts +35 -0
  452. package/dist/schemas/tts-pcm.js +69 -0
  453. package/dist/schemas/vector-index.d.ts +47 -1
  454. package/dist/schemas/vector-index.js +37 -10
  455. package/dist/schemas/vla.d.ts +3 -3
  456. package/dist/surface.d.ts +4 -3
  457. package/dist/surface.js +4 -3
  458. package/dist/utils/addon-responses.d.ts +71 -2
  459. package/dist/utils/aggregate-events.d.ts +1 -1
  460. package/dist/utils/aggregate-events.js +2 -2
  461. package/dist/utils/archive.d.ts +7 -0
  462. package/dist/utils/archive.js +89 -30
  463. package/dist/utils/asr-events.d.ts +5 -0
  464. package/dist/utils/asr-events.js +10 -1
  465. package/dist/utils/audio/decoder.js +3 -1
  466. package/dist/utils/audio-input.js +1 -1
  467. package/dist/utils/cache/auto-history.d.ts +2 -1
  468. package/dist/utils/cache/auto-history.js +5 -3
  469. package/dist/utils/cache/layout.d.ts +48 -0
  470. package/dist/utils/cache/layout.js +190 -0
  471. package/dist/utils/cache/paths.js +8 -10
  472. package/dist/utils/cache/types.d.ts +2 -1
  473. package/dist/utils/checksum.d.ts +7 -0
  474. package/dist/utils/checksum.js +59 -2
  475. package/dist/utils/completion-normalizer.js +287 -88
  476. package/dist/utils/gguf-architecture.d.ts +7 -0
  477. package/dist/utils/gguf-architecture.js +142 -0
  478. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  479. package/dist/utils/gguf-tensor-extractor.js +1 -50
  480. package/dist/utils/index.d.ts +13 -13
  481. package/dist/utils/index.js +13 -13
  482. package/dist/utils/mcp-adapter.d.ts +1 -1
  483. package/dist/utils/mcp-adapter.js +1 -1
  484. package/dist/utils/path-security.js +1 -1
  485. package/dist/utils/shard-utils.d.ts +0 -16
  486. package/dist/utils/shard-utils.js +14 -48
  487. package/dist/utils/tool-integration.d.ts +3 -2
  488. package/dist/utils/tool-integration.js +3 -6
  489. package/dist/utils/tools/defer.d.ts +50 -31
  490. package/dist/utils/tools/defer.js +115 -51
  491. package/dist/utils/tools/dialect.d.ts +15 -4
  492. package/dist/utils/tools/dialect.js +34 -5
  493. package/dist/utils/tools/index.d.ts +3 -3
  494. package/dist/utils/tools/index.js +3 -3
  495. package/dist/utils/tools/parser.js +8 -8
  496. package/dist/utils/tools/parsers/dsml.d.ts +1 -1
  497. package/dist/utils/tools/parsers/dsml.js +1 -1
  498. package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
  499. package/dist/utils/tools/parsers/gemma4native.js +1 -1
  500. package/dist/utils/tools/parsers/harmony.d.ts +1 -1
  501. package/dist/utils/tools/parsers/harmony.js +1 -1
  502. package/dist/utils/tools/parsers/hermes.d.ts +1 -1
  503. package/dist/utils/tools/parsers/hermes.js +1 -1
  504. package/dist/utils/tools/parsers/json.d.ts +1 -1
  505. package/dist/utils/tools/parsers/json.js +1 -1
  506. package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
  507. package/dist/utils/tools/parsers/pythonic.js +1 -1
  508. package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
  509. package/dist/utils/tools/parsers/qwen35.js +1 -1
  510. package/dist/utils/transcribe-metadata.d.ts +28 -2
  511. package/dist/utils/transcribe-metadata.js +16 -0
  512. package/dist/utils/tts-stats.d.ts +43 -1
  513. package/dist/utils/tts-stats.js +120 -22
  514. package/dist/utils/until-aborted.d.ts +7 -0
  515. package/dist/utils/until-aborted.js +34 -0
  516. package/package.json +67 -55
  517. package/dist/models/registry/resource-profiles.d.ts +0 -17
  518. package/dist/models/registry/resource-profiles.js +0 -5657
  519. package/dist/resources/model-fit/assess.d.ts +0 -72
  520. package/dist/resources/model-fit/assess.js +0 -679
  521. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
  522. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
  523. package/dist/resources/model-fit/floor.d.ts +0 -31
  524. package/dist/resources/model-fit/floor.js +0 -65
  525. package/dist/resources/model-fit/kv-cache.d.ts +0 -23
  526. package/dist/resources/model-fit/kv-cache.js +0 -85
  527. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
  528. package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
  529. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
  530. package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
  531. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
  532. package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
  533. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
  534. package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
  535. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
  536. package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
  537. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
  538. package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
  539. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
  540. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
  541. package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
  542. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
  543. package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
  544. package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
  545. package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
  546. package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
  547. package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
  548. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
  549. package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
  550. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
  551. package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
  552. package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
  553. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
  554. package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
  555. package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
  556. package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
  557. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
  558. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
  559. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
  560. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
  561. package/dist/resources/model-fit/platform.d.ts +0 -9
  562. package/dist/resources/model-fit/platform.js +0 -26
  563. package/dist/resources/model-fit/types.d.ts +0 -6
  564. package/dist/resources/model-fit/types.js +0 -1
  565. package/dist/schemas/model-resource-profile.d.ts +0 -108
  566. package/dist/schemas/model-resource-profile.js +0 -80
  567. /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
  568. /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
  569. /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
@@ -50,6 +50,9 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
50
50
  var e = new Error(message);
51
51
  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
52
  });
53
+ import fs from 'bare-fs';
54
+ import Buffer from 'bare-buffer';
55
+ import { requestParent } from '../../schemas/plugin.js';
53
56
  import { getModel } from '../../runtime/model-registry.js';
54
57
  import { embedParamsSchema } from '../../schemas/index.js';
55
58
  import { buildUnaryResult } from '../../profiling/model-execution.js';
@@ -58,25 +61,35 @@ import { nowMs } from '../../profiling/index.js';
58
61
  import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
59
62
  import { generateRandomRequestId } from '../../runtime/request-id.js';
60
63
  import { getEngineLogger } from '../../logging/index.js';
61
- export async function embed(params, requestId) {
64
+ const modelsWithProjector = new WeakSet();
65
+ export function markEmbeddingProjector(model) {
66
+ modelsWithProjector.add(model);
67
+ }
68
+ export async function embed(params, requestId, sessionId, context) {
62
69
  const env_1 = { stack: [], error: void 0, hasError: false };
63
70
  try {
64
- const { modelId, text } = embedParamsSchema.parse(params);
71
+ const parsed = embedParamsSchema.parse(params);
72
+ const inputs = parsed.input !== undefined ? [parsed.input] : parsed.inputs;
73
+ const mediaInputs = inputs?.map((input) => {
74
+ if (typeof input === 'object' && !Array.isArray(input)) {
75
+ throw new EmbedFailedError('This model accepts text and ordered media parts, not raw RGB image inputs');
76
+ }
77
+ return input;
78
+ });
79
+ const { modelId } = parsed;
65
80
  // Open a request-scoped lifecycle. The registry routes
66
81
  // `cancel({ requestId })` and broad `cancel({ modelId, kind: "embeddings" })`
67
82
  // straight to this context's signal. Falls back to a generated
68
83
  // id if the caller didn't send one.
69
84
  const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
70
85
  requestId: requestId ?? generateRandomRequestId(),
86
+ sessionId,
87
+ ...requestParent(context),
71
88
  kind: 'embeddings',
72
89
  modelId
73
- })
74
- // `requestLogger` is intentionally referenced once so the
75
- // request-scoped logger is built at handler entry per the canonical
76
- // shape, even when this op has no per-step emits beyond the
77
- // registry's own lifecycle lines. Future addon-level warns inside
78
- // this body should route through `requestLogger`.
79
- , true);
90
+ }), true);
91
+ if (ctx.signal.aborted)
92
+ throw new InferenceCancelledError(ctx.requestId);
80
93
  // `requestLogger` is intentionally referenced once so the
81
94
  // request-scoped logger is built at handler entry per the canonical
82
95
  // shape, even when this op has no per-step emits beyond the
@@ -84,42 +97,47 @@ export async function embed(params, requestId) {
84
97
  // this body should route through `requestLogger`.
85
98
  const requestLogger = withRequestContext(getEngineLogger(), ctx);
86
99
  const model = getModel(modelId);
87
- // Hard-cancel wiring: llamacpp-embedding declares
88
- // `cancel: { scope: "model", hard: true }`. The signal listener
89
- // forwards an abort to the addon so C++ work stops as soon as the
90
- // user cancels — mirrors the canonical handler shape documented in
91
- // `request-lifecycle-primitives.mdc`. The post-await
92
- // `signal.aborted` checks below are the soft-cancel safety net for
93
- // the case where the abort fires between `model.run(...)` returning
94
- // and `response.await()` resolving (both can take seconds on large
95
- // batches).
100
+ const hasMedia = mediaInputs?.some((input) => Array.isArray(input) && input.some((part) => typeof part !== 'string'));
101
+ if (hasMedia && !modelsWithProjector.has(model)) {
102
+ throw new EmbedFailedError('Media inputs require modelConfig.projectionModelSrc when loading the embedding model');
103
+ }
104
+ const input = mediaInputs !== undefined
105
+ ? await prepareEmbedInputs(mediaInputs, ctx.signal, ctx.requestId)
106
+ : parsed.text;
107
+ if (ctx.signal.aborted)
108
+ throw new InferenceCancelledError(ctx.requestId);
109
+ // Admission keeps model-wide cancellation scoped to this request. Media
110
+ // preparation has no native job to cancel, so wire the addon only afterward.
111
+ let cancelPromise;
96
112
  const onAbort = () => {
97
113
  const addon = model.addon;
98
114
  if (addon?.cancel) {
99
- addon.cancel.call(addon).catch((err) => {
115
+ cancelPromise ??= addon.cancel.call(addon).catch((err) => {
100
116
  requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
101
117
  });
102
118
  }
103
119
  };
104
120
  ctx.signal.addEventListener('abort', onAbort, { once: true });
105
- // `{ once: true }` does not fire if the signal is already aborted at
106
- // register time. The registry may abort synchronously when a
107
- // cancel-before-begin race resolves or the parent signal was pre-
108
- // aborted — re-use `onAbort` so the listener body is the single
109
- // source of truth for "what cancel does."
110
- if (ctx.signal.aborted)
111
- onAbort();
112
- ctx.scope.defer(() => {
121
+ ctx.scope.defer(async () => {
113
122
  ctx.signal.removeEventListener('abort', onAbort);
123
+ // A delayed native cancel must finish before the next job owns this model slot.
124
+ await cancelPromise;
114
125
  });
115
126
  const modelStart = nowMs();
116
- const response = (await model.run(text));
117
- if (ctx.signal.aborted) {
118
- throw new InferenceCancelledError(ctx.requestId);
127
+ let response;
128
+ let rawEmbeddings;
129
+ try {
130
+ response = (await model.run(input));
131
+ if (ctx.signal.aborted)
132
+ throw new InferenceCancelledError(ctx.requestId);
133
+ rawEmbeddings = await response.await();
134
+ if (ctx.signal.aborted)
135
+ throw new InferenceCancelledError(ctx.requestId);
119
136
  }
120
- const rawEmbeddings = await response.await();
121
- if (ctx.signal.aborted) {
122
- throw new InferenceCancelledError(ctx.requestId);
137
+ catch (cause) {
138
+ if (ctx.signal.aborted)
139
+ throw new InferenceCancelledError(ctx.requestId, {}, cause);
140
+ throw cause;
123
141
  }
124
142
  const modelExecutionMs = nowMs() - modelStart;
125
143
  const stats = {
@@ -134,7 +152,7 @@ export async function embed(params, requestId) {
134
152
  ...(response.stats?.context_size !== undefined && { contextSize: response.stats.context_size })
135
153
  };
136
154
  const embeddingsArray = rawEmbeddings[0];
137
- if (Array.isArray(text)) {
155
+ if (parsed.input === undefined && (parsed.inputs !== undefined || Array.isArray(parsed.text))) {
138
156
  if (!embeddingsArray || embeddingsArray.length === 0) {
139
157
  throw new EmbedNoEmbeddingsError();
140
158
  }
@@ -191,3 +209,40 @@ export function normalizeVector(vector) {
191
209
  }
192
210
  return normalizedVector;
193
211
  }
212
+ async function prepareEmbedInputs(inputs, signal, requestId) {
213
+ const result = [];
214
+ for (const input of inputs) {
215
+ if (signal.aborted)
216
+ throw new InferenceCancelledError(requestId);
217
+ if (typeof input === 'string') {
218
+ result.push(input);
219
+ continue;
220
+ }
221
+ const parts = [];
222
+ for (const part of input) {
223
+ if (signal.aborted)
224
+ throw new InferenceCancelledError(requestId);
225
+ if (typeof part === 'string') {
226
+ parts.push(part);
227
+ }
228
+ else {
229
+ const { type, value } = part.media;
230
+ try {
231
+ const media = type === 'base64'
232
+ ? Buffer.from(value, 'base64')
233
+ : await fs.promises.readFile(value, 'buffer');
234
+ parts.push({ media });
235
+ }
236
+ catch (cause) {
237
+ if (signal.aborted)
238
+ throw new InferenceCancelledError(requestId, {}, cause);
239
+ throw new EmbedFailedError('Unable to read embedding media', cause);
240
+ }
241
+ }
242
+ }
243
+ result.push(parts);
244
+ }
245
+ if (signal.aborted)
246
+ throw new InferenceCancelledError(requestId);
247
+ return { inputs: result };
248
+ }
@@ -1,13 +1,15 @@
1
1
  import { type CacheMessage } from '../../utils/index.js';
2
+ import type { Tool } from '../../schemas/tools.js';
2
3
  export declare function extractSystemPrompt(messages: CacheMessage[]): string | null;
3
4
  export declare function generateConfigHash(systemPrompt: string | null): string;
4
- export declare function generateCacheKey(messages: CacheMessage[]): string;
5
+ export declare function generateCacheKey(messages: CacheMessage[], tools?: readonly Tool[]): string;
6
+ export declare function resolveCacheFilePath(modelId: string, configHash: string, cacheKey: string): string;
5
7
  export declare function getCacheFilePath(modelId: string, configHash: string, cacheKey: string): Promise<string>;
6
- export declare function findMatchingCache(modelId: string, configHash: string, currentHistory: CacheMessage[]): Promise<{
8
+ export declare function findMatchingCache(modelId: string, configHash: string, currentHistory: CacheMessage[], tools?: readonly Tool[]): Promise<{
7
9
  cacheKey: string;
8
10
  cachePath: string;
9
11
  } | null>;
10
- export declare function getCurrentCacheInfo(modelId: string, configHash: string, currentHistory: CacheMessage[]): Promise<{
12
+ export declare function getCurrentCacheInfo(modelId: string, configHash: string, currentHistory: CacheMessage[], tools?: readonly Tool[]): Promise<{
11
13
  cacheKey: string;
12
14
  cachePath: string;
13
15
  }>;
@@ -5,14 +5,16 @@ import { getAutoCacheLookupHistory, getKVCacheDir, validateAndJoinPath } from '.
5
5
  import { getEngineLogger } from '../../logging/index.js';
6
6
  import { PathTraversalError } from '../../errors/index.js';
7
7
  import { Buffer } from 'bare-buffer';
8
- import { markAutoCacheKey } from './kv-cache-retention.js';
8
+ import { markAutoCacheKey } from '../../plugins/ops/kv-cache-retention.js';
9
9
  const logger = getEngineLogger();
10
10
  // In-memory KV-cache state lives in `KvCacheSession`, its single mutation
11
11
  // point. This module keeps only the pure path / hash utilities that don't
12
12
  // touch in-memory state.
13
13
  export function extractSystemPrompt(messages) {
14
14
  const systemMessage = messages.find((msg) => msg.role === 'system');
15
- return systemMessage ? systemMessage.content : null;
15
+ if (!systemMessage || typeof systemMessage.content !== 'string')
16
+ return null;
17
+ return systemMessage.content;
16
18
  }
17
19
  // Names the cache file by system prompt only. Tools stay out on purpose: the
18
20
  // addon compares the rendered prompt against the tokens the file holds, so a
@@ -24,15 +26,22 @@ export function generateConfigHash(systemPrompt) {
24
26
  hash.update(Buffer.from(JSON.stringify({ systemPrompt, tools: [] }), 'utf8'));
25
27
  return hash.digest('hex').substring(0, 16);
26
28
  }
27
- export function generateCacheKey(messages) {
29
+ export function generateCacheKey(messages, tools) {
28
30
  const hash = crypto.createHash('sha-256');
29
- const historyString = JSON.stringify(messages);
31
+ // Before the first assistant reply, the rendered tools are part of the
32
+ // reusable prefix. Keep those primes isolated; established conversations
33
+ // retain history-only keys so the addon can trim a changed tool block.
34
+ const historyString = JSON.stringify(tools?.length && !messages.some((message) => message.role === 'assistant')
35
+ ? { messages, tools }
36
+ : messages);
30
37
  const historyBuffer = Buffer.from(historyString, 'utf8');
31
38
  hash.update(historyBuffer);
32
39
  const hashString = hash.digest('hex');
33
40
  return hashString.substring(0, 16);
34
41
  }
35
- function resolveCacheFilePath(modelId, configHash, cacheKey) {
42
+ // Resolves the path without creating its directories, for a cache that may
43
+ // not exist (a seed source).
44
+ export function resolveCacheFilePath(modelId, configHash, cacheKey) {
36
45
  const cacheDir = getKVCacheDir();
37
46
  const sessionCacheDir = validateAndJoinPath(cacheDir, cacheKey);
38
47
  const modelCacheDir = validateAndJoinPath(sessionCacheDir, modelId);
@@ -50,12 +59,14 @@ export async function getCacheFilePath(modelId, configHash, cacheKey) {
50
59
  return cachePath;
51
60
  }
52
61
  // Used for auto-generated cache key
53
- export async function findMatchingCache(modelId, configHash, currentHistory) {
54
- if (currentHistory.length <= 1) {
62
+ export async function findMatchingCache(modelId, configHash, currentHistory, tools) {
63
+ if (currentHistory.length === 0) {
55
64
  return null;
56
65
  }
57
66
  const previousHistory = getAutoCacheLookupHistory(currentHistory);
58
- const cacheKey = generateCacheKey(previousHistory);
67
+ if (previousHistory.length === 0 && !tools?.length)
68
+ return null;
69
+ const cacheKey = generateCacheKey(previousHistory, tools);
59
70
  const cachePath = resolveCacheFilePath(modelId, configHash, cacheKey);
60
71
  try {
61
72
  await fsPromises.access(cachePath);
@@ -66,8 +77,8 @@ export async function findMatchingCache(modelId, configHash, currentHistory) {
66
77
  return null;
67
78
  }
68
79
  }
69
- export async function getCurrentCacheInfo(modelId, configHash, currentHistory) {
70
- const cacheKey = generateCacheKey(currentHistory);
80
+ export async function getCurrentCacheInfo(modelId, configHash, currentHistory, tools) {
81
+ const cacheKey = generateCacheKey(currentHistory, tools);
71
82
  await markAutoCacheKey(cacheKey);
72
83
  const cachePath = await getCacheFilePath(modelId, configHash, cacheKey);
73
84
  return { cacheKey, cachePath };
@@ -1,8 +1,16 @@
1
+ import type { AbortSignal } from 'bare-abort-controller';
1
2
  import { type LoadModelServerParams } from '../../schemas/index.js';
2
3
  import type { LoadTimingStats } from '../../profiling/types.js';
3
4
  export interface LoadModelResult {
4
5
  timing?: LoadTimingStats;
5
6
  }
6
- export declare function loadModel(params: LoadModelServerParams, options?: {
7
+ export interface LoadModelOptions {
7
8
  collectTiming?: boolean;
8
- }): Promise<LoadModelResult>;
9
+ request?: {
10
+ signal: AbortSignal;
11
+ requestId: string;
12
+ };
13
+ /** Session that takes a lease on the model. Defaults to the owner session. */
14
+ sessionId?: string;
15
+ }
16
+ export declare function loadModel(params: LoadModelServerParams, options?: LoadModelOptions): Promise<LoadModelResult>;
@@ -1,26 +1,46 @@
1
1
  import { loadModelServerParamsSchema, normalizeModelType } from '../../schemas/index.js';
2
2
  import { nowMs } from '../../profiling/clock.js';
3
3
  import { forwardBackendDiagnostics } from '../../profiling/backend-diagnostics.js';
4
- import { isModelLoaded, registerModel } from '../../runtime/model-registry.js';
4
+ import { addModelLease, getModelEntry, isModelLoaded, isModelReconfigured, registerModel } from '../../runtime/model-registry.js';
5
+ import { OWNER_SESSION_ID } from '../../runtime/session-id.js';
6
+ import { withModelWorkLock } from '../../runtime/model-work-lock.js';
5
7
  import { startLogBuffering, stopLogBufferingWithTimeout } from '../../runtime/logging-stream-registry.js';
6
- import { detectShardedModel, generateShardFilenames, validateShardedModelCache } from '../../utils/index.js';
7
- import { PluginNotFoundError, ModelFileNotFoundError, ModelFileNotFoundInDirError, ModelFileLocateFailedError, ModelFitRefusedError } from '../../errors/index.js';
8
- import { ensureAddonLoggerReady, getPlugin } from '../index.js';
9
- import { runAdvisoryFitCheck } from '../../resources/model-fit/native-probe/advisory-fit.js';
8
+ import { detectShardedModel, checkAllShardsExist } from '../../utils/index.js';
9
+ import { PluginNotFoundError, ModelFileNotFoundError, ModelFileNotFoundInDirError, ModelFileLocateFailedError, ModelFitRefusedError, ModelReconfiguredError, InferenceCancelledError } from '../../errors/index.js';
10
+ import { ensureAddonLoggerReady, getPlugin } from '../../plugins/index.js';
11
+ import { runLoadFitCheck } from '../../model-fit/load-check.js';
10
12
  import { refusesLoad, resolveModelFitPolicy, runsProbe } from '../../schemas/model-fit-policy.js';
11
13
  import { getConfig } from '../../runtime/state.js';
14
+ import { resolveToolDialect } from '../../utils/tools/dialect.js';
12
15
  import { promises as fsPromises } from 'bare-fs';
13
16
  import path from 'bare-path';
14
17
  import { getEngineLogger } from '../../logging/index.js';
15
18
  const logger = getEngineLogger();
16
19
  export async function loadModel(params, options) {
17
- const { modelId, modelPath, options: modelOptions, artifacts, modelName } = loadModelServerParamsSchema.parse(params);
20
+ const parsed = loadModelServerParamsSchema.parse(params);
21
+ return withModelWorkLock(parsed.modelId, () => loadModelLocked(parsed, options), options?.request);
22
+ }
23
+ async function loadModelLocked(params, options) {
24
+ const { modelId, modelPath, options: modelOptions, artifacts, modelName } = params;
18
25
  const { modelConfig, modelType: rawModelType, modelFitPolicy } = modelOptions;
19
26
  // Normalize modelType to canonical form (handles aliases and custom types)
20
27
  const modelType = normalizeModelType(rawModelType);
21
- // Check if model is already loaded
28
+ const sessionId = options?.sessionId ?? OWNER_SESSION_ID;
29
+ // A cancel that landed while this load waited for another load of the same
30
+ // model must not take a lease the caller no longer wants.
31
+ if (options?.request?.signal.aborted === true) {
32
+ throw new InferenceCancelledError(options.request.requestId);
33
+ }
34
+ // An already loaded model is shared: the caller takes a lease on it. A
35
+ // session that holds no lease on a reconfigured model is refused: it asked
36
+ // for the configuration the id hashes, which the model no longer has.
22
37
  if (isModelLoaded(modelId)) {
23
- logger.info(`${modelType} model ${modelId} is already loaded`);
38
+ const entry = getModelEntry(modelId);
39
+ if (entry && !entry.leases.has(sessionId) && isModelReconfigured(modelId)) {
40
+ throw new ModelReconfiguredError(modelId);
41
+ }
42
+ addModelLease(modelId, sessionId);
43
+ logger.info(`${modelType} model ${modelId} is already loaded; lease added for ${sessionId}`);
24
44
  return {};
25
45
  }
26
46
  // Detect if sharded model
@@ -32,12 +52,9 @@ export async function loadModel(params, options) {
32
52
  throw new PluginNotFoundError(modelType);
33
53
  }
34
54
  if (isShardedModel) {
35
- // For sharded models, validate all shards and tensors.txt exist
36
55
  const shardDir = path.dirname(modelPath);
37
- const isValid = await validateShardedModelCache(shardDir, modelFileName);
38
- if (!isValid) {
39
- const numberedShards = generateShardFilenames(modelFileName);
40
- throw new ModelFileNotFoundError(`Missing shards or ${shardInfo.baseFilename}.tensors.txt. Expected ${numberedShards.length} shard files + tensors.txt in ${shardDir}`);
56
+ if (!(await checkAllShardsExist(shardDir, modelFileName))) {
57
+ throw new ModelFileNotFoundError(`Missing shards. Expected ${shardInfo.totalShards} shard files in ${shardDir}`);
41
58
  }
42
59
  }
43
60
  else if (!plugin.skipPrimaryModelPathValidation) {
@@ -59,18 +76,23 @@ export async function loadModel(params, options) {
59
76
  // the real load uses, and before `createModel()` so it never competes with
60
77
  // the native load for device memory.
61
78
  const fitPolicy = resolveModelFitPolicy(modelFitPolicy, getConfig().modelFitPolicy);
62
- const fitProbe = runsProbe(fitPolicy)
63
- ? await runAdvisoryFitCheck({
79
+ const request = options?.request;
80
+ const fitCheck = runsProbe(fitPolicy)
81
+ ? await runLoadFitCheck({
64
82
  modelId,
65
83
  modelType: modelType,
66
84
  modelPath,
67
85
  modelConfig,
68
86
  artifacts,
69
87
  isShardedModel
70
- })
88
+ }, request !== undefined ? { signal: request.signal } : {})
71
89
  : undefined;
90
+ const fitProbe = fitCheck?.probe;
91
+ if (request?.signal.aborted === true) {
92
+ throw new InferenceCancelledError(request.requestId);
93
+ }
72
94
  if (refusesLoad(fitPolicy, fitProbe?.verdict)) {
73
- throw new ModelFitRefusedError(modelType, fitProbe.reason, fitProbe.message);
95
+ throw new ModelFitRefusedError(modelType, 'does-not-fit', fitCheck?.detail);
74
96
  }
75
97
  // Load the addon's logger before the addon itself runs. Plugins that defer
76
98
  // their logging module do so to keep registration free of native loads, so
@@ -90,14 +112,18 @@ export async function loadModel(params, options) {
90
112
  await result.model.load(false);
91
113
  const modelInitializationTimeMs = options?.collectTiming ? nowMs() - initStart : undefined;
92
114
  logger.info(`${modelType} model ${modelId} loaded`);
115
+ const toolDialect = 'completionStream' in plugin.handlers
116
+ ? await resolveToolDialect(modelName, modelPath)
117
+ : undefined;
93
118
  registerModel(modelId, {
94
119
  model: result.model,
95
120
  path: modelPath,
96
121
  config: modelConfig,
97
122
  modelType: modelType,
98
123
  name: modelName,
99
- fitProbe
100
- });
124
+ fitProbe,
125
+ toolDialect
126
+ }, sessionId);
101
127
  const loadResult = modelInitializationTimeMs !== undefined ? { timing: { modelInitializationTimeMs } } : {};
102
128
  return forwardBackendDiagnostics(loadResult, result);
103
129
  }
@@ -1,3 +1,4 @@
1
+ import { type PluginHandlerContext } from '../../schemas/plugin.js';
1
2
  import { type TranscribeParams, type TranscribeSegment, type TranscribeStats, type TranscribeStreamEvent, type ParakeetStreamingRunConfig } from '../../schemas/index.js';
2
3
  import type Buffer from 'bare-buffer';
3
4
  import type { InferenceBackendDiagnostics } from '../../schemas/index.js';
@@ -10,13 +11,13 @@ type TranscribeReturn = {
10
11
  };
11
12
  export declare function transcribe(params: TranscribeParams & {
12
13
  metadata: true;
13
- }, requestId?: string): AsyncGenerator<TranscribeSegment, TranscribeReturn, void>;
14
- export declare function transcribe(params: TranscribeParams, requestId?: string): AsyncGenerator<string, TranscribeReturn, void>;
14
+ }, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<TranscribeSegment, TranscribeReturn, void>;
15
+ export declare function transcribe(params: TranscribeParams, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<string, TranscribeReturn, void>;
15
16
  export interface TranscribeStreamOpts {
16
17
  emitVadEvents?: boolean;
17
18
  endOfTurnSilenceMs?: number;
18
19
  vadRunIntervalMs?: number;
19
20
  parakeetStreamingConfig?: ParakeetStreamingRunConfig;
20
21
  }
21
- export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent, TranscribeReturn, void>;
22
- export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<string | TranscribeStreamEvent, TranscribeReturn, void>;
22
+ export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent, TranscribeReturn, void>;
23
+ export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<string | TranscribeStreamEvent, TranscribeReturn, void>;
@@ -50,11 +50,13 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
50
50
  var e = new Error(message);
51
51
  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
52
  });
53
+ import { requestParent } from '../../schemas/plugin.js';
53
54
  import { getModel, getModelConfig, getModelEntry } from '../../runtime/model-registry.js';
54
55
  import { ModelType } from '../../schemas/index.js';
55
56
  import { createAudioStream } from '../../utils/audio-input.js';
57
+ import { untilAborted } from '../../utils/until-aborted.js';
56
58
  import { getEngineLogger } from '../../logging/index.js';
57
- import { TranscriptionFailedError } from '../../errors/index.js';
59
+ import { InferenceCancelledError, TranscriptionFailedError } from '../../errors/index.js';
58
60
  import { nowMs } from '../../profiling/index.js';
59
61
  import { buildStreamResult } from '../../profiling/model-execution.js';
60
62
  import { buildAsrBackendDiagnostics } from '../../utils/asr-diagnostics.js';
@@ -62,8 +64,11 @@ import { assertMetadataSupported, toTranscribeSegment } from '../../utils/transc
62
64
  import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
63
65
  import { generateRandomRequestId } from '../../runtime/request-id.js';
64
66
  import { isEndOfTurnEvent, isVadEvent, toEndOfTurnEvent, toVadStateEvent } from '../../utils/asr-events.js';
65
- import { buildWhisperReloadConfig } from '../builtin/asr-ggml/config.js';
67
+ import { buildWhisperReloadConfig } from '../../plugins/builtin/asr-ggml/config.js';
66
68
  export { assertMetadataSupported, toTranscribeSegment };
69
+ async function* iterateAll(source) {
70
+ yield* source;
71
+ }
67
72
  function hasRunStreaming(model) {
68
73
  return 'runStreaming' in model && typeof model.runStreaming === 'function';
69
74
  }
@@ -173,10 +178,23 @@ function mapTranscribeStats(stats) {
173
178
  }),
174
179
  ...(stats?.encoderUsedCoreml !== undefined && {
175
180
  encoderUsedCoreml: stats.encoderUsedCoreml
181
+ }),
182
+ ...(stats?.aoscActive !== undefined && { aoscActive: stats.aoscActive }),
183
+ // MOSS-Transcribe-Diarize reports its counters under its own names. Its
184
+ // stage timings land on the shared encoder/decoder fields and its segment
185
+ // count on `totalSegments`; the token counts and prefill are MOSS-only.
186
+ ...(stats?.segments !== undefined && { totalSegments: stats.segments }),
187
+ ...(stats?.encodeMs !== undefined && { encoderTime: stats.encodeMs }),
188
+ ...(stats?.decodeMs !== undefined && { decoderTime: stats.decodeMs }),
189
+ ...(stats?.prefillMs !== undefined && { prefillTime: stats.prefillMs }),
190
+ ...(stats?.audioTokens !== undefined && { audioTokens: stats.audioTokens }),
191
+ ...(stats?.promptTokens !== undefined && { promptTokens: stats.promptTokens }),
192
+ ...(stats?.generatedTokens !== undefined && {
193
+ generatedTokens: stats.generatedTokens
176
194
  })
177
195
  };
178
196
  }
179
- export async function* transcribe(params, requestId) {
197
+ export async function* transcribe(params, requestId, sessionId, context) {
180
198
  const env_1 = { stack: [], error: void 0, hasError: false };
181
199
  try {
182
200
  const { modelId, metadata } = params;
@@ -186,9 +204,18 @@ export async function* transcribe(params, requestId) {
186
204
  // caller didn't send one.
187
205
  const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
188
206
  requestId: requestId ?? generateRandomRequestId(),
207
+ sessionId,
208
+ ...requestParent(context),
189
209
  kind: 'transcribe',
190
210
  modelId
191
- }), true);
211
+ })
212
+ // A request cancelled while it queued holds no slot; the model-wide
213
+ // `addon.cancel()` below would stop the job that does.
214
+ , true);
215
+ // A request cancelled while it queued holds no slot; the model-wide
216
+ // `addon.cancel()` below would stop the job that does.
217
+ if (ctx.signal.aborted)
218
+ throw new InferenceCancelledError(ctx.requestId);
192
219
  const requestLogger = withRequestContext(getEngineLogger(), ctx);
193
220
  const engineType = getEngineModelType(modelId);
194
221
  const hasMossOptions = params.hotwords !== undefined || params.maxNewTokens !== undefined;
@@ -248,9 +275,7 @@ export async function* transcribe(params, requestId) {
248
275
  const chunks = (Array.isArray(output) ? output : [output]);
249
276
  if (metadata) {
250
277
  for (const chunk of chunks) {
251
- if (!chunk.text)
252
- continue;
253
- if (silenceMarker && chunk.text.includes(silenceMarker))
278
+ if (!carriesSegment(chunk, silenceMarker))
254
279
  continue;
255
280
  yield toTranscribeSegment(chunk);
256
281
  }
@@ -297,7 +322,7 @@ function buildRunStreamingOpts(engineType, opts) {
297
322
  }
298
323
  return runOpts;
299
324
  }
300
- export async function* transcribeStream(modelId, audioInputStream, prompt, metadata, opts, requestId) {
325
+ export async function* transcribeStream(modelId, audioInputStream, prompt, metadata, opts, requestId, sessionId, context) {
301
326
  const env_2 = { stack: [], error: void 0, hasError: false };
302
327
  try {
303
328
  // Same `kind: "transcribe"` as the unary variant — the registry
@@ -306,9 +331,13 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
306
331
  // either shape.
307
332
  const ctx = __addDisposableResource(env_2, await getRequestRegistry().begin({
308
333
  requestId: requestId ?? generateRandomRequestId(),
334
+ sessionId,
335
+ ...requestParent(context),
309
336
  kind: 'transcribe',
310
337
  modelId
311
338
  }), true);
339
+ if (ctx.signal.aborted)
340
+ throw new InferenceCancelledError(ctx.requestId);
312
341
  const requestLogger = withRequestContext(getEngineLogger(), ctx);
313
342
  const engineType = getEngineModelType(modelId);
314
343
  assertMetadataSupported(modelId, engineType, metadata);
@@ -321,26 +350,19 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
321
350
  if (!hasRunStreaming(model)) {
322
351
  throw new TranscriptionFailedError(`Model ${modelId} does not support streaming transcription`);
323
352
  }
324
- const onAbort = () => {
325
- const addon = model.addon;
326
- if (addon?.cancel) {
327
- addon.cancel.call(addon).catch((err) => {
328
- requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
329
- });
330
- }
331
- };
332
- ctx.signal.addEventListener('abort', onAbort, { once: true });
333
- if (ctx.signal.aborted)
334
- onAbort();
335
- ctx.scope.defer(() => {
336
- ctx.signal.removeEventListener('abort', onAbort);
337
- });
353
+ // A streaming session ends when its input ends, so an abort ends the input
354
+ // and the session drains the audio it already has, yielding nothing more.
355
+ // The model-wide `addon.cancel()` the unary path uses is not safe here: a
356
+ // parakeet stream cancelled that way leaves its stream state to the next
357
+ // session, whose output starts with the cancelled session's audio and never
358
+ // ends.
359
+ const input = untilAborted(iterateAll(audioInputStream), ctx.signal);
338
360
  const runOpts = buildRunStreamingOpts(engineType, opts);
339
361
  const modelStart = nowMs();
340
- const response = await model.runStreaming(audioInputStream, runOpts);
362
+ const response = await model.runStreaming(input, runOpts);
341
363
  for await (const output of response.iterate()) {
342
364
  if (ctx.signal.aborted)
343
- break;
365
+ continue;
344
366
  requestLogger.debug('Live Transcription Update:', output);
345
367
  if (!Array.isArray(output)) {
346
368
  if (isVadEvent(output)) {
@@ -386,17 +408,30 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
386
408
  await result_2;
387
409
  }
388
410
  }
411
+ /**
412
+ * Whether a native segment becomes a metadata segment. Text-less segments are
413
+ * dropped, except an explicit final marker: parakeet streaming closes a chunk
414
+ * with `isFinal: true` and no text, and that marker is the only finality signal
415
+ * the caller gets for the chunk.
416
+ */
417
+ function carriesSegment(segment, silenceMarker) {
418
+ if (!segment.text)
419
+ return segment.isFinal === true;
420
+ return !silenceMarker || !segment.text.includes(silenceMarker);
421
+ }
389
422
  function* emitSegment(segment, metadata, silenceMarker) {
423
+ if (metadata) {
424
+ if (carriesSegment(segment, silenceMarker))
425
+ yield toTranscribeSegment(segment);
426
+ return;
427
+ }
390
428
  if (!segment.text) {
391
429
  return;
392
430
  }
393
431
  if (silenceMarker && segment.text.includes(silenceMarker)) {
394
432
  return;
395
433
  }
396
- if (metadata) {
397
- yield toTranscribeSegment(segment);
398
- }
399
- else if (segment.text.trim()) {
434
+ if (segment.text.trim()) {
400
435
  yield segment.text;
401
436
  }
402
437
  }
@@ -1,7 +1,8 @@
1
+ import { type PluginHandlerContext } from '../../schemas/plugin.js';
1
2
  import { type TranslateParams, type TranslationStats } from '../../schemas/index.js';
2
3
  export declare function getLanguage(code: string | undefined): string;
3
4
  export declare function isAfrican(code: string | undefined): boolean;
4
- export declare function translate(params: TranslateParams, requestId?: string): AsyncGenerator<string, {
5
+ export declare function translate(params: TranslateParams, requestId?: string, sessionId?: string, invoke?: PluginHandlerContext): AsyncGenerator<string, {
5
6
  modelExecutionMs: number;
6
7
  stats?: TranslationStats;
7
8
  }, unknown>;