@qvac/inference 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/NOTICE +60 -28
  2. package/README.md +3 -1
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +135 -44
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +1 -1
  13. package/dist/api/completion-stream.js +2 -1
  14. package/dist/api/delete-cache.d.ts +7 -1
  15. package/dist/api/delete-cache.js +11 -1
  16. package/dist/api/diffusion.d.ts +14 -1
  17. package/dist/api/diffusion.js +14 -1
  18. package/dist/api/embed.d.ts +82 -29
  19. package/dist/api/embed.js +64 -18
  20. package/dist/api/get-model-info.d.ts +1 -1
  21. package/dist/api/index.d.ts +2 -1
  22. package/dist/api/index.js +2 -1
  23. package/dist/api/rag.d.ts +1 -1
  24. package/dist/api/rag.js +1 -1
  25. package/dist/api/registry.d.ts +16 -6
  26. package/dist/api/registry.js +32 -9
  27. package/dist/api/text-to-speech.d.ts +14 -5
  28. package/dist/api/text-to-speech.js +83 -29
  29. package/dist/api/transcribe.d.ts +6 -2
  30. package/dist/api/transcribe.js +7 -1
  31. package/dist/api/vector-index.d.ts +25 -3
  32. package/dist/api/vector-index.js +17 -2
  33. package/dist/api/video-request.d.ts +15 -15
  34. package/dist/dispatch.d.ts +10 -4
  35. package/dist/dispatch.js +65 -15
  36. package/dist/engine.d.ts +1 -1
  37. package/dist/errors/index.d.ts +70 -0
  38. package/dist/errors/index.js +128 -0
  39. package/dist/handlers/assess-model-fit.d.ts +3 -21
  40. package/dist/handlers/assess-model-fit.js +4 -303
  41. package/dist/handlers/cancelHandler.d.ts +10 -9
  42. package/dist/handlers/cancelHandler.js +17 -13
  43. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  44. package/dist/handlers/completion-orchestrate.js +164 -22
  45. package/dist/handlers/delete-cache.d.ts +2 -1
  46. package/dist/handlers/delete-cache.js +6 -1
  47. package/dist/handlers/download-asset.js +11 -4
  48. package/dist/handlers/get-loaded-model-info.js +1 -0
  49. package/dist/handlers/get-model-info.js +71 -2
  50. package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
  51. package/dist/handlers/load-model/coreml-sidecars.js +16 -119
  52. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  53. package/dist/handlers/load-model/download-lock.js +213 -0
  54. package/dist/handlers/load-model/download-manager.d.ts +10 -2
  55. package/dist/handlers/load-model/download-manager.js +62 -11
  56. package/dist/handlers/load-model/handler.js +88 -35
  57. package/dist/handlers/load-model/http.js +327 -205
  58. package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
  59. package/dist/handlers/load-model/hyperdrive.js +123 -50
  60. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  61. package/dist/handlers/load-model/registry-client-options.js +4 -2
  62. package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
  63. package/dist/handlers/load-model/registry-companion-set.js +19 -1
  64. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  65. package/dist/handlers/load-model/registry-delete.js +269 -0
  66. package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
  67. package/dist/handlers/load-model/registry-download-utils.js +93 -22
  68. package/dist/handlers/load-model/registry.d.ts +20 -0
  69. package/dist/handlers/load-model/registry.js +308 -175
  70. package/dist/handlers/load-model/resolve.js +4 -1
  71. package/dist/handlers/load-model/types.d.ts +2 -0
  72. package/dist/handlers/logging-stream.js +17 -5
  73. package/dist/handlers/plugin-invoke.js +152 -39
  74. package/dist/handlers/registry.d.ts +2 -2
  75. package/dist/handlers/registry.js +105 -28
  76. package/dist/handlers/rpc-server.js +7 -7
  77. package/dist/handlers/session.d.ts +11 -0
  78. package/dist/handlers/session.js +40 -0
  79. package/dist/handlers/unload-model.js +8 -3
  80. package/dist/handlers/vector-index.js +118 -19
  81. package/dist/index.d.ts +1 -1
  82. package/dist/index.js +1 -1
  83. package/dist/logging/utils.js +10 -2
  84. package/dist/model-cache-layout.d.ts +1 -0
  85. package/dist/model-cache-layout.js +2 -0
  86. package/dist/model-fit/assess.d.ts +18 -67
  87. package/dist/model-fit/assess.js +96 -665
  88. package/dist/model-fit/describe.d.ts +12 -0
  89. package/dist/model-fit/describe.js +29 -0
  90. package/dist/model-fit/description/fetch.d.ts +1 -1
  91. package/dist/model-fit/description/fetch.js +95 -16
  92. package/dist/model-fit/description/resolve-load.d.ts +5 -3
  93. package/dist/model-fit/description/resolve-load.js +14 -14
  94. package/dist/model-fit/fit-load.d.ts +23 -0
  95. package/dist/model-fit/fit-load.js +29 -0
  96. package/dist/model-fit/load-check.d.ts +15 -33
  97. package/dist/model-fit/load-check.js +50 -195
  98. package/dist/model-fit/run/engine.d.ts +12 -72
  99. package/dist/model-fit/run/engine.js +4 -8
  100. package/dist/model-fit/run/run.d.ts +10 -7
  101. package/dist/model-fit/run/run.js +22 -18
  102. package/dist/model-fit/sum.d.ts +32 -0
  103. package/dist/model-fit/sum.js +60 -0
  104. package/dist/models/registry/models.d.ts +865 -128
  105. package/dist/models/registry/models.js +3737 -2963
  106. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  107. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  108. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  109. package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
  110. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  111. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  112. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  113. package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  114. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  115. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
  116. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  117. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  118. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  119. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  120. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  121. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
  122. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  123. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
  124. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
  125. package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
  126. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  127. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  128. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
  129. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
  130. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
  131. package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
  132. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  133. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  134. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  135. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  136. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  137. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  138. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  139. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
  140. package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
  141. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  142. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  143. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
  144. package/dist/plugins/builtin/index.d.ts +1 -0
  145. package/dist/plugins/builtin/index.js +1 -0
  146. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  147. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  148. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
  149. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  150. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  151. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  152. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  153. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
  154. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
  155. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  156. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  157. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
  158. package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
  159. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  160. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
  161. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
  162. package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
  163. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  164. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  165. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
  166. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
  167. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  168. package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
  169. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
  170. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
  171. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
  172. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
  173. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
  174. package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
  175. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  176. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  177. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
  178. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
  179. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
  180. package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
  181. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  182. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  183. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  184. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  185. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
  186. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
  187. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  188. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  189. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
  190. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
  191. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
  192. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
  193. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
  194. package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
  195. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  196. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  197. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  198. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  199. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  200. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
  201. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
  202. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  203. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
  204. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  205. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  206. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
  207. package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
  208. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
  209. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
  210. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  211. package/dist/plugins/ops/bci-transcribe.js +18 -4
  212. package/dist/plugins/ops/cancel.d.ts +1 -0
  213. package/dist/plugins/ops/cancel.js +11 -2
  214. package/dist/plugins/ops/embed.d.ts +5 -3
  215. package/dist/plugins/ops/embed.js +89 -34
  216. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
  217. package/dist/plugins/ops/kv-cache-utils.js +6 -2
  218. package/dist/plugins/ops/load-model.d.ts +2 -0
  219. package/dist/plugins/ops/load-model.js +33 -16
  220. package/dist/plugins/ops/transcribe.d.ts +5 -4
  221. package/dist/plugins/ops/transcribe.js +62 -27
  222. package/dist/plugins/ops/translate.d.ts +2 -1
  223. package/dist/plugins/ops/translate.js +4 -1
  224. package/dist/plugins/ops/unload-model.d.ts +17 -1
  225. package/dist/plugins/ops/unload-model.js +31 -2
  226. package/dist/plugins/turbovec-backend.d.ts +1 -1
  227. package/dist/plugins/turbovec-backend.js +16 -1
  228. package/dist/profiling/operation-metrics.js +22 -19
  229. package/dist/rag/embedding-scope.d.ts +16 -0
  230. package/dist/rag/embedding-scope.js +81 -0
  231. package/dist/rag/handler.d.ts +25 -0
  232. package/dist/rag/handler.js +50 -16
  233. package/dist/rag/handlers/ingest.js +2 -6
  234. package/dist/rag/handlers/search.d.ts +10 -1
  235. package/dist/rag/handlers/search.js +16 -7
  236. package/dist/rag/rag-operation-manager.d.ts +27 -3
  237. package/dist/rag/rag-operation-manager.js +10 -27
  238. package/dist/registry.js +5 -0
  239. package/dist/resources/collector.d.ts +70 -36
  240. package/dist/resources/collector.js +8 -1
  241. package/dist/resources/types.d.ts +2 -0
  242. package/dist/rpc/inventory.js +1 -12
  243. package/dist/runtime/env.d.ts +5 -1
  244. package/dist/runtime/env.js +6 -21
  245. package/dist/runtime/index.d.ts +4 -1
  246. package/dist/runtime/index.js +2 -0
  247. package/dist/runtime/lifecycle.d.ts +6 -4
  248. package/dist/runtime/lifecycle.js +9 -5
  249. package/dist/runtime/model-config-utils.js +5 -1
  250. package/dist/runtime/model-registry.d.ts +28 -4
  251. package/dist/runtime/model-registry.js +46 -1
  252. package/dist/runtime/model-work-lock.d.ts +13 -0
  253. package/dist/runtime/model-work-lock.js +48 -0
  254. package/dist/runtime/registry-client.d.ts +47 -1
  255. package/dist/runtime/registry-client.js +446 -88
  256. package/dist/runtime/request-context.d.ts +9 -1
  257. package/dist/runtime/request-context.js +25 -1
  258. package/dist/runtime/request-registry.d.ts +55 -7
  259. package/dist/runtime/request-registry.js +189 -71
  260. package/dist/runtime/runtime-lifecycle.js +10 -1
  261. package/dist/runtime/session-id.d.ts +18 -0
  262. package/dist/runtime/session-id.js +20 -0
  263. package/dist/runtime/session-registry.d.ts +39 -0
  264. package/dist/runtime/session-registry.js +114 -0
  265. package/dist/runtime/vector-index-backend.d.ts +4 -0
  266. package/dist/runtime/vector-index-registry.d.ts +21 -0
  267. package/dist/runtime/vector-index-registry.js +56 -18
  268. package/dist/schemas/abort-signal.d.ts +8 -0
  269. package/dist/schemas/abort-signal.js +1 -0
  270. package/dist/schemas/assess-model-fit.d.ts +132 -282
  271. package/dist/schemas/assess-model-fit.js +82 -216
  272. package/dist/schemas/audio-gen.d.ts +278 -24
  273. package/dist/schemas/audio-gen.js +127 -23
  274. package/dist/schemas/batch-completion-stream.d.ts +105 -13
  275. package/dist/schemas/batch-completion-stream.js +8 -4
  276. package/dist/schemas/bci-config.d.ts +2 -1
  277. package/dist/schemas/bci.d.ts +54 -3
  278. package/dist/schemas/bci.js +11 -16
  279. package/dist/schemas/calc-tokens.d.ts +141 -0
  280. package/dist/schemas/calc-tokens.js +53 -0
  281. package/dist/schemas/cancel.d.ts +26 -0
  282. package/dist/schemas/cancel.js +10 -2
  283. package/dist/schemas/classification.d.ts +3 -0
  284. package/dist/schemas/classification.js +6 -2
  285. package/dist/schemas/common.d.ts +1831 -348
  286. package/dist/schemas/common.js +18 -2
  287. package/dist/schemas/completion-event.d.ts +3 -0
  288. package/dist/schemas/completion-event.js +1 -1
  289. package/dist/schemas/completion-stream.d.ts +162 -17
  290. package/dist/schemas/completion-stream.js +83 -13
  291. package/dist/schemas/config.d.ts +1212 -169
  292. package/dist/schemas/config.js +9 -13
  293. package/dist/schemas/decision.d.ts +4 -2
  294. package/dist/schemas/decision.js +2 -1
  295. package/dist/schemas/download-asset.d.ts +10 -5
  296. package/dist/schemas/download-asset.js +3 -6
  297. package/dist/schemas/embed.d.ts +286 -3
  298. package/dist/schemas/embed.js +162 -16
  299. package/dist/schemas/engine-addon-map.d.ts +1 -0
  300. package/dist/schemas/engine-addon-map.js +1 -0
  301. package/dist/schemas/errors.d.ts +12 -0
  302. package/dist/schemas/errors.js +66 -1
  303. package/dist/schemas/finetune.d.ts +14 -4
  304. package/dist/schemas/finetune.js +5 -6
  305. package/dist/schemas/heartbeat.d.ts +1 -0
  306. package/dist/schemas/heartbeat.js +5 -2
  307. package/dist/schemas/image-embedding.d.ts +27 -0
  308. package/dist/schemas/image-embedding.js +107 -0
  309. package/dist/schemas/index.d.ts +7 -3
  310. package/dist/schemas/index.js +7 -3
  311. package/dist/schemas/lifecycle.d.ts +3 -0
  312. package/dist/schemas/lifecycle.js +13 -6
  313. package/dist/schemas/llamacpp-config.d.ts +269 -22
  314. package/dist/schemas/llamacpp-config.js +86 -7
  315. package/dist/schemas/load-model.d.ts +6418 -1960
  316. package/dist/schemas/load-model.js +39 -8
  317. package/dist/schemas/logging-stream.d.ts +1 -0
  318. package/dist/schemas/logging-stream.js +5 -2
  319. package/dist/schemas/model-config-schemas.d.ts +542 -116
  320. package/dist/schemas/model-config-schemas.js +2 -0
  321. package/dist/schemas/model-fit-policy.d.ts +2 -2
  322. package/dist/schemas/model-info.d.ts +54 -56
  323. package/dist/schemas/model-info.js +17 -6
  324. package/dist/schemas/model-ops.d.ts +12 -0
  325. package/dist/schemas/model-ops.js +33 -5
  326. package/dist/schemas/model-src-utils.d.ts +10 -6
  327. package/dist/schemas/model-types.d.ts +8 -4
  328. package/dist/schemas/model-types.js +1 -0
  329. package/dist/schemas/ocr.d.ts +9 -4
  330. package/dist/schemas/ocr.js +6 -2
  331. package/dist/schemas/plugin.d.ts +62 -6
  332. package/dist/schemas/plugin.js +26 -4
  333. package/dist/schemas/profiling.d.ts +32 -32
  334. package/dist/schemas/rag.d.ts +9 -0
  335. package/dist/schemas/rag.js +10 -16
  336. package/dist/schemas/registry.d.ts +30 -10
  337. package/dist/schemas/registry.js +14 -6
  338. package/dist/schemas/reload-config.d.ts +41 -6
  339. package/dist/schemas/reload-config.js +11 -4
  340. package/dist/schemas/request-base.d.ts +39 -0
  341. package/dist/schemas/request-base.js +64 -0
  342. package/dist/schemas/rerank.d.ts +8 -7
  343. package/dist/schemas/rerank.js +6 -4
  344. package/dist/schemas/rpc-server.d.ts +3 -0
  345. package/dist/schemas/rpc-server.js +15 -15
  346. package/dist/schemas/sdcpp-config.d.ts +90 -20
  347. package/dist/schemas/sdcpp-config.js +96 -19
  348. package/dist/schemas/session.d.ts +37 -0
  349. package/dist/schemas/session.js +53 -0
  350. package/dist/schemas/system-resources.d.ts +364 -179
  351. package/dist/schemas/system-resources.js +8 -3
  352. package/dist/schemas/text-to-speech.d.ts +1343 -271
  353. package/dist/schemas/text-to-speech.js +547 -33
  354. package/dist/schemas/tools.d.ts +7 -0
  355. package/dist/schemas/tools.js +29 -0
  356. package/dist/schemas/transcription-config.d.ts +262 -2
  357. package/dist/schemas/transcription-config.js +122 -6
  358. package/dist/schemas/transcription.d.ts +93 -6
  359. package/dist/schemas/transcription.js +98 -21
  360. package/dist/schemas/translate.d.ts +2 -0
  361. package/dist/schemas/translate.js +5 -9
  362. package/dist/schemas/translation-config.d.ts +30 -20
  363. package/dist/schemas/tts-pcm.d.ts +35 -0
  364. package/dist/schemas/tts-pcm.js +69 -0
  365. package/dist/schemas/vector-index.d.ts +47 -1
  366. package/dist/schemas/vector-index.js +37 -10
  367. package/dist/schemas/vla.d.ts +3 -3
  368. package/dist/surface.d.ts +3 -3
  369. package/dist/surface.js +3 -3
  370. package/dist/utils/addon-responses.d.ts +69 -2
  371. package/dist/utils/archive.d.ts +7 -0
  372. package/dist/utils/archive.js +88 -29
  373. package/dist/utils/asr-events.d.ts +5 -0
  374. package/dist/utils/asr-events.js +10 -1
  375. package/dist/utils/cache/auto-history.js +3 -1
  376. package/dist/utils/cache/layout.d.ts +48 -0
  377. package/dist/utils/cache/layout.js +190 -0
  378. package/dist/utils/cache/paths.js +2 -4
  379. package/dist/utils/cache/types.d.ts +2 -1
  380. package/dist/utils/checksum.d.ts +5 -0
  381. package/dist/utils/checksum.js +18 -0
  382. package/dist/utils/completion-normalizer.js +286 -87
  383. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  384. package/dist/utils/gguf-tensor-extractor.js +1 -50
  385. package/dist/utils/shard-utils.d.ts +0 -21
  386. package/dist/utils/shard-utils.js +12 -62
  387. package/dist/utils/tool-integration.d.ts +2 -2
  388. package/dist/utils/tools/defer.d.ts +39 -10
  389. package/dist/utils/tools/defer.js +72 -17
  390. package/dist/utils/tools/index.d.ts +1 -1
  391. package/dist/utils/tools/index.js +1 -1
  392. package/dist/utils/transcribe-metadata.d.ts +28 -2
  393. package/dist/utils/transcribe-metadata.js +16 -0
  394. package/dist/utils/tts-stats.d.ts +42 -0
  395. package/dist/utils/tts-stats.js +120 -22
  396. package/package.json +43 -39
  397. package/dist/model-fit/classify.d.ts +0 -6
  398. package/dist/model-fit/classify.js +0 -166
  399. package/dist/model-fit/description/project.d.ts +0 -27
  400. package/dist/model-fit/description/project.js +0 -33
  401. package/dist/model-fit/floor.d.ts +0 -31
  402. package/dist/model-fit/floor.js +0 -65
  403. package/dist/model-fit/kv-cache.d.ts +0 -23
  404. package/dist/model-fit/kv-cache.js +0 -85
  405. package/dist/model-fit/platform.d.ts +0 -14
  406. package/dist/model-fit/platform.js +0 -26
  407. package/dist/model-fit/request/create.d.ts +0 -31
  408. package/dist/model-fit/request/create.js +0 -61
  409. package/dist/model-fit/request/engines/asr.d.ts +0 -14
  410. package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
  411. package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
  412. package/dist/model-fit/request/engines/diffusion.js +0 -93
  413. package/dist/model-fit/request/engines/llama.d.ts +0 -12
  414. package/dist/model-fit/request/engines/llama.js +0 -44
  415. package/dist/model-fit/request/engines/nmt.d.ts +0 -8
  416. package/dist/model-fit/request/engines/nmt.js +0 -11
  417. package/dist/model-fit/request/engines/ocr.d.ts +0 -9
  418. package/dist/model-fit/request/engines/ocr.js +0 -13
  419. package/dist/model-fit/request/engines/tts.d.ts +0 -10
  420. package/dist/model-fit/request/engines/tts.js +0 -152
  421. package/dist/models/registry/resource-profiles.d.ts +0 -17
  422. package/dist/models/registry/resource-profiles.js +0 -5859
  423. package/dist/schemas/model-resource-profile.d.ts +0 -109
  424. package/dist/schemas/model-resource-profile.js +0 -80
  425. /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
  426. /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
@@ -4,6 +4,7 @@ import { transcribe, transcribeStream } from '../../../plugins/ops/transcribe.js
4
4
  import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
5
5
  import { attachBackendDiagnostics } from '../../../profiling/backend-diagnostics.js';
6
6
  import { buildWhisperEngineConfig } from '../../../plugins/builtin/asr-ggml/config.js';
7
+ import { whisperFitRequest } from '../../../plugins/builtin/asr-ggml/fit.js';
7
8
  import { createAsrModelLogger } from '../../../plugins/builtin/asr-ggml/logging.js';
8
9
  function createWhisperModel(modelId, modelPath, whisperConfig, vadModelPath) {
9
10
  const logger = createAsrModelLogger(modelId, ModelType.whispercppTranscription);
@@ -23,6 +24,7 @@ export const whisperPlugin = definePlugin({
23
24
  displayName: 'Whisper (whisper.cpp)',
24
25
  addonPackage: ADDON_ASR,
25
26
  loadConfigSchema: whisperConfigSchema,
27
+ fitRequest: whisperFitRequest,
26
28
  assessFit: ASRGgml.assessFit,
27
29
  async resolveConfig(cfg, ctx) {
28
30
  const { vadModelSrc, ...whisperConfig } = cfg;
@@ -48,7 +50,7 @@ export const whisperPlugin = definePlugin({
48
50
  // whisper.cpp addon exposes a model-wide hard cancel — compute
49
51
  // is interrupted on the currently-running transcription.
50
52
  cancel: { scope: 'model', hard: true },
51
- handler: async function* (request) {
53
+ handler: async function* (request, _inputStream, context) {
52
54
  const metadata = request.metadata === true;
53
55
  const stream = metadata
54
56
  ? transcribe({
@@ -58,14 +60,14 @@ export const whisperPlugin = definePlugin({
58
60
  hotwords: request.hotwords,
59
61
  maxNewTokens: request.maxNewTokens,
60
62
  metadata: true
61
- }, request.requestId)
63
+ }, request.requestId, request.sessionId, context)
62
64
  : transcribe({
63
65
  modelId: request.modelId,
64
66
  audioChunk: request.audioChunk,
65
67
  prompt: request.prompt,
66
68
  hotwords: request.hotwords,
67
69
  maxNewTokens: request.maxNewTokens
68
- }, request.requestId);
70
+ }, request.requestId, request.sessionId, context);
69
71
  try {
70
72
  let result = await stream.next();
71
73
  while (!result.done) {
@@ -105,7 +107,7 @@ export const whisperPlugin = definePlugin({
105
107
  // Same model-wide hard cancel surface as `transcribe` — both
106
108
  // route through the whisper.cpp addon.
107
109
  cancel: { scope: 'model', hard: true },
108
- handler: async function* (request, inputStream) {
110
+ handler: async function* (request, inputStream, context) {
109
111
  const streamOpts = {
110
112
  ...(request.emitVadEvents !== undefined && {
111
113
  emitVadEvents: request.emitVadEvents
@@ -119,8 +121,8 @@ export const whisperPlugin = definePlugin({
119
121
  };
120
122
  const metadata = request.metadata === true;
121
123
  const iterator = metadata
122
- ? transcribeStream(request.modelId, inputStream, request.prompt, true, streamOpts, request.requestId)
123
- : transcribeStream(request.modelId, inputStream, request.prompt, false, streamOpts, request.requestId);
124
+ ? transcribeStream(request.modelId, inputStream, request.prompt, true, streamOpts, request.requestId, request.sessionId, context)
125
+ : transcribeStream(request.modelId, inputStream, request.prompt, false, streamOpts, request.requestId, request.sessionId, context);
124
126
  try {
125
127
  let result = await iterator.next();
126
128
  while (!result.done) {
@@ -1,3 +1,4 @@
1
+ import { type PluginHandlerContext } from '../../schemas/plugin.js';
1
2
  import { type BciStreamOpts, type BciTranscribeParams, type TranscribeSegment, type TranscribeStats } from '../../schemas/index.js';
2
3
  import Buffer from 'bare-buffer';
3
4
  import { type StreamResult } from '../../profiling/model-execution.js';
@@ -9,9 +10,9 @@ type BciTranscribeReturn = {
9
10
  };
10
11
  export declare function bciTranscribe(params: BciTranscribeParams & {
11
12
  metadata: true;
12
- }, requestId?: string): AsyncGenerator<TranscribeSegment, BciTranscribeReturn, void>;
13
- export declare function bciTranscribe(params: BciTranscribeParams, requestId?: string): AsyncGenerator<string, BciTranscribeReturn, void>;
13
+ }, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<TranscribeSegment, BciTranscribeReturn, void>;
14
+ export declare function bciTranscribe(params: BciTranscribeParams, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<string, BciTranscribeReturn, void>;
14
15
  type BciStreamReturn = StreamResult;
15
- export declare function bciTranscribeStream(modelId: string, neuralStream: AsyncIterable<Buffer>, metadata: true, opts?: BciStreamOpts, requestId?: string): AsyncGenerator<TranscribeSegment, BciStreamReturn, void>;
16
- export declare function bciTranscribeStream(modelId: string, neuralStream: AsyncIterable<Buffer>, metadata?: boolean, opts?: BciStreamOpts, requestId?: string): AsyncGenerator<string, BciStreamReturn, void>;
16
+ export declare function bciTranscribeStream(modelId: string, neuralStream: AsyncIterable<Buffer>, metadata: true, opts?: BciStreamOpts, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<TranscribeSegment, BciStreamReturn, void>;
17
+ export declare function bciTranscribeStream(modelId: string, neuralStream: AsyncIterable<Buffer>, metadata?: boolean, opts?: BciStreamOpts, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<string, BciStreamReturn, void>;
17
18
  export {};
@@ -50,10 +50,11 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
50
50
  var e = new Error(message);
51
51
  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
52
  });
53
+ import { requestParent } from '../../schemas/plugin.js';
53
54
  import { getModel } from '../../runtime/model-registry.js';
54
55
  import Buffer from 'bare-buffer';
55
56
  import { getEngineLogger } from '../../logging/index.js';
56
- import { TranscriptionFailedError } from '../../errors/index.js';
57
+ import { InferenceCancelledError, TranscriptionFailedError } from '../../errors/index.js';
57
58
  import { nowMs } from '../../profiling/index.js';
58
59
  import { buildStreamResult } from '../../profiling/model-execution.js';
59
60
  import { buildAsrBackendDiagnostics } from '../../utils/asr-diagnostics.js';
@@ -70,7 +71,7 @@ async function runBci(model, input) {
70
71
  throw new TranscriptionFailedError('Invalid neural input');
71
72
  }
72
73
  }
73
- export async function* bciTranscribe(params, requestId) {
74
+ export async function* bciTranscribe(params, requestId, sessionId, context) {
74
75
  const env_1 = { stack: [], error: void 0, hasError: false };
75
76
  try {
76
77
  const { modelId, metadata } = params;
@@ -80,9 +81,18 @@ export async function* bciTranscribe(params, requestId) {
80
81
  // caller didn't send one.
81
82
  const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
82
83
  requestId: requestId ?? generateRandomRequestId(),
84
+ sessionId,
85
+ ...requestParent(context),
83
86
  kind: 'transcribe',
84
87
  modelId
85
- }), true);
88
+ })
89
+ // Cancelled while queued for the model: it holds no slot, so wiring the
90
+ // model-wide cancel below would stop the job of whoever runs now.
91
+ , true);
92
+ // Cancelled while queued for the model: it holds no slot, so wiring the
93
+ // model-wide cancel below would stop the job of whoever runs now.
94
+ if (ctx.signal.aborted)
95
+ throw new InferenceCancelledError(ctx.requestId);
86
96
  const requestLogger = withRequestContext(getEngineLogger(), ctx);
87
97
  const model = getModel(modelId);
88
98
  // Hard-cancel wiring: the BCI addon exposes a model-wide `cancel()`
@@ -181,7 +191,7 @@ export async function* bciTranscribe(params, requestId) {
181
191
  await result_1;
182
192
  }
183
193
  }
184
- export async function* bciTranscribeStream(modelId, neuralStream, metadata, opts, requestId) {
194
+ export async function* bciTranscribeStream(modelId, neuralStream, metadata, opts, requestId, sessionId, context) {
185
195
  const env_2 = { stack: [], error: void 0, hasError: false };
186
196
  try {
187
197
  // Same `kind: "transcribe"` as the unary BCI variant — the registry
@@ -190,9 +200,13 @@ export async function* bciTranscribeStream(modelId, neuralStream, metadata, opts
190
200
  // either shape.
191
201
  const ctx = __addDisposableResource(env_2, await getRequestRegistry().begin({
192
202
  requestId: requestId ?? generateRandomRequestId(),
203
+ sessionId,
204
+ ...requestParent(context),
193
205
  kind: 'transcribe',
194
206
  modelId
195
207
  }), true);
208
+ if (ctx.signal.aborted)
209
+ throw new InferenceCancelledError(ctx.requestId);
196
210
  const requestLogger = withRequestContext(getEngineLogger(), ctx);
197
211
  const model = getModel(modelId);
198
212
  // Hard-cancel wiring: `cancel()` tears down the active stream driver
@@ -22,4 +22,5 @@ import type { RequestKind } from '../../runtime/index.js';
22
22
  */
23
23
  export declare function cancel(params: CancelInferenceBaseParams, opts?: {
24
24
  kind?: RequestKind;
25
+ sessionId?: string;
25
26
  }): number;
@@ -2,6 +2,8 @@ import { getModel } from '../../runtime/model-registry.js';
2
2
  import { cancelInferenceBaseSchema } from '../../schemas/index.js';
3
3
  import { ModelNotLoadedError } from '../../errors/index.js';
4
4
  import { getRequestRegistry } from '../../runtime/index.js';
5
+ import { OWNER_SESSION_ID } from '../../runtime/session-id.js';
6
+ import { isOwnerSession } from '../../runtime/session-registry.js';
5
7
  import { getEngineLogger } from '../../logging/index.js';
6
8
  const logger = getEngineLogger();
7
9
  /**
@@ -30,9 +32,16 @@ export function cancel(params, opts) {
30
32
  if (!model) {
31
33
  throw new ModelNotLoadedError(modelId);
32
34
  }
35
+ // The owner sweeps every session's requests on the model; a session only
36
+ // reaches its own.
37
+ const sessionId = opts?.sessionId ?? OWNER_SESSION_ID;
33
38
  const registry = getRequestRegistry();
34
- const target = opts?.kind ? { modelId, kind: opts.kind } : { modelId };
35
- const cancelled = registry.cancel(target);
39
+ const cancelled = registry.cancel({
40
+ modelId,
41
+ ...(opts?.kind && { kind: opts.kind }),
42
+ sessionId,
43
+ allSessions: isOwnerSession(sessionId)
44
+ });
36
45
  if (cancelled === 0) {
37
46
  // Callers (workbench "Stop" button, app shutdown sweeps) often
38
47
  // fire-and-forget; log so operators can see when a broad cancel
@@ -1,15 +1,17 @@
1
+ import { type PluginHandlerContext } from '../../schemas/plugin.js';
1
2
  import { type EmbedParams, type EmbedStats } from '../../schemas/index.js';
2
3
  export interface EmbedResult {
3
4
  embedding: number[] | number[][];
4
5
  stats?: EmbedStats;
5
6
  }
7
+ export declare function markEmbeddingProjector(model: object): void;
6
8
  export declare function embed(params: {
7
9
  modelId: string;
8
10
  text: string;
9
- }, requestId?: string): Promise<EmbedResult>;
11
+ }, requestId?: string, sessionId?: string, context?: PluginHandlerContext): Promise<EmbedResult>;
10
12
  export declare function embed(params: {
11
13
  modelId: string;
12
14
  text: string[];
13
- }, requestId?: string): Promise<EmbedResult>;
14
- export declare function embed(params: EmbedParams, requestId?: string): Promise<EmbedResult>;
15
+ }, requestId?: string, sessionId?: string, context?: PluginHandlerContext): Promise<EmbedResult>;
16
+ export declare function embed(params: EmbedParams, requestId?: string, sessionId?: string, context?: PluginHandlerContext): Promise<EmbedResult>;
15
17
  export declare function normalizeVector(vector: Float32Array): number[];
@@ -50,6 +50,9 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
50
50
  var e = new Error(message);
51
51
  return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
52
52
  });
53
+ import fs from 'bare-fs';
54
+ import Buffer from 'bare-buffer';
55
+ import { requestParent } from '../../schemas/plugin.js';
53
56
  import { getModel } from '../../runtime/model-registry.js';
54
57
  import { embedParamsSchema } from '../../schemas/index.js';
55
58
  import { buildUnaryResult } from '../../profiling/model-execution.js';
@@ -58,25 +61,35 @@ import { nowMs } from '../../profiling/index.js';
58
61
  import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
59
62
  import { generateRandomRequestId } from '../../runtime/request-id.js';
60
63
  import { getEngineLogger } from '../../logging/index.js';
61
- export async function embed(params, requestId) {
64
+ const modelsWithProjector = new WeakSet();
65
+ export function markEmbeddingProjector(model) {
66
+ modelsWithProjector.add(model);
67
+ }
68
+ export async function embed(params, requestId, sessionId, context) {
62
69
  const env_1 = { stack: [], error: void 0, hasError: false };
63
70
  try {
64
- const { modelId, text } = embedParamsSchema.parse(params);
71
+ const parsed = embedParamsSchema.parse(params);
72
+ const inputs = parsed.input !== undefined ? [parsed.input] : parsed.inputs;
73
+ const mediaInputs = inputs?.map((input) => {
74
+ if (typeof input === 'object' && !Array.isArray(input)) {
75
+ throw new EmbedFailedError('This model accepts text and ordered media parts, not raw RGB image inputs');
76
+ }
77
+ return input;
78
+ });
79
+ const { modelId } = parsed;
65
80
  // Open a request-scoped lifecycle. The registry routes
66
81
  // `cancel({ requestId })` and broad `cancel({ modelId, kind: "embeddings" })`
67
82
  // straight to this context's signal. Falls back to a generated
68
83
  // id if the caller didn't send one.
69
84
  const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
70
85
  requestId: requestId ?? generateRandomRequestId(),
86
+ sessionId,
87
+ ...requestParent(context),
71
88
  kind: 'embeddings',
72
89
  modelId
73
- })
74
- // `requestLogger` is intentionally referenced once so the
75
- // request-scoped logger is built at handler entry per the canonical
76
- // shape, even when this op has no per-step emits beyond the
77
- // registry's own lifecycle lines. Future addon-level warns inside
78
- // this body should route through `requestLogger`.
79
- , true);
90
+ }), true);
91
+ if (ctx.signal.aborted)
92
+ throw new InferenceCancelledError(ctx.requestId);
80
93
  // `requestLogger` is intentionally referenced once so the
81
94
  // request-scoped logger is built at handler entry per the canonical
82
95
  // shape, even when this op has no per-step emits beyond the
@@ -84,42 +97,47 @@ export async function embed(params, requestId) {
84
97
  // this body should route through `requestLogger`.
85
98
  const requestLogger = withRequestContext(getEngineLogger(), ctx);
86
99
  const model = getModel(modelId);
87
- // Hard-cancel wiring: llamacpp-embedding declares
88
- // `cancel: { scope: "model", hard: true }`. The signal listener
89
- // forwards an abort to the addon so C++ work stops as soon as the
90
- // user cancels — mirrors the canonical handler shape documented in
91
- // `request-lifecycle-primitives.mdc`. The post-await
92
- // `signal.aborted` checks below are the soft-cancel safety net for
93
- // the case where the abort fires between `model.run(...)` returning
94
- // and `response.await()` resolving (both can take seconds on large
95
- // batches).
100
+ const hasMedia = mediaInputs?.some((input) => Array.isArray(input) && input.some((part) => typeof part !== 'string'));
101
+ if (hasMedia && !modelsWithProjector.has(model)) {
102
+ throw new EmbedFailedError('Media inputs require modelConfig.projectionModelSrc when loading the embedding model');
103
+ }
104
+ const input = mediaInputs !== undefined
105
+ ? await prepareEmbedInputs(mediaInputs, ctx.signal, ctx.requestId)
106
+ : parsed.text;
107
+ if (ctx.signal.aborted)
108
+ throw new InferenceCancelledError(ctx.requestId);
109
+ // Admission keeps model-wide cancellation scoped to this request. Media
110
+ // preparation has no native job to cancel, so wire the addon only afterward.
111
+ let cancelPromise;
96
112
  const onAbort = () => {
97
113
  const addon = model.addon;
98
114
  if (addon?.cancel) {
99
- addon.cancel.call(addon).catch((err) => {
115
+ cancelPromise ??= addon.cancel.call(addon).catch((err) => {
100
116
  requestLogger.warn(`[cancel] addon.cancel() rejected during abort for modelId=${modelId}: ${err instanceof Error ? err.message : String(err)}`);
101
117
  });
102
118
  }
103
119
  };
104
120
  ctx.signal.addEventListener('abort', onAbort, { once: true });
105
- // `{ once: true }` does not fire if the signal is already aborted at
106
- // register time. The registry may abort synchronously when a
107
- // cancel-before-begin race resolves or the parent signal was pre-
108
- // aborted — re-use `onAbort` so the listener body is the single
109
- // source of truth for "what cancel does."
110
- if (ctx.signal.aborted)
111
- onAbort();
112
- ctx.scope.defer(() => {
121
+ ctx.scope.defer(async () => {
113
122
  ctx.signal.removeEventListener('abort', onAbort);
123
+ // A delayed native cancel must finish before the next job owns this model slot.
124
+ await cancelPromise;
114
125
  });
115
126
  const modelStart = nowMs();
116
- const response = (await model.run(text));
117
- if (ctx.signal.aborted) {
118
- throw new InferenceCancelledError(ctx.requestId);
127
+ let response;
128
+ let rawEmbeddings;
129
+ try {
130
+ response = (await model.run(input));
131
+ if (ctx.signal.aborted)
132
+ throw new InferenceCancelledError(ctx.requestId);
133
+ rawEmbeddings = await response.await();
134
+ if (ctx.signal.aborted)
135
+ throw new InferenceCancelledError(ctx.requestId);
119
136
  }
120
- const rawEmbeddings = await response.await();
121
- if (ctx.signal.aborted) {
122
- throw new InferenceCancelledError(ctx.requestId);
137
+ catch (cause) {
138
+ if (ctx.signal.aborted)
139
+ throw new InferenceCancelledError(ctx.requestId, {}, cause);
140
+ throw cause;
123
141
  }
124
142
  const modelExecutionMs = nowMs() - modelStart;
125
143
  const stats = {
@@ -134,7 +152,7 @@ export async function embed(params, requestId) {
134
152
  ...(response.stats?.context_size !== undefined && { contextSize: response.stats.context_size })
135
153
  };
136
154
  const embeddingsArray = rawEmbeddings[0];
137
- if (Array.isArray(text)) {
155
+ if (parsed.input === undefined && (parsed.inputs !== undefined || Array.isArray(parsed.text))) {
138
156
  if (!embeddingsArray || embeddingsArray.length === 0) {
139
157
  throw new EmbedNoEmbeddingsError();
140
158
  }
@@ -191,3 +209,40 @@ export function normalizeVector(vector) {
191
209
  }
192
210
  return normalizedVector;
193
211
  }
212
+ async function prepareEmbedInputs(inputs, signal, requestId) {
213
+ const result = [];
214
+ for (const input of inputs) {
215
+ if (signal.aborted)
216
+ throw new InferenceCancelledError(requestId);
217
+ if (typeof input === 'string') {
218
+ result.push(input);
219
+ continue;
220
+ }
221
+ const parts = [];
222
+ for (const part of input) {
223
+ if (signal.aborted)
224
+ throw new InferenceCancelledError(requestId);
225
+ if (typeof part === 'string') {
226
+ parts.push(part);
227
+ }
228
+ else {
229
+ const { type, value } = part.media;
230
+ try {
231
+ const media = type === 'base64'
232
+ ? Buffer.from(value, 'base64')
233
+ : await fs.promises.readFile(value, 'buffer');
234
+ parts.push({ media });
235
+ }
236
+ catch (cause) {
237
+ if (signal.aborted)
238
+ throw new InferenceCancelledError(requestId, {}, cause);
239
+ throw new EmbedFailedError('Unable to read embedding media', cause);
240
+ }
241
+ }
242
+ }
243
+ result.push(parts);
244
+ }
245
+ if (signal.aborted)
246
+ throw new InferenceCancelledError(requestId);
247
+ return { inputs: result };
248
+ }
@@ -3,6 +3,7 @@ import type { Tool } from '../../schemas/tools.js';
3
3
  export declare function extractSystemPrompt(messages: CacheMessage[]): string | null;
4
4
  export declare function generateConfigHash(systemPrompt: string | null): string;
5
5
  export declare function generateCacheKey(messages: CacheMessage[], tools?: readonly Tool[]): string;
6
+ export declare function resolveCacheFilePath(modelId: string, configHash: string, cacheKey: string): string;
6
7
  export declare function getCacheFilePath(modelId: string, configHash: string, cacheKey: string): Promise<string>;
7
8
  export declare function findMatchingCache(modelId: string, configHash: string, currentHistory: CacheMessage[], tools?: readonly Tool[]): Promise<{
8
9
  cacheKey: string;
@@ -12,7 +12,9 @@ const logger = getEngineLogger();
12
12
  // touch in-memory state.
13
13
  export function extractSystemPrompt(messages) {
14
14
  const systemMessage = messages.find((msg) => msg.role === 'system');
15
- return systemMessage ? systemMessage.content : null;
15
+ if (!systemMessage || typeof systemMessage.content !== 'string')
16
+ return null;
17
+ return systemMessage.content;
16
18
  }
17
19
  // Names the cache file by system prompt only. Tools stay out on purpose: the
18
20
  // addon compares the rendered prompt against the tokens the file holds, so a
@@ -37,7 +39,9 @@ export function generateCacheKey(messages, tools) {
37
39
  const hashString = hash.digest('hex');
38
40
  return hashString.substring(0, 16);
39
41
  }
40
- function resolveCacheFilePath(modelId, configHash, cacheKey) {
42
+ // Resolves the path without creating its directories, for a cache that may
43
+ // not exist (a seed source).
44
+ export function resolveCacheFilePath(modelId, configHash, cacheKey) {
41
45
  const cacheDir = getKVCacheDir();
42
46
  const sessionCacheDir = validateAndJoinPath(cacheDir, cacheKey);
43
47
  const modelCacheDir = validateAndJoinPath(sessionCacheDir, modelId);
@@ -10,5 +10,7 @@ export interface LoadModelOptions {
10
10
  signal: AbortSignal;
11
11
  requestId: string;
12
12
  };
13
+ /** Session that takes a lease on the model. Defaults to the owner session. */
14
+ sessionId?: string;
13
15
  }
14
16
  export declare function loadModel(params: LoadModelServerParams, options?: LoadModelOptions): Promise<LoadModelResult>;
@@ -1,12 +1,14 @@
1
1
  import { loadModelServerParamsSchema, normalizeModelType } from '../../schemas/index.js';
2
2
  import { nowMs } from '../../profiling/clock.js';
3
3
  import { forwardBackendDiagnostics } from '../../profiling/backend-diagnostics.js';
4
- import { isModelLoaded, registerModel } from '../../runtime/model-registry.js';
4
+ import { addModelLease, getModelEntry, isModelLoaded, isModelReconfigured, registerModel } from '../../runtime/model-registry.js';
5
+ import { OWNER_SESSION_ID } from '../../runtime/session-id.js';
6
+ import { withModelWorkLock } from '../../runtime/model-work-lock.js';
5
7
  import { startLogBuffering, stopLogBufferingWithTimeout } from '../../runtime/logging-stream-registry.js';
6
- import { detectShardedModel, generateShardFilenames, validateShardedModelCache } from '../../utils/index.js';
7
- import { PluginNotFoundError, ModelFileNotFoundError, ModelFileNotFoundInDirError, ModelFileLocateFailedError, ModelFitRefusedError, InferenceCancelledError } from '../../errors/index.js';
8
+ import { detectShardedModel, checkAllShardsExist } from '../../utils/index.js';
9
+ import { PluginNotFoundError, ModelFileNotFoundError, ModelFileNotFoundInDirError, ModelFileLocateFailedError, ModelFitRefusedError, ModelReconfiguredError, InferenceCancelledError } from '../../errors/index.js';
8
10
  import { ensureAddonLoggerReady, getPlugin } from '../../plugins/index.js';
9
- import { runAdvisoryFitCheck } from '../../model-fit/load-check.js';
11
+ import { runLoadFitCheck } from '../../model-fit/load-check.js';
10
12
  import { refusesLoad, resolveModelFitPolicy, runsProbe } from '../../schemas/model-fit-policy.js';
11
13
  import { getConfig } from '../../runtime/state.js';
12
14
  import { resolveToolDialect } from '../../utils/tools/dialect.js';
@@ -15,13 +17,30 @@ import path from 'bare-path';
15
17
  import { getEngineLogger } from '../../logging/index.js';
16
18
  const logger = getEngineLogger();
17
19
  export async function loadModel(params, options) {
18
- const { modelId, modelPath, options: modelOptions, artifacts, modelName } = loadModelServerParamsSchema.parse(params);
20
+ const parsed = loadModelServerParamsSchema.parse(params);
21
+ return withModelWorkLock(parsed.modelId, () => loadModelLocked(parsed, options), options?.request);
22
+ }
23
+ async function loadModelLocked(params, options) {
24
+ const { modelId, modelPath, options: modelOptions, artifacts, modelName } = params;
19
25
  const { modelConfig, modelType: rawModelType, modelFitPolicy } = modelOptions;
20
26
  // Normalize modelType to canonical form (handles aliases and custom types)
21
27
  const modelType = normalizeModelType(rawModelType);
22
- // Check if model is already loaded
28
+ const sessionId = options?.sessionId ?? OWNER_SESSION_ID;
29
+ // A cancel that landed while this load waited for another load of the same
30
+ // model must not take a lease the caller no longer wants.
31
+ if (options?.request?.signal.aborted === true) {
32
+ throw new InferenceCancelledError(options.request.requestId);
33
+ }
34
+ // An already loaded model is shared: the caller takes a lease on it. A
35
+ // session that holds no lease on a reconfigured model is refused: it asked
36
+ // for the configuration the id hashes, which the model no longer has.
23
37
  if (isModelLoaded(modelId)) {
24
- logger.info(`${modelType} model ${modelId} is already loaded`);
38
+ const entry = getModelEntry(modelId);
39
+ if (entry && !entry.leases.has(sessionId) && isModelReconfigured(modelId)) {
40
+ throw new ModelReconfiguredError(modelId);
41
+ }
42
+ addModelLease(modelId, sessionId);
43
+ logger.info(`${modelType} model ${modelId} is already loaded; lease added for ${sessionId}`);
25
44
  return {};
26
45
  }
27
46
  // Detect if sharded model
@@ -33,12 +52,9 @@ export async function loadModel(params, options) {
33
52
  throw new PluginNotFoundError(modelType);
34
53
  }
35
54
  if (isShardedModel) {
36
- // For sharded models, validate all shards and tensors.txt exist
37
55
  const shardDir = path.dirname(modelPath);
38
- const isValid = await validateShardedModelCache(shardDir, modelFileName);
39
- if (!isValid) {
40
- const numberedShards = generateShardFilenames(modelFileName);
41
- throw new ModelFileNotFoundError(`Missing shards or ${shardInfo.baseFilename}.tensors.txt. Expected ${numberedShards.length} shard files + tensors.txt in ${shardDir}`);
56
+ if (!(await checkAllShardsExist(shardDir, modelFileName))) {
57
+ throw new ModelFileNotFoundError(`Missing shards. Expected ${shardInfo.totalShards} shard files in ${shardDir}`);
42
58
  }
43
59
  }
44
60
  else if (!plugin.skipPrimaryModelPathValidation) {
@@ -61,8 +77,8 @@ export async function loadModel(params, options) {
61
77
  // the native load for device memory.
62
78
  const fitPolicy = resolveModelFitPolicy(modelFitPolicy, getConfig().modelFitPolicy);
63
79
  const request = options?.request;
64
- const fitProbe = runsProbe(fitPolicy)
65
- ? await runAdvisoryFitCheck({
80
+ const fitCheck = runsProbe(fitPolicy)
81
+ ? await runLoadFitCheck({
66
82
  modelId,
67
83
  modelType: modelType,
68
84
  modelPath,
@@ -71,11 +87,12 @@ export async function loadModel(params, options) {
71
87
  isShardedModel
72
88
  }, request !== undefined ? { signal: request.signal } : {})
73
89
  : undefined;
90
+ const fitProbe = fitCheck?.probe;
74
91
  if (request?.signal.aborted === true) {
75
92
  throw new InferenceCancelledError(request.requestId);
76
93
  }
77
94
  if (refusesLoad(fitPolicy, fitProbe?.verdict)) {
78
- throw new ModelFitRefusedError(modelType, fitProbe.reason, fitProbe.message);
95
+ throw new ModelFitRefusedError(modelType, 'does-not-fit', fitCheck?.detail);
79
96
  }
80
97
  // Load the addon's logger before the addon itself runs. Plugins that defer
81
98
  // their logging module do so to keep registration free of native loads, so
@@ -106,7 +123,7 @@ export async function loadModel(params, options) {
106
123
  name: modelName,
107
124
  fitProbe,
108
125
  toolDialect
109
- });
126
+ }, sessionId);
110
127
  const loadResult = modelInitializationTimeMs !== undefined ? { timing: { modelInitializationTimeMs } } : {};
111
128
  return forwardBackendDiagnostics(loadResult, result);
112
129
  }
@@ -1,3 +1,4 @@
1
+ import { type PluginHandlerContext } from '../../schemas/plugin.js';
1
2
  import { type TranscribeParams, type TranscribeSegment, type TranscribeStats, type TranscribeStreamEvent, type ParakeetStreamingRunConfig } from '../../schemas/index.js';
2
3
  import type Buffer from 'bare-buffer';
3
4
  import type { InferenceBackendDiagnostics } from '../../schemas/index.js';
@@ -10,13 +11,13 @@ type TranscribeReturn = {
10
11
  };
11
12
  export declare function transcribe(params: TranscribeParams & {
12
13
  metadata: true;
13
- }, requestId?: string): AsyncGenerator<TranscribeSegment, TranscribeReturn, void>;
14
- export declare function transcribe(params: TranscribeParams, requestId?: string): AsyncGenerator<string, TranscribeReturn, void>;
14
+ }, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<TranscribeSegment, TranscribeReturn, void>;
15
+ export declare function transcribe(params: TranscribeParams, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<string, TranscribeReturn, void>;
15
16
  export interface TranscribeStreamOpts {
16
17
  emitVadEvents?: boolean;
17
18
  endOfTurnSilenceMs?: number;
18
19
  vadRunIntervalMs?: number;
19
20
  parakeetStreamingConfig?: ParakeetStreamingRunConfig;
20
21
  }
21
- export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent, TranscribeReturn, void>;
22
- export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string): AsyncGenerator<string | TranscribeStreamEvent, TranscribeReturn, void>;
22
+ export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt: string | undefined, metadata: true, opts?: TranscribeStreamOpts, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<TranscribeSegment | TranscribeStreamEvent, TranscribeReturn, void>;
23
+ export declare function transcribeStream(modelId: string, audioInputStream: AsyncIterable<Buffer>, prompt?: string, metadata?: boolean, opts?: TranscribeStreamOpts, requestId?: string, sessionId?: string, context?: PluginHandlerContext): AsyncGenerator<string | TranscribeStreamEvent, TranscribeReturn, void>;