@qvac/inference 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (426) hide show
  1. package/NOTICE +60 -28
  2. package/README.md +3 -1
  3. package/dist/api/assess-model-fit.d.ts +14 -14
  4. package/dist/api/assess-model-fit.js +14 -14
  5. package/dist/api/audio-gen.d.ts +2 -2
  6. package/dist/api/audio-gen.js +135 -44
  7. package/dist/api/batch-completion.js +2 -1
  8. package/dist/api/calc-tokens.d.ts +30 -0
  9. package/dist/api/calc-tokens.js +41 -0
  10. package/dist/api/classify.d.ts +5 -2
  11. package/dist/api/classify.js +5 -2
  12. package/dist/api/completion-stream.d.ts +1 -1
  13. package/dist/api/completion-stream.js +2 -1
  14. package/dist/api/delete-cache.d.ts +7 -1
  15. package/dist/api/delete-cache.js +11 -1
  16. package/dist/api/diffusion.d.ts +14 -1
  17. package/dist/api/diffusion.js +14 -1
  18. package/dist/api/embed.d.ts +82 -29
  19. package/dist/api/embed.js +64 -18
  20. package/dist/api/get-model-info.d.ts +1 -1
  21. package/dist/api/index.d.ts +2 -1
  22. package/dist/api/index.js +2 -1
  23. package/dist/api/rag.d.ts +1 -1
  24. package/dist/api/rag.js +1 -1
  25. package/dist/api/registry.d.ts +16 -6
  26. package/dist/api/registry.js +32 -9
  27. package/dist/api/text-to-speech.d.ts +14 -5
  28. package/dist/api/text-to-speech.js +83 -29
  29. package/dist/api/transcribe.d.ts +6 -2
  30. package/dist/api/transcribe.js +7 -1
  31. package/dist/api/vector-index.d.ts +25 -3
  32. package/dist/api/vector-index.js +17 -2
  33. package/dist/api/video-request.d.ts +15 -15
  34. package/dist/dispatch.d.ts +10 -4
  35. package/dist/dispatch.js +65 -15
  36. package/dist/engine.d.ts +1 -1
  37. package/dist/errors/index.d.ts +70 -0
  38. package/dist/errors/index.js +128 -0
  39. package/dist/handlers/assess-model-fit.d.ts +3 -21
  40. package/dist/handlers/assess-model-fit.js +4 -303
  41. package/dist/handlers/cancelHandler.d.ts +10 -9
  42. package/dist/handlers/cancelHandler.js +17 -13
  43. package/dist/handlers/completion-orchestrate.d.ts +14 -2
  44. package/dist/handlers/completion-orchestrate.js +164 -22
  45. package/dist/handlers/delete-cache.d.ts +2 -1
  46. package/dist/handlers/delete-cache.js +6 -1
  47. package/dist/handlers/download-asset.js +11 -4
  48. package/dist/handlers/get-loaded-model-info.js +1 -0
  49. package/dist/handlers/get-model-info.js +71 -2
  50. package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
  51. package/dist/handlers/load-model/coreml-sidecars.js +16 -119
  52. package/dist/handlers/load-model/download-lock.d.ts +20 -0
  53. package/dist/handlers/load-model/download-lock.js +213 -0
  54. package/dist/handlers/load-model/download-manager.d.ts +10 -2
  55. package/dist/handlers/load-model/download-manager.js +62 -11
  56. package/dist/handlers/load-model/handler.js +88 -35
  57. package/dist/handlers/load-model/http.js +327 -205
  58. package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
  59. package/dist/handlers/load-model/hyperdrive.js +123 -50
  60. package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
  61. package/dist/handlers/load-model/registry-client-options.js +4 -2
  62. package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
  63. package/dist/handlers/load-model/registry-companion-set.js +19 -1
  64. package/dist/handlers/load-model/registry-delete.d.ts +5 -0
  65. package/dist/handlers/load-model/registry-delete.js +269 -0
  66. package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
  67. package/dist/handlers/load-model/registry-download-utils.js +93 -22
  68. package/dist/handlers/load-model/registry.d.ts +20 -0
  69. package/dist/handlers/load-model/registry.js +308 -175
  70. package/dist/handlers/load-model/resolve.js +4 -1
  71. package/dist/handlers/load-model/types.d.ts +2 -0
  72. package/dist/handlers/logging-stream.js +17 -5
  73. package/dist/handlers/plugin-invoke.js +152 -39
  74. package/dist/handlers/registry.d.ts +2 -2
  75. package/dist/handlers/registry.js +105 -28
  76. package/dist/handlers/rpc-server.js +7 -7
  77. package/dist/handlers/session.d.ts +11 -0
  78. package/dist/handlers/session.js +40 -0
  79. package/dist/handlers/unload-model.js +8 -3
  80. package/dist/handlers/vector-index.js +118 -19
  81. package/dist/index.d.ts +1 -1
  82. package/dist/index.js +1 -1
  83. package/dist/logging/utils.js +10 -2
  84. package/dist/model-cache-layout.d.ts +1 -0
  85. package/dist/model-cache-layout.js +2 -0
  86. package/dist/model-fit/assess.d.ts +18 -67
  87. package/dist/model-fit/assess.js +96 -665
  88. package/dist/model-fit/describe.d.ts +12 -0
  89. package/dist/model-fit/describe.js +29 -0
  90. package/dist/model-fit/description/fetch.d.ts +1 -1
  91. package/dist/model-fit/description/fetch.js +95 -16
  92. package/dist/model-fit/description/resolve-load.d.ts +5 -3
  93. package/dist/model-fit/description/resolve-load.js +14 -14
  94. package/dist/model-fit/fit-load.d.ts +23 -0
  95. package/dist/model-fit/fit-load.js +29 -0
  96. package/dist/model-fit/load-check.d.ts +15 -33
  97. package/dist/model-fit/load-check.js +50 -195
  98. package/dist/model-fit/run/engine.d.ts +12 -72
  99. package/dist/model-fit/run/engine.js +4 -8
  100. package/dist/model-fit/run/run.d.ts +10 -7
  101. package/dist/model-fit/run/run.js +22 -18
  102. package/dist/model-fit/sum.d.ts +32 -0
  103. package/dist/model-fit/sum.js +60 -0
  104. package/dist/models/registry/models.d.ts +865 -128
  105. package/dist/models/registry/models.js +3737 -2963
  106. package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
  107. package/dist/plugins/builtin/asr-ggml/config.js +39 -3
  108. package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
  109. package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
  110. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
  111. package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
  112. package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
  113. package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
  114. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
  115. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
  116. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
  117. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
  118. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
  119. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
  120. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
  121. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
  122. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
  123. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
  124. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
  125. package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
  126. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
  127. package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
  128. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
  129. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
  130. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
  131. package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
  132. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
  133. package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
  134. package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
  135. package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
  136. package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
  137. package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
  138. package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
  139. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
  140. package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
  141. package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
  142. package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
  143. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
  144. package/dist/plugins/builtin/index.d.ts +1 -0
  145. package/dist/plugins/builtin/index.js +1 -0
  146. package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
  147. package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
  148. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
  149. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
  150. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
  151. package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
  152. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  153. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
  154. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
  155. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
  156. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
  157. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
  158. package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
  159. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
  160. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
  161. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
  162. package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
  163. package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
  164. package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
  165. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
  166. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
  167. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
  168. package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
  169. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
  170. package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
  171. package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
  172. package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
  173. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
  174. package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
  175. package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
  176. package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
  177. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
  178. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
  179. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
  180. package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
  181. package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
  182. package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
  183. package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
  184. package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
  185. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
  186. package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
  187. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
  188. package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
  189. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
  190. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
  191. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
  192. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
  193. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
  194. package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
  195. package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
  196. package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
  197. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
  198. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
  199. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
  200. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
  201. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
  202. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
  203. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
  204. package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
  205. package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
  206. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
  207. package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
  208. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
  209. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
  210. package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
  211. package/dist/plugins/ops/bci-transcribe.js +18 -4
  212. package/dist/plugins/ops/cancel.d.ts +1 -0
  213. package/dist/plugins/ops/cancel.js +11 -2
  214. package/dist/plugins/ops/embed.d.ts +5 -3
  215. package/dist/plugins/ops/embed.js +89 -34
  216. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
  217. package/dist/plugins/ops/kv-cache-utils.js +6 -2
  218. package/dist/plugins/ops/load-model.d.ts +2 -0
  219. package/dist/plugins/ops/load-model.js +33 -16
  220. package/dist/plugins/ops/transcribe.d.ts +5 -4
  221. package/dist/plugins/ops/transcribe.js +62 -27
  222. package/dist/plugins/ops/translate.d.ts +2 -1
  223. package/dist/plugins/ops/translate.js +4 -1
  224. package/dist/plugins/ops/unload-model.d.ts +17 -1
  225. package/dist/plugins/ops/unload-model.js +31 -2
  226. package/dist/plugins/turbovec-backend.d.ts +1 -1
  227. package/dist/plugins/turbovec-backend.js +16 -1
  228. package/dist/profiling/operation-metrics.js +22 -19
  229. package/dist/rag/embedding-scope.d.ts +16 -0
  230. package/dist/rag/embedding-scope.js +81 -0
  231. package/dist/rag/handler.d.ts +25 -0
  232. package/dist/rag/handler.js +50 -16
  233. package/dist/rag/handlers/ingest.js +2 -6
  234. package/dist/rag/handlers/search.d.ts +10 -1
  235. package/dist/rag/handlers/search.js +16 -7
  236. package/dist/rag/rag-operation-manager.d.ts +27 -3
  237. package/dist/rag/rag-operation-manager.js +10 -27
  238. package/dist/registry.js +5 -0
  239. package/dist/resources/collector.d.ts +70 -36
  240. package/dist/resources/collector.js +8 -1
  241. package/dist/resources/types.d.ts +2 -0
  242. package/dist/rpc/inventory.js +1 -12
  243. package/dist/runtime/env.d.ts +5 -1
  244. package/dist/runtime/env.js +6 -21
  245. package/dist/runtime/index.d.ts +4 -1
  246. package/dist/runtime/index.js +2 -0
  247. package/dist/runtime/lifecycle.d.ts +6 -4
  248. package/dist/runtime/lifecycle.js +9 -5
  249. package/dist/runtime/model-config-utils.js +5 -1
  250. package/dist/runtime/model-registry.d.ts +28 -4
  251. package/dist/runtime/model-registry.js +46 -1
  252. package/dist/runtime/model-work-lock.d.ts +13 -0
  253. package/dist/runtime/model-work-lock.js +48 -0
  254. package/dist/runtime/registry-client.d.ts +47 -1
  255. package/dist/runtime/registry-client.js +446 -88
  256. package/dist/runtime/request-context.d.ts +9 -1
  257. package/dist/runtime/request-context.js +25 -1
  258. package/dist/runtime/request-registry.d.ts +55 -7
  259. package/dist/runtime/request-registry.js +189 -71
  260. package/dist/runtime/runtime-lifecycle.js +10 -1
  261. package/dist/runtime/session-id.d.ts +18 -0
  262. package/dist/runtime/session-id.js +20 -0
  263. package/dist/runtime/session-registry.d.ts +39 -0
  264. package/dist/runtime/session-registry.js +114 -0
  265. package/dist/runtime/vector-index-backend.d.ts +4 -0
  266. package/dist/runtime/vector-index-registry.d.ts +21 -0
  267. package/dist/runtime/vector-index-registry.js +56 -18
  268. package/dist/schemas/abort-signal.d.ts +8 -0
  269. package/dist/schemas/abort-signal.js +1 -0
  270. package/dist/schemas/assess-model-fit.d.ts +132 -282
  271. package/dist/schemas/assess-model-fit.js +82 -216
  272. package/dist/schemas/audio-gen.d.ts +278 -24
  273. package/dist/schemas/audio-gen.js +127 -23
  274. package/dist/schemas/batch-completion-stream.d.ts +105 -13
  275. package/dist/schemas/batch-completion-stream.js +8 -4
  276. package/dist/schemas/bci-config.d.ts +2 -1
  277. package/dist/schemas/bci.d.ts +54 -3
  278. package/dist/schemas/bci.js +11 -16
  279. package/dist/schemas/calc-tokens.d.ts +141 -0
  280. package/dist/schemas/calc-tokens.js +53 -0
  281. package/dist/schemas/cancel.d.ts +26 -0
  282. package/dist/schemas/cancel.js +10 -2
  283. package/dist/schemas/classification.d.ts +3 -0
  284. package/dist/schemas/classification.js +6 -2
  285. package/dist/schemas/common.d.ts +1831 -348
  286. package/dist/schemas/common.js +18 -2
  287. package/dist/schemas/completion-event.d.ts +3 -0
  288. package/dist/schemas/completion-event.js +1 -1
  289. package/dist/schemas/completion-stream.d.ts +162 -17
  290. package/dist/schemas/completion-stream.js +83 -13
  291. package/dist/schemas/config.d.ts +1212 -169
  292. package/dist/schemas/config.js +9 -13
  293. package/dist/schemas/decision.d.ts +4 -2
  294. package/dist/schemas/decision.js +2 -1
  295. package/dist/schemas/download-asset.d.ts +10 -5
  296. package/dist/schemas/download-asset.js +3 -6
  297. package/dist/schemas/embed.d.ts +286 -3
  298. package/dist/schemas/embed.js +162 -16
  299. package/dist/schemas/engine-addon-map.d.ts +1 -0
  300. package/dist/schemas/engine-addon-map.js +1 -0
  301. package/dist/schemas/errors.d.ts +12 -0
  302. package/dist/schemas/errors.js +66 -1
  303. package/dist/schemas/finetune.d.ts +14 -4
  304. package/dist/schemas/finetune.js +5 -6
  305. package/dist/schemas/heartbeat.d.ts +1 -0
  306. package/dist/schemas/heartbeat.js +5 -2
  307. package/dist/schemas/image-embedding.d.ts +27 -0
  308. package/dist/schemas/image-embedding.js +107 -0
  309. package/dist/schemas/index.d.ts +7 -3
  310. package/dist/schemas/index.js +7 -3
  311. package/dist/schemas/lifecycle.d.ts +3 -0
  312. package/dist/schemas/lifecycle.js +13 -6
  313. package/dist/schemas/llamacpp-config.d.ts +269 -22
  314. package/dist/schemas/llamacpp-config.js +86 -7
  315. package/dist/schemas/load-model.d.ts +6418 -1960
  316. package/dist/schemas/load-model.js +39 -8
  317. package/dist/schemas/logging-stream.d.ts +1 -0
  318. package/dist/schemas/logging-stream.js +5 -2
  319. package/dist/schemas/model-config-schemas.d.ts +542 -116
  320. package/dist/schemas/model-config-schemas.js +2 -0
  321. package/dist/schemas/model-fit-policy.d.ts +2 -2
  322. package/dist/schemas/model-info.d.ts +54 -56
  323. package/dist/schemas/model-info.js +17 -6
  324. package/dist/schemas/model-ops.d.ts +12 -0
  325. package/dist/schemas/model-ops.js +33 -5
  326. package/dist/schemas/model-src-utils.d.ts +10 -6
  327. package/dist/schemas/model-types.d.ts +8 -4
  328. package/dist/schemas/model-types.js +1 -0
  329. package/dist/schemas/ocr.d.ts +9 -4
  330. package/dist/schemas/ocr.js +6 -2
  331. package/dist/schemas/plugin.d.ts +62 -6
  332. package/dist/schemas/plugin.js +26 -4
  333. package/dist/schemas/profiling.d.ts +32 -32
  334. package/dist/schemas/rag.d.ts +9 -0
  335. package/dist/schemas/rag.js +10 -16
  336. package/dist/schemas/registry.d.ts +30 -10
  337. package/dist/schemas/registry.js +14 -6
  338. package/dist/schemas/reload-config.d.ts +41 -6
  339. package/dist/schemas/reload-config.js +11 -4
  340. package/dist/schemas/request-base.d.ts +39 -0
  341. package/dist/schemas/request-base.js +64 -0
  342. package/dist/schemas/rerank.d.ts +8 -7
  343. package/dist/schemas/rerank.js +6 -4
  344. package/dist/schemas/rpc-server.d.ts +3 -0
  345. package/dist/schemas/rpc-server.js +15 -15
  346. package/dist/schemas/sdcpp-config.d.ts +90 -20
  347. package/dist/schemas/sdcpp-config.js +96 -19
  348. package/dist/schemas/session.d.ts +37 -0
  349. package/dist/schemas/session.js +53 -0
  350. package/dist/schemas/system-resources.d.ts +364 -179
  351. package/dist/schemas/system-resources.js +8 -3
  352. package/dist/schemas/text-to-speech.d.ts +1343 -271
  353. package/dist/schemas/text-to-speech.js +547 -33
  354. package/dist/schemas/tools.d.ts +7 -0
  355. package/dist/schemas/tools.js +29 -0
  356. package/dist/schemas/transcription-config.d.ts +262 -2
  357. package/dist/schemas/transcription-config.js +122 -6
  358. package/dist/schemas/transcription.d.ts +93 -6
  359. package/dist/schemas/transcription.js +98 -21
  360. package/dist/schemas/translate.d.ts +2 -0
  361. package/dist/schemas/translate.js +5 -9
  362. package/dist/schemas/translation-config.d.ts +30 -20
  363. package/dist/schemas/tts-pcm.d.ts +35 -0
  364. package/dist/schemas/tts-pcm.js +69 -0
  365. package/dist/schemas/vector-index.d.ts +47 -1
  366. package/dist/schemas/vector-index.js +37 -10
  367. package/dist/schemas/vla.d.ts +3 -3
  368. package/dist/surface.d.ts +3 -3
  369. package/dist/surface.js +3 -3
  370. package/dist/utils/addon-responses.d.ts +69 -2
  371. package/dist/utils/archive.d.ts +7 -0
  372. package/dist/utils/archive.js +88 -29
  373. package/dist/utils/asr-events.d.ts +5 -0
  374. package/dist/utils/asr-events.js +10 -1
  375. package/dist/utils/cache/auto-history.js +3 -1
  376. package/dist/utils/cache/layout.d.ts +48 -0
  377. package/dist/utils/cache/layout.js +190 -0
  378. package/dist/utils/cache/paths.js +2 -4
  379. package/dist/utils/cache/types.d.ts +2 -1
  380. package/dist/utils/checksum.d.ts +5 -0
  381. package/dist/utils/checksum.js +18 -0
  382. package/dist/utils/completion-normalizer.js +286 -87
  383. package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
  384. package/dist/utils/gguf-tensor-extractor.js +1 -50
  385. package/dist/utils/shard-utils.d.ts +0 -21
  386. package/dist/utils/shard-utils.js +12 -62
  387. package/dist/utils/tool-integration.d.ts +2 -2
  388. package/dist/utils/tools/defer.d.ts +39 -10
  389. package/dist/utils/tools/defer.js +72 -17
  390. package/dist/utils/tools/index.d.ts +1 -1
  391. package/dist/utils/tools/index.js +1 -1
  392. package/dist/utils/transcribe-metadata.d.ts +28 -2
  393. package/dist/utils/transcribe-metadata.js +16 -0
  394. package/dist/utils/tts-stats.d.ts +42 -0
  395. package/dist/utils/tts-stats.js +120 -22
  396. package/package.json +43 -39
  397. package/dist/model-fit/classify.d.ts +0 -6
  398. package/dist/model-fit/classify.js +0 -166
  399. package/dist/model-fit/description/project.d.ts +0 -27
  400. package/dist/model-fit/description/project.js +0 -33
  401. package/dist/model-fit/floor.d.ts +0 -31
  402. package/dist/model-fit/floor.js +0 -65
  403. package/dist/model-fit/kv-cache.d.ts +0 -23
  404. package/dist/model-fit/kv-cache.js +0 -85
  405. package/dist/model-fit/platform.d.ts +0 -14
  406. package/dist/model-fit/platform.js +0 -26
  407. package/dist/model-fit/request/create.d.ts +0 -31
  408. package/dist/model-fit/request/create.js +0 -61
  409. package/dist/model-fit/request/engines/asr.d.ts +0 -14
  410. package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
  411. package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
  412. package/dist/model-fit/request/engines/diffusion.js +0 -93
  413. package/dist/model-fit/request/engines/llama.d.ts +0 -12
  414. package/dist/model-fit/request/engines/llama.js +0 -44
  415. package/dist/model-fit/request/engines/nmt.d.ts +0 -8
  416. package/dist/model-fit/request/engines/nmt.js +0 -11
  417. package/dist/model-fit/request/engines/ocr.d.ts +0 -9
  418. package/dist/model-fit/request/engines/ocr.js +0 -13
  419. package/dist/model-fit/request/engines/tts.d.ts +0 -10
  420. package/dist/model-fit/request/engines/tts.js +0 -152
  421. package/dist/models/registry/resource-profiles.d.ts +0 -17
  422. package/dist/models/registry/resource-profiles.js +0 -5859
  423. package/dist/schemas/model-resource-profile.d.ts +0 -109
  424. package/dist/schemas/model-resource-profile.js +0 -80
  425. /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
  426. /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
@@ -1,3 +1,4 @@
1
+ import { encodeTtsPcm } from '../schemas/tts-pcm.js';
1
2
  /**
2
3
  * Everything an addon chunk carries beyond its PCM. Shared by both TTS ops so
3
4
  * the plain `stream: true` path and the sentence-stream path emit the same
@@ -11,9 +12,19 @@ export function chunkMetadata(chunk) {
11
12
  ...(typeof chunk.sentenceChunk === 'string' && chunk.sentenceChunk.length > 0
12
13
  ? { sentenceChunk: chunk.sentenceChunk }
13
14
  : {}),
14
- ...(chunk.isLast !== undefined ? { isLast: chunk.isLast } : {})
15
+ ...(chunk.isLast !== undefined ? { isLast: chunk.isLast } : {}),
16
+ ...(chunk.speechTokenIds !== undefined ? { speechTokenIds: chunk.speechTokenIds } : {})
15
17
  };
16
18
  }
19
+ /**
20
+ * Whether a chunk has anything to forward besides PCM. CosyVoice3's native
21
+ * chunk streaming ends with an empty-PCM `isLast` chunk that carries the
22
+ * run's `speechTokenIds`; Audio8's native end markers are empty too but carry
23
+ * nothing, so the ops skip those and forward these.
24
+ */
25
+ export function carriesSpeechTokenIds(chunk) {
26
+ return chunk.speechTokenIds !== undefined;
27
+ }
17
28
  /**
18
29
  * Appends a chunk's samples to `target` in place, which keeps collecting an
19
30
  * utterance linear in its length — a natively chunked MOSS run emits dozens of
@@ -23,29 +34,116 @@ export function appendPcm(target, samples) {
23
34
  for (let i = 0; i < samples.length; i++)
24
35
  target.push(samples[i]);
25
36
  }
26
- // Field-for-field, apart from `audioDurationMs` -> `audioDuration`, which keeps
27
- // the name the SDK has always exposed.
37
+ /**
38
+ * One contiguous buffer from the engine's per-chunk output arrays: an
39
+ * `Int16Array` when every part is one (the engines' PCM), else a plain array.
40
+ */
41
+ export function concatPcm(parts) {
42
+ if (parts.length === 1 && parts[0] instanceof Int16Array)
43
+ return parts[0];
44
+ if (!parts.every((p) => p instanceof Int16Array)) {
45
+ const out = [];
46
+ for (const p of parts)
47
+ appendPcm(out, p);
48
+ return out;
49
+ }
50
+ let total = 0;
51
+ for (const p of parts)
52
+ total += p.length;
53
+ const out = new Int16Array(total);
54
+ let offset = 0;
55
+ for (const p of parts) {
56
+ out.set(p, offset);
57
+ offset += p.length;
58
+ }
59
+ return out;
60
+ }
61
+ /**
62
+ * The audio fields of a frame: int16 engine output rides the wire as base64
63
+ * `pcm`; anything else (test doubles, non-int16 engines) stays a `buffer`.
64
+ */
65
+ export function pcmFrame(samples) {
66
+ if (samples instanceof Int16Array)
67
+ return { buffer: [], pcm: encodeTtsPcm(samples) };
68
+ return { buffer: Array.from(samples) };
69
+ }
70
+ /**
71
+ * The addon stats that keep their name on the wire. `audioDurationMs` is the
72
+ * one rename (`audioDuration`, the name the SDK has always exposed) and is
73
+ * handled on its own below.
74
+ *
75
+ * Two compile-time checks keep this table honest. `satisfies` requires every
76
+ * entry to be a key of both the addon's stats and `ttsStatsSchema`, and
77
+ * `UnforwardedTtsStats` below requires every key of either type to be an
78
+ * entry. A stat added to one side only, or to both without a line here, fails
79
+ * to compile rather than being dropped silently.
80
+ */
81
+ const TTS_FORWARDED_STATS = [
82
+ 'totalTime',
83
+ 'realTimeFactor',
84
+ 'tokensPerSecond',
85
+ 'totalSamples',
86
+ 'generatedFrames',
87
+ 'firstAudioMs',
88
+ 'backendDevice',
89
+ 'backendId',
90
+ 'gpuUnsupported',
91
+ 'enhancerBackendDevice',
92
+ 'enhancerBackendId',
93
+ 'denoiserBackendDevice',
94
+ 'denoiserBackendId',
95
+ 'codecSidecarLoaded',
96
+ 'codecOnCoreml',
97
+ // Parler Core ML sidecars
98
+ 'decoderSidecarLoaded',
99
+ 'dacSidecarLoaded',
100
+ 'decoderOnCoreml',
101
+ 'dacOnCoreml',
102
+ // Chatterbox
103
+ 't3Ms',
104
+ 's3genMs',
105
+ 't3Tokens',
106
+ // CosyVoice3 / Audio8
107
+ 'stageTotalMs',
108
+ // CosyVoice3
109
+ 'lmPrefillMs',
110
+ 'lmDecodeMs',
111
+ 'flowFrontendMs',
112
+ 'ditEulerMs',
113
+ 'hiftF0Ms',
114
+ 'hiftSourceMs',
115
+ 'hiftStftMs',
116
+ 'hiftDecodeMs',
117
+ 'decodeSteps',
118
+ 'speechTokens',
119
+ 'flowFrames',
120
+ 'melFrames',
121
+ 'textIds',
122
+ 'promptSpeechTokens',
123
+ // Audio8
124
+ 'voiceEncodeMs',
125
+ 'promptMs',
126
+ 'prefillMs',
127
+ 'sampleMs',
128
+ 'fastDecodeMs',
129
+ 'slowDecodeMs',
130
+ 'codecLatentMs',
131
+ 'codecSynthMs',
132
+ 'resampleMs'
133
+ ];
134
+ // Presence, not truthiness: `backendDevice: 0` (CPU) and a zero stage time are
135
+ // real reports, so only `undefined` is dropped.
28
136
  export function collectTtsStats(response) {
29
137
  const stats = response.stats;
30
138
  if (!stats)
31
139
  return {};
32
- return {
33
- ...(stats.audioDurationMs !== undefined && { audioDuration: stats.audioDurationMs }),
34
- ...(stats.totalTime !== undefined && { totalTime: stats.totalTime }),
35
- ...(stats.realTimeFactor !== undefined && { realTimeFactor: stats.realTimeFactor }),
36
- ...(stats.tokensPerSecond !== undefined && { tokensPerSecond: stats.tokensPerSecond }),
37
- ...(stats.totalSamples !== undefined && { totalSamples: stats.totalSamples }),
38
- ...(stats.generatedFrames !== undefined && { generatedFrames: stats.generatedFrames }),
39
- ...(stats.backendDevice !== undefined && { backendDevice: stats.backendDevice }),
40
- ...(stats.backendId !== undefined && { backendId: stats.backendId }),
41
- ...(stats.gpuUnsupported !== undefined && { gpuUnsupported: stats.gpuUnsupported }),
42
- ...(stats.enhancerBackendDevice !== undefined && {
43
- enhancerBackendDevice: stats.enhancerBackendDevice
44
- }),
45
- ...(stats.enhancerBackendId !== undefined && { enhancerBackendId: stats.enhancerBackendId }),
46
- ...(stats.codecSidecarLoaded !== undefined && {
47
- codecSidecarLoaded: stats.codecSidecarLoaded
48
- }),
49
- ...(stats.codecOnCoreml !== undefined && { codecOnCoreml: stats.codecOnCoreml })
50
- };
140
+ const out = {};
141
+ if (stats.audioDurationMs !== undefined)
142
+ out.audioDuration = stats.audioDurationMs;
143
+ for (const key of TTS_FORWARDED_STATS) {
144
+ const value = stats[key];
145
+ if (value !== undefined)
146
+ out[key] = value;
147
+ }
148
+ return out;
51
149
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@qvac/inference",
3
- "version": "0.23.0",
3
+ "version": "0.24.0",
4
4
  "description": "Bare-only in-process core of the QVAC SDK. Runs inference directly on the Bare runtime with explicit plugin assembly and no RPC, worker, or subprocess layer.",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {
@@ -82,9 +82,9 @@
82
82
  "types": "./dist/models/registry/index.d.ts",
83
83
  "import": "./dist/models/registry/index.js"
84
84
  },
85
- "./model-resource-profiles": {
86
- "types": "./dist/models/registry/resource-profiles.d.ts",
87
- "import": "./dist/models/registry/resource-profiles.js"
85
+ "./model-cache-layout": {
86
+ "types": "./dist/model-cache-layout.d.ts",
87
+ "import": "./dist/model-cache-layout.js"
88
88
  },
89
89
  "./llamacpp-completion/plugin": {
90
90
  "types": "./dist/plugins/builtin/llamacpp-completion/plugin.d.ts",
@@ -153,6 +153,10 @@
153
153
  "./llamacpp-rerank/plugin": {
154
154
  "types": "./dist/plugins/builtin/llamacpp-rerank/plugin.d.ts",
155
155
  "import": "./dist/plugins/builtin/llamacpp-rerank/plugin.js"
156
+ },
157
+ "./ggml-image-embedding/plugin": {
158
+ "types": "./dist/plugins/builtin/ggml-image-embedding/plugin.d.ts",
159
+ "import": "./dist/plugins/builtin/ggml-image-embedding/plugin.js"
156
160
  }
157
161
  },
158
162
  "files": [
@@ -168,26 +172,26 @@
168
172
  "@qvac/error": "^0.1.1",
169
173
  "@qvac/logging": "^0.1.1",
170
174
  "@qvac/rag": "^0.9.0",
171
- "@qvac/registry-client": "^0.7.0",
175
+ "@qvac/registry-client": "^0.8.0",
172
176
  "bare-abort-controller": "^1.1.2",
173
177
  "bare-buffer": "^3.6.2",
174
- "bare-cpu-info": "0.1.1",
178
+ "bare-cpu-info": "^0.1.1",
175
179
  "bare-crypto": "^1.15.3",
176
180
  "bare-env": "^3.0.1",
177
- "bare-fs": "^4.7.4",
178
- "bare-gpu-info": "0.1.1",
181
+ "bare-fs": "^4.8.4",
182
+ "bare-gpu-info": "^0.1.3",
179
183
  "bare-http1": "^4.6.2",
180
184
  "bare-https": "^3.0.0",
181
185
  "bare-net": "^2.3.2",
182
186
  "bare-os": "^3.9.3",
183
187
  "bare-path": "^3.1.1",
184
- "bare-rpc": "^1.3.8",
185
188
  "bare-stream": "^2.13.3",
186
189
  "bare-url": "^2.4.5",
187
190
  "bare-zlib": "^1.4.0",
188
191
  "compact-encoding": "^3.3.0",
189
192
  "corestore": "^7.11.0",
190
193
  "fast-safe-stringify": "2.1.1",
194
+ "fs-native-extensions": "^1.5.0",
191
195
  "hyperdrive": "^13.3.3",
192
196
  "hyperswarm": "^4.17.0",
193
197
  "semver": "^7.8.5",
@@ -196,21 +200,21 @@
196
200
  "zod": "4.6.5"
197
201
  },
198
202
  "peerDependencies": {
199
- "@qvac/asr-ggml": "^0.11.1",
200
- "@qvac/audiogen-ggml": "^0.9.0",
201
- "@qvac/bci-whispercpp": "^0.14.0",
202
- "@qvac/classification-ggml": "^0.34.0",
203
- "@qvac/decoder-audio": "^0.8.1",
204
- "@qvac/diffusion-cpp": "^0.31.0",
205
- "@qvac/embed-llamacpp": "^0.53.0",
203
+ "@qvac/asr-ggml": "^0.12.0",
204
+ "@qvac/audiogen-ggml": "^0.10.1",
205
+ "@qvac/bci-whispercpp": "^0.15.0",
206
+ "@qvac/classification-ggml": "^0.36.0",
207
+ "@qvac/decoder-audio": "^0.9.0",
208
+ "@qvac/diffusion-cpp": "^0.35.0",
209
+ "@qvac/embed-llamacpp": "^0.56.0",
206
210
  "@qvac/langdetect-text": "^0.1.2",
207
- "@qvac/llm-llamacpp": "^0.65.0",
208
- "@qvac/ocr-ggml": "^0.31.0",
209
- "@qvac/translation-nmtcpp": "^0.23.0",
210
- "@qvac/tts-ggml": "^0.14.0",
211
- "@qvac/turbovec-ggml": "^0.3.0",
212
- "@qvac/vla-ggml": "^0.34.0",
213
- "@qvac/ggml-rpc-server": "^0.7.0"
211
+ "@qvac/llm-llamacpp": "^0.70.0",
212
+ "@qvac/ocr-ggml": "^0.33.0",
213
+ "@qvac/translation-nmtcpp": "^0.25.0",
214
+ "@qvac/tts-ggml": "^0.15.0",
215
+ "@qvac/turbovec-ggml": "^0.5.0",
216
+ "@qvac/vla-ggml": "^0.36.0",
217
+ "@qvac/ggml-rpc-server": "^0.9.0"
214
218
  },
215
219
  "peerDependenciesMeta": {
216
220
  "@qvac/asr-ggml": {
@@ -260,20 +264,20 @@
260
264
  }
261
265
  },
262
266
  "devDependencies": {
263
- "@qvac/asr-ggml": "^0.11.1",
264
- "@qvac/audiogen-ggml": "^0.9.0",
265
- "@qvac/bci-whispercpp": "^0.14.0",
266
- "@qvac/classification-ggml": "^0.34.0",
267
- "@qvac/decoder-audio": "^0.8.1",
268
- "@qvac/diffusion-cpp": "^0.31.0",
269
- "@qvac/embed-llamacpp": "^0.53.0",
267
+ "@qvac/asr-ggml": "^0.12.0",
268
+ "@qvac/audiogen-ggml": "^0.10.1",
269
+ "@qvac/bci-whispercpp": "^0.15.0",
270
+ "@qvac/classification-ggml": "^0.36.0",
271
+ "@qvac/decoder-audio": "^0.9.0",
272
+ "@qvac/diffusion-cpp": "^0.35.0",
273
+ "@qvac/embed-llamacpp": "^0.56.0",
270
274
  "@qvac/langdetect-text": "^0.1.2",
271
- "@qvac/llm-llamacpp": "^0.65.0",
272
- "@qvac/ocr-ggml": "^0.31.0",
273
- "@qvac/translation-nmtcpp": "^0.23.0",
274
- "@qvac/tts-ggml": "^0.14.0",
275
- "@qvac/turbovec-ggml": "^0.3.0",
276
- "@qvac/vla-ggml": "^0.34.0",
275
+ "@qvac/llm-llamacpp": "^0.70.0",
276
+ "@qvac/ocr-ggml": "^0.33.0",
277
+ "@qvac/translation-nmtcpp": "^0.25.0",
278
+ "@qvac/tts-ggml": "^0.15.0",
279
+ "@qvac/turbovec-ggml": "^0.5.0",
280
+ "@qvac/vla-ggml": "^0.36.0",
277
281
  "@types/brittle": "^3.5.0",
278
282
  "bare": "1.34.1 || >=1.35.1",
279
283
  "bare-console": "*",
@@ -288,9 +292,9 @@
288
292
  "prettier-config-holepunch": "^2.0.0",
289
293
  "tsc-alias": "^1.8.17",
290
294
  "typescript": "^5.9.3",
291
- "@qvac/ggml-rpc-server": "^0.7.0"
295
+ "@qvac/ggml-rpc-server": "^0.9.0"
292
296
  },
293
- "gitHead": "4badf6ff1900bbf099b63792fe663a33cc0e6c82",
297
+ "gitHead": "1f559be4e208ee7e58e775bde4dd078263ca740c",
294
298
  "scripts": {
295
299
  "lint": "lunte",
296
300
  "lint:fix": "lunte --fix",
@@ -304,6 +308,6 @@
304
308
  "test": "npm run build:test && brittle-bare 'test/dist/test/**/*.test.js'",
305
309
  "test:bare": "npm run test",
306
310
  "check-models": "bare ./dist/models/update-models/index.js --check",
307
- "update-models": "bare ./dist/models/update-models/index.js && prettier --write src/models/registry/models.ts src/models/registry/resource-profiles.ts"
311
+ "update-models": "bare ./dist/models/update-models/index.js && prettier --write src/models/registry/models.ts"
308
312
  }
309
313
  }
@@ -1,6 +0,0 @@
1
- import type { NativeProbeFit } from '../schemas/index.js';
2
- import type { FitProbeResult } from '../model-fit/run/engine.js';
3
- export declare function classifyFit(probe: FitProbeResult, provenance: {
4
- basis: 'native-probe';
5
- estimatorVersion: string;
6
- }): NativeProbeFit;
@@ -1,166 +0,0 @@
1
- /** The llama fitters append a host row after the devices they assigned to. */
2
- const HOST_ROW = 'host';
3
- /** ggml's own name for the CPU backend device, which the fitter may assign to. */
4
- const CPU_DEVICE = 'CPU';
5
- function isLlama(probe) {
6
- return probe.engine === 'llm-llamacpp' || probe.engine === 'embed-llamacpp';
7
- }
8
- /**
9
- * `error` is the engine reporting it could not read the model or find a backend
10
- * device: absence of evidence, not evidence of insufficiency.
11
- */
12
- function verdictFor(status) {
13
- if (status === 'fits')
14
- return 'fit';
15
- if (status === 'does-not-fit')
16
- return 'does-not-fit';
17
- return 'unknown';
18
- }
19
- /**
20
- * The diffusion fitter reports a placement alongside its reason. `changed`
21
- * means it reached a fitting arrangement only by moving modules between
22
- * backends, which is why the status is already `does-not-fit`.
23
- */
24
- function diffusionReason(result) {
25
- if (result.status === 'fits')
26
- return 'fits';
27
- if (result.status === 'error')
28
- return result.reason;
29
- return result.changed ? `does-not-fit-as-configured (${result.backend})` : 'does-not-fit';
30
- }
31
- function reasonFor(probe) {
32
- return probe.engine === 'diffusion-cpp' ? diffusionReason(probe.result) : probe.result.reason;
33
- }
34
- function sumOf(...values) {
35
- const present = values.filter((value) => value !== undefined);
36
- return present.length === 0 ? undefined : present.reduce((total, value) => total + value, 0);
37
- }
38
- function breakdown(weights, context, compute) {
39
- return {
40
- ...(weights !== undefined && { weightsBytes: weights }),
41
- ...(context !== undefined && { contextBytes: context }),
42
- ...(compute !== undefined && { computeBytes: compute })
43
- };
44
- }
45
- /**
46
- * The parts of `deviceBytes` the engine separates. Audiogen reports a peak
47
- * across pipeline phases and diffusion a per-module table, neither of which
48
- * divides into these three, so both come back empty. OCR sizes its weights
49
- * alone and leaves the image-dependent memory uncounted.
50
- *
51
- * The llama rows are summed over the same devices the engine's own
52
- * `deviceBytes` covers, which is every row but `host`.
53
- */
54
- function breakdownFor(probe) {
55
- if (isLlama(probe)) {
56
- const devices = probe.result.devices.filter((device) => device.name !== HOST_ROW);
57
- return breakdown(sumOf(...devices.map((device) => device.modelBytes)), sumOf(...devices.map((device) => device.contextBytes)), sumOf(...devices.map((device) => device.computeBytes)));
58
- }
59
- // Whisper names its parts `kvBytes` and `computeBytes`, parakeet splits
60
- // compute across encoder and decoder; an engine fills one set or the other.
61
- if (probe.engine === 'asr-ggml') {
62
- const result = probe.result;
63
- // A VAD model's weights are inside `deviceBytes`, so they belong in the
64
- // weights the breakdown divides it into.
65
- return breakdown(sumOf(result.weightsBytes, result.vadBytes), sumOf(result.kvBytes, result.decoderStateBytes), sumOf(result.computeBytes, result.encoderComputeBytes, result.decoderComputeBytes));
66
- }
67
- if (probe.engine === 'bci-whispercpp') {
68
- const result = probe.result;
69
- return breakdown(result.weightsBytes, result.kvBytes, result.computeBytes);
70
- }
71
- if (probe.engine === 'tts-ggml') {
72
- const result = probe.result;
73
- return breakdown(result.weightsBytes, result.stateBytes, sumOf(result.lmComputeBytes, result.codecComputeBytes));
74
- }
75
- if (probe.engine === 'ocr-ggml') {
76
- return breakdown(probe.result.weightsBytes, undefined, undefined);
77
- }
78
- return {};
79
- }
80
- function llamaProjection(probe) {
81
- const first = probe.result.devices.find((device) => device.name !== HOST_ROW && device.name !== CPU_DEVICE);
82
- const devices = probe.result.devices
83
- .filter((device) => device.name !== HOST_ROW)
84
- .map((device) => ({
85
- name: device.name,
86
- totalBytes: device.totalBytes,
87
- freeBytes: device.freeBytes,
88
- weightsBytes: device.modelBytes,
89
- contextBytes: device.contextBytes,
90
- computeBytes: device.computeBytes
91
- }));
92
- return {
93
- deviceBytes: probe.result.deviceBytes,
94
- hostBytes: probe.result.hostBytes,
95
- ...breakdownFor(probe),
96
- ...(devices.length > 0 && { devices }),
97
- ...(first !== undefined && {
98
- deviceName: first.name,
99
- deviceFreeBytes: first.freeBytes,
100
- deviceTotalBytes: first.totalBytes
101
- })
102
- };
103
- }
104
- /**
105
- * The translation fitter reports one allowance against the free memory of the
106
- * device it ran on. Only the file bytes inside it are weights; the rest does
107
- * not divide into context and compute.
108
- */
109
- function nmtProjection(result) {
110
- return {
111
- deviceName: result.backend,
112
- deviceBytes: result.requiredBytes,
113
- deviceFreeBytes: result.freeBytes,
114
- weightsBytes: result.modelBytes,
115
- report: result.report
116
- };
117
- }
118
- function projectionFor(probe) {
119
- if (probe.engine === 'diffusion-cpp') {
120
- return { deviceName: probe.result.backend, report: probe.result.report };
121
- }
122
- if (probe.engine === 'translation-nmtcpp')
123
- return nmtProjection(probe.result);
124
- if (isLlama(probe))
125
- return llamaProjection(probe);
126
- const result = probe.result;
127
- return {
128
- deviceName: result.deviceName,
129
- deviceBytes: result.deviceBytes,
130
- hostBytes: result.hostBytes,
131
- deviceFreeBytes: result.deviceFreeBytes,
132
- deviceTotalBytes: result.deviceTotalBytes,
133
- report: result.report,
134
- ...breakdownFor(probe)
135
- };
136
- }
137
- /**
138
- * Only the llama fitters resolve a placement; the other engines' loads are not
139
- * layer-sliced across devices.
140
- *
141
- * `nGpuDevices` counts the devices the weights were actually placed on, less
142
- * the CPU backend device, which the fitter assigns to like any other but whose
143
- * demand is host memory.
144
- */
145
- function planFor(probe) {
146
- if (!isLlama(probe))
147
- return undefined;
148
- const assigned = probe.result.devices.filter((device) => device.name !== HOST_ROW && device.name !== CPU_DEVICE && device.modelBytes > 0);
149
- return {
150
- nCtx: probe.result.ctxSize,
151
- nGpuLayers: probe.result.gpuLayers,
152
- nGpuDevices: assigned.length
153
- };
154
- }
155
- export function classifyFit(probe, provenance) {
156
- const verdict = verdictFor(probe.result.status);
157
- const plan = verdict === 'fit' ? planFor(probe) : undefined;
158
- return {
159
- ...provenance,
160
- engine: probe.engine,
161
- verdict,
162
- reason: reasonFor(probe),
163
- ...(plan !== undefined && { plan }),
164
- ...(verdict !== 'unknown' && { projection: projectionFor(probe) })
165
- };
166
- }
@@ -1,27 +0,0 @@
1
- import { type LoadDescription } from '../../model-fit/description/resolve-load.js';
2
- import type { FitStubOptions, FitStubUnavailableReason } from '../../model-fit/description/fetch.js';
3
- import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '../../model-fit/load-check.js';
4
- export type LoadFitOutcome = {
5
- status: 'projected';
6
- fit: AdvisoryFitOutcome;
7
- } | {
8
- status: 'no-stub';
9
- reason: FitStubUnavailableReason;
10
- message?: string;
11
- } | {
12
- status: 'unsupported-load';
13
- detail: string;
14
- };
15
- export interface LoadFitOptions {
16
- stub?: FitStubOptions;
17
- fit?: AdvisoryFitOptions;
18
- }
19
- /**
20
- * Projects the load a caller describes, before its weights exist locally.
21
- *
22
- * The same probe `loadModel` runs, over the same config and artifacts that
23
- * load's own plugin resolves, pointed at files that are tens of KB instead of
24
- * gigabytes. It inherits the probe's fail-open contract: every failure is an
25
- * `unknown` verdict or a non-projected outcome, never a throw.
26
- */
27
- export declare function projectFitFromLoad(load: LoadDescription, modelId: string, options?: LoadFitOptions): Promise<LoadFitOutcome>;
@@ -1,33 +0,0 @@
1
- import { resolveLoadFromStubs } from '../../model-fit/description/resolve-load.js';
2
- import { runAdvisoryFitCheck } from '../../model-fit/load-check.js';
3
- /**
4
- * Projects the load a caller describes, before its weights exist locally.
5
- *
6
- * The same probe `loadModel` runs, over the same config and artifacts that
7
- * load's own plugin resolves, pointed at files that are tens of KB instead of
8
- * gigabytes. It inherits the probe's fail-open contract: every failure is an
9
- * `unknown` verdict or a non-projected outcome, never a throw.
10
- */
11
- export async function projectFitFromLoad(load, modelId, options = {}) {
12
- const resolved = await resolveLoadFromStubs(load, options.stub);
13
- if (resolved.status !== 'resolved')
14
- return resolved;
15
- try {
16
- const fit = await runAdvisoryFitCheck({
17
- modelId,
18
- modelType: load.modelType,
19
- modelPath: resolved.modelPath,
20
- modelConfig: resolved.modelConfig,
21
- transcriptionWorkload: load.transcriptionWorkload,
22
- ttsWorkload: load.ttsWorkload,
23
- artifacts: resolved.artifacts,
24
- // Nothing here assembles a shard set, so no load may claim the fitter
25
- // is looking at a split model.
26
- isShardedModel: false
27
- }, options.fit);
28
- return { status: 'projected', fit };
29
- }
30
- finally {
31
- await resolved.release();
32
- }
33
- }
@@ -1,31 +0,0 @@
1
- import type { ModelFitWorkload } from '../schemas/assess-model-fit.js';
2
- import type { ModelResourceProfile } from '../schemas/model-resource-profile.js';
3
- export declare const FLOOR_VERSION = "floor-v1";
4
- export interface FloorInput {
5
- profile: ModelResourceProfile;
6
- workload: ModelFitWorkload;
7
- /** Artifact bytes of companion constants the caller passed alongside. */
8
- extraArtifactBytes: number;
9
- }
10
- /** A lower bound with no upper bound, and how it was assembled. */
11
- export interface FloorResult {
12
- bytes: number;
13
- reasons: readonly string[];
14
- assumptions: readonly string[];
15
- }
16
- /**
17
- * The smallest resident footprint the catalog facts prove: artifact bytes, plus
18
- * the KV cache for a llama.cpp model at the cheapest element width the engine
19
- * can default to.
20
- *
21
- * Every term the real load adds on top — weight slack, engine overhead, compute
22
- * buffers, a completion's working peak — is non-negative, so a floor with all of
23
- * them at zero is unconditionally valid on any platform and any backend. It can
24
- * therefore refuse a model (`likely-too-large` when even the floor is over
25
- * budget) but never confirm one, since how far above the floor the real load
26
- * lands is unmeasured here.
27
- *
28
- * This is the zero-fetch gate. It reads nothing but the catalog and stays the
29
- * fallback wherever no engine measurement is available.
30
- */
31
- export declare function computeFloor(input: FloorInput): FloorResult;
@@ -1,65 +0,0 @@
1
- import { LLAMA_WEIGHTS_ASSUMPTION, kvCacheFloorBytes, narrowestKvElementBytes } from '../model-fit/kv-cache.js';
2
- export const FLOOR_VERSION = 'floor-v1';
3
- /**
4
- * Engines whose load builds a KV cache the file sizes exactly, so the floor can
5
- * count it. Every other engine's working memory is engine-owned and unsized
6
- * here; the floor for those is the weights alone.
7
- */
8
- const KV_CACHE_ENGINES = new Set(['llamacpp-completion', 'llamacpp-embedding']);
9
- /**
10
- * The smallest resident footprint the catalog facts prove: artifact bytes, plus
11
- * the KV cache for a llama.cpp model at the cheapest element width the engine
12
- * can default to.
13
- *
14
- * Every term the real load adds on top — weight slack, engine overhead, compute
15
- * buffers, a completion's working peak — is non-negative, so a floor with all of
16
- * them at zero is unconditionally valid on any platform and any backend. It can
17
- * therefore refuse a model (`likely-too-large` when even the floor is over
18
- * budget) but never confirm one, since how far above the floor the real load
19
- * lands is unmeasured here.
20
- *
21
- * This is the zero-fetch gate. It reads nothing but the catalog and stays the
22
- * fallback wherever no engine measurement is available.
23
- */
24
- export function computeFloor(input) {
25
- const { profile, workload, extraArtifactBytes } = input;
26
- const reasons = [];
27
- const assumptions = [
28
- KV_CACHE_ENGINES.has(profile.engine)
29
- ? LLAMA_WEIGHTS_ASSUMPTION
30
- : 'weights are counted at full artifact size; whatever the engine maps or copies, the file has to be resident to run',
31
- 'the floor omits engine overhead, compute buffers and working memory, all of which are non-negative, so it cannot overstate the cost — and says nothing about how much more the load needs'
32
- ];
33
- const artifactBytes = profile.artifactBytes + extraArtifactBytes;
34
- if (extraArtifactBytes > 0) {
35
- assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
36
- }
37
- if (!KV_CACHE_ENGINES.has(profile.engine)) {
38
- reasons.push(`the floor for engine '${profile.engine}' counts weights only; its working memory is engine-owned and not sized here`);
39
- return { bytes: artifactBytes, reasons, assumptions };
40
- }
41
- if (workload.kind !== 'llm') {
42
- reasons.push(`workload kind '${workload.kind}' carries no context for a llama.cpp model, so the floor counts weights only`);
43
- return { bytes: artifactBytes, reasons, assumptions };
44
- }
45
- const facts = profile.ggufFacts;
46
- if (!facts) {
47
- reasons.push('no GGUF metadata for this model in the catalog, so the floor counts weights only');
48
- return { bytes: artifactBytes, reasons, assumptions };
49
- }
50
- let contextTokens = workload.contextTokens ?? facts.contextLength;
51
- if (contextTokens > facts.contextLength) {
52
- assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
53
- contextTokens = facts.contextLength;
54
- }
55
- // The narrowest default the engine can pick on any backend, so the floor
56
- // holds whether the model lands on a GPU (q8_0 with flash attention) or on
57
- // the CPU (f16). Architectures that disable flash attention stay f16 anyway.
58
- const element = narrowestKvElementBytes(facts);
59
- const kv = kvCacheFloorBytes(facts, contextTokens, element);
60
- assumptions.push(element < 2
61
- ? 'the KV cache is floored at the q8_0 element width, the narrowest default the engine picks on any backend; a CPU load allocates an f16 cache above it'
62
- : `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not read here and would change these numbers');
63
- reasons.push(`KV cache floored for ${contextTokens} tokens at the narrowest default element width`);
64
- return { bytes: Math.ceil(artifactBytes + kv), reasons, assumptions };
65
- }