@qvac/inference 0.21.1 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (246) hide show
  1. package/NOTICE +30 -1
  2. package/README.md +44 -1
  3. package/dist/api/assess-model-fit.d.ts +4 -4
  4. package/dist/api/assess-model-fit.js +4 -4
  5. package/dist/api/audio-gen.d.ts +4 -1
  6. package/dist/api/audio-gen.js +4 -1
  7. package/dist/api/batch-completion.js +1 -1
  8. package/dist/api/completion-stream.js +1 -1
  9. package/dist/api/decide.d.ts +26 -0
  10. package/dist/api/decide.js +26 -0
  11. package/dist/api/get-model-info.d.ts +1 -1
  12. package/dist/api/index.d.ts +1 -0
  13. package/dist/api/index.js +1 -0
  14. package/dist/api/text-to-speech.d.ts +1 -0
  15. package/dist/api/text-to-speech.js +15 -19
  16. package/dist/api/transcribe.js +5 -2
  17. package/dist/api/video-request.d.ts +5 -5
  18. package/dist/api/video.d.ts +5 -4
  19. package/dist/api/video.js +5 -4
  20. package/dist/errors/index.d.ts +3 -0
  21. package/dist/errors/index.js +5 -0
  22. package/dist/handlers/assess-model-fit.d.ts +5 -3
  23. package/dist/handlers/assess-model-fit.js +80 -28
  24. package/dist/handlers/get-loaded-model-info.js +1 -2
  25. package/dist/handlers/load-model/safe-fetch.js +5 -6
  26. package/dist/handlers/registry.d.ts +3 -1
  27. package/dist/handlers/registry.js +3 -3
  28. package/dist/index.d.ts +1 -1
  29. package/dist/index.js +1 -1
  30. package/dist/logging/logger.js +1 -1
  31. package/dist/logging/utils.js +1 -1
  32. package/dist/models/registry/models.d.ts +563 -816
  33. package/dist/models/registry/models.js +2873 -3110
  34. package/dist/models/registry/resource-profiles.d.ts +1 -2
  35. package/dist/models/registry/resource-profiles.js +203 -2
  36. package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
  37. package/dist/plugins/builtin/asr-ggml/config.js +6 -0
  38. package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
  39. package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
  40. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +28 -10
  41. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
  42. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
  43. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +6 -27
  44. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +147 -60
  45. package/dist/plugins/builtin/audiogen-ggml/plugin.js +1 -0
  46. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +40 -24
  47. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -1
  48. package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
  49. package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
  50. package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
  51. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +18 -10
  52. package/dist/plugins/builtin/ggml-ocr/plugin.js +16 -38
  53. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +21 -14
  54. package/dist/plugins/builtin/ggml-vla/plugin.js +1 -1
  55. package/dist/plugins/builtin/index.d.ts +2 -0
  56. package/dist/plugins/builtin/index.js +2 -0
  57. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -7
  58. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +109 -189
  59. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +32 -90
  60. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +60 -211
  61. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
  62. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
  63. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +185 -87
  64. package/dist/plugins/builtin/llamacpp-completion/plugin.js +4 -5
  65. package/dist/plugins/builtin/llamacpp-completion/transform.js +1 -1
  66. package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +6 -0
  67. package/dist/plugins/builtin/llamacpp-decision/helpers.js +97 -0
  68. package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
  69. package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
  70. package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +216 -0
  71. package/dist/plugins/builtin/llamacpp-decision/plugin.js +43 -0
  72. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +12 -5
  73. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +1 -1
  74. package/dist/plugins/builtin/llamacpp-embedding/transform.js +3 -0
  75. package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
  76. package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
  77. package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
  78. package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
  79. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +25 -13
  80. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +9 -73
  81. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +52 -31
  82. package/dist/plugins/builtin/parakeet-transcription/plugin.js +18 -3
  83. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +10 -6
  84. package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
  85. package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
  86. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +59 -33
  87. package/dist/plugins/builtin/sdcpp-generation/plugin.js +12 -9
  88. package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
  89. package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
  90. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +4 -1
  91. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +17 -5
  92. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +164 -29
  93. package/dist/plugins/builtin/tts-ggml/plugin.js +38 -1
  94. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +58 -35
  95. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +6 -2
  96. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
  97. package/dist/plugins/ops/kv-cache-utils.js +10 -25
  98. package/dist/plugins/ops/load-model.js +21 -16
  99. package/dist/plugins/ops/transcribe.js +17 -1
  100. package/dist/registry.js +1 -0
  101. package/dist/resources/model-fit/assess.d.ts +15 -23
  102. package/dist/resources/model-fit/assess.js +207 -271
  103. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +3 -1
  104. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +6 -1
  105. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +2 -0
  106. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +5 -3
  107. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +9 -1
  108. package/dist/resources/model-fit/floor.d.ts +10 -10
  109. package/dist/resources/model-fit/floor.js +13 -13
  110. package/dist/resources/model-fit/kv-cache.d.ts +23 -0
  111. package/dist/resources/model-fit/kv-cache.js +85 -0
  112. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +11 -6
  113. package/dist/resources/model-fit/native-probe/advisory-fit.js +12 -27
  114. package/dist/resources/model-fit/native-probe/child-engine-fit.js +8 -0
  115. package/dist/resources/model-fit/native-probe/classify-fit.js +32 -1
  116. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +4 -1
  117. package/dist/resources/model-fit/native-probe/create-fit-request.js +24 -14
  118. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +14 -0
  119. package/dist/resources/model-fit/native-probe/engine-fit.js +1 -1
  120. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +4 -0
  121. package/dist/resources/model-fit/native-probe/engines/asr.js +23 -0
  122. package/dist/resources/model-fit/native-probe/engines/audiogen.js +29 -1
  123. package/dist/resources/model-fit/native-probe/engines/diffusion.js +16 -6
  124. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -1
  125. package/dist/resources/model-fit/native-probe/engines/llama.js +19 -123
  126. package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +8 -0
  127. package/dist/resources/model-fit/native-probe/engines/nmt.js +11 -0
  128. package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +9 -0
  129. package/dist/resources/model-fit/native-probe/engines/ocr.js +13 -0
  130. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +2 -0
  131. package/dist/resources/model-fit/native-probe/engines/tts.js +49 -3
  132. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +5 -3
  133. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +38 -23
  134. package/dist/resources/model-fit/platform.d.ts +4 -4
  135. package/dist/resources/model-fit/platform.js +6 -8
  136. package/dist/resources/model-fit/types.d.ts +3 -117
  137. package/dist/rpc/inventory.js +5 -7
  138. package/dist/runtime/env.d.ts +0 -2
  139. package/dist/runtime/env.js +2 -9
  140. package/dist/runtime/model-config-utils.js +6 -1
  141. package/dist/runtime/request-context.d.ts +1 -1
  142. package/dist/runtime/request-context.js +8 -0
  143. package/dist/runtime/state.js +5 -0
  144. package/dist/schemas/assess-model-fit.d.ts +100 -43
  145. package/dist/schemas/assess-model-fit.js +66 -29
  146. package/dist/schemas/audio-gen.d.ts +186 -18
  147. package/dist/schemas/audio-gen.js +180 -48
  148. package/dist/schemas/batch-completion-stream.d.ts +202 -34
  149. package/dist/schemas/bci-config.d.ts +2 -0
  150. package/dist/schemas/bci.d.ts +23 -2
  151. package/dist/schemas/cancel.d.ts +5 -0
  152. package/dist/schemas/cancel.js +1 -0
  153. package/dist/schemas/common.d.ts +1083 -158
  154. package/dist/schemas/common.js +5 -2
  155. package/dist/schemas/completion-stream.d.ts +232 -37
  156. package/dist/schemas/completion-stream.js +64 -20
  157. package/dist/schemas/config.d.ts +477 -12
  158. package/dist/schemas/config.js +21 -1
  159. package/dist/schemas/decision.d.ts +602 -0
  160. package/dist/schemas/decision.js +205 -0
  161. package/dist/schemas/download-asset.d.ts +7 -3
  162. package/dist/schemas/engine-addon-map.d.ts +7 -0
  163. package/dist/schemas/engine-addon-map.js +23 -3
  164. package/dist/schemas/errors.d.ts +1 -0
  165. package/dist/schemas/errors.js +5 -0
  166. package/dist/schemas/index.d.ts +2 -0
  167. package/dist/schemas/index.js +2 -0
  168. package/dist/schemas/llamacpp-config.d.ts +42 -2
  169. package/dist/schemas/llamacpp-config.js +27 -0
  170. package/dist/schemas/load-model.d.ts +5247 -1309
  171. package/dist/schemas/load-model.js +88 -9
  172. package/dist/schemas/model-config-schemas.d.ts +233 -8
  173. package/dist/schemas/model-config-schemas.js +4 -1
  174. package/dist/schemas/model-fit-policy.d.ts +17 -0
  175. package/dist/schemas/model-fit-policy.js +17 -0
  176. package/dist/schemas/model-info.d.ts +20 -2
  177. package/dist/schemas/model-info.js +2 -1
  178. package/dist/schemas/model-resource-profile.d.ts +3 -4
  179. package/dist/schemas/model-resource-profile.js +1 -4
  180. package/dist/schemas/model-src-utils.d.ts +10 -2
  181. package/dist/schemas/model-types.d.ts +10 -1
  182. package/dist/schemas/model-types.js +3 -0
  183. package/dist/schemas/ocr.d.ts +74 -0
  184. package/dist/schemas/ocr.js +42 -0
  185. package/dist/schemas/plugin.d.ts +8 -2
  186. package/dist/schemas/plugin.js +6 -0
  187. package/dist/schemas/profiling.d.ts +4 -4
  188. package/dist/schemas/registry.d.ts +26 -0
  189. package/dist/schemas/registry.js +3 -0
  190. package/dist/schemas/reload-config.d.ts +10 -2
  191. package/dist/schemas/rpc-server.js +3 -3
  192. package/dist/schemas/sdcpp-config.d.ts +26 -0
  193. package/dist/schemas/sdcpp-config.js +8 -7
  194. package/dist/schemas/system-resources.d.ts +4 -4
  195. package/dist/schemas/text-to-speech.d.ts +402 -3
  196. package/dist/schemas/text-to-speech.js +129 -11
  197. package/dist/schemas/tools.d.ts +13 -4
  198. package/dist/schemas/tools.js +18 -6
  199. package/dist/schemas/transcription-config.d.ts +15 -0
  200. package/dist/schemas/transcription-config.js +38 -3
  201. package/dist/schemas/transcription.d.ts +46 -4
  202. package/dist/schemas/transcription.js +51 -4
  203. package/dist/schemas/translation-config.d.ts +30 -10
  204. package/dist/schemas/vla.d.ts +9 -2
  205. package/dist/schemas/vla.js +9 -5
  206. package/dist/surface.d.ts +1 -1
  207. package/dist/surface.js +1 -1
  208. package/dist/utils/addon-responses.d.ts +1 -0
  209. package/dist/utils/asr-events.d.ts +1 -0
  210. package/dist/utils/asr-events.js +2 -1
  211. package/dist/utils/cache/types.d.ts +7 -0
  212. package/dist/utils/mcp-adapter.js +12 -8
  213. package/dist/utils/tool-helpers.d.ts +1 -1
  214. package/dist/utils/tool-helpers.js +20 -52
  215. package/dist/utils/tools/parsers/dsml.js +8 -5
  216. package/dist/utils/tools/parsers/qwen35.js +7 -4
  217. package/dist/utils/tools/shared.d.ts +3 -0
  218. package/dist/utils/tools/shared.js +62 -0
  219. package/dist/utils/transcribe-metadata.d.ts +8 -1
  220. package/dist/utils/transcribe-metadata.js +13 -3
  221. package/dist/utils/tts-stats.d.ts +1 -0
  222. package/package.json +41 -32
  223. package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
  224. package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
  225. package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
  226. package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
  227. package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
  228. package/dist/resources/model-fit/calibration/fit.js +0 -152
  229. package/dist/resources/model-fit/calibration/index.d.ts +0 -37
  230. package/dist/resources/model-fit/calibration/index.js +0 -75
  231. package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
  232. package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
  233. package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
  234. package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
  235. package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
  236. package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
  237. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
  238. package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
  239. package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
  240. package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
  241. package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
  242. package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
  243. package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
  244. package/dist/resources/model-fit/estimators/llm.js +0 -222
  245. package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
  246. package/dist/resources/model-fit/estimators/whisper.js +0 -76
@@ -1,4 +1,27 @@
1
1
  import { gpuLayersFromGate } from './gpu.js';
2
+ export function createMossTranscribeFitRequest(params) {
3
+ if (params.transcriptionWorkload === undefined) {
4
+ return {
5
+ supported: false,
6
+ detail: 'MOSS fit requires transcriptionWorkload with explicit audioSeconds'
7
+ };
8
+ }
9
+ const config = (params.modelConfig ?? {});
10
+ const workload = params.transcriptionWorkload;
11
+ const request = {
12
+ engine: 'moss-transcribe',
13
+ modelPath: params.modelPath,
14
+ audioSeconds: workload.audioSeconds,
15
+ ...(workload.prompt !== undefined && { prompt: workload.prompt }),
16
+ ...(workload.hotwords !== undefined && { hotwords: workload.hotwords }),
17
+ ...(workload.maxNewTokens !== undefined && { maxNewTokens: workload.maxNewTokens }),
18
+ gpuLayers: gpuLayersFromGate(config.useGPU),
19
+ ...(config.maxThreads !== undefined && { threads: config.maxThreads }),
20
+ ...(config.backendsDir !== undefined && { backendsDir: config.backendsDir }),
21
+ ...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
22
+ };
23
+ return { supported: true, probe: { engine: 'asr-ggml', request } };
24
+ }
2
25
  /**
3
26
  * The longest single transcribe the projection has to cover. Whisper's KV cache
4
27
  * and decode graph are sized from it; unset leaves the engine's own default,
@@ -5,11 +5,39 @@ import { gpuLayersFromGate } from './gpu.js';
5
5
  * matched by its literal rather than imported.
6
6
  */
7
7
  const MINIMAX = 'minimax';
8
+ /**
9
+ * MiniMax-Music3 is fitted from its LM and synthesis pair. The load config
10
+ * carries no frame budget, so the fitter projects the default generation
11
+ * `run()` falls back to (300 frames, 12 s).
12
+ *
13
+ * `device` is passed through as the load takes it. Without one, the GPU switch
14
+ * maps to `auto` as it does on load: the fitter reads a positive layer count
15
+ * that way. The plugin refuses MiniMax on mobile before this is reached, and
16
+ * mobile builds carry no MiniMax fitter.
17
+ */
18
+ function minimaxFitRequest(config, artifacts, marginBytes) {
19
+ const lmPath = artifacts['lmModelPath'];
20
+ const synthPath = artifacts['synthModelPath'];
21
+ if (lmPath === undefined || synthPath === undefined) {
22
+ return { supported: false, detail: 'the minimax model pair is not resolved' };
23
+ }
24
+ const request = {
25
+ engine: MINIMAX,
26
+ lmPath,
27
+ synthPath,
28
+ ...(config.device !== undefined
29
+ ? { device: config.device }
30
+ : { gpuLayers: gpuLayersFromGate(config.useGPU) }),
31
+ ...(config.threads !== undefined && { threads: config.threads }),
32
+ ...(marginBytes !== undefined && { marginBytes })
33
+ };
34
+ return { supported: true, probe: { engine: 'audiogen-ggml', request } };
35
+ }
8
36
  export function createAudiogenFitRequest(params) {
9
37
  const config = (params.modelConfig ?? {});
10
38
  const artifacts = params.artifacts ?? {};
11
39
  if (config.engine === MINIMAX) {
12
- return { supported: false, detail: 'the minimax pipeline has no fitter' };
40
+ return minimaxFitRequest(config, artifacts, params.marginBytes);
13
41
  }
14
42
  const textEncoderPath = artifacts['textEncModelPath'];
15
43
  const lmPath = artifacts['lmModelPath'];
@@ -1,16 +1,19 @@
1
1
  /**
2
2
  * Modes whose load the projection would not describe.
3
3
  *
4
- * `upscale` is an ESRGAN checkpoint, which has no fitter. `video` and `world`
5
- * assemble file sets — the audio VAE, the embeddings connectors, the TAEHV
6
- * preview decoder, a seed scene — that the fit request has no place for, and
7
- * their peak is driven by a frame count the load config does not carry.
4
+ * `upscale` is an ESRGAN checkpoint, which has no fitter. `world` assembles a
5
+ * TAEHV preview decoder and a seed scene that the fit request has no place for.
8
6
  */
9
7
  const UNFITTABLE_MODES = {
10
8
  upscale: 'a standalone upscaler load has no fitter',
11
- video: 'video loads are not representable',
12
9
  world: 'world loads are not representable'
13
10
  };
11
+ /**
12
+ * Frames to project a video load at. The fitter reads one frame when nothing
13
+ * is passed, which projects image generation; `run` generates 33. The frame
14
+ * count is a generation parameter, and this assessment sees load config alone.
15
+ */
16
+ const DEFAULT_VIDEO_FRAMES = 33;
14
17
  export function createDiffusionFitRequest(params) {
15
18
  const config = (params.modelConfig ?? {});
16
19
  const artifacts = params.artifacts ?? {};
@@ -34,6 +37,12 @@ export function createDiffusionFitRequest(params) {
34
37
  }),
35
38
  ...(artifacts['clipVisionModelPath'] !== undefined && {
36
39
  clipVision: artifacts['clipVisionModelPath']
40
+ }),
41
+ ...(artifacts['audioVaeModelPath'] !== undefined && {
42
+ audioVae: artifacts['audioVaeModelPath']
43
+ }),
44
+ ...(artifacts['embeddingsConnectorsModelPath'] !== undefined && {
45
+ embeddingsConnectors: artifacts['embeddingsConnectorsModelPath']
37
46
  })
38
47
  };
39
48
  // `mode` and `upscaler` are this SDK's own keys; the engine config has no
@@ -47,7 +56,8 @@ export function createDiffusionFitRequest(params) {
47
56
  files,
48
57
  config: engineConfig,
49
58
  workload: {
50
- ...(config.vae_tiling !== undefined && { vaeTiling: config.vae_tiling })
59
+ ...(config.vae_tiling !== undefined && { vaeTiling: config.vae_tiling }),
60
+ ...(config.mode === 'video' && { videoFrames: DEFAULT_VIDEO_FRAMES })
51
61
  }
52
62
  }
53
63
  }
@@ -7,7 +7,6 @@ export interface LlamaFitRequestParams {
7
7
  modelPath: string;
8
8
  modelConfig: unknown;
9
9
  artifacts?: Record<string, string> | undefined;
10
- isShardedModel: boolean;
11
10
  marginBytes?: number | undefined;
12
11
  }
13
12
  export declare function createLlamaFitRequest(params: LlamaFitRequestParams): FitRequestPlan;
@@ -1,56 +1,6 @@
1
1
  import { ModelType } from '../../../../schemas/index.js';
2
2
  import { transformLlmConfig } from '../../../../plugins/builtin/llamacpp-completion/transform.js';
3
3
  import { transformEmbedConfig } from '../../../../plugins/builtin/llamacpp-embedding/transform.js';
4
- const BYTES_PER_MIB = 1024 * 1024;
5
- /**
6
- * The transforms emit some settings with underscores where llama's own
7
- * argument table spells them with hyphens.
8
- */
9
- const CANONICAL_KEY = {
10
- ctx_size: 'ctx-size',
11
- gpu_layers: 'gpu-layers',
12
- batch_size: 'batch-size',
13
- ubatch_size: 'ubatch-size',
14
- flash_attn: 'flash-attn',
15
- split_mode: 'split-mode',
16
- main_gpu: 'main-gpu',
17
- load_mode: 'load-mode'
18
- };
19
- /**
20
- * Settings that decide how much device memory the load needs, in the spelling
21
- * each transform emits. `device` and `fit-target` are absent because they are
22
- * read before this point.
23
- */
24
- const MEMORY_KEYS = {
25
- completion: [
26
- 'ctx_size',
27
- 'gpu_layers',
28
- 'load_mode',
29
- 'parallel',
30
- 'cache-type-k',
31
- 'cache-type-v',
32
- 'main-gpu',
33
- 'split-mode',
34
- 'flash-attn',
35
- 'tensor-split',
36
- 'batch-size',
37
- 'ubatch-size',
38
- 'cpu-moe',
39
- 'n-cpu-moe',
40
- 'kv-offload',
41
- 'no-kv-offload',
42
- 'override-tensor',
43
- 'n-cpu-ffn',
44
- 'moe-cache-mib',
45
- 'prefetch-weights',
46
- 'tensor-read-lazy',
47
- 'fit',
48
- 'fit-ctx'
49
- ],
50
- embedding: ['gpu_layers', 'batch_size', 'flash_attn', 'main-gpu', 'split-mode', 'tensor-split']
51
- };
52
- /** Shapes the SDK resolves elsewhere; reaching here means they were not. */
53
- const UNSUPPORTED_KEYS = ['lora', 'projection_model_src'];
54
4
  export function llamaLoadKindFor(modelType) {
55
5
  if (modelType === ModelType.llamacppCompletion)
56
6
  return 'completion';
@@ -58,88 +8,34 @@ export function llamaLoadKindFor(modelType) {
58
8
  return 'embedding';
59
9
  return undefined;
60
10
  }
61
- /**
62
- * A CPU load's weights stay file-backed and evictable, so the fitter projects
63
- * `fits` for nearly any model the OS could page through. Measured on a 24 GiB
64
- * M4 Pro: an 18.3 GiB model at 32k context still reports `fits` on
65
- * `device: 'cpu'`, and it does decode — at 0.1 tok/s. That answer carries no
66
- * admission information.
67
- */
68
- function isCpuLoad(transformed) {
69
- return transformed['device']?.toLowerCase() === 'cpu';
70
- }
71
- /**
72
- * `fit-target` is the free margin the real load asks the engine to leave, in
73
- * MiB, and a comma-separated list sets it per device. The fit request carries
74
- * one value for every device, so a list resolves to its largest entry, which is
75
- * the only reading that cannot project a device as roomier than the load will
76
- * leave it.
77
- */
78
- function fitTargetBytes(transformed) {
79
- const value = transformed['fit-target'];
80
- if (value === undefined)
81
- return undefined;
82
- const targets = value
83
- .split(',')
84
- .map((entry) => Number(entry.trim()))
85
- .filter((entry) => Number.isFinite(entry) && entry >= 0);
86
- return targets.length === 0 ? undefined : Math.max(...targets) * BYTES_PER_MIB;
87
- }
88
- /**
89
- * Selects the settings that describe the load and spells them the way llama's
90
- * argument table does. Everything forwarded is parsed by llama itself, so this
91
- * layer classifies rather than interprets.
92
- */
93
- function loadParams(loadKind, transformed) {
94
- const carried = new Set(MEMORY_KEYS[loadKind]);
95
- const params = {};
96
- for (const [key, value] of Object.entries(transformed)) {
97
- if (UNSUPPORTED_KEYS.includes(key)) {
98
- return { detail: `unsupported load setting: ${key}` };
99
- }
100
- if (!carried.has(key))
101
- continue;
102
- params[CANONICAL_KEY[key] ?? key] = value;
103
- }
104
- return { params };
105
- }
106
11
  export function createLlamaFitRequest(params) {
107
12
  const { loadKind } = params;
108
- if (params.isShardedModel) {
109
- return { supported: false, detail: 'sharded models are not representable' };
110
- }
111
13
  if (loadKind === 'completion' && params.artifacts?.['projectionModelPath'] !== undefined) {
112
14
  return { supported: false, detail: 'multimodal projection loads are not representable' };
113
15
  }
114
16
  const modelConfig = (params.modelConfig ?? {});
115
- const transformed = loadKind === 'completion'
116
- ? transformLlmConfig(modelConfig)
117
- : transformEmbedConfig(modelConfig);
118
- if (isCpuLoad(transformed)) {
119
- return { supported: false, detail: 'cpu loads carry no device-memory evidence' };
17
+ // The assessment budgets against this machine's devices, so a load spread
18
+ // over remote GPUs is refused rather than measured against the wrong ones.
19
+ if (modelConfig['rpc-servers'] !== undefined) {
20
+ return { supported: false, detail: 'unsupported load setting: rpc-servers' };
120
21
  }
121
- const selected = loadParams(loadKind, transformed);
122
- if ('detail' in selected)
123
- return { supported: false, detail: selected.detail };
124
22
  const request = {
125
23
  modelPath: params.modelPath,
126
- params: selected.params
24
+ ...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
127
25
  };
128
- // A pinned context is also the floor, so the fitter reports on the load the
129
- // caller asked for rather than one it reduced.
130
- const ctxSize = Number(selected.params['ctx-size']);
131
- if (Number.isSafeInteger(ctxSize) && ctxSize > 0)
132
- request.minCtxSize = ctxSize;
133
- // The caller's own target and the advisory margin answer different questions
134
- // — what this load leaves free, and what the models already resident need —
135
- // so the projection holds to whichever is stricter.
136
- const margins = [params.marginBytes, fitTargetBytes(transformed)].filter((value) => value !== undefined);
137
- if (margins.length > 0)
138
- request.marginBytes = Math.max(...margins);
139
- return {
140
- supported: true,
141
- probe: loadKind === 'completion'
142
- ? { engine: 'llm-llamacpp', request }
143
- : { engine: 'embed-llamacpp', request }
26
+ if (loadKind === 'completion') {
27
+ // The image limit gates images at inference and does not change load
28
+ // memory. The addon's fitter rejects it as an unknown argument.
29
+ const { 'image-max-megapixels': _imageMaxMegapixels, ...fitConfig } = modelConfig;
30
+ const llm = {
31
+ ...request,
32
+ config: transformLlmConfig(fitConfig)
33
+ };
34
+ return { supported: true, probe: { engine: 'llm-llamacpp', request: llm } };
35
+ }
36
+ const embed = {
37
+ ...request,
38
+ config: transformEmbedConfig(modelConfig)
144
39
  };
40
+ return { supported: true, probe: { engine: 'embed-llamacpp', request: embed } };
145
41
  }
@@ -0,0 +1,8 @@
1
+ import type { FitRequestPlan } from '../create-fit-request.js';
2
+ export interface NmtFitRequestParams {
3
+ modelPath: string;
4
+ modelConfig: unknown;
5
+ artifacts?: Record<string, string> | undefined;
6
+ marginBytes?: number | undefined;
7
+ }
8
+ export declare function createNmtFitRequest(params: NmtFitRequestParams): FitRequestPlan;
@@ -0,0 +1,11 @@
1
+ import { nmtConfigFor, nmtFilesFor } from '../../../../plugins/builtin/nmtcpp-translation/load-files.js';
2
+ export function createNmtFitRequest(params) {
3
+ const config = (params.modelConfig ?? {});
4
+ const files = nmtFilesFor(params.modelPath, params.artifacts ?? {});
5
+ const request = {
6
+ files,
7
+ config: nmtConfigFor(config, files.pivotModel),
8
+ ...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
9
+ };
10
+ return { supported: true, probe: { engine: 'translation-nmtcpp', request } };
11
+ }
@@ -0,0 +1,9 @@
1
+ import type { FitRequestPlan } from '../create-fit-request.js';
2
+ export interface OcrFitRequestParams {
3
+ /** The recognizer, which is the load's primary model. */
4
+ modelPath: string;
5
+ modelConfig: unknown;
6
+ artifacts?: Record<string, string> | undefined;
7
+ marginBytes?: number | undefined;
8
+ }
9
+ export declare function createOcrFitRequest(params: OcrFitRequestParams): FitRequestPlan;
@@ -0,0 +1,13 @@
1
+ import { ocrParamsFor } from '../../../../plugins/builtin/ggml-ocr/params.js';
2
+ export function createOcrFitRequest(params) {
3
+ const detectorPath = params.artifacts?.['detectorModelPath'];
4
+ if (detectorPath === undefined) {
5
+ return { supported: false, detail: 'the ocr detector is not resolved' };
6
+ }
7
+ const config = (params.modelConfig ?? {});
8
+ const request = {
9
+ ...ocrParamsFor(detectorPath, params.modelPath, config),
10
+ ...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
11
+ };
12
+ return { supported: true, probe: { engine: 'ocr-ggml', request } };
13
+ }
@@ -1,5 +1,7 @@
1
+ import type { TtsFitWorkload } from '../../../../schemas/assess-model-fit.js';
1
2
  import type { FitRequestPlan } from '../create-fit-request.js';
2
3
  export interface TtsFitRequestParams {
4
+ ttsWorkload?: TtsFitWorkload | undefined;
3
5
  modelPath: string;
4
6
  modelConfig: unknown;
5
7
  artifacts?: Record<string, string> | undefined;
@@ -46,9 +46,14 @@ function cosyvoiceRequest(modelPath, common) {
46
46
  export function createTtsFitRequest(params) {
47
47
  const config = (params.modelConfig ?? {});
48
48
  const artifacts = params.artifacts ?? {};
49
+ const lavasrEnhancerPath = artifacts['lavasrEnhancerPath'];
50
+ const lavasrDenoiserPath = artifacts['lavasrDenoiserPath'];
49
51
  const common = {
50
52
  nGpuLayers: gpuLayersFromCount(config.useGPU, config.nGpuLayers),
51
- ...(params.marginBytes !== undefined && { marginBytes: params.marginBytes })
53
+ ...(config.backendsDir !== undefined && { backendsDir: config.backendsDir }),
54
+ ...(params.marginBytes !== undefined && { marginBytes: params.marginBytes }),
55
+ ...(lavasrEnhancerPath !== undefined && { lavasrEnhancerPath }),
56
+ ...(lavasrDenoiserPath !== undefined && { lavasrDenoiserPath })
52
57
  };
53
58
  const supported = (request) => ({
54
59
  supported: true,
@@ -86,8 +91,49 @@ export function createTtsFitRequest(params) {
86
91
  }
87
92
  case 'cosyvoice3':
88
93
  return cosyvoiceRequest(params.modelPath, common);
89
- case 'moss':
90
- return unsupported('the moss engine takes no fit request');
94
+ case 'moss': {
95
+ const decoder = artifacts['mossCodecDecoderPath'];
96
+ const encoder = artifacts['mossCodecEncoderPath'];
97
+ if (!decoder)
98
+ return unsupported('the moss codec decoder is not resolved');
99
+ if (!params.ttsWorkload || !('promptRows' in params.ttsWorkload)) {
100
+ return unsupported('MOSS fit requires ttsWorkload with promptRows, referenceSamples and streaming');
101
+ }
102
+ if (params.ttsWorkload.referenceSamples > 0 && !encoder) {
103
+ return unsupported('MOSS reference encoding requires the codec encoder');
104
+ }
105
+ return supported({
106
+ engineType: 'moss',
107
+ mossBackbonePath: params.modelPath,
108
+ mossCodecDecoderPath: decoder,
109
+ ...(encoder ? { mossCodecEncoderPath: encoder } : {}),
110
+ ...common,
111
+ ...params.ttsWorkload,
112
+ ...(config.threads !== undefined ? { threads: config.threads } : {}),
113
+ ...(config.streamChunkTokens !== undefined
114
+ ? { streamChunkTokens: config.streamChunkTokens }
115
+ : {}),
116
+ ...(config.durationTokens !== undefined ? { durationTokens: config.durationTokens } : {})
117
+ });
118
+ }
119
+ case 'moss-sfx': {
120
+ if (!params.ttsWorkload || !('prompt' in params.ttsWorkload)) {
121
+ return unsupported('MOSS-SoundEffect fit requires ttsWorkload with prompt and seconds');
122
+ }
123
+ return supported({
124
+ engineType: 'moss-sfx',
125
+ mossSoundEffectPath: params.modelPath,
126
+ ...common,
127
+ ...params.ttsWorkload,
128
+ ...(config.negativePrompt !== undefined ? { negativePrompt: config.negativePrompt } : {}),
129
+ ...(config.steps !== undefined ? { steps: config.steps } : {}),
130
+ ...(config.guidance !== undefined ? { guidance: config.guidance } : {}),
131
+ ...(config.shift !== undefined ? { shift: config.shift } : {}),
132
+ ...(config.threads !== undefined ? { threads: config.threads } : {})
133
+ });
134
+ }
135
+ case 'moss-speech':
136
+ return unsupported('the addon does not provide MOSS-Speech memory fit');
91
137
  default: {
92
138
  const s3genPath = artifacts['s3genPath'];
93
139
  if (s3genPath === undefined) {
@@ -10,13 +10,15 @@ export interface RunInProcessFitOptions {
10
10
  export declare function crashMarkerPath(stateDir: string, probe: FitProbeRequest): string;
11
11
  export declare function crashedMarkerPath(stateDir: string, probe: FitProbeRequest): string;
12
12
  /**
13
- * Runs one engine's fitter on the calling thread, for hosts that cannot spawn a
13
+ * Runs one engine's fitter in this process, for hosts that cannot spawn a
14
14
  * disposable child.
15
15
  *
16
16
  * There is no boundary here to contain a fitter that aborts: it takes the host
17
17
  * with it. A leftover `.running` marker is that abort, observed on the next
18
18
  * launch, and it is promoted to `.crashed` so the same request skips native
19
- * rather than repeating the abort. `timeoutMs` has no meaning on this path,
20
- * since the native call blocks the thread until it returns.
19
+ * rather than repeating the abort. `timeoutMs` is not applied on this path.
20
+ *
21
+ * Fits of the same request can be in flight together. A `.running` marker this
22
+ * process holds is not an abort, so it is shared and removed by the last fit.
21
23
  */
22
24
  export declare function runInProcessFit(probe: FitProbeRequest, options?: RunInProcessFitOptions): Promise<FitRunResult>;
@@ -10,6 +10,8 @@ function describe(error) {
10
10
  }
11
11
  /** How long a marker keeps a model off the native path. */
12
12
  const MARKER_TTL_MS = 7 * 24 * 60 * 60 * 1000;
13
+ /** Fits this process has in flight, per `.running` marker path. */
14
+ const inFlight = new Map();
13
15
  /**
14
16
  * `marginBytes` is excluded: it carries the bytes of every model resident at
15
17
  * the time, so it differs between two attempts at the same load.
@@ -56,14 +58,16 @@ async function resolveStateDir(explicit) {
56
58
  return getCacheDir('model-fit');
57
59
  }
58
60
  /**
59
- * Runs one engine's fitter on the calling thread, for hosts that cannot spawn a
61
+ * Runs one engine's fitter in this process, for hosts that cannot spawn a
60
62
  * disposable child.
61
63
  *
62
64
  * There is no boundary here to contain a fitter that aborts: it takes the host
63
65
  * with it. A leftover `.running` marker is that abort, observed on the next
64
66
  * launch, and it is promoted to `.crashed` so the same request skips native
65
- * rather than repeating the abort. `timeoutMs` has no meaning on this path,
66
- * since the native call blocks the thread until it returns.
67
+ * rather than repeating the abort. `timeoutMs` is not applied on this path.
68
+ *
69
+ * Fits of the same request can be in flight together. A `.running` marker this
70
+ * process holds is not an abort, so it is shared and removed by the last fit.
67
71
  */
68
72
  export async function runInProcessFit(probe, options = {}) {
69
73
  if (options.signal?.aborted === true) {
@@ -72,29 +76,33 @@ export async function runInProcessFit(probe, options = {}) {
72
76
  const stateDir = await resolveStateDir(options.stateDir);
73
77
  const running = crashMarkerPath(stateDir, probe);
74
78
  const crashed = crashedMarkerPath(stateDir, probe);
75
- for (const marker of [crashed, running]) {
76
- if (exists(marker) && expired(marker))
77
- clearRunning(marker);
78
- }
79
- if (exists(crashed) || exists(running)) {
79
+ const held = inFlight.get(running) ?? 0;
80
+ if (held === 0) {
81
+ for (const marker of [crashed, running]) {
82
+ if (exists(marker) && expired(marker))
83
+ clearRunning(marker);
84
+ }
85
+ if (exists(crashed) || exists(running)) {
86
+ try {
87
+ fs.mkdirSync(path.dirname(crashed), { recursive: true });
88
+ fs.writeFileSync(crashed, '');
89
+ clearRunning(running);
90
+ }
91
+ catch {
92
+ // Keep `.running` if `.crashed` could not be written so the next launch
93
+ // still skips native.
94
+ }
95
+ return unknown('crashed', 'Previous fit did not finish; treating this model as unknown');
96
+ }
80
97
  try {
81
- fs.mkdirSync(path.dirname(crashed), { recursive: true });
82
- fs.writeFileSync(crashed, '');
83
- clearRunning(running);
98
+ fs.mkdirSync(path.dirname(running), { recursive: true });
99
+ fs.writeFileSync(running, '');
84
100
  }
85
- catch {
86
- // Keep `.running` if `.crashed` could not be written so the next launch
87
- // still skips native.
101
+ catch (error) {
102
+ return unknown('invocation-error', `Fit crash marker could not be written: ${describe(error)}`);
88
103
  }
89
- return unknown('crashed', 'Previous fit did not finish; treating this model as unknown');
90
- }
91
- try {
92
- fs.mkdirSync(path.dirname(running), { recursive: true });
93
- fs.writeFileSync(running, '');
94
- }
95
- catch (error) {
96
- return unknown('invocation-error', `Fit crash marker could not be written: ${describe(error)}`);
97
104
  }
105
+ inFlight.set(running, held + 1);
98
106
  try {
99
107
  const callFit = options.callFit ?? callEngineFit;
100
108
  return { status: 'completed', probe: await callFit(probe) };
@@ -103,6 +111,13 @@ export async function runInProcessFit(probe, options = {}) {
103
111
  return unknown('invocation-error', describe(error));
104
112
  }
105
113
  finally {
106
- clearRunning(running);
114
+ const remaining = (inFlight.get(running) ?? 1) - 1;
115
+ if (remaining === 0) {
116
+ inFlight.delete(running);
117
+ clearRunning(running);
118
+ }
119
+ else {
120
+ inFlight.set(running, remaining);
121
+ }
107
122
  }
108
123
  }
@@ -1,9 +1,9 @@
1
1
  import type { ModelFitPlatform } from './types.js';
2
2
  /**
3
- * Maps the runtime's platform and architecture onto a calibration target.
3
+ * Maps the runtime's platform and architecture onto a platform this assessment
4
+ * covers.
4
5
  *
5
- * @returns `undefined` for any pair this feature has no calibration target for,
6
- * which assesses as `unknown` rather than borrowing another platform's
7
- * coefficients.
6
+ * @returns `undefined` for any pair outside that set, which assesses as
7
+ * `unknown`.
8
8
  */
9
9
  export declare function detectPlatform(): ModelFitPlatform | undefined;
@@ -1,10 +1,10 @@
1
1
  import os from 'bare-os';
2
2
  /**
3
- * Maps the runtime's platform and architecture onto a calibration target.
3
+ * Maps the runtime's platform and architecture onto a platform this assessment
4
+ * covers.
4
5
  *
5
- * @returns `undefined` for any pair this feature has no calibration target for,
6
- * which assesses as `unknown` rather than borrowing another platform's
7
- * coefficients.
6
+ * @returns `undefined` for any pair outside that set, which assesses as
7
+ * `unknown`.
8
8
  */
9
9
  export function detectPlatform() {
10
10
  const key = `${os.platform()}-${os.arch()}`;
@@ -14,10 +14,8 @@ export function detectPlatform() {
14
14
  case 'linux-arm64':
15
15
  case 'linux-x64':
16
16
  case 'win32-x64':
17
- // Listed for completeness rather than reach: no engine addon is built for
18
- // win32-arm64 (`@qvac/llm-llamacpp/prebuilds` has no such target, and no
19
- // windows-arm runner exists to build one), so nothing on that platform can
20
- // load a model today. Calibrating it needs that build first, not a fixture.
17
+ // No engine addon is built for win32-arm64 (`@qvac/llm-llamacpp/prebuilds`
18
+ // has no such target), so no load on it reaches a fitter.
21
19
  case 'win32-arm64':
22
20
  case 'android-arm64':
23
21
  case 'ios-arm64':