@qvac/inference 0.20.1 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (212) hide show
  1. package/NOTICE +90 -24
  2. package/README.md +8 -5
  3. package/dist/api/bci-transcribe.d.ts +4 -1
  4. package/dist/api/index.d.ts +1 -0
  5. package/dist/api/index.js +1 -0
  6. package/dist/api/rpc-server.d.ts +19 -0
  7. package/dist/api/rpc-server.js +44 -0
  8. package/dist/api/text-to-speech.d.ts +9 -0
  9. package/dist/api/text-to-speech.js +9 -0
  10. package/dist/api/transcribe.d.ts +11 -7
  11. package/dist/api/transcribe.js +47 -15
  12. package/dist/api/video-request.d.ts +5 -5
  13. package/dist/dispatch.js +39 -7
  14. package/dist/errors/index.d.ts +7 -0
  15. package/dist/errors/index.js +15 -0
  16. package/dist/handlers/assess-model-fit.d.ts +8 -20
  17. package/dist/handlers/assess-model-fit.js +195 -77
  18. package/dist/handlers/completion-orchestrate.js +17 -0
  19. package/dist/handlers/get-loaded-model-info.js +4 -1
  20. package/dist/handlers/get-model-info.js +29 -1
  21. package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
  22. package/dist/handlers/load-model/coreml-sidecars.js +274 -0
  23. package/dist/handlers/load-model/hyperdrive.js +2 -2
  24. package/dist/handlers/load-model/registry.js +59 -0
  25. package/dist/handlers/rpc-server.d.ts +23 -0
  26. package/dist/handlers/rpc-server.js +141 -0
  27. package/dist/index.d.ts +2 -1
  28. package/dist/index.js +2 -1
  29. package/dist/models/registry/models.d.ts +7731 -5672
  30. package/dist/models/registry/models.js +4652 -2526
  31. package/dist/models/registry/resource-profiles.js +481 -94
  32. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
  33. package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
  34. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
  35. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
  36. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
  37. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
  38. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
  40. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -11
  41. package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
  42. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
  43. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
  44. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
  45. package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
  46. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
  47. package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
  48. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
  49. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
  50. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
  51. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
  52. package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
  53. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
  54. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
  55. package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
  56. package/dist/plugins/ops/bci-transcribe.js +18 -13
  57. package/dist/plugins/ops/load-model.js +2 -2
  58. package/dist/plugins/ops/transcribe.d.ts +4 -2
  59. package/dist/plugins/ops/transcribe.js +39 -2
  60. package/dist/plugins/ops/translate-stats.d.ts +2 -1
  61. package/dist/plugins/ops/translate-stats.js +64 -7
  62. package/dist/plugins/ops/translate.js +20 -4
  63. package/dist/profiling/operation-metrics.js +56 -0
  64. package/dist/registry.js +4 -0
  65. package/dist/resources/model-fit/assess.d.ts +37 -2
  66. package/dist/resources/model-fit/assess.js +24 -3
  67. package/dist/resources/model-fit/estimators/llm.js +2 -2
  68. package/dist/resources/model-fit/estimators/whisper.js +3 -4
  69. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +20 -1
  70. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +126 -47
  71. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
  72. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
  73. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
  74. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
  75. package/dist/resources/model-fit/floor.js +2 -2
  76. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
  77. package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
  78. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
  79. package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
  80. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
  81. package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
  82. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
  83. package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
  84. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
  85. package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
  86. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
  87. package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
  88. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
  89. package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
  90. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
  91. package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
  92. package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
  93. package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
  94. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
  95. package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
  96. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
  97. package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
  98. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
  99. package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
  100. package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
  101. package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
  102. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
  103. package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
  104. package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
  105. package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
  106. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
  107. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
  108. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
  109. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
  110. package/dist/resources/model-fit/platform.d.ts +9 -0
  111. package/dist/resources/model-fit/platform.js +28 -0
  112. package/dist/rpc/device-map.d.ts +13 -0
  113. package/dist/rpc/device-map.js +22 -0
  114. package/dist/rpc/discovery.d.ts +12 -0
  115. package/dist/rpc/discovery.js +225 -0
  116. package/dist/rpc/ggml-provider.d.ts +4 -0
  117. package/dist/rpc/ggml-provider.js +25 -0
  118. package/dist/rpc/index.d.ts +2 -0
  119. package/dist/rpc/index.js +1 -0
  120. package/dist/rpc/instance.d.ts +6 -0
  121. package/dist/rpc/instance.js +21 -0
  122. package/dist/rpc/inventory.d.ts +5 -0
  123. package/dist/rpc/inventory.js +144 -0
  124. package/dist/rpc/manager.d.ts +14 -0
  125. package/dist/rpc/manager.js +174 -0
  126. package/dist/rpc/network.d.ts +11 -0
  127. package/dist/rpc/network.js +85 -0
  128. package/dist/rpc/provider.d.ts +6 -0
  129. package/dist/rpc/provider.js +24 -0
  130. package/dist/runtime/lifecycle.js +31 -13
  131. package/dist/runtime/request-context.d.ts +1 -1
  132. package/dist/runtime/state.js +5 -0
  133. package/dist/schemas/assess-model-fit.d.ts +269 -85
  134. package/dist/schemas/assess-model-fit.js +78 -54
  135. package/dist/schemas/audio-gen.d.ts +6 -6
  136. package/dist/schemas/audio-gen.js +2 -2
  137. package/dist/schemas/batch-completion-stream.d.ts +13 -5
  138. package/dist/schemas/batch-completion-stream.js +5 -3
  139. package/dist/schemas/bci-config.js +1 -1
  140. package/dist/schemas/bci.d.ts +86 -1
  141. package/dist/schemas/bci.js +21 -2
  142. package/dist/schemas/cancel.d.ts +10 -0
  143. package/dist/schemas/cancel.js +2 -0
  144. package/dist/schemas/common.d.ts +708 -66
  145. package/dist/schemas/common.js +7 -0
  146. package/dist/schemas/completion-event.d.ts +6 -6
  147. package/dist/schemas/completion-stream.d.ts +27 -8
  148. package/dist/schemas/completion-stream.js +41 -4
  149. package/dist/schemas/config.d.ts +104 -36
  150. package/dist/schemas/config.js +10 -0
  151. package/dist/schemas/embed.d.ts +2 -2
  152. package/dist/schemas/error.js +13 -0
  153. package/dist/schemas/errors.d.ts +1 -0
  154. package/dist/schemas/errors.js +5 -0
  155. package/dist/schemas/index.d.ts +1 -0
  156. package/dist/schemas/index.js +1 -0
  157. package/dist/schemas/llamacpp-config.d.ts +25 -18
  158. package/dist/schemas/llamacpp-config.js +26 -16
  159. package/dist/schemas/load-model.d.ts +2608 -469
  160. package/dist/schemas/load-model.js +2 -2
  161. package/dist/schemas/mcp-adapter.d.ts +4 -0
  162. package/dist/schemas/model-config-schemas.d.ts +183 -6
  163. package/dist/schemas/model-info.d.ts +56 -0
  164. package/dist/schemas/model-info.js +5 -1
  165. package/dist/schemas/plugin.d.ts +8 -2
  166. package/dist/schemas/profiling.d.ts +4 -4
  167. package/dist/schemas/reload-config.d.ts +30 -0
  168. package/dist/schemas/rpc-server.d.ts +101 -0
  169. package/dist/schemas/rpc-server.js +87 -0
  170. package/dist/schemas/sdcpp-config.d.ts +7 -3
  171. package/dist/schemas/sdcpp-config.js +37 -2
  172. package/dist/schemas/system-resources.d.ts +4 -4
  173. package/dist/schemas/text-to-speech.d.ts +529 -1
  174. package/dist/schemas/text-to-speech.js +122 -12
  175. package/dist/schemas/tools.d.ts +2 -0
  176. package/dist/schemas/tools.js +8 -0
  177. package/dist/schemas/transcription-config.d.ts +5 -0
  178. package/dist/schemas/transcription-config.js +20 -2
  179. package/dist/schemas/transcription.d.ts +164 -0
  180. package/dist/schemas/transcription.js +71 -4
  181. package/dist/schemas/vla.d.ts +1 -1
  182. package/dist/surface.d.ts +4 -1
  183. package/dist/surface.js +3 -1
  184. package/dist/utils/addon-responses.d.ts +5 -1
  185. package/dist/utils/archive.js +5 -1
  186. package/dist/utils/asr-diagnostics.d.ts +12 -0
  187. package/dist/utils/asr-diagnostics.js +42 -0
  188. package/dist/utils/asr-events.d.ts +1 -0
  189. package/dist/utils/asr-events.js +3 -2
  190. package/dist/utils/cache/paths.js +3 -2
  191. package/dist/utils/mcp-adapter.js +7 -1
  192. package/dist/utils/multi-gpu-mobile.d.ts +1 -1
  193. package/dist/utils/multi-gpu-mobile.js +8 -6
  194. package/dist/utils/path-security.d.ts +5 -0
  195. package/dist/utils/path-security.js +14 -0
  196. package/dist/utils/tool-helpers.d.ts +4 -0
  197. package/dist/utils/tool-helpers.js +2 -0
  198. package/dist/utils/tools/defer.d.ts +93 -0
  199. package/dist/utils/tools/defer.js +290 -0
  200. package/dist/utils/tools/index.d.ts +1 -0
  201. package/dist/utils/tools/index.js +1 -0
  202. package/dist/utils/transcribe-metadata.d.ts +21 -7
  203. package/dist/utils/transcribe-metadata.js +24 -8
  204. package/dist/utils/tts-stats.d.ts +6 -0
  205. package/dist/utils/tts-stats.js +14 -1
  206. package/package.json +43 -29
  207. package/dist/resources/model-fit/fit-stub/index.d.ts +0 -2
  208. package/dist/resources/model-fit/fit-stub/index.js +0 -2
  209. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +0 -31
  210. package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +0 -36
  211. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
  212. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -191
@@ -1,5 +1,7 @@
1
1
  import { getEngineLogger } from '../../../logging/index.js';
2
- import { createLlamaFitRequest } from './create-llama-fit-request.js';
2
+ import { classifyFit } from './classify-fit.js';
3
+ import { createFitRequest } from './create-fit-request.js';
4
+ import { runFit as runFitDefault } from './run-fit.js';
3
5
  /**
4
6
  * Shorter than the supervisor's own 60s default: this check sits in front of a
5
7
  * real load, so a wedged child must not hold the load for a full minute. Every
@@ -7,30 +9,26 @@ import { createLlamaFitRequest } from './create-llama-fit-request.js';
7
9
  */
8
10
  const ADVISORY_FIT_TIMEOUT_MS = 30_000;
9
11
  /**
10
- * `@qvac/model-fit`'s own default margin. Made explicit here because the
11
- * resident-model reserve below is *added* to it: setting `marginMiB` at all
12
- * replaces the package default, so the base has to travel with the reserve.
12
+ * The base every engine's fitter withholds by default. Made explicit here
13
+ * because the resident-model reserve is added to it: setting a margin at all
14
+ * replaces the engine default, so the base has to travel with it. It is also
15
+ * the headroom `withMachineBudget` leaves on the system budget.
13
16
  */
14
17
  const ADVISORY_FIT_BASE_MARGIN_MIB = 1024;
15
18
  /**
16
- * Names the headroom policy these two numbers add up to: withhold
17
- * `ADVISORY_FIT_BASE_MARGIN_MIB`, plus the on-disk bytes of every model already
18
- * resident in this worker.
19
- *
20
- * Deliberately *not* `interactive-v1` (withhold 20%, cap 2 GiB desktop / 1 GiB
21
- * mobile), which `assessModelFit` applies. That policy is applied by the SDK to
22
- * a budget the SDK computed; here the child owns the budget (total − wired −
23
- * compressor, measured inside the disposable process) and `marginMiB` is the
24
- * only lever this side has. Reconciling the two means one entry point owning
25
- * both bases — see the precedence note on `nativeProbeFitSchema`.
19
+ * Names the headroom policy: the fitter withholds
20
+ * `ADVISORY_FIT_BASE_MARGIN_MIB` plus the on-disk bytes of every model resident
21
+ * in this worker, and a `fit` is then judged against the budget
22
+ * `assessModelFit` reports, under that platform's basis and less the
23
+ * `interactive-v1` reserve.
26
24
  */
27
- const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-v1';
25
+ const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-v2';
28
26
  const BYTES_PER_MIB = 1024 * 1024;
29
27
  const DISABLED_VALUES = new Set(['0', 'false', 'off', 'no']);
30
28
  /**
31
29
  * On by default; `QVAC_ADVISORY_MODEL_FIT=0` (or `false`/`off`/`no`) is the
32
30
  * operator opt-out — the escape hatch for a load-heavy startup path or a
33
- * runtime where the disposable child cannot spawn. Any other value, including
31
+ * runtime whose operator would rather not pay for it. Any other value, including
34
32
  * unset, leaves the check on.
35
33
  *
36
34
  * The worker environment and the mobile runtime flag are imported lazily. Both
@@ -52,19 +50,9 @@ async function resolveMobile(explicit) {
52
50
  }
53
51
  /**
54
52
  * Sums the on-disk weight sizes of every model currently registered in this
55
- * worker. Since `@qvac/model-fit@0.8.0` (qvac-fabric#214) the fit child budgets
56
- * against system-wide availability (total − wired − compressor), so weights the
57
- * worker holds *wired* on the GPU are already visible to it. What the child
58
- * still cannot see is the unwired remainder — mmap'd host-side layers and KV of
59
- * resident models — which the OS reports as evictable right up until a decode
60
- * needs it resident. Measured consequence on 0.7.0 (where the child saw a fully
61
- * idle device): a verdict correct on an idle machine admitted a load that could
62
- * not decode with another model loaded.
63
- *
64
- * Reserving the resident weight bytes through `marginMiB` keeps that footprint
65
- * in the child's budget. Where those weights are wired this now double-counts
66
- * and the verdict turns conservative — deliberately so: the measured failure
67
- * modes punish optimism (loads that cannot decode), not caution.
53
+ * worker. The fallback for a host whose memory sample is unavailable, where
54
+ * models this worker loaded are the one part of the machine's usage it can
55
+ * still account for.
68
56
  *
69
57
  * Advisory and fail-open like everything else here: any failure to stat a file
70
58
  * contributes zero rather than an error.
@@ -89,20 +77,6 @@ async function defaultResidentModelBytes() {
89
77
  }
90
78
  return bytes;
91
79
  }
92
- /**
93
- * Also lazy: the desktop supervisor pulls the Bare process launcher, and a
94
- * disabled check must not load it. Mobile uses in-process `fitParamsAsync` instead
95
- * of spawn (`bare-runtime/spawn` stays deferred from the mobile pack).
96
- */
97
- async function resolveRunFit(explicit, mobile) {
98
- if (explicit !== undefined)
99
- return explicit;
100
- if (mobile) {
101
- const { runInProcessFit } = await import('./run-in-process-fit.js');
102
- return runInProcessFit;
103
- }
104
- return (await import('./run-isolated-fit.js')).runIsolatedFit;
105
- }
106
80
  const PROVENANCE = {
107
81
  basis: 'native-probe',
108
82
  estimatorVersion: NATIVE_PROBE_ESTIMATOR_VERSION
@@ -112,44 +86,116 @@ function unknown(reason, message) {
112
86
  ? { ...PROVENANCE, verdict: 'unknown', reason }
113
87
  : { ...PROVENANCE, verdict: 'unknown', reason, message };
114
88
  }
115
- function classify(result) {
116
- if (result.status === 0) {
117
- return {
118
- ...PROVENANCE,
119
- verdict: 'fit',
120
- reason: result.reason,
121
- plan: {
122
- nCtx: result.nCtx,
123
- nGpuLayers: result.nGpuLayers,
124
- nGpuDevices: result.nGpuDevices
125
- }
126
- };
127
- }
128
- if (result.status === 1) {
129
- return { ...PROVENANCE, verdict: 'does-not-fit', reason: result.reason };
130
- }
131
- // `model-unreadable`, `no-backend-device`, and `unsupported-config` are all
132
- // absence of evidence, not evidence of insufficiency.
133
- return unknown(result.reason);
89
+ function mib(bytes) {
90
+ return `${Math.round(bytes / BYTES_PER_MIB)} MiB`;
91
+ }
92
+ /**
93
+ * What the operating system would give back, which no backend reports. Metal
94
+ * answers with its working-set allowance minus this process's own allocations,
95
+ * so another application's resident pages read as free, and the host figure
96
+ * counts little beyond wired and compressed memory.
97
+ */
98
+ async function defaultAvailableSystemBytes() {
99
+ const [{ getResourceCollector }, { resolveBasis, resolveBudget }, { detectPlatform }] = await Promise.all([
100
+ import('../../instance.js'),
101
+ import('../assess.js'),
102
+ import('../platform.js')
103
+ ]);
104
+ const collector = getResourceCollector();
105
+ if (!collector)
106
+ return undefined;
107
+ const platform = detectPlatform();
108
+ const resources = { capabilities: collector.getCapabilities(), sample: collector.sample() };
109
+ const budget = resolveBudget(resources, platform, resolveBasis(platform), []);
110
+ if (budget === undefined)
111
+ return undefined;
112
+ return Math.max(0, budget.availableAfterReserveBytes);
113
+ }
114
+ /**
115
+ * Whether the device's allocations come out of the pool the system budget
116
+ * measures. Host bytes always do; device bytes do only where every allocation
117
+ * is system RAM, which is what `boundBySystemMemory` decides.
118
+ */
119
+ async function deviceBytesCountAgainstSystem() {
120
+ const [{ getResourceCollector }, { boundBySystemMemory }, { detectPlatform }] = await Promise.all([
121
+ import('../../instance.js'),
122
+ import('../assess.js'),
123
+ import('../platform.js')
124
+ ]);
125
+ const collector = getResourceCollector();
126
+ // Unknown counts them: admitting a load that cannot decode is the failure
127
+ // this check exists to stop, and refusing one that would have fitted is
128
+ // advisory only.
129
+ if (!collector)
130
+ return true;
131
+ const resources = { capabilities: collector.getCapabilities(), sample: collector.sample() };
132
+ return boundBySystemMemory(resources, detectPlatform());
133
+ }
134
+ /**
135
+ * Peak the projection places on the pool the system budget measures. A
136
+ * discrete card's memory is not that pool, so charging its bytes here would
137
+ * refuse a load the machine can hold.
138
+ */
139
+ function projectedDemandBytes(outcome, countDevice) {
140
+ const projection = outcome.projection;
141
+ if (projection === undefined)
142
+ return undefined;
143
+ if (projection.deviceBytes === undefined && projection.hostBytes === undefined)
144
+ return undefined;
145
+ return (countDevice ? (projection.deviceBytes ?? 0) : 0) + (projection.hostBytes ?? 0);
146
+ }
147
+ /**
148
+ * Judges a `fit` against the machine. The fitter decides placement and reports
149
+ * what the load costs; only this side knows what is free, so only this side can
150
+ * refuse on capacity.
151
+ *
152
+ * The margin cannot carry this. It is subtracted from each device's own free,
153
+ * and a device whose total sits below the machine's — Metal caps its allowance
154
+ * near 74% of system memory — would be charged the shortfall twice.
155
+ *
156
+ * A projection with no byte totals leaves the engine's verdict standing.
157
+ */
158
+ function withMachineBudget(outcome, availableBytes, countDevice) {
159
+ if (outcome.verdict !== 'fit' || availableBytes === undefined)
160
+ return outcome;
161
+ const demand = projectedDemandBytes(outcome, countDevice);
162
+ if (demand === undefined)
163
+ return outcome;
164
+ if (demand <= availableBytes)
165
+ return outcome;
166
+ return {
167
+ ...outcome,
168
+ verdict: 'does-not-fit',
169
+ reason: 'exceeds-available-memory',
170
+ message: `needs ${mib(demand)} against a ${mib(availableBytes)} budget`
171
+ };
172
+ }
173
+ /** The device and host totals, for the engines that measure them. */
174
+ function footprint(outcome) {
175
+ const projection = outcome.projection;
176
+ if (projection?.deviceBytes === undefined)
177
+ return '';
178
+ const host = projection.hostBytes === undefined ? '' : `, host ${mib(projection.hostBytes)}`;
179
+ return ` — device ${mib(projection.deviceBytes)}${host}`;
134
180
  }
135
181
  function report(logger, input, outcome) {
136
182
  const prefix = `[advisory-fit:${input.modelType}:${input.modelId}]`;
137
183
  if (outcome.verdict === 'fit') {
138
184
  const plan = outcome.plan;
139
185
  logger.info(`${prefix} projected to fit (advisory only)${plan === undefined
140
- ? ''
186
+ ? footprint(outcome)
141
187
  : ` — nCtx ${plan.nCtx}, nGpuLayers ${plan.nGpuLayers} across ${plan.nGpuDevices} GPU device(s)`}`);
142
188
  return;
143
189
  }
144
190
  if (outcome.verdict === 'does-not-fit') {
145
- logger.warn(`${prefix} projected not to fit (advisory only — the load continues unchanged)`);
191
+ logger.warn(`${prefix} projected not to fit (advisory only — the load continues unchanged)${footprint(outcome)}: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`);
146
192
  return;
147
193
  }
148
- // `unsupported-load` covers every load this check refuses up front — all
149
- // non-llama.cpp model types — so at `info` it would tag every whisper/tts/ocr
150
- // load on every start. `debug` for those; `info` where a fitter ran (or
151
- // should have) and produced no verdict, because there "why was there no
152
- // evidence" is the most useful thing this check can report.
194
+ // `unsupported-load` covers every load this check refuses up front — a model
195
+ // type with no fitter, a companion set that did not resolve — so at `info` it
196
+ // would tag those loads on every start. `debug` for those; `info` where a
197
+ // fitter ran (or should have) and produced no verdict, because there "why was
198
+ // there no evidence" is the most useful thing this check can report.
153
199
  const line = `${prefix} no fit evidence: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`;
154
200
  if (outcome.reason === 'unsupported-load') {
155
201
  logger.debug(line);
@@ -158,7 +204,8 @@ function report(logger, input, outcome) {
158
204
  logger.info(line);
159
205
  }
160
206
  /**
161
- * Runs the advisory llama.cpp fit check for a load that is about to start.
207
+ * Runs the advisory fit check for a load that is about to start, against the
208
+ * fitter belonging to the engine that will run it.
162
209
  *
163
210
  * Fail-open by construction: an unsupported shape, a crashed or wedged child, a
164
211
  * malformed response, and an unexpected internal error all resolve to `unknown`
@@ -172,30 +219,32 @@ export async function runAdvisoryFitCheck(input, options = {}) {
172
219
  if (!(await resolveEnabled(options.enabled)))
173
220
  return unknown('disabled');
174
221
  const mobile = await resolveMobile(options.mobile);
175
- const plan = createLlamaFitRequest({
222
+ const availableBytes = await (options.availableSystemBytes ?? defaultAvailableSystemBytes)();
223
+ const residentBytes = await (options.residentModelBytes ?? defaultResidentModelBytes)();
224
+ const residentReserveMiB = Math.ceil(residentBytes / BYTES_PER_MIB);
225
+ // Always sent, even with a zero reserve: relying on the engine default for
226
+ // the base margin would leave two sources of truth that match today and
227
+ // diverge silently if an engine default moves.
228
+ const plan = createFitRequest({
176
229
  modelType: input.modelType,
177
230
  modelPath: input.modelPath,
178
231
  modelConfig: input.modelConfig,
179
232
  artifacts: input.artifacts,
180
- isShardedModel: input.isShardedModel
233
+ isShardedModel: input.isShardedModel,
234
+ marginBytes: (ADVISORY_FIT_BASE_MARGIN_MIB + residentReserveMiB) * BYTES_PER_MIB
181
235
  });
182
236
  if (!plan.supported) {
183
237
  const outcome = unknown('unsupported-load', plan.detail);
184
238
  report(logger, input, outcome);
185
239
  return outcome;
186
240
  }
187
- const residentBytes = await (options.residentModelBytes ?? defaultResidentModelBytes)();
188
- const residentReserveMiB = Math.ceil(residentBytes / BYTES_PER_MIB);
189
- const runFit = await resolveRunFit(options.runFit, mobile);
190
- // Always sent, even with a zero reserve: relying on the addon default for
191
- // the base margin would leave two sources of truth that match today and
192
- // diverge silently if the addon default moves.
193
- const result = await runFit(plan.loadKind, { ...plan.config, marginMiB: ADVISORY_FIT_BASE_MARGIN_MIB + residentReserveMiB }, {
241
+ const result = await (options.runFit ?? runFitDefault)(plan.probe, {
242
+ mobile,
194
243
  timeoutMs: options.timeoutMs ?? ADVISORY_FIT_TIMEOUT_MS,
195
244
  ...(options.signal !== undefined && { signal: options.signal })
196
245
  });
197
246
  const outcome = result.status === 'completed'
198
- ? classify(result.result)
247
+ ? withMachineBudget(classifyFit(result.probe, PROVENANCE), availableBytes, await (options.countsDeviceRows ?? deviceBytesCountAgainstSystem)())
199
248
  : unknown(result.reason, result.message);
200
249
  report(logger, input, outcome);
201
250
  return outcome;
@@ -0,0 +1,2 @@
1
+ import type { FitProbeRequest, FitProbeResult } from './engine-fit.js';
2
+ export declare function callChildEngineFit(probe: FitProbeRequest): Promise<FitProbeResult>;
@@ -0,0 +1,41 @@
1
+ /**
2
+ * Loads the one addon a probe names and asks it for a projection. The child
3
+ * registers no plugins, so the fitter cannot come off the registry here. Only
4
+ * the child reaches this module, and the supervisor spawns it by path rather
5
+ * than importing it, so these specifiers stay out of the graph a host bundles.
6
+ */
7
+ async function assessFitOf(probe) {
8
+ switch (probe.engine) {
9
+ case 'llm-llamacpp': {
10
+ const { default: LlmLlamacpp } = await import('@qvac/llm-llamacpp');
11
+ return LlmLlamacpp.assessFit(probe.request);
12
+ }
13
+ case 'embed-llamacpp': {
14
+ const { assessFit } = await import('@qvac/embed-llamacpp');
15
+ return assessFit(probe.request);
16
+ }
17
+ case 'asr-ggml': {
18
+ const { default: ASRGgml } = await import('@qvac/asr-ggml');
19
+ return ASRGgml.assessFit(probe.request);
20
+ }
21
+ case 'bci-whispercpp': {
22
+ const { assessFit } = await import('@qvac/bci-whispercpp');
23
+ return assessFit(probe.request);
24
+ }
25
+ case 'tts-ggml': {
26
+ const { default: TTSGgml } = await import('@qvac/tts-ggml');
27
+ return TTSGgml.assessFit(probe.request);
28
+ }
29
+ case 'audiogen-ggml': {
30
+ const { assessFit } = await import('@qvac/audiogen-ggml');
31
+ return assessFit(probe.request);
32
+ }
33
+ case 'diffusion-cpp': {
34
+ const { assessFit } = await import('@qvac/diffusion-cpp');
35
+ return assessFit(probe.request);
36
+ }
37
+ }
38
+ }
39
+ export async function callChildEngineFit(probe) {
40
+ return { engine: probe.engine, result: await assessFitOf(probe) };
41
+ }
@@ -0,0 +1,6 @@
1
+ import type { NativeProbeFit } from '../../../schemas/index.js';
2
+ import type { FitProbeResult } from './engine-fit.js';
3
+ export declare function classifyFit(probe: FitProbeResult, provenance: {
4
+ basis: 'native-probe';
5
+ estimatorVersion: string;
6
+ }): NativeProbeFit;
@@ -0,0 +1,135 @@
1
+ /** The llama fitters append a host row after the devices they assigned to. */
2
+ const HOST_ROW = 'host';
3
+ /** ggml's own name for the CPU backend device, which the fitter may assign to. */
4
+ const CPU_DEVICE = 'CPU';
5
+ function isLlama(probe) {
6
+ return probe.engine === 'llm-llamacpp' || probe.engine === 'embed-llamacpp';
7
+ }
8
+ /**
9
+ * `error` is the engine reporting it could not read the model or find a backend
10
+ * device: absence of evidence, not evidence of insufficiency.
11
+ */
12
+ function verdictFor(status) {
13
+ if (status === 'fits')
14
+ return 'fit';
15
+ if (status === 'does-not-fit')
16
+ return 'does-not-fit';
17
+ return 'unknown';
18
+ }
19
+ /**
20
+ * The diffusion fitter reports a placement rather than a reason string.
21
+ * `changed` means it reached a fitting arrangement only by moving modules
22
+ * between backends, which is why the status is already `does-not-fit`.
23
+ */
24
+ function diffusionReason(result) {
25
+ if (result.status === 'fits')
26
+ return 'fits';
27
+ if (result.status === 'error')
28
+ return 'model-unreadable';
29
+ return result.changed ? `does-not-fit-as-configured (${result.backend})` : 'does-not-fit';
30
+ }
31
+ function reasonFor(probe) {
32
+ return probe.engine === 'diffusion-cpp' ? diffusionReason(probe.result) : probe.result.reason;
33
+ }
34
+ function sumOf(...values) {
35
+ const present = values.filter((value) => value !== undefined);
36
+ return present.length === 0 ? undefined : present.reduce((total, value) => total + value, 0);
37
+ }
38
+ function breakdown(weights, context, compute) {
39
+ return {
40
+ ...(weights !== undefined && { weightsBytes: weights }),
41
+ ...(context !== undefined && { contextBytes: context }),
42
+ ...(compute !== undefined && { computeBytes: compute })
43
+ };
44
+ }
45
+ /**
46
+ * The parts of `deviceBytes` the engine separates. Audiogen reports a peak
47
+ * across pipeline phases and diffusion a per-module table, neither of which
48
+ * divides into these three, so both come back empty.
49
+ *
50
+ * The llama rows are summed over the same devices the engine's own
51
+ * `deviceBytes` covers, which is every row but `host`.
52
+ */
53
+ function breakdownFor(probe) {
54
+ if (isLlama(probe)) {
55
+ const devices = probe.result.devices.filter((device) => device.name !== HOST_ROW);
56
+ return breakdown(sumOf(...devices.map((device) => device.modelBytes)), sumOf(...devices.map((device) => device.contextBytes)), sumOf(...devices.map((device) => device.computeBytes)));
57
+ }
58
+ // Whisper names its parts `kvBytes` and `computeBytes`, parakeet splits
59
+ // compute across encoder and decoder; an engine fills one set or the other.
60
+ if (probe.engine === 'asr-ggml') {
61
+ const result = probe.result;
62
+ // A VAD model's weights are inside `deviceBytes`, so they belong in the
63
+ // weights the breakdown divides it into.
64
+ return breakdown(sumOf(result.weightsBytes, result.vadBytes), sumOf(result.kvBytes, result.decoderStateBytes), sumOf(result.computeBytes, result.encoderComputeBytes, result.decoderComputeBytes));
65
+ }
66
+ if (probe.engine === 'bci-whispercpp') {
67
+ const result = probe.result;
68
+ return breakdown(result.weightsBytes, result.kvBytes, result.computeBytes);
69
+ }
70
+ if (probe.engine === 'tts-ggml') {
71
+ const result = probe.result;
72
+ return breakdown(result.weightsBytes, result.stateBytes, sumOf(result.lmComputeBytes, result.codecComputeBytes));
73
+ }
74
+ return {};
75
+ }
76
+ function llamaProjection(probe) {
77
+ const first = probe.result.devices.find((device) => device.name !== HOST_ROW && device.name !== CPU_DEVICE);
78
+ return {
79
+ deviceBytes: probe.result.deviceBytes,
80
+ hostBytes: probe.result.hostBytes,
81
+ ...breakdownFor(probe),
82
+ ...(first !== undefined && {
83
+ deviceName: first.name,
84
+ deviceFreeBytes: first.freeBytes,
85
+ deviceTotalBytes: first.totalBytes
86
+ })
87
+ };
88
+ }
89
+ function projectionFor(probe) {
90
+ if (probe.engine === 'diffusion-cpp') {
91
+ return { deviceName: probe.result.backend, report: probe.result.report };
92
+ }
93
+ if (isLlama(probe))
94
+ return llamaProjection(probe);
95
+ const result = probe.result;
96
+ return {
97
+ deviceName: result.deviceName,
98
+ deviceBytes: result.deviceBytes,
99
+ hostBytes: result.hostBytes,
100
+ deviceFreeBytes: result.deviceFreeBytes,
101
+ deviceTotalBytes: result.deviceTotalBytes,
102
+ report: result.report,
103
+ ...breakdownFor(probe)
104
+ };
105
+ }
106
+ /**
107
+ * Only the llama fitters resolve a placement; the other engines' loads are not
108
+ * layer-sliced across devices.
109
+ *
110
+ * `nGpuDevices` counts the devices the weights were actually placed on, less
111
+ * the CPU backend device, which the fitter assigns to like any other but whose
112
+ * demand is host memory.
113
+ */
114
+ function planFor(probe) {
115
+ if (!isLlama(probe))
116
+ return undefined;
117
+ const assigned = probe.result.devices.filter((device) => device.name !== HOST_ROW && device.name !== CPU_DEVICE && device.modelBytes > 0);
118
+ return {
119
+ nCtx: probe.result.ctxSize,
120
+ nGpuLayers: probe.result.gpuLayers,
121
+ nGpuDevices: assigned.length
122
+ };
123
+ }
124
+ export function classifyFit(probe, provenance) {
125
+ const verdict = verdictFor(probe.result.status);
126
+ const plan = verdict === 'fit' ? planFor(probe) : undefined;
127
+ return {
128
+ ...provenance,
129
+ engine: probe.engine,
130
+ verdict,
131
+ reason: reasonFor(probe),
132
+ ...(plan !== undefined && { plan }),
133
+ ...(verdict !== 'unknown' && { projection: projectionFor(probe) })
134
+ };
135
+ }
@@ -0,0 +1,28 @@
1
+ import { type CanonicalModelType } from '../../../schemas/index.js';
2
+ import type { FitProbeRequest } from './engine-fit.js';
3
+ export type FitRequestPlan = {
4
+ supported: true;
5
+ probe: FitProbeRequest;
6
+ } | {
7
+ supported: false;
8
+ detail: string;
9
+ };
10
+ export interface CreateFitRequestParams {
11
+ modelType: CanonicalModelType;
12
+ modelPath: string;
13
+ modelConfig: unknown;
14
+ artifacts?: Record<string, string> | undefined;
15
+ isShardedModel: boolean;
16
+ /** Free memory that must remain for the projection to count as fitting. */
17
+ marginBytes?: number | undefined;
18
+ }
19
+ /**
20
+ * Builds a fit request from the same resolved config and artifacts the real
21
+ * load is about to use, or explains why this load cannot be answered for.
22
+ *
23
+ * Structural shapes the SDK owns — a load with no fitter behind it, a companion
24
+ * set that did not resolve, a setting whose effect on the projection is unknown
25
+ * — are refused here so no engine is asked a question it cannot answer.
26
+ * Value-level policy stays inside each engine's own fitter.
27
+ */
28
+ export declare function createFitRequest(params: CreateFitRequestParams): FitRequestPlan;
@@ -0,0 +1,51 @@
1
+ import { ModelType } from '../../../schemas/index.js';
2
+ import { createBciFitRequest, createParakeetFitRequest, createWhisperFitRequest } from './engines/asr.js';
3
+ import { createAudiogenFitRequest } from './engines/audiogen.js';
4
+ import { createDiffusionFitRequest } from './engines/diffusion.js';
5
+ import { createLlamaFitRequest, llamaLoadKindFor } from './engines/llama.js';
6
+ import { createTtsFitRequest } from './engines/tts.js';
7
+ /**
8
+ * Builds a fit request from the same resolved config and artifacts the real
9
+ * load is about to use, or explains why this load cannot be answered for.
10
+ *
11
+ * Structural shapes the SDK owns — a load with no fitter behind it, a companion
12
+ * set that did not resolve, a setting whose effect on the projection is unknown
13
+ * — are refused here so no engine is asked a question it cannot answer.
14
+ * Value-level policy stays inside each engine's own fitter.
15
+ */
16
+ export function createFitRequest(params) {
17
+ const { modelType, modelPath, modelConfig, artifacts, marginBytes } = params;
18
+ const loadKind = llamaLoadKindFor(modelType);
19
+ if (loadKind !== undefined) {
20
+ return createLlamaFitRequest({
21
+ loadKind,
22
+ modelPath,
23
+ modelConfig,
24
+ artifacts,
25
+ isShardedModel: params.isShardedModel,
26
+ marginBytes
27
+ });
28
+ }
29
+ // Every engine below reads whole files rather than a shard set, so a split
30
+ // model is not a shape any of them can be pointed at.
31
+ if (params.isShardedModel) {
32
+ return { supported: false, detail: 'sharded models are not representable' };
33
+ }
34
+ const common = { modelPath, modelConfig, artifacts, marginBytes };
35
+ switch (modelType) {
36
+ case ModelType.whispercppTranscription:
37
+ return createWhisperFitRequest(common);
38
+ case ModelType.parakeetTranscription:
39
+ return createParakeetFitRequest(common);
40
+ case ModelType.bciWhispercppTranscription:
41
+ return createBciFitRequest(common);
42
+ case ModelType.ttsGgml:
43
+ return createTtsFitRequest(common);
44
+ case ModelType.audiogenGgml:
45
+ return createAudiogenFitRequest(common);
46
+ case ModelType.sdcppGeneration:
47
+ return createDiffusionFitRequest({ modelPath, modelConfig, artifacts });
48
+ default:
49
+ return { supported: false, detail: `no fitter for model type: ${modelType}` };
50
+ }
51
+ }
@@ -0,0 +1,59 @@
1
+ import type { AsrFitRequest, AsrFitResult } from '@qvac/asr-ggml';
2
+ import type { AudiogenFitRequest, AudiogenFitResult } from '@qvac/audiogen-ggml';
3
+ import type { BciFitRequest, BciFitResult } from '@qvac/bci-whispercpp';
4
+ import type { DiffusionFitRequest, DiffusionFitResult } from '@qvac/diffusion-cpp';
5
+ import type { EmbedFitRequest, EmbedFitResult } from '@qvac/embed-llamacpp';
6
+ import type LlmLlamacpp from '@qvac/llm-llamacpp';
7
+ import type { TtsFitRequest, TtsFitResult } from '@qvac/tts-ggml';
8
+ /**
9
+ * The engine packages each expose their own `assessFit`, taking the same fields
10
+ * their loader takes and returning the figures their fitter measures. Every
11
+ * member is plain JSON, so a request can cross a process boundary unchanged.
12
+ */
13
+ export type FitProbeRequest = {
14
+ engine: 'llm-llamacpp';
15
+ request: LlmLlamacpp.FitRequest;
16
+ } | {
17
+ engine: 'embed-llamacpp';
18
+ request: EmbedFitRequest;
19
+ } | {
20
+ engine: 'asr-ggml';
21
+ request: AsrFitRequest;
22
+ } | {
23
+ engine: 'bci-whispercpp';
24
+ request: BciFitRequest;
25
+ } | {
26
+ engine: 'tts-ggml';
27
+ request: TtsFitRequest;
28
+ } | {
29
+ engine: 'audiogen-ggml';
30
+ request: AudiogenFitRequest;
31
+ } | {
32
+ engine: 'diffusion-cpp';
33
+ request: DiffusionFitRequest;
34
+ };
35
+ export type FitProbeEngine = FitProbeRequest['engine'];
36
+ export type FitProbeResult = {
37
+ engine: 'llm-llamacpp';
38
+ result: LlmLlamacpp.FitResult;
39
+ } | {
40
+ engine: 'embed-llamacpp';
41
+ result: EmbedFitResult;
42
+ } | {
43
+ engine: 'asr-ggml';
44
+ result: AsrFitResult;
45
+ } | {
46
+ engine: 'bci-whispercpp';
47
+ result: BciFitResult;
48
+ } | {
49
+ engine: 'tts-ggml';
50
+ result: TtsFitResult;
51
+ } | {
52
+ engine: 'audiogen-ggml';
53
+ result: AudiogenFitResult;
54
+ } | {
55
+ engine: 'diffusion-cpp';
56
+ result: DiffusionFitResult;
57
+ };
58
+ /** Calls one engine's fitter, taken from the plugin that owns its addon. */
59
+ export declare function callEngineFit(probe: FitProbeRequest): Promise<FitProbeResult>;
@@ -0,0 +1,13 @@
1
+ /** Calls one engine's fitter, taken from the plugin that owns its addon. */
2
+ export async function callEngineFit(probe) {
3
+ const { getAllPlugins } = await import('../../../plugins/registry.js');
4
+ const addonPackage = `@qvac/${probe.engine}`;
5
+ const plugin = getAllPlugins().find((candidate) => candidate.addonPackage === addonPackage);
6
+ if (plugin?.assessFit === undefined) {
7
+ throw new TypeError(`no registered plugin exposes assessFit for ${addonPackage}`);
8
+ }
9
+ return {
10
+ engine: probe.engine,
11
+ result: plugin.assessFit(probe.request)
12
+ };
13
+ }
@@ -0,0 +1,10 @@
1
+ import type { FitRequestPlan } from '../create-fit-request.js';
2
+ export interface AsrFitRequestParams {
3
+ modelPath: string;
4
+ modelConfig: unknown;
5
+ artifacts?: Record<string, string> | undefined;
6
+ marginBytes?: number | undefined;
7
+ }
8
+ export declare function createWhisperFitRequest(params: AsrFitRequestParams): FitRequestPlan;
9
+ export declare function createParakeetFitRequest(params: AsrFitRequestParams): FitRequestPlan;
10
+ export declare function createBciFitRequest(params: AsrFitRequestParams): FitRequestPlan;