@qvac/inference 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +107 -33
- package/README.md +8 -5
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +18 -6
- package/dist/handlers/assess-model-fit.js +215 -35
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +80 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +46 -2
- package/dist/resources/model-fit/assess.js +67 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +102 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +212 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +284 -77
- package/dist/schemas/assess-model-fit.js +93 -55
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +765 -56
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +236 -30
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +105 -14
- package/dist/schemas/llamacpp-config.js +113 -3
- package/dist/schemas/load-model.d.ts +3102 -627
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +205 -5
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +44 -30
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -171
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { getEngineLogger } from '../../../logging/index.js';
|
|
2
|
-
import {
|
|
2
|
+
import { classifyFit } from './classify-fit.js';
|
|
3
|
+
import { createFitRequest } from './create-fit-request.js';
|
|
4
|
+
import { runFit as runFitDefault } from './run-fit.js';
|
|
3
5
|
/**
|
|
4
6
|
* Shorter than the supervisor's own 60s default: this check sits in front of a
|
|
5
7
|
* real load, so a wedged child must not hold the load for a full minute. Every
|
|
@@ -7,30 +9,26 @@ import { createLlamaFitRequest } from './create-llama-fit-request.js';
|
|
|
7
9
|
*/
|
|
8
10
|
const ADVISORY_FIT_TIMEOUT_MS = 30_000;
|
|
9
11
|
/**
|
|
10
|
-
*
|
|
11
|
-
* resident-model reserve
|
|
12
|
-
* replaces the
|
|
12
|
+
* The base every engine's fitter withholds by default. Made explicit here
|
|
13
|
+
* because the resident-model reserve is added to it: setting a margin at all
|
|
14
|
+
* replaces the engine default, so the base has to travel with it. It is also
|
|
15
|
+
* the headroom `withMachineBudget` leaves on the system budget.
|
|
13
16
|
*/
|
|
14
17
|
const ADVISORY_FIT_BASE_MARGIN_MIB = 1024;
|
|
15
18
|
/**
|
|
16
|
-
* Names the headroom policy
|
|
17
|
-
* `ADVISORY_FIT_BASE_MARGIN_MIB
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
* mobile), which `assessModelFit` applies. That policy is applied by the SDK to
|
|
22
|
-
* a budget the SDK computed; here the child owns the budget (total − wired −
|
|
23
|
-
* compressor, measured inside the disposable process) and `marginMiB` is the
|
|
24
|
-
* only lever this side has. Reconciling the two means one entry point owning
|
|
25
|
-
* both bases — see the precedence note on `nativeProbeFitSchema`.
|
|
19
|
+
* Names the headroom policy: the fitter withholds
|
|
20
|
+
* `ADVISORY_FIT_BASE_MARGIN_MIB` plus the on-disk bytes of every model resident
|
|
21
|
+
* in this worker, and a `fit` is then judged against the budget
|
|
22
|
+
* `assessModelFit` reports, under that platform's basis and less the
|
|
23
|
+
* `interactive-v1` reserve.
|
|
26
24
|
*/
|
|
27
|
-
const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-
|
|
25
|
+
const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-v2';
|
|
28
26
|
const BYTES_PER_MIB = 1024 * 1024;
|
|
29
27
|
const DISABLED_VALUES = new Set(['0', 'false', 'off', 'no']);
|
|
30
28
|
/**
|
|
31
29
|
* On by default; `QVAC_ADVISORY_MODEL_FIT=0` (or `false`/`off`/`no`) is the
|
|
32
30
|
* operator opt-out — the escape hatch for a load-heavy startup path or a
|
|
33
|
-
* runtime
|
|
31
|
+
* runtime whose operator would rather not pay for it. Any other value, including
|
|
34
32
|
* unset, leaves the check on.
|
|
35
33
|
*
|
|
36
34
|
* The worker environment and the mobile runtime flag are imported lazily. Both
|
|
@@ -52,19 +50,9 @@ async function resolveMobile(explicit) {
|
|
|
52
50
|
}
|
|
53
51
|
/**
|
|
54
52
|
* Sums the on-disk weight sizes of every model currently registered in this
|
|
55
|
-
* worker.
|
|
56
|
-
*
|
|
57
|
-
*
|
|
58
|
-
* still cannot see is the unwired remainder — mmap'd host-side layers and KV of
|
|
59
|
-
* resident models — which the OS reports as evictable right up until a decode
|
|
60
|
-
* needs it resident. Measured consequence on 0.7.0 (where the child saw a fully
|
|
61
|
-
* idle device): a verdict correct on an idle machine admitted a load that could
|
|
62
|
-
* not decode with another model loaded.
|
|
63
|
-
*
|
|
64
|
-
* Reserving the resident weight bytes through `marginMiB` keeps that footprint
|
|
65
|
-
* in the child's budget. Where those weights are wired this now double-counts
|
|
66
|
-
* and the verdict turns conservative — deliberately so: the measured failure
|
|
67
|
-
* modes punish optimism (loads that cannot decode), not caution.
|
|
53
|
+
* worker. The fallback for a host whose memory sample is unavailable, where
|
|
54
|
+
* models this worker loaded are the one part of the machine's usage it can
|
|
55
|
+
* still account for.
|
|
68
56
|
*
|
|
69
57
|
* Advisory and fail-open like everything else here: any failure to stat a file
|
|
70
58
|
* contributes zero rather than an error.
|
|
@@ -89,20 +77,6 @@ async function defaultResidentModelBytes() {
|
|
|
89
77
|
}
|
|
90
78
|
return bytes;
|
|
91
79
|
}
|
|
92
|
-
/**
|
|
93
|
-
* Also lazy: the desktop supervisor pulls the Bare process launcher, and a
|
|
94
|
-
* disabled check must not load it. Mobile uses in-process `fitParamsAsync` instead
|
|
95
|
-
* of spawn (`bare-runtime/spawn` stays deferred from the mobile pack).
|
|
96
|
-
*/
|
|
97
|
-
async function resolveRunFit(explicit, mobile) {
|
|
98
|
-
if (explicit !== undefined)
|
|
99
|
-
return explicit;
|
|
100
|
-
if (mobile) {
|
|
101
|
-
const { runInProcessFit } = await import('./run-in-process-fit.js');
|
|
102
|
-
return runInProcessFit;
|
|
103
|
-
}
|
|
104
|
-
return (await import('./run-isolated-fit.js')).runIsolatedFit;
|
|
105
|
-
}
|
|
106
80
|
const PROVENANCE = {
|
|
107
81
|
basis: 'native-probe',
|
|
108
82
|
estimatorVersion: NATIVE_PROBE_ESTIMATOR_VERSION
|
|
@@ -112,44 +86,116 @@ function unknown(reason, message) {
|
|
|
112
86
|
? { ...PROVENANCE, verdict: 'unknown', reason }
|
|
113
87
|
: { ...PROVENANCE, verdict: 'unknown', reason, message };
|
|
114
88
|
}
|
|
115
|
-
function
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
89
|
+
function mib(bytes) {
|
|
90
|
+
return `${Math.round(bytes / BYTES_PER_MIB)} MiB`;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* What the operating system would give back, which no backend reports. Metal
|
|
94
|
+
* answers with its working-set allowance minus this process's own allocations,
|
|
95
|
+
* so another application's resident pages read as free, and the host figure
|
|
96
|
+
* counts little beyond wired and compressed memory.
|
|
97
|
+
*/
|
|
98
|
+
async function defaultAvailableSystemBytes() {
|
|
99
|
+
const [{ getResourceCollector }, { resolveBasis, resolveBudget }, { detectPlatform }] = await Promise.all([
|
|
100
|
+
import('../../instance.js'),
|
|
101
|
+
import('../assess.js'),
|
|
102
|
+
import('../platform.js')
|
|
103
|
+
]);
|
|
104
|
+
const collector = getResourceCollector();
|
|
105
|
+
if (!collector)
|
|
106
|
+
return undefined;
|
|
107
|
+
const platform = detectPlatform();
|
|
108
|
+
const resources = { capabilities: collector.getCapabilities(), sample: collector.sample() };
|
|
109
|
+
const budget = resolveBudget(resources, platform, resolveBasis(platform), []);
|
|
110
|
+
if (budget === undefined)
|
|
111
|
+
return undefined;
|
|
112
|
+
return Math.max(0, budget.availableAfterReserveBytes);
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Whether the device's allocations come out of the pool the system budget
|
|
116
|
+
* measures. Host bytes always do; device bytes do only where every allocation
|
|
117
|
+
* is system RAM, which is what `boundBySystemMemory` decides.
|
|
118
|
+
*/
|
|
119
|
+
async function deviceBytesCountAgainstSystem() {
|
|
120
|
+
const [{ getResourceCollector }, { boundBySystemMemory }, { detectPlatform }] = await Promise.all([
|
|
121
|
+
import('../../instance.js'),
|
|
122
|
+
import('../assess.js'),
|
|
123
|
+
import('../platform.js')
|
|
124
|
+
]);
|
|
125
|
+
const collector = getResourceCollector();
|
|
126
|
+
// Unknown counts them: admitting a load that cannot decode is the failure
|
|
127
|
+
// this check exists to stop, and refusing one that would have fitted is
|
|
128
|
+
// advisory only.
|
|
129
|
+
if (!collector)
|
|
130
|
+
return true;
|
|
131
|
+
const resources = { capabilities: collector.getCapabilities(), sample: collector.sample() };
|
|
132
|
+
return boundBySystemMemory(resources, detectPlatform());
|
|
133
|
+
}
|
|
134
|
+
/**
|
|
135
|
+
* Peak the projection places on the pool the system budget measures. A
|
|
136
|
+
* discrete card's memory is not that pool, so charging its bytes here would
|
|
137
|
+
* refuse a load the machine can hold.
|
|
138
|
+
*/
|
|
139
|
+
function projectedDemandBytes(outcome, countDevice) {
|
|
140
|
+
const projection = outcome.projection;
|
|
141
|
+
if (projection === undefined)
|
|
142
|
+
return undefined;
|
|
143
|
+
if (projection.deviceBytes === undefined && projection.hostBytes === undefined)
|
|
144
|
+
return undefined;
|
|
145
|
+
return (countDevice ? (projection.deviceBytes ?? 0) : 0) + (projection.hostBytes ?? 0);
|
|
146
|
+
}
|
|
147
|
+
/**
|
|
148
|
+
* Judges a `fit` against the machine. The fitter decides placement and reports
|
|
149
|
+
* what the load costs; only this side knows what is free, so only this side can
|
|
150
|
+
* refuse on capacity.
|
|
151
|
+
*
|
|
152
|
+
* The margin cannot carry this. It is subtracted from each device's own free,
|
|
153
|
+
* and a device whose total sits below the machine's — Metal caps its allowance
|
|
154
|
+
* near 74% of system memory — would be charged the shortfall twice.
|
|
155
|
+
*
|
|
156
|
+
* A projection with no byte totals leaves the engine's verdict standing.
|
|
157
|
+
*/
|
|
158
|
+
function withMachineBudget(outcome, availableBytes, countDevice) {
|
|
159
|
+
if (outcome.verdict !== 'fit' || availableBytes === undefined)
|
|
160
|
+
return outcome;
|
|
161
|
+
const demand = projectedDemandBytes(outcome, countDevice);
|
|
162
|
+
if (demand === undefined)
|
|
163
|
+
return outcome;
|
|
164
|
+
if (demand <= availableBytes)
|
|
165
|
+
return outcome;
|
|
166
|
+
return {
|
|
167
|
+
...outcome,
|
|
168
|
+
verdict: 'does-not-fit',
|
|
169
|
+
reason: 'exceeds-available-memory',
|
|
170
|
+
message: `needs ${mib(demand)} against a ${mib(availableBytes)} budget`
|
|
171
|
+
};
|
|
172
|
+
}
|
|
173
|
+
/** The device and host totals, for the engines that measure them. */
|
|
174
|
+
function footprint(outcome) {
|
|
175
|
+
const projection = outcome.projection;
|
|
176
|
+
if (projection?.deviceBytes === undefined)
|
|
177
|
+
return '';
|
|
178
|
+
const host = projection.hostBytes === undefined ? '' : `, host ${mib(projection.hostBytes)}`;
|
|
179
|
+
return ` — device ${mib(projection.deviceBytes)}${host}`;
|
|
134
180
|
}
|
|
135
181
|
function report(logger, input, outcome) {
|
|
136
182
|
const prefix = `[advisory-fit:${input.modelType}:${input.modelId}]`;
|
|
137
183
|
if (outcome.verdict === 'fit') {
|
|
138
184
|
const plan = outcome.plan;
|
|
139
185
|
logger.info(`${prefix} projected to fit (advisory only)${plan === undefined
|
|
140
|
-
?
|
|
186
|
+
? footprint(outcome)
|
|
141
187
|
: ` — nCtx ${plan.nCtx}, nGpuLayers ${plan.nGpuLayers} across ${plan.nGpuDevices} GPU device(s)`}`);
|
|
142
188
|
return;
|
|
143
189
|
}
|
|
144
190
|
if (outcome.verdict === 'does-not-fit') {
|
|
145
|
-
logger.warn(`${prefix} projected not to fit (advisory only — the load continues unchanged)`);
|
|
191
|
+
logger.warn(`${prefix} projected not to fit (advisory only — the load continues unchanged)${footprint(outcome)}: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`);
|
|
146
192
|
return;
|
|
147
193
|
}
|
|
148
|
-
// `unsupported-load` covers every load this check refuses up front —
|
|
149
|
-
//
|
|
150
|
-
//
|
|
151
|
-
// should have) and produced no verdict, because there "why was
|
|
152
|
-
// evidence" is the most useful thing this check can report.
|
|
194
|
+
// `unsupported-load` covers every load this check refuses up front — a model
|
|
195
|
+
// type with no fitter, a companion set that did not resolve — so at `info` it
|
|
196
|
+
// would tag those loads on every start. `debug` for those; `info` where a
|
|
197
|
+
// fitter ran (or should have) and produced no verdict, because there "why was
|
|
198
|
+
// there no evidence" is the most useful thing this check can report.
|
|
153
199
|
const line = `${prefix} no fit evidence: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`;
|
|
154
200
|
if (outcome.reason === 'unsupported-load') {
|
|
155
201
|
logger.debug(line);
|
|
@@ -158,7 +204,8 @@ function report(logger, input, outcome) {
|
|
|
158
204
|
logger.info(line);
|
|
159
205
|
}
|
|
160
206
|
/**
|
|
161
|
-
* Runs the advisory
|
|
207
|
+
* Runs the advisory fit check for a load that is about to start, against the
|
|
208
|
+
* fitter belonging to the engine that will run it.
|
|
162
209
|
*
|
|
163
210
|
* Fail-open by construction: an unsupported shape, a crashed or wedged child, a
|
|
164
211
|
* malformed response, and an unexpected internal error all resolve to `unknown`
|
|
@@ -172,30 +219,32 @@ export async function runAdvisoryFitCheck(input, options = {}) {
|
|
|
172
219
|
if (!(await resolveEnabled(options.enabled)))
|
|
173
220
|
return unknown('disabled');
|
|
174
221
|
const mobile = await resolveMobile(options.mobile);
|
|
175
|
-
const
|
|
222
|
+
const availableBytes = await (options.availableSystemBytes ?? defaultAvailableSystemBytes)();
|
|
223
|
+
const residentBytes = await (options.residentModelBytes ?? defaultResidentModelBytes)();
|
|
224
|
+
const residentReserveMiB = Math.ceil(residentBytes / BYTES_PER_MIB);
|
|
225
|
+
// Always sent, even with a zero reserve: relying on the engine default for
|
|
226
|
+
// the base margin would leave two sources of truth that match today and
|
|
227
|
+
// diverge silently if an engine default moves.
|
|
228
|
+
const plan = createFitRequest({
|
|
176
229
|
modelType: input.modelType,
|
|
177
230
|
modelPath: input.modelPath,
|
|
178
231
|
modelConfig: input.modelConfig,
|
|
179
232
|
artifacts: input.artifacts,
|
|
180
|
-
isShardedModel: input.isShardedModel
|
|
233
|
+
isShardedModel: input.isShardedModel,
|
|
234
|
+
marginBytes: (ADVISORY_FIT_BASE_MARGIN_MIB + residentReserveMiB) * BYTES_PER_MIB
|
|
181
235
|
});
|
|
182
236
|
if (!plan.supported) {
|
|
183
237
|
const outcome = unknown('unsupported-load', plan.detail);
|
|
184
238
|
report(logger, input, outcome);
|
|
185
239
|
return outcome;
|
|
186
240
|
}
|
|
187
|
-
const
|
|
188
|
-
|
|
189
|
-
const runFit = await resolveRunFit(options.runFit, mobile);
|
|
190
|
-
// Always sent, even with a zero reserve: relying on the addon default for
|
|
191
|
-
// the base margin would leave two sources of truth that match today and
|
|
192
|
-
// diverge silently if the addon default moves.
|
|
193
|
-
const result = await runFit(plan.loadKind, { ...plan.config, marginMiB: ADVISORY_FIT_BASE_MARGIN_MIB + residentReserveMiB }, {
|
|
241
|
+
const result = await (options.runFit ?? runFitDefault)(plan.probe, {
|
|
242
|
+
mobile,
|
|
194
243
|
timeoutMs: options.timeoutMs ?? ADVISORY_FIT_TIMEOUT_MS,
|
|
195
244
|
...(options.signal !== undefined && { signal: options.signal })
|
|
196
245
|
});
|
|
197
246
|
const outcome = result.status === 'completed'
|
|
198
|
-
?
|
|
247
|
+
? withMachineBudget(classifyFit(result.probe, PROVENANCE), availableBytes, await (options.countsDeviceRows ?? deviceBytesCountAgainstSystem)())
|
|
199
248
|
: unknown(result.reason, result.message);
|
|
200
249
|
report(logger, input, outcome);
|
|
201
250
|
return outcome;
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Loads the one addon a probe names and asks it for a projection. The child
|
|
3
|
+
* registers no plugins, so the fitter cannot come off the registry here. Only
|
|
4
|
+
* the child reaches this module, and the supervisor spawns it by path rather
|
|
5
|
+
* than importing it, so these specifiers stay out of the graph a host bundles.
|
|
6
|
+
*/
|
|
7
|
+
async function assessFitOf(probe) {
|
|
8
|
+
switch (probe.engine) {
|
|
9
|
+
case 'llm-llamacpp': {
|
|
10
|
+
const { default: LlmLlamacpp } = await import('@qvac/llm-llamacpp');
|
|
11
|
+
return LlmLlamacpp.assessFit(probe.request);
|
|
12
|
+
}
|
|
13
|
+
case 'embed-llamacpp': {
|
|
14
|
+
const { assessFit } = await import('@qvac/embed-llamacpp');
|
|
15
|
+
return assessFit(probe.request);
|
|
16
|
+
}
|
|
17
|
+
case 'asr-ggml': {
|
|
18
|
+
const { default: ASRGgml } = await import('@qvac/asr-ggml');
|
|
19
|
+
return ASRGgml.assessFit(probe.request);
|
|
20
|
+
}
|
|
21
|
+
case 'bci-whispercpp': {
|
|
22
|
+
const { assessFit } = await import('@qvac/bci-whispercpp');
|
|
23
|
+
return assessFit(probe.request);
|
|
24
|
+
}
|
|
25
|
+
case 'tts-ggml': {
|
|
26
|
+
const { default: TTSGgml } = await import('@qvac/tts-ggml');
|
|
27
|
+
return TTSGgml.assessFit(probe.request);
|
|
28
|
+
}
|
|
29
|
+
case 'audiogen-ggml': {
|
|
30
|
+
const { assessFit } = await import('@qvac/audiogen-ggml');
|
|
31
|
+
return assessFit(probe.request);
|
|
32
|
+
}
|
|
33
|
+
case 'diffusion-cpp': {
|
|
34
|
+
const { assessFit } = await import('@qvac/diffusion-cpp');
|
|
35
|
+
return assessFit(probe.request);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
export async function callChildEngineFit(probe) {
|
|
40
|
+
return { engine: probe.engine, result: await assessFitOf(probe) };
|
|
41
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { NativeProbeFit } from '../../../schemas/index.js';
|
|
2
|
+
import type { FitProbeResult } from './engine-fit.js';
|
|
3
|
+
export declare function classifyFit(probe: FitProbeResult, provenance: {
|
|
4
|
+
basis: 'native-probe';
|
|
5
|
+
estimatorVersion: string;
|
|
6
|
+
}): NativeProbeFit;
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/** The llama fitters append a host row after the devices they assigned to. */
|
|
2
|
+
const HOST_ROW = 'host';
|
|
3
|
+
/** ggml's own name for the CPU backend device, which the fitter may assign to. */
|
|
4
|
+
const CPU_DEVICE = 'CPU';
|
|
5
|
+
function isLlama(probe) {
|
|
6
|
+
return probe.engine === 'llm-llamacpp' || probe.engine === 'embed-llamacpp';
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* `error` is the engine reporting it could not read the model or find a backend
|
|
10
|
+
* device: absence of evidence, not evidence of insufficiency.
|
|
11
|
+
*/
|
|
12
|
+
function verdictFor(status) {
|
|
13
|
+
if (status === 'fits')
|
|
14
|
+
return 'fit';
|
|
15
|
+
if (status === 'does-not-fit')
|
|
16
|
+
return 'does-not-fit';
|
|
17
|
+
return 'unknown';
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* The diffusion fitter reports a placement rather than a reason string.
|
|
21
|
+
* `changed` means it reached a fitting arrangement only by moving modules
|
|
22
|
+
* between backends, which is why the status is already `does-not-fit`.
|
|
23
|
+
*/
|
|
24
|
+
function diffusionReason(result) {
|
|
25
|
+
if (result.status === 'fits')
|
|
26
|
+
return 'fits';
|
|
27
|
+
if (result.status === 'error')
|
|
28
|
+
return 'model-unreadable';
|
|
29
|
+
return result.changed ? `does-not-fit-as-configured (${result.backend})` : 'does-not-fit';
|
|
30
|
+
}
|
|
31
|
+
function reasonFor(probe) {
|
|
32
|
+
return probe.engine === 'diffusion-cpp' ? diffusionReason(probe.result) : probe.result.reason;
|
|
33
|
+
}
|
|
34
|
+
function sumOf(...values) {
|
|
35
|
+
const present = values.filter((value) => value !== undefined);
|
|
36
|
+
return present.length === 0 ? undefined : present.reduce((total, value) => total + value, 0);
|
|
37
|
+
}
|
|
38
|
+
function breakdown(weights, context, compute) {
|
|
39
|
+
return {
|
|
40
|
+
...(weights !== undefined && { weightsBytes: weights }),
|
|
41
|
+
...(context !== undefined && { contextBytes: context }),
|
|
42
|
+
...(compute !== undefined && { computeBytes: compute })
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* The parts of `deviceBytes` the engine separates. Audiogen reports a peak
|
|
47
|
+
* across pipeline phases and diffusion a per-module table, neither of which
|
|
48
|
+
* divides into these three, so both come back empty.
|
|
49
|
+
*
|
|
50
|
+
* The llama rows are summed over the same devices the engine's own
|
|
51
|
+
* `deviceBytes` covers, which is every row but `host`.
|
|
52
|
+
*/
|
|
53
|
+
function breakdownFor(probe) {
|
|
54
|
+
if (isLlama(probe)) {
|
|
55
|
+
const devices = probe.result.devices.filter((device) => device.name !== HOST_ROW);
|
|
56
|
+
return breakdown(sumOf(...devices.map((device) => device.modelBytes)), sumOf(...devices.map((device) => device.contextBytes)), sumOf(...devices.map((device) => device.computeBytes)));
|
|
57
|
+
}
|
|
58
|
+
// Whisper names its parts `kvBytes` and `computeBytes`, parakeet splits
|
|
59
|
+
// compute across encoder and decoder; an engine fills one set or the other.
|
|
60
|
+
if (probe.engine === 'asr-ggml') {
|
|
61
|
+
const result = probe.result;
|
|
62
|
+
// A VAD model's weights are inside `deviceBytes`, so they belong in the
|
|
63
|
+
// weights the breakdown divides it into.
|
|
64
|
+
return breakdown(sumOf(result.weightsBytes, result.vadBytes), sumOf(result.kvBytes, result.decoderStateBytes), sumOf(result.computeBytes, result.encoderComputeBytes, result.decoderComputeBytes));
|
|
65
|
+
}
|
|
66
|
+
if (probe.engine === 'bci-whispercpp') {
|
|
67
|
+
const result = probe.result;
|
|
68
|
+
return breakdown(result.weightsBytes, result.kvBytes, result.computeBytes);
|
|
69
|
+
}
|
|
70
|
+
if (probe.engine === 'tts-ggml') {
|
|
71
|
+
const result = probe.result;
|
|
72
|
+
return breakdown(result.weightsBytes, result.stateBytes, sumOf(result.lmComputeBytes, result.codecComputeBytes));
|
|
73
|
+
}
|
|
74
|
+
return {};
|
|
75
|
+
}
|
|
76
|
+
function llamaProjection(probe) {
|
|
77
|
+
const first = probe.result.devices.find((device) => device.name !== HOST_ROW && device.name !== CPU_DEVICE);
|
|
78
|
+
return {
|
|
79
|
+
deviceBytes: probe.result.deviceBytes,
|
|
80
|
+
hostBytes: probe.result.hostBytes,
|
|
81
|
+
...breakdownFor(probe),
|
|
82
|
+
...(first !== undefined && {
|
|
83
|
+
deviceName: first.name,
|
|
84
|
+
deviceFreeBytes: first.freeBytes,
|
|
85
|
+
deviceTotalBytes: first.totalBytes
|
|
86
|
+
})
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
function projectionFor(probe) {
|
|
90
|
+
if (probe.engine === 'diffusion-cpp') {
|
|
91
|
+
return { deviceName: probe.result.backend, report: probe.result.report };
|
|
92
|
+
}
|
|
93
|
+
if (isLlama(probe))
|
|
94
|
+
return llamaProjection(probe);
|
|
95
|
+
const result = probe.result;
|
|
96
|
+
return {
|
|
97
|
+
deviceName: result.deviceName,
|
|
98
|
+
deviceBytes: result.deviceBytes,
|
|
99
|
+
hostBytes: result.hostBytes,
|
|
100
|
+
deviceFreeBytes: result.deviceFreeBytes,
|
|
101
|
+
deviceTotalBytes: result.deviceTotalBytes,
|
|
102
|
+
report: result.report,
|
|
103
|
+
...breakdownFor(probe)
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Only the llama fitters resolve a placement; the other engines' loads are not
|
|
108
|
+
* layer-sliced across devices.
|
|
109
|
+
*
|
|
110
|
+
* `nGpuDevices` counts the devices the weights were actually placed on, less
|
|
111
|
+
* the CPU backend device, which the fitter assigns to like any other but whose
|
|
112
|
+
* demand is host memory.
|
|
113
|
+
*/
|
|
114
|
+
function planFor(probe) {
|
|
115
|
+
if (!isLlama(probe))
|
|
116
|
+
return undefined;
|
|
117
|
+
const assigned = probe.result.devices.filter((device) => device.name !== HOST_ROW && device.name !== CPU_DEVICE && device.modelBytes > 0);
|
|
118
|
+
return {
|
|
119
|
+
nCtx: probe.result.ctxSize,
|
|
120
|
+
nGpuLayers: probe.result.gpuLayers,
|
|
121
|
+
nGpuDevices: assigned.length
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
export function classifyFit(probe, provenance) {
|
|
125
|
+
const verdict = verdictFor(probe.result.status);
|
|
126
|
+
const plan = verdict === 'fit' ? planFor(probe) : undefined;
|
|
127
|
+
return {
|
|
128
|
+
...provenance,
|
|
129
|
+
engine: probe.engine,
|
|
130
|
+
verdict,
|
|
131
|
+
reason: reasonFor(probe),
|
|
132
|
+
...(plan !== undefined && { plan }),
|
|
133
|
+
...(verdict !== 'unknown' && { projection: projectionFor(probe) })
|
|
134
|
+
};
|
|
135
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { type CanonicalModelType } from '../../../schemas/index.js';
|
|
2
|
+
import type { FitProbeRequest } from './engine-fit.js';
|
|
3
|
+
export type FitRequestPlan = {
|
|
4
|
+
supported: true;
|
|
5
|
+
probe: FitProbeRequest;
|
|
6
|
+
} | {
|
|
7
|
+
supported: false;
|
|
8
|
+
detail: string;
|
|
9
|
+
};
|
|
10
|
+
export interface CreateFitRequestParams {
|
|
11
|
+
modelType: CanonicalModelType;
|
|
12
|
+
modelPath: string;
|
|
13
|
+
modelConfig: unknown;
|
|
14
|
+
artifacts?: Record<string, string> | undefined;
|
|
15
|
+
isShardedModel: boolean;
|
|
16
|
+
/** Free memory that must remain for the projection to count as fitting. */
|
|
17
|
+
marginBytes?: number | undefined;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Builds a fit request from the same resolved config and artifacts the real
|
|
21
|
+
* load is about to use, or explains why this load cannot be answered for.
|
|
22
|
+
*
|
|
23
|
+
* Structural shapes the SDK owns — a load with no fitter behind it, a companion
|
|
24
|
+
* set that did not resolve, a setting whose effect on the projection is unknown
|
|
25
|
+
* — are refused here so no engine is asked a question it cannot answer.
|
|
26
|
+
* Value-level policy stays inside each engine's own fitter.
|
|
27
|
+
*/
|
|
28
|
+
export declare function createFitRequest(params: CreateFitRequestParams): FitRequestPlan;
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { ModelType } from '../../../schemas/index.js';
|
|
2
|
+
import { createBciFitRequest, createParakeetFitRequest, createWhisperFitRequest } from './engines/asr.js';
|
|
3
|
+
import { createAudiogenFitRequest } from './engines/audiogen.js';
|
|
4
|
+
import { createDiffusionFitRequest } from './engines/diffusion.js';
|
|
5
|
+
import { createLlamaFitRequest, llamaLoadKindFor } from './engines/llama.js';
|
|
6
|
+
import { createTtsFitRequest } from './engines/tts.js';
|
|
7
|
+
/**
|
|
8
|
+
* Builds a fit request from the same resolved config and artifacts the real
|
|
9
|
+
* load is about to use, or explains why this load cannot be answered for.
|
|
10
|
+
*
|
|
11
|
+
* Structural shapes the SDK owns — a load with no fitter behind it, a companion
|
|
12
|
+
* set that did not resolve, a setting whose effect on the projection is unknown
|
|
13
|
+
* — are refused here so no engine is asked a question it cannot answer.
|
|
14
|
+
* Value-level policy stays inside each engine's own fitter.
|
|
15
|
+
*/
|
|
16
|
+
export function createFitRequest(params) {
|
|
17
|
+
const { modelType, modelPath, modelConfig, artifacts, marginBytes } = params;
|
|
18
|
+
const loadKind = llamaLoadKindFor(modelType);
|
|
19
|
+
if (loadKind !== undefined) {
|
|
20
|
+
return createLlamaFitRequest({
|
|
21
|
+
loadKind,
|
|
22
|
+
modelPath,
|
|
23
|
+
modelConfig,
|
|
24
|
+
artifacts,
|
|
25
|
+
isShardedModel: params.isShardedModel,
|
|
26
|
+
marginBytes
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
// Every engine below reads whole files rather than a shard set, so a split
|
|
30
|
+
// model is not a shape any of them can be pointed at.
|
|
31
|
+
if (params.isShardedModel) {
|
|
32
|
+
return { supported: false, detail: 'sharded models are not representable' };
|
|
33
|
+
}
|
|
34
|
+
const common = { modelPath, modelConfig, artifacts, marginBytes };
|
|
35
|
+
switch (modelType) {
|
|
36
|
+
case ModelType.whispercppTranscription:
|
|
37
|
+
return createWhisperFitRequest(common);
|
|
38
|
+
case ModelType.parakeetTranscription:
|
|
39
|
+
return createParakeetFitRequest(common);
|
|
40
|
+
case ModelType.bciWhispercppTranscription:
|
|
41
|
+
return createBciFitRequest(common);
|
|
42
|
+
case ModelType.ttsGgml:
|
|
43
|
+
return createTtsFitRequest(common);
|
|
44
|
+
case ModelType.audiogenGgml:
|
|
45
|
+
return createAudiogenFitRequest(common);
|
|
46
|
+
case ModelType.sdcppGeneration:
|
|
47
|
+
return createDiffusionFitRequest({ modelPath, modelConfig, artifacts });
|
|
48
|
+
default:
|
|
49
|
+
return { supported: false, detail: `no fitter for model type: ${modelType}` };
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import type { AsrFitRequest, AsrFitResult } from '@qvac/asr-ggml';
|
|
2
|
+
import type { AudiogenFitRequest, AudiogenFitResult } from '@qvac/audiogen-ggml';
|
|
3
|
+
import type { BciFitRequest, BciFitResult } from '@qvac/bci-whispercpp';
|
|
4
|
+
import type { DiffusionFitRequest, DiffusionFitResult } from '@qvac/diffusion-cpp';
|
|
5
|
+
import type { EmbedFitRequest, EmbedFitResult } from '@qvac/embed-llamacpp';
|
|
6
|
+
import type LlmLlamacpp from '@qvac/llm-llamacpp';
|
|
7
|
+
import type { TtsFitRequest, TtsFitResult } from '@qvac/tts-ggml';
|
|
8
|
+
/**
|
|
9
|
+
* The engine packages each expose their own `assessFit`, taking the same fields
|
|
10
|
+
* their loader takes and returning the figures their fitter measures. Every
|
|
11
|
+
* member is plain JSON, so a request can cross a process boundary unchanged.
|
|
12
|
+
*/
|
|
13
|
+
export type FitProbeRequest = {
|
|
14
|
+
engine: 'llm-llamacpp';
|
|
15
|
+
request: LlmLlamacpp.FitRequest;
|
|
16
|
+
} | {
|
|
17
|
+
engine: 'embed-llamacpp';
|
|
18
|
+
request: EmbedFitRequest;
|
|
19
|
+
} | {
|
|
20
|
+
engine: 'asr-ggml';
|
|
21
|
+
request: AsrFitRequest;
|
|
22
|
+
} | {
|
|
23
|
+
engine: 'bci-whispercpp';
|
|
24
|
+
request: BciFitRequest;
|
|
25
|
+
} | {
|
|
26
|
+
engine: 'tts-ggml';
|
|
27
|
+
request: TtsFitRequest;
|
|
28
|
+
} | {
|
|
29
|
+
engine: 'audiogen-ggml';
|
|
30
|
+
request: AudiogenFitRequest;
|
|
31
|
+
} | {
|
|
32
|
+
engine: 'diffusion-cpp';
|
|
33
|
+
request: DiffusionFitRequest;
|
|
34
|
+
};
|
|
35
|
+
export type FitProbeEngine = FitProbeRequest['engine'];
|
|
36
|
+
export type FitProbeResult = {
|
|
37
|
+
engine: 'llm-llamacpp';
|
|
38
|
+
result: LlmLlamacpp.FitResult;
|
|
39
|
+
} | {
|
|
40
|
+
engine: 'embed-llamacpp';
|
|
41
|
+
result: EmbedFitResult;
|
|
42
|
+
} | {
|
|
43
|
+
engine: 'asr-ggml';
|
|
44
|
+
result: AsrFitResult;
|
|
45
|
+
} | {
|
|
46
|
+
engine: 'bci-whispercpp';
|
|
47
|
+
result: BciFitResult;
|
|
48
|
+
} | {
|
|
49
|
+
engine: 'tts-ggml';
|
|
50
|
+
result: TtsFitResult;
|
|
51
|
+
} | {
|
|
52
|
+
engine: 'audiogen-ggml';
|
|
53
|
+
result: AudiogenFitResult;
|
|
54
|
+
} | {
|
|
55
|
+
engine: 'diffusion-cpp';
|
|
56
|
+
result: DiffusionFitResult;
|
|
57
|
+
};
|
|
58
|
+
/** Calls one engine's fitter, taken from the plugin that owns its addon. */
|
|
59
|
+
export declare function callEngineFit(probe: FitProbeRequest): Promise<FitProbeResult>;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/** Calls one engine's fitter, taken from the plugin that owns its addon. */
|
|
2
|
+
export async function callEngineFit(probe) {
|
|
3
|
+
const { getAllPlugins } = await import('../../../plugins/registry.js');
|
|
4
|
+
const addonPackage = `@qvac/${probe.engine}`;
|
|
5
|
+
const plugin = getAllPlugins().find((candidate) => candidate.addonPackage === addonPackage);
|
|
6
|
+
if (plugin?.assessFit === undefined) {
|
|
7
|
+
throw new TypeError(`no registered plugin exposes assessFit for ${addonPackage}`);
|
|
8
|
+
}
|
|
9
|
+
return {
|
|
10
|
+
engine: probe.engine,
|
|
11
|
+
result: plugin.assessFit(probe.request)
|
|
12
|
+
};
|
|
13
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { FitRequestPlan } from '../create-fit-request.js';
|
|
2
|
+
export interface AsrFitRequestParams {
|
|
3
|
+
modelPath: string;
|
|
4
|
+
modelConfig: unknown;
|
|
5
|
+
artifacts?: Record<string, string> | undefined;
|
|
6
|
+
marginBytes?: number | undefined;
|
|
7
|
+
}
|
|
8
|
+
export declare function createWhisperFitRequest(params: AsrFitRequestParams): FitRequestPlan;
|
|
9
|
+
export declare function createParakeetFitRequest(params: AsrFitRequestParams): FitRequestPlan;
|
|
10
|
+
export declare function createBciFitRequest(params: AsrFitRequestParams): FitRequestPlan;
|