@qvac/inference 0.19.1 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +57 -17
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/assess-model-fit.d.ts +30 -6
- package/dist/handlers/assess-model-fit.js +68 -6
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +69 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +10 -1
- package/dist/resources/model-fit/assess.js +43 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +83 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +133 -0
- package/dist/resources/model-fit/fit-stub/index.d.ts +2 -0
- package/dist/resources/model-fit/fit-stub/index.js +2 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +31 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +36 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +191 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +67 -0
- package/dist/schemas/assess-model-fit.js +56 -2
- package/dist/schemas/audio-gen.d.ts +479 -4
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +7 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/common.d.ts +816 -15
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +11 -1
- package/dist/schemas/completion-event.js +3 -0
- package/dist/schemas/completion-stream.d.ts +20 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +186 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +88 -8
- package/dist/schemas/llamacpp-config.js +108 -8
- package/dist/schemas/load-model.d.ts +2906 -206
- package/dist/schemas/model-config-schemas.d.ts +251 -5
- package/dist/schemas/plugin.d.ts +10 -0
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +29 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +29 -26
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { fetchFitStub, removeStub } from './fetch-fit-stub.js';
|
|
2
|
+
import { runAdvisoryFitCheck } from '../native-probe/advisory-fit.js';
|
|
3
|
+
/**
|
|
4
|
+
* Projects a load from the registry's fit stub, before the weights exist
|
|
5
|
+
* locally.
|
|
6
|
+
*
|
|
7
|
+
* The stub is an ordinary GGUF as far as the fitter is concerned, so this is
|
|
8
|
+
* the same probe `loadModel` runs, pointed at a file that is tens of KB instead
|
|
9
|
+
* of gigabytes. It inherits that probe's fail-open contract: every failure is
|
|
10
|
+
* an `unknown` verdict or a `no-stub` outcome, never a throw.
|
|
11
|
+
*/
|
|
12
|
+
export async function projectFitFromStub(input, options = {}) {
|
|
13
|
+
const stub = await fetchFitStub(input.model, options.stub);
|
|
14
|
+
if (stub.status !== 'ready') {
|
|
15
|
+
return stub.message === undefined
|
|
16
|
+
? { status: 'no-stub', reason: stub.reason }
|
|
17
|
+
: { status: 'no-stub', reason: stub.reason, message: stub.message };
|
|
18
|
+
}
|
|
19
|
+
try {
|
|
20
|
+
const fit = await runAdvisoryFitCheck({
|
|
21
|
+
modelId: input.model.name,
|
|
22
|
+
modelType: input.modelType,
|
|
23
|
+
modelPath: stub.path,
|
|
24
|
+
modelConfig: input.modelConfig,
|
|
25
|
+
// One stub is one artifact: `fetchFitStub` does not assemble a shard set,
|
|
26
|
+
// so nothing here may claim the fitter is looking at a split model.
|
|
27
|
+
isShardedModel: false
|
|
28
|
+
}, options.fit);
|
|
29
|
+
return { status: 'projected', fit };
|
|
30
|
+
}
|
|
31
|
+
finally {
|
|
32
|
+
// The stub is a per-call payload. Nothing else knows the path, so nothing
|
|
33
|
+
// else could remove it.
|
|
34
|
+
await removeStub(stub.path);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { Logger } from '../../../logging/types.js';
|
|
3
|
+
import type { CanonicalModelType, NativeProbeFit, NativeProbeVerdict } from '../../../schemas/index.js';
|
|
4
|
+
import type { runIsolatedFit } from './run-isolated-fit.js';
|
|
5
|
+
/**
|
|
6
|
+
* `fit` and `does-not-fit` are projections of the load the SDK is about to run,
|
|
7
|
+
* not admission decisions. `@qvac/model-fit` duplicates the loader's policy for
|
|
8
|
+
* this experiment and the real loader neither consumes nor verifies the fitted
|
|
9
|
+
* plan, so neither verdict is denial-grade and no verdict changes the load.
|
|
10
|
+
*
|
|
11
|
+
* The outcome is the wire shape: `loadModel` hands it to the model registry and
|
|
12
|
+
* `getLoadedModelInfo` returns it, so no caller has to read a verdict out of a
|
|
13
|
+
* log line.
|
|
14
|
+
*/
|
|
15
|
+
export type AdvisoryFitVerdict = NativeProbeVerdict;
|
|
16
|
+
export type AdvisoryFitOutcome = NativeProbeFit;
|
|
17
|
+
export interface AdvisoryFitInput {
|
|
18
|
+
modelId: string;
|
|
19
|
+
modelType: CanonicalModelType;
|
|
20
|
+
modelPath: string;
|
|
21
|
+
modelConfig: unknown;
|
|
22
|
+
artifacts?: Record<string, string> | undefined;
|
|
23
|
+
isShardedModel: boolean;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Injection seams. Every field defaults to the real runtime dependency; tests
|
|
27
|
+
* substitute them rather than mocking modules.
|
|
28
|
+
*/
|
|
29
|
+
export interface AdvisoryFitOptions {
|
|
30
|
+
signal?: AbortSignal;
|
|
31
|
+
enabled?: boolean;
|
|
32
|
+
mobile?: boolean;
|
|
33
|
+
timeoutMs?: number;
|
|
34
|
+
runFit?: typeof runIsolatedFit;
|
|
35
|
+
logger?: Logger;
|
|
36
|
+
residentModelBytes?: () => Promise<number>;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Runs the advisory llama.cpp fit check for a load that is about to start.
|
|
40
|
+
*
|
|
41
|
+
* Fail-open by construction: an unsupported shape, a crashed or wedged child, a
|
|
42
|
+
* malformed response, and an unexpected internal error all resolve to `unknown`
|
|
43
|
+
* and the caller proceeds with the ordinary load path. This function never
|
|
44
|
+
* throws and never rejects.
|
|
45
|
+
*/
|
|
46
|
+
export declare function runAdvisoryFitCheck(input: AdvisoryFitInput, options?: AdvisoryFitOptions): Promise<AdvisoryFitOutcome>;
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import { getEngineLogger } from '../../../logging/index.js';
|
|
2
|
+
import { createLlamaFitRequest } from './create-llama-fit-request.js';
|
|
3
|
+
/**
|
|
4
|
+
* Shorter than the supervisor's own 60s default: this check sits in front of a
|
|
5
|
+
* real load, so a wedged child must not hold the load for a full minute. Every
|
|
6
|
+
* expiry is `unknown` and the load continues.
|
|
7
|
+
*/
|
|
8
|
+
const ADVISORY_FIT_TIMEOUT_MS = 30_000;
|
|
9
|
+
/**
|
|
10
|
+
* `@qvac/model-fit`'s own default margin. Made explicit here because the
|
|
11
|
+
* resident-model reserve below is *added* to it: setting `marginMiB` at all
|
|
12
|
+
* replaces the package default, so the base has to travel with the reserve.
|
|
13
|
+
*/
|
|
14
|
+
const ADVISORY_FIT_BASE_MARGIN_MIB = 1024;
|
|
15
|
+
/**
|
|
16
|
+
* Names the headroom policy these two numbers add up to: withhold
|
|
17
|
+
* `ADVISORY_FIT_BASE_MARGIN_MIB`, plus the on-disk bytes of every model already
|
|
18
|
+
* resident in this worker.
|
|
19
|
+
*
|
|
20
|
+
* Deliberately *not* `interactive-v1` (withhold 20%, cap 2 GiB desktop / 1 GiB
|
|
21
|
+
* mobile), which `assessModelFit` applies. That policy is applied by the SDK to
|
|
22
|
+
* a budget the SDK computed; here the child owns the budget (total − wired −
|
|
23
|
+
* compressor, measured inside the disposable process) and `marginMiB` is the
|
|
24
|
+
* only lever this side has. Reconciling the two means one entry point owning
|
|
25
|
+
* both bases — see the precedence note on `nativeProbeFitSchema`.
|
|
26
|
+
*/
|
|
27
|
+
const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-v1';
|
|
28
|
+
const BYTES_PER_MIB = 1024 * 1024;
|
|
29
|
+
const DISABLED_VALUES = new Set(['0', 'false', 'off', 'no']);
|
|
30
|
+
/**
|
|
31
|
+
* On by default; `QVAC_ADVISORY_MODEL_FIT=0` (or `false`/`off`/`no`) is the
|
|
32
|
+
* operator opt-out — the escape hatch for a load-heavy startup path or a
|
|
33
|
+
* runtime where the disposable child cannot spawn. Any other value, including
|
|
34
|
+
* unset, leaves the check on.
|
|
35
|
+
*
|
|
36
|
+
* The worker environment and the mobile runtime flag are imported lazily. Both
|
|
37
|
+
* modules reach Bare-only bindings, and resolving them eagerly would make this
|
|
38
|
+
* orchestration untestable outside a Bare runtime.
|
|
39
|
+
*/
|
|
40
|
+
async function resolveEnabled(explicit) {
|
|
41
|
+
if (explicit !== undefined)
|
|
42
|
+
return explicit;
|
|
43
|
+
const { getValidatedEnv } = await import('../../../runtime/env.js');
|
|
44
|
+
const value = getValidatedEnv().QVAC_ADVISORY_MODEL_FIT;
|
|
45
|
+
return value === undefined || !DISABLED_VALUES.has(value.toLowerCase());
|
|
46
|
+
}
|
|
47
|
+
async function resolveMobile(explicit) {
|
|
48
|
+
if (explicit !== undefined)
|
|
49
|
+
return explicit;
|
|
50
|
+
const { isMobile } = await import('../../../runtime/state.js');
|
|
51
|
+
return isMobile();
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Sums the on-disk weight sizes of every model currently registered in this
|
|
55
|
+
* worker. Since `@qvac/model-fit@0.8.0` (qvac-fabric#214) the fit child budgets
|
|
56
|
+
* against system-wide availability (total − wired − compressor), so weights the
|
|
57
|
+
* worker holds *wired* on the GPU are already visible to it. What the child
|
|
58
|
+
* still cannot see is the unwired remainder — mmap'd host-side layers and KV of
|
|
59
|
+
* resident models — which the OS reports as evictable right up until a decode
|
|
60
|
+
* needs it resident. Measured consequence on 0.7.0 (where the child saw a fully
|
|
61
|
+
* idle device): a verdict correct on an idle machine admitted a load that could
|
|
62
|
+
* not decode with another model loaded.
|
|
63
|
+
*
|
|
64
|
+
* Reserving the resident weight bytes through `marginMiB` keeps that footprint
|
|
65
|
+
* in the child's budget. Where those weights are wired this now double-counts
|
|
66
|
+
* and the verdict turns conservative — deliberately so: the measured failure
|
|
67
|
+
* modes punish optimism (loads that cannot decode), not caution.
|
|
68
|
+
*
|
|
69
|
+
* Advisory and fail-open like everything else here: any failure to stat a file
|
|
70
|
+
* contributes zero rather than an error.
|
|
71
|
+
*/
|
|
72
|
+
async function defaultResidentModelBytes() {
|
|
73
|
+
const [{ getAllModelIds, getModelInfo }, { promises: fsPromises }] = await Promise.all([
|
|
74
|
+
import('../../../runtime/model-registry.js'),
|
|
75
|
+
import('bare-fs')
|
|
76
|
+
]);
|
|
77
|
+
let bytes = 0;
|
|
78
|
+
for (const id of getAllModelIds()) {
|
|
79
|
+
const info = getModelInfo(id);
|
|
80
|
+
if (info === null)
|
|
81
|
+
continue;
|
|
82
|
+
try {
|
|
83
|
+
const stats = (await fsPromises.stat(info.path));
|
|
84
|
+
bytes += stats.size;
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
// Unreadable path: contribute nothing rather than fail the check.
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return bytes;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Also lazy: the desktop supervisor pulls the Bare process launcher, and a
|
|
94
|
+
* disabled check must not load it. Mobile uses in-process `fitParamsAsync` instead
|
|
95
|
+
* of spawn (`bare-runtime/spawn` stays deferred from the mobile pack).
|
|
96
|
+
*/
|
|
97
|
+
async function resolveRunFit(explicit, mobile) {
|
|
98
|
+
if (explicit !== undefined)
|
|
99
|
+
return explicit;
|
|
100
|
+
if (mobile) {
|
|
101
|
+
const { runInProcessFit } = await import('./run-in-process-fit.js');
|
|
102
|
+
return runInProcessFit;
|
|
103
|
+
}
|
|
104
|
+
return (await import('./run-isolated-fit.js')).runIsolatedFit;
|
|
105
|
+
}
|
|
106
|
+
const PROVENANCE = {
|
|
107
|
+
basis: 'native-probe',
|
|
108
|
+
estimatorVersion: NATIVE_PROBE_ESTIMATOR_VERSION
|
|
109
|
+
};
|
|
110
|
+
function unknown(reason, message) {
|
|
111
|
+
return message === undefined
|
|
112
|
+
? { ...PROVENANCE, verdict: 'unknown', reason }
|
|
113
|
+
: { ...PROVENANCE, verdict: 'unknown', reason, message };
|
|
114
|
+
}
|
|
115
|
+
function classify(result) {
|
|
116
|
+
if (result.status === 0) {
|
|
117
|
+
return {
|
|
118
|
+
...PROVENANCE,
|
|
119
|
+
verdict: 'fit',
|
|
120
|
+
reason: result.reason,
|
|
121
|
+
plan: {
|
|
122
|
+
nCtx: result.nCtx,
|
|
123
|
+
nGpuLayers: result.nGpuLayers,
|
|
124
|
+
nGpuDevices: result.nGpuDevices
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
if (result.status === 1) {
|
|
129
|
+
return { ...PROVENANCE, verdict: 'does-not-fit', reason: result.reason };
|
|
130
|
+
}
|
|
131
|
+
// `model-unreadable`, `no-backend-device`, and `unsupported-config` are all
|
|
132
|
+
// absence of evidence, not evidence of insufficiency.
|
|
133
|
+
return unknown(result.reason);
|
|
134
|
+
}
|
|
135
|
+
function report(logger, input, outcome) {
|
|
136
|
+
const prefix = `[advisory-fit:${input.modelType}:${input.modelId}]`;
|
|
137
|
+
if (outcome.verdict === 'fit') {
|
|
138
|
+
const plan = outcome.plan;
|
|
139
|
+
logger.info(`${prefix} projected to fit (advisory only)${plan === undefined
|
|
140
|
+
? ''
|
|
141
|
+
: ` — nCtx ${plan.nCtx}, nGpuLayers ${plan.nGpuLayers} across ${plan.nGpuDevices} GPU device(s)`}`);
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
if (outcome.verdict === 'does-not-fit') {
|
|
145
|
+
logger.warn(`${prefix} projected not to fit (advisory only — the load continues unchanged)`);
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
// `unsupported-load` covers every load this check refuses up front — all
|
|
149
|
+
// non-llama.cpp model types — so at `info` it would tag every whisper/tts/ocr
|
|
150
|
+
// load on every start. `debug` for those; `info` where a fitter ran (or
|
|
151
|
+
// should have) and produced no verdict, because there "why was there no
|
|
152
|
+
// evidence" is the most useful thing this check can report.
|
|
153
|
+
const line = `${prefix} no fit evidence: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`;
|
|
154
|
+
if (outcome.reason === 'unsupported-load') {
|
|
155
|
+
logger.debug(line);
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
logger.info(line);
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Runs the advisory llama.cpp fit check for a load that is about to start.
|
|
162
|
+
*
|
|
163
|
+
* Fail-open by construction: an unsupported shape, a crashed or wedged child, a
|
|
164
|
+
* malformed response, and an unexpected internal error all resolve to `unknown`
|
|
165
|
+
* and the caller proceeds with the ordinary load path. This function never
|
|
166
|
+
* throws and never rejects.
|
|
167
|
+
*/
|
|
168
|
+
export async function runAdvisoryFitCheck(input, options = {}) {
|
|
169
|
+
let logger = options.logger;
|
|
170
|
+
try {
|
|
171
|
+
logger ??= getEngineLogger();
|
|
172
|
+
if (!(await resolveEnabled(options.enabled)))
|
|
173
|
+
return unknown('disabled');
|
|
174
|
+
const mobile = await resolveMobile(options.mobile);
|
|
175
|
+
const plan = createLlamaFitRequest({
|
|
176
|
+
modelType: input.modelType,
|
|
177
|
+
modelPath: input.modelPath,
|
|
178
|
+
modelConfig: input.modelConfig,
|
|
179
|
+
artifacts: input.artifacts,
|
|
180
|
+
isShardedModel: input.isShardedModel
|
|
181
|
+
});
|
|
182
|
+
if (!plan.supported) {
|
|
183
|
+
const outcome = unknown('unsupported-load', plan.detail);
|
|
184
|
+
report(logger, input, outcome);
|
|
185
|
+
return outcome;
|
|
186
|
+
}
|
|
187
|
+
const residentBytes = await (options.residentModelBytes ?? defaultResidentModelBytes)();
|
|
188
|
+
const residentReserveMiB = Math.ceil(residentBytes / BYTES_PER_MIB);
|
|
189
|
+
const runFit = await resolveRunFit(options.runFit, mobile);
|
|
190
|
+
// Always sent, even with a zero reserve: relying on the addon default for
|
|
191
|
+
// the base margin would leave two sources of truth that match today and
|
|
192
|
+
// diverge silently if the addon default moves.
|
|
193
|
+
const result = await runFit(plan.loadKind, { ...plan.config, marginMiB: ADVISORY_FIT_BASE_MARGIN_MIB + residentReserveMiB }, {
|
|
194
|
+
timeoutMs: options.timeoutMs ?? ADVISORY_FIT_TIMEOUT_MS,
|
|
195
|
+
...(options.signal !== undefined && { signal: options.signal })
|
|
196
|
+
});
|
|
197
|
+
const outcome = result.status === 'completed'
|
|
198
|
+
? classify(result.result)
|
|
199
|
+
: unknown(result.reason, result.message);
|
|
200
|
+
report(logger, input, outcome);
|
|
201
|
+
return outcome;
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
const outcome = unknown('internal-error', error instanceof Error ? `${error.name}: ${error.message}` : String(error));
|
|
205
|
+
try {
|
|
206
|
+
if (logger !== undefined)
|
|
207
|
+
report(logger, input, outcome);
|
|
208
|
+
}
|
|
209
|
+
catch {
|
|
210
|
+
// A failing logger must not turn an advisory check into a load failure.
|
|
211
|
+
}
|
|
212
|
+
return outcome;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { FitLlamaProcessConfig, LlamaLoadKind } from '@qvac/model-fit/process';
|
|
2
|
+
import { type CanonicalModelType } from '../../../schemas/index.js';
|
|
3
|
+
export type LlamaFitRequestPlan = {
|
|
4
|
+
supported: true;
|
|
5
|
+
loadKind: LlamaLoadKind;
|
|
6
|
+
config: FitLlamaProcessConfig;
|
|
7
|
+
} | {
|
|
8
|
+
supported: false;
|
|
9
|
+
detail: string;
|
|
10
|
+
};
|
|
11
|
+
export interface CreateLlamaFitRequestParams {
|
|
12
|
+
modelType: CanonicalModelType;
|
|
13
|
+
modelPath: string;
|
|
14
|
+
modelConfig: unknown;
|
|
15
|
+
artifacts?: Record<string, string> | undefined;
|
|
16
|
+
isShardedModel: boolean;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Builds a protocol-v2 fit request from the same resolved model config the real
|
|
20
|
+
* load is about to use, or explains why this load cannot be answered for.
|
|
21
|
+
*
|
|
22
|
+
* Structural shapes the SDK owns are refused here so the fitter is not asked
|
|
23
|
+
* a question it cannot answer. Value-level policy (device names, symbolic GPU
|
|
24
|
+
* selection, context bounds) stays inside `@qvac/model-fit`.
|
|
25
|
+
*/
|
|
26
|
+
export declare function createLlamaFitRequest(params: CreateLlamaFitRequestParams): LlamaFitRequestPlan;
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
import { ModelType } from '../../../schemas/index.js';
|
|
2
|
+
import { transformLlmConfig } from '../../../plugins/builtin/llamacpp-completion/transform.js';
|
|
3
|
+
import { transformEmbedConfig } from '../../../plugins/builtin/llamacpp-embedding/transform.js';
|
|
4
|
+
/**
|
|
5
|
+
* Keys forwarded to model-fit, in the spelling the SDK's completion and
|
|
6
|
+
* embedding transforms emit. model-fit canonicalizes `_` to `-`, so `ctx_size`
|
|
7
|
+
* and `ctx-size` reach the same native setting.
|
|
8
|
+
*
|
|
9
|
+
* A key belongs here when it could change a load's device-memory footprint.
|
|
10
|
+
*
|
|
11
|
+
* model-fit answers unsupported for a key it does not recognize, and the
|
|
12
|
+
* advisory then reports unknown for every load that sets it. A key added here
|
|
13
|
+
* stays in that state until model-fit learns it.
|
|
14
|
+
*/
|
|
15
|
+
// NOTE: adding a key here needs a follow-up in model-fit to recognize it and
|
|
16
|
+
// account for it in the projection. Until that lands, every load setting the
|
|
17
|
+
// key gets no fit verdict.
|
|
18
|
+
const FIT_LOAD_KEYS = {
|
|
19
|
+
completion: [
|
|
20
|
+
'device',
|
|
21
|
+
'ctx_size',
|
|
22
|
+
'gpu_layers',
|
|
23
|
+
'load_mode',
|
|
24
|
+
'parallel',
|
|
25
|
+
'cache-type-k',
|
|
26
|
+
'cache-type-v',
|
|
27
|
+
'flash-attn',
|
|
28
|
+
'main-gpu',
|
|
29
|
+
'split-mode',
|
|
30
|
+
'tensor-split',
|
|
31
|
+
'batch-size',
|
|
32
|
+
'ubatch-size',
|
|
33
|
+
'cpu-moe',
|
|
34
|
+
'n-cpu-moe',
|
|
35
|
+
'n-cpu-ffn',
|
|
36
|
+
'override-tensor',
|
|
37
|
+
'moe-cache-mib',
|
|
38
|
+
'kv-offload',
|
|
39
|
+
'no-kv-offload',
|
|
40
|
+
'prefetch-weights',
|
|
41
|
+
'tensor-read-lazy',
|
|
42
|
+
'fit',
|
|
43
|
+
'fit-target',
|
|
44
|
+
'fit-ctx',
|
|
45
|
+
'image-max-tokens',
|
|
46
|
+
'image-min-tokens'
|
|
47
|
+
],
|
|
48
|
+
embedding: [
|
|
49
|
+
'device',
|
|
50
|
+
'gpu_layers',
|
|
51
|
+
'batch_size',
|
|
52
|
+
'flash_attn',
|
|
53
|
+
'main-gpu',
|
|
54
|
+
'split-mode',
|
|
55
|
+
'tensor-split'
|
|
56
|
+
]
|
|
57
|
+
};
|
|
58
|
+
/**
|
|
59
|
+
* Keys withheld from model-fit — sampling, generation, logging, CPU scheduling
|
|
60
|
+
* and JS-side presentation settings. A key belongs here when it cannot change
|
|
61
|
+
* a load's device-memory footprint, so withholding it leaves the verdict
|
|
62
|
+
* intact.
|
|
63
|
+
*/
|
|
64
|
+
const NON_FIT_KEYS = {
|
|
65
|
+
completion: [
|
|
66
|
+
'temp',
|
|
67
|
+
'top_p',
|
|
68
|
+
'top_k',
|
|
69
|
+
'seed',
|
|
70
|
+
'predict',
|
|
71
|
+
'presence_penalty',
|
|
72
|
+
'frequency_penalty',
|
|
73
|
+
'repeat_penalty',
|
|
74
|
+
'reverse_prompt',
|
|
75
|
+
'n_discarded',
|
|
76
|
+
'tools',
|
|
77
|
+
'verbosity',
|
|
78
|
+
'reasoning_budget',
|
|
79
|
+
'image_tile_mode',
|
|
80
|
+
'image_no_upscale',
|
|
81
|
+
'mmproj-use-gpu',
|
|
82
|
+
'openclCacheDir',
|
|
83
|
+
'threads',
|
|
84
|
+
'threads-batch',
|
|
85
|
+
'cpu-mask',
|
|
86
|
+
'cpu-mask-batch'
|
|
87
|
+
],
|
|
88
|
+
embedding: ['pooling', 'attention', 'embd_normalize', 'verbosity', 'openclCacheDir']
|
|
89
|
+
};
|
|
90
|
+
/**
|
|
91
|
+
* Load-config keys that describe a shape the fitter cannot answer for.
|
|
92
|
+
* `projection_model_src` reaches here when the config still carries the
|
|
93
|
+
* unresolved source, ahead of the resolved-artifact refusal below.
|
|
94
|
+
*/
|
|
95
|
+
const UNSUPPORTED_KEYS = ['lora', 'projection_model_src'];
|
|
96
|
+
/**
|
|
97
|
+
* A CPU load's weights stay file-backed and evictable, so the fitter projects
|
|
98
|
+
* `fits` for nearly any model the OS could page through. Measured on a 24 GiB
|
|
99
|
+
* M4 Pro against `@qvac/model-fit@0.8.0`: an 18.3 GiB model at 32k context
|
|
100
|
+
* still reports `fits` on `device: 'cpu'` (and it does decode — at 0.1 tok/s).
|
|
101
|
+
* That answer carries no admission information, so it is refused here rather
|
|
102
|
+
* than spending a fit call to produce it.
|
|
103
|
+
*/
|
|
104
|
+
function isCpuLoad(params) {
|
|
105
|
+
return params['device']?.toLowerCase() === 'cpu';
|
|
106
|
+
}
|
|
107
|
+
function loadKindFor(modelType) {
|
|
108
|
+
if (modelType === ModelType.llamacppCompletion)
|
|
109
|
+
return 'completion';
|
|
110
|
+
if (modelType === ModelType.llamacppEmbedding)
|
|
111
|
+
return 'embedding';
|
|
112
|
+
return undefined;
|
|
113
|
+
}
|
|
114
|
+
function unsupported(detail) {
|
|
115
|
+
return { supported: false, detail };
|
|
116
|
+
}
|
|
117
|
+
function partitionParams(loadKind, transformed) {
|
|
118
|
+
const forwarded = new Set(FIT_LOAD_KEYS[loadKind]);
|
|
119
|
+
const dropped = new Set(NON_FIT_KEYS[loadKind]);
|
|
120
|
+
const params = {};
|
|
121
|
+
for (const [key, value] of Object.entries(transformed)) {
|
|
122
|
+
if (UNSUPPORTED_KEYS.includes(key)) {
|
|
123
|
+
return { detail: `unsupported load setting: ${key}` };
|
|
124
|
+
}
|
|
125
|
+
if (forwarded.has(key)) {
|
|
126
|
+
params[key] = value;
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
if (dropped.has(key))
|
|
130
|
+
continue;
|
|
131
|
+
// In neither list, so its effect on the projection is unknown.
|
|
132
|
+
return { detail: `unclassified load setting: ${key}` };
|
|
133
|
+
}
|
|
134
|
+
return { params };
|
|
135
|
+
}
|
|
136
|
+
/**
|
|
137
|
+
* Pins the requested context so the fitter reports on the exact load the SDK
|
|
138
|
+
* is about to run. Left unset for an auto context (`0`, "use the model's
|
|
139
|
+
* trained context"), where the package's own floor applies and the fitter
|
|
140
|
+
* stays free to reduce.
|
|
141
|
+
*/
|
|
142
|
+
function contextFloor(params) {
|
|
143
|
+
const ctxSize = params['ctx_size'];
|
|
144
|
+
if (ctxSize === undefined)
|
|
145
|
+
return undefined;
|
|
146
|
+
const parsed = Number(ctxSize);
|
|
147
|
+
if (!Number.isSafeInteger(parsed) || parsed <= 0)
|
|
148
|
+
return undefined;
|
|
149
|
+
return parsed;
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Builds a protocol-v2 fit request from the same resolved model config the real
|
|
153
|
+
* load is about to use, or explains why this load cannot be answered for.
|
|
154
|
+
*
|
|
155
|
+
* Structural shapes the SDK owns are refused here so the fitter is not asked
|
|
156
|
+
* a question it cannot answer. Value-level policy (device names, symbolic GPU
|
|
157
|
+
* selection, context bounds) stays inside `@qvac/model-fit`.
|
|
158
|
+
*/
|
|
159
|
+
export function createLlamaFitRequest(params) {
|
|
160
|
+
const loadKind = loadKindFor(params.modelType);
|
|
161
|
+
if (loadKind === undefined) {
|
|
162
|
+
return unsupported(`model type is not a llama.cpp load: ${params.modelType}`);
|
|
163
|
+
}
|
|
164
|
+
if (params.isShardedModel) {
|
|
165
|
+
return unsupported('sharded models are not representable');
|
|
166
|
+
}
|
|
167
|
+
if (loadKind === 'completion' && params.artifacts?.['projectionModelPath'] !== undefined) {
|
|
168
|
+
return unsupported('multimodal projection loads are not representable');
|
|
169
|
+
}
|
|
170
|
+
const modelConfig = (params.modelConfig ?? {});
|
|
171
|
+
const transformed = loadKind === 'completion'
|
|
172
|
+
? transformLlmConfig(modelConfig)
|
|
173
|
+
: transformEmbedConfig(modelConfig);
|
|
174
|
+
const partitioned = partitionParams(loadKind, transformed);
|
|
175
|
+
if ('detail' in partitioned)
|
|
176
|
+
return unsupported(partitioned.detail);
|
|
177
|
+
if (isCpuLoad(partitioned.params)) {
|
|
178
|
+
return unsupported('cpu loads carry no device-memory evidence');
|
|
179
|
+
}
|
|
180
|
+
const nCtxMin = contextFloor(partitioned.params);
|
|
181
|
+
return {
|
|
182
|
+
supported: true,
|
|
183
|
+
loadKind,
|
|
184
|
+
config: {
|
|
185
|
+
// Sharded loads are refused above, so the resolved path is the whole model.
|
|
186
|
+
modelPath: params.modelPath,
|
|
187
|
+
params: partitioned.params,
|
|
188
|
+
...(nCtxMin !== undefined && { nCtxMin })
|
|
189
|
+
}
|
|
190
|
+
};
|
|
191
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { FitConfig, FitResult } from '@qvac/model-fit';
|
|
2
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
3
|
+
import type { FitLlamaProcessConfig, LlamaLoadKind } from '@qvac/model-fit/process';
|
|
4
|
+
import type { IsolatedFitUnknownReason } from './run-isolated-fit.js';
|
|
5
|
+
export interface RunInProcessFitOptions {
|
|
6
|
+
signal?: AbortSignal;
|
|
7
|
+
timeoutMs?: number;
|
|
8
|
+
fit?: (config: FitConfig) => FitResult | Promise<FitResult>;
|
|
9
|
+
stateDir?: string;
|
|
10
|
+
}
|
|
11
|
+
export declare function toFitConfig(config: FitLlamaProcessConfig): {
|
|
12
|
+
splitMode?: number;
|
|
13
|
+
flashAttnType?: number;
|
|
14
|
+
typeV?: number;
|
|
15
|
+
typeK?: number;
|
|
16
|
+
mainGpu?: number;
|
|
17
|
+
nBatch?: number;
|
|
18
|
+
nGpuLayers?: number;
|
|
19
|
+
nCtx?: number;
|
|
20
|
+
nCtxMin?: number;
|
|
21
|
+
marginMiB?: number;
|
|
22
|
+
backendsDir?: string;
|
|
23
|
+
modelPath: string;
|
|
24
|
+
};
|
|
25
|
+
export declare function crashMarkerPath(stateDir: string, config: FitLlamaProcessConfig): string;
|
|
26
|
+
export declare function crashedMarkerPath(stateDir: string, config: FitLlamaProcessConfig): string;
|
|
27
|
+
/**
|
|
28
|
+
* In-process llama.cpp fit for hosts with no disposable child (Android/iOS).
|
|
29
|
+
*
|
|
30
|
+
* The fit runs on a model-fit worker thread, so the JS loop stays free, but a
|
|
31
|
+
* thread is not a process boundary: a leftover `.running` marker means a
|
|
32
|
+
* previous call aborted the process. That is converted to `.crashed` so later
|
|
33
|
+
* launches skip native for the same path+config instead of retrying the abort.
|
|
34
|
+
*/
|
|
35
|
+
export declare function runInProcessFit(_loadKind: LlamaLoadKind, config: FitLlamaProcessConfig, options?: RunInProcessFitOptions): Promise<{
|
|
36
|
+
status: string;
|
|
37
|
+
reason: IsolatedFitUnknownReason;
|
|
38
|
+
message: string;
|
|
39
|
+
} | {
|
|
40
|
+
status: "completed";
|
|
41
|
+
result: FitResult;
|
|
42
|
+
}>;
|