@qvac/inference 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +95 -39
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/delete-cache.js +3 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.js +169 -68
- package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
- package/dist/resources/model-fit/estimators/llm.js +3 -1
- package/dist/resources/model-fit/floor.d.ts +31 -0
- package/dist/resources/model-fit/floor.js +65 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/resources/native.js +5 -4
- package/dist/resources/types.d.ts +1 -3
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +84 -0
- package/dist/schemas/assess-model-fit.js +72 -3
- package/dist/schemas/audio-gen.d.ts +482 -7
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +10 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci.d.ts +1 -1
- package/dist/schemas/common.d.ts +782 -19
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +14 -1
- package/dist/schemas/completion-event.js +4 -0
- package/dist/schemas/completion-stream.d.ts +22 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2604 -180
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/model-ops.d.ts +3 -0
- package/dist/schemas/model-ops.js +6 -0
- package/dist/schemas/plugin.d.ts +21 -11
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +30 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +29 -26
|
@@ -19,6 +19,8 @@ const SSM_STATE_BYTES_PER_ELEMENT = 4;
|
|
|
19
19
|
function disablesFlashAttention(architecture) {
|
|
20
20
|
return architecture.startsWith('bitnet');
|
|
21
21
|
}
|
|
22
|
+
// Shared with the computed floor so a mixed set carries one weights assumption, not two.
|
|
23
|
+
export const LLAMA_WEIGHTS_ASSUMPTION = 'weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM';
|
|
22
24
|
/**
|
|
23
25
|
* Estimates memory for a llama.cpp completion or embedding model from catalog
|
|
24
26
|
* metadata alone.
|
|
@@ -52,7 +54,7 @@ export function estimateLlm(input) {
|
|
|
52
54
|
// The upper coefficient covers the allocator's copy-on-write and alignment
|
|
53
55
|
// slack measured during calibration.
|
|
54
56
|
const artifactBytes = profile.artifactBytes + extraArtifactBytes;
|
|
55
|
-
assumptions.push(
|
|
57
|
+
assumptions.push(LLAMA_WEIGHTS_ASSUMPTION);
|
|
56
58
|
if (extraArtifactBytes > 0) {
|
|
57
59
|
assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
|
|
58
60
|
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { ModelFitWorkload } from '../../schemas/assess-model-fit.js';
|
|
2
|
+
import type { ModelResourceProfile } from '../../schemas/model-resource-profile.js';
|
|
3
|
+
export declare const FLOOR_VERSION = "floor-v1";
|
|
4
|
+
export interface FloorInput {
|
|
5
|
+
profile: ModelResourceProfile;
|
|
6
|
+
workload: ModelFitWorkload;
|
|
7
|
+
/** Artifact bytes of companion constants the caller passed alongside. */
|
|
8
|
+
extraArtifactBytes: number;
|
|
9
|
+
}
|
|
10
|
+
/** A lower bound with no upper bound, and how it was assembled. */
|
|
11
|
+
export interface FloorResult {
|
|
12
|
+
bytes: number;
|
|
13
|
+
reasons: readonly string[];
|
|
14
|
+
assumptions: readonly string[];
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* The smallest resident footprint the catalog facts prove, with no coefficient
|
|
18
|
+
* measured anywhere: artifact bytes, plus the KV cache for a llama.cpp model at
|
|
19
|
+
* the cheapest element width the engine can default to.
|
|
20
|
+
*
|
|
21
|
+
* Every term a calibrated estimate adds on top — weight slack, engine overhead,
|
|
22
|
+
* compute buffers, a completion's working peak — is non-negative, so a floor
|
|
23
|
+
* with all of them at zero is unconditionally valid on any platform and any
|
|
24
|
+
* backend. It can therefore refuse a model (`likely-too-large` when even the
|
|
25
|
+
* floor is over budget) but never confirm one: how far above the floor the real
|
|
26
|
+
* load lands is exactly what calibration exists to measure.
|
|
27
|
+
*
|
|
28
|
+
* This is the zero-fetch gate. It reads nothing but the catalog and stays the
|
|
29
|
+
* fallback wherever no calibration or engine projection is available.
|
|
30
|
+
*/
|
|
31
|
+
export declare function computeFloor(input: FloorInput): FloorResult;
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { LLAMA_WEIGHTS_ASSUMPTION, kvCacheBytesForWidth, kvElementBytes } from './estimators/llm.js';
|
|
2
|
+
export const FLOOR_VERSION = 'floor-v1';
|
|
3
|
+
/**
|
|
4
|
+
* Engines whose load builds a KV cache the file sizes exactly, so the floor can
|
|
5
|
+
* count it. Every other engine's working memory is engine-owned and unsized
|
|
6
|
+
* here; the floor for those is the weights alone.
|
|
7
|
+
*/
|
|
8
|
+
const KV_CACHE_ENGINES = new Set(['llamacpp-completion', 'llamacpp-embedding']);
|
|
9
|
+
/**
|
|
10
|
+
* The smallest resident footprint the catalog facts prove, with no coefficient
|
|
11
|
+
* measured anywhere: artifact bytes, plus the KV cache for a llama.cpp model at
|
|
12
|
+
* the cheapest element width the engine can default to.
|
|
13
|
+
*
|
|
14
|
+
* Every term a calibrated estimate adds on top — weight slack, engine overhead,
|
|
15
|
+
* compute buffers, a completion's working peak — is non-negative, so a floor
|
|
16
|
+
* with all of them at zero is unconditionally valid on any platform and any
|
|
17
|
+
* backend. It can therefore refuse a model (`likely-too-large` when even the
|
|
18
|
+
* floor is over budget) but never confirm one: how far above the floor the real
|
|
19
|
+
* load lands is exactly what calibration exists to measure.
|
|
20
|
+
*
|
|
21
|
+
* This is the zero-fetch gate. It reads nothing but the catalog and stays the
|
|
22
|
+
* fallback wherever no calibration or engine projection is available.
|
|
23
|
+
*/
|
|
24
|
+
export function computeFloor(input) {
|
|
25
|
+
const { profile, workload, extraArtifactBytes } = input;
|
|
26
|
+
const reasons = [];
|
|
27
|
+
const assumptions = [
|
|
28
|
+
KV_CACHE_ENGINES.has(profile.engine)
|
|
29
|
+
? LLAMA_WEIGHTS_ASSUMPTION
|
|
30
|
+
: 'weights are counted at full artifact size; whatever the engine maps or copies, the file has to be resident to run',
|
|
31
|
+
'the floor omits engine overhead, compute buffers and working memory, all of which are non-negative, so it cannot overstate the cost — and says nothing about how much more the load needs'
|
|
32
|
+
];
|
|
33
|
+
const artifactBytes = profile.artifactBytes + extraArtifactBytes;
|
|
34
|
+
if (extraArtifactBytes > 0) {
|
|
35
|
+
assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
|
|
36
|
+
}
|
|
37
|
+
if (!KV_CACHE_ENGINES.has(profile.engine)) {
|
|
38
|
+
reasons.push(`the floor for engine '${profile.engine}' counts weights only; its working memory is engine-owned and not sized here`);
|
|
39
|
+
return { bytes: artifactBytes, reasons, assumptions };
|
|
40
|
+
}
|
|
41
|
+
if (workload.kind !== 'llm') {
|
|
42
|
+
reasons.push(`workload kind '${workload.kind}' carries no context for a llama.cpp model, so the floor counts weights only`);
|
|
43
|
+
return { bytes: artifactBytes, reasons, assumptions };
|
|
44
|
+
}
|
|
45
|
+
const facts = profile.ggufFacts;
|
|
46
|
+
if (!facts) {
|
|
47
|
+
reasons.push('no GGUF metadata for this model in the catalog, so the floor counts weights only');
|
|
48
|
+
return { bytes: artifactBytes, reasons, assumptions };
|
|
49
|
+
}
|
|
50
|
+
let contextTokens = workload.contextTokens;
|
|
51
|
+
if (contextTokens > facts.contextLength) {
|
|
52
|
+
assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
|
|
53
|
+
contextTokens = facts.contextLength;
|
|
54
|
+
}
|
|
55
|
+
// The narrowest default the engine can pick on any backend, so the floor
|
|
56
|
+
// holds whether the model lands on a GPU (q8_0 with flash attention) or on
|
|
57
|
+
// the CPU (f16). Architectures that disable flash attention stay f16 anyway.
|
|
58
|
+
const element = kvElementBytes(facts, true).bytes.lower;
|
|
59
|
+
const kv = kvCacheBytesForWidth(facts, contextTokens, element).lower;
|
|
60
|
+
assumptions.push(element < 2
|
|
61
|
+
? 'the KV cache is floored at the q8_0 element width, the narrowest default the engine picks on any backend; a CPU load allocates an f16 cache above it'
|
|
62
|
+
: `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not expressible in a workload and would change these numbers');
|
|
63
|
+
reasons.push(`KV cache floored for ${contextTokens} tokens at the narrowest default element width`);
|
|
64
|
+
return { bytes: Math.ceil(artifactBytes + kv), reasons, assumptions };
|
|
65
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { Logger } from '../../../logging/types.js';
|
|
3
|
+
import type { CanonicalModelType, NativeProbeFit, NativeProbeVerdict } from '../../../schemas/index.js';
|
|
4
|
+
import type { runIsolatedFit } from './run-isolated-fit.js';
|
|
5
|
+
/**
|
|
6
|
+
* `fit` and `does-not-fit` are projections of the load the SDK is about to run,
|
|
7
|
+
* not admission decisions. `@qvac/model-fit` duplicates the loader's policy for
|
|
8
|
+
* this experiment and the real loader neither consumes nor verifies the fitted
|
|
9
|
+
* plan, so neither verdict is denial-grade and no verdict changes the load.
|
|
10
|
+
*
|
|
11
|
+
* The outcome is the wire shape: `loadModel` hands it to the model registry and
|
|
12
|
+
* `getLoadedModelInfo` returns it, so no caller has to read a verdict out of a
|
|
13
|
+
* log line.
|
|
14
|
+
*/
|
|
15
|
+
export type AdvisoryFitVerdict = NativeProbeVerdict;
|
|
16
|
+
export type AdvisoryFitOutcome = NativeProbeFit;
|
|
17
|
+
export interface AdvisoryFitInput {
|
|
18
|
+
modelId: string;
|
|
19
|
+
modelType: CanonicalModelType;
|
|
20
|
+
modelPath: string;
|
|
21
|
+
modelConfig: unknown;
|
|
22
|
+
artifacts?: Record<string, string> | undefined;
|
|
23
|
+
isShardedModel: boolean;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Injection seams. Every field defaults to the real runtime dependency; tests
|
|
27
|
+
* substitute them rather than mocking modules.
|
|
28
|
+
*/
|
|
29
|
+
export interface AdvisoryFitOptions {
|
|
30
|
+
signal?: AbortSignal;
|
|
31
|
+
enabled?: boolean;
|
|
32
|
+
mobile?: boolean;
|
|
33
|
+
timeoutMs?: number;
|
|
34
|
+
runFit?: typeof runIsolatedFit;
|
|
35
|
+
logger?: Logger;
|
|
36
|
+
residentModelBytes?: () => Promise<number>;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Runs the advisory llama.cpp fit check for a load that is about to start.
|
|
40
|
+
*
|
|
41
|
+
* Fail-open by construction: an unsupported shape, a crashed or wedged child, a
|
|
42
|
+
* malformed response, and an unexpected internal error all resolve to `unknown`
|
|
43
|
+
* and the caller proceeds with the ordinary load path. This function never
|
|
44
|
+
* throws and never rejects.
|
|
45
|
+
*/
|
|
46
|
+
export declare function runAdvisoryFitCheck(input: AdvisoryFitInput, options?: AdvisoryFitOptions): Promise<AdvisoryFitOutcome>;
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import { getEngineLogger } from '../../../logging/index.js';
|
|
2
|
+
import { createLlamaFitRequest } from './create-llama-fit-request.js';
|
|
3
|
+
/**
|
|
4
|
+
* Shorter than the supervisor's own 60s default: this check sits in front of a
|
|
5
|
+
* real load, so a wedged child must not hold the load for a full minute. Every
|
|
6
|
+
* expiry is `unknown` and the load continues.
|
|
7
|
+
*/
|
|
8
|
+
const ADVISORY_FIT_TIMEOUT_MS = 30_000;
|
|
9
|
+
/**
|
|
10
|
+
* `@qvac/model-fit`'s own default margin. Made explicit here because the
|
|
11
|
+
* resident-model reserve below is *added* to it: setting `marginMiB` at all
|
|
12
|
+
* replaces the package default, so the base has to travel with the reserve.
|
|
13
|
+
*/
|
|
14
|
+
const ADVISORY_FIT_BASE_MARGIN_MIB = 1024;
|
|
15
|
+
/**
|
|
16
|
+
* Names the headroom policy these two numbers add up to: withhold
|
|
17
|
+
* `ADVISORY_FIT_BASE_MARGIN_MIB`, plus the on-disk bytes of every model already
|
|
18
|
+
* resident in this worker.
|
|
19
|
+
*
|
|
20
|
+
* Deliberately *not* `interactive-v1` (withhold 20%, cap 2 GiB desktop / 1 GiB
|
|
21
|
+
* mobile), which `assessModelFit` applies. That policy is applied by the SDK to
|
|
22
|
+
* a budget the SDK computed; here the child owns the budget (total − wired −
|
|
23
|
+
* compressor, measured inside the disposable process) and `marginMiB` is the
|
|
24
|
+
* only lever this side has. Reconciling the two means one entry point owning
|
|
25
|
+
* both bases — see the precedence note on `nativeProbeFitSchema`.
|
|
26
|
+
*/
|
|
27
|
+
const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-v1';
|
|
28
|
+
const BYTES_PER_MIB = 1024 * 1024;
|
|
29
|
+
const DISABLED_VALUES = new Set(['0', 'false', 'off', 'no']);
|
|
30
|
+
/**
|
|
31
|
+
* On by default; `QVAC_ADVISORY_MODEL_FIT=0` (or `false`/`off`/`no`) is the
|
|
32
|
+
* operator opt-out — the escape hatch for a load-heavy startup path or a
|
|
33
|
+
* runtime where the disposable child cannot spawn. Any other value, including
|
|
34
|
+
* unset, leaves the check on.
|
|
35
|
+
*
|
|
36
|
+
* The worker environment and the mobile runtime flag are imported lazily. Both
|
|
37
|
+
* modules reach Bare-only bindings, and resolving them eagerly would make this
|
|
38
|
+
* orchestration untestable outside a Bare runtime.
|
|
39
|
+
*/
|
|
40
|
+
async function resolveEnabled(explicit) {
|
|
41
|
+
if (explicit !== undefined)
|
|
42
|
+
return explicit;
|
|
43
|
+
const { getValidatedEnv } = await import('../../../runtime/env.js');
|
|
44
|
+
const value = getValidatedEnv().QVAC_ADVISORY_MODEL_FIT;
|
|
45
|
+
return value === undefined || !DISABLED_VALUES.has(value.toLowerCase());
|
|
46
|
+
}
|
|
47
|
+
async function resolveMobile(explicit) {
|
|
48
|
+
if (explicit !== undefined)
|
|
49
|
+
return explicit;
|
|
50
|
+
const { isMobile } = await import('../../../runtime/state.js');
|
|
51
|
+
return isMobile();
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Sums the on-disk weight sizes of every model currently registered in this
|
|
55
|
+
* worker. Since `@qvac/model-fit@0.8.0` (qvac-fabric#214) the fit child budgets
|
|
56
|
+
* against system-wide availability (total − wired − compressor), so weights the
|
|
57
|
+
* worker holds *wired* on the GPU are already visible to it. What the child
|
|
58
|
+
* still cannot see is the unwired remainder — mmap'd host-side layers and KV of
|
|
59
|
+
* resident models — which the OS reports as evictable right up until a decode
|
|
60
|
+
* needs it resident. Measured consequence on 0.7.0 (where the child saw a fully
|
|
61
|
+
* idle device): a verdict correct on an idle machine admitted a load that could
|
|
62
|
+
* not decode with another model loaded.
|
|
63
|
+
*
|
|
64
|
+
* Reserving the resident weight bytes through `marginMiB` keeps that footprint
|
|
65
|
+
* in the child's budget. Where those weights are wired this now double-counts
|
|
66
|
+
* and the verdict turns conservative — deliberately so: the measured failure
|
|
67
|
+
* modes punish optimism (loads that cannot decode), not caution.
|
|
68
|
+
*
|
|
69
|
+
* Advisory and fail-open like everything else here: any failure to stat a file
|
|
70
|
+
* contributes zero rather than an error.
|
|
71
|
+
*/
|
|
72
|
+
async function defaultResidentModelBytes() {
|
|
73
|
+
const [{ getAllModelIds, getModelInfo }, { promises: fsPromises }] = await Promise.all([
|
|
74
|
+
import('../../../runtime/model-registry.js'),
|
|
75
|
+
import('bare-fs')
|
|
76
|
+
]);
|
|
77
|
+
let bytes = 0;
|
|
78
|
+
for (const id of getAllModelIds()) {
|
|
79
|
+
const info = getModelInfo(id);
|
|
80
|
+
if (info === null)
|
|
81
|
+
continue;
|
|
82
|
+
try {
|
|
83
|
+
const stats = (await fsPromises.stat(info.path));
|
|
84
|
+
bytes += stats.size;
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
// Unreadable path: contribute nothing rather than fail the check.
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return bytes;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Also lazy: the desktop supervisor pulls the Bare process launcher, and a
|
|
94
|
+
* disabled check must not load it. Mobile uses in-process `fitParamsAsync` instead
|
|
95
|
+
* of spawn (`bare-runtime/spawn` stays deferred from the mobile pack).
|
|
96
|
+
*/
|
|
97
|
+
async function resolveRunFit(explicit, mobile) {
|
|
98
|
+
if (explicit !== undefined)
|
|
99
|
+
return explicit;
|
|
100
|
+
if (mobile) {
|
|
101
|
+
const { runInProcessFit } = await import('./run-in-process-fit.js');
|
|
102
|
+
return runInProcessFit;
|
|
103
|
+
}
|
|
104
|
+
return (await import('./run-isolated-fit.js')).runIsolatedFit;
|
|
105
|
+
}
|
|
106
|
+
const PROVENANCE = {
|
|
107
|
+
basis: 'native-probe',
|
|
108
|
+
estimatorVersion: NATIVE_PROBE_ESTIMATOR_VERSION
|
|
109
|
+
};
|
|
110
|
+
function unknown(reason, message) {
|
|
111
|
+
return message === undefined
|
|
112
|
+
? { ...PROVENANCE, verdict: 'unknown', reason }
|
|
113
|
+
: { ...PROVENANCE, verdict: 'unknown', reason, message };
|
|
114
|
+
}
|
|
115
|
+
function classify(result) {
|
|
116
|
+
if (result.status === 0) {
|
|
117
|
+
return {
|
|
118
|
+
...PROVENANCE,
|
|
119
|
+
verdict: 'fit',
|
|
120
|
+
reason: result.reason,
|
|
121
|
+
plan: {
|
|
122
|
+
nCtx: result.nCtx,
|
|
123
|
+
nGpuLayers: result.nGpuLayers,
|
|
124
|
+
nGpuDevices: result.nGpuDevices
|
|
125
|
+
}
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
if (result.status === 1) {
|
|
129
|
+
return { ...PROVENANCE, verdict: 'does-not-fit', reason: result.reason };
|
|
130
|
+
}
|
|
131
|
+
// `model-unreadable`, `no-backend-device`, and `unsupported-config` are all
|
|
132
|
+
// absence of evidence, not evidence of insufficiency.
|
|
133
|
+
return unknown(result.reason);
|
|
134
|
+
}
|
|
135
|
+
function report(logger, input, outcome) {
|
|
136
|
+
const prefix = `[advisory-fit:${input.modelType}:${input.modelId}]`;
|
|
137
|
+
if (outcome.verdict === 'fit') {
|
|
138
|
+
const plan = outcome.plan;
|
|
139
|
+
logger.info(`${prefix} projected to fit (advisory only)${plan === undefined
|
|
140
|
+
? ''
|
|
141
|
+
: ` — nCtx ${plan.nCtx}, nGpuLayers ${plan.nGpuLayers} across ${plan.nGpuDevices} GPU device(s)`}`);
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
if (outcome.verdict === 'does-not-fit') {
|
|
145
|
+
logger.warn(`${prefix} projected not to fit (advisory only — the load continues unchanged)`);
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
// `unsupported-load` covers every load this check refuses up front — all
|
|
149
|
+
// non-llama.cpp model types — so at `info` it would tag every whisper/tts/ocr
|
|
150
|
+
// load on every start. `debug` for those; `info` where a fitter ran (or
|
|
151
|
+
// should have) and produced no verdict, because there "why was there no
|
|
152
|
+
// evidence" is the most useful thing this check can report.
|
|
153
|
+
const line = `${prefix} no fit evidence: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`;
|
|
154
|
+
if (outcome.reason === 'unsupported-load') {
|
|
155
|
+
logger.debug(line);
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
logger.info(line);
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Runs the advisory llama.cpp fit check for a load that is about to start.
|
|
162
|
+
*
|
|
163
|
+
* Fail-open by construction: an unsupported shape, a crashed or wedged child, a
|
|
164
|
+
* malformed response, and an unexpected internal error all resolve to `unknown`
|
|
165
|
+
* and the caller proceeds with the ordinary load path. This function never
|
|
166
|
+
* throws and never rejects.
|
|
167
|
+
*/
|
|
168
|
+
export async function runAdvisoryFitCheck(input, options = {}) {
|
|
169
|
+
let logger = options.logger;
|
|
170
|
+
try {
|
|
171
|
+
logger ??= getEngineLogger();
|
|
172
|
+
if (!(await resolveEnabled(options.enabled)))
|
|
173
|
+
return unknown('disabled');
|
|
174
|
+
const mobile = await resolveMobile(options.mobile);
|
|
175
|
+
const plan = createLlamaFitRequest({
|
|
176
|
+
modelType: input.modelType,
|
|
177
|
+
modelPath: input.modelPath,
|
|
178
|
+
modelConfig: input.modelConfig,
|
|
179
|
+
artifacts: input.artifacts,
|
|
180
|
+
isShardedModel: input.isShardedModel
|
|
181
|
+
});
|
|
182
|
+
if (!plan.supported) {
|
|
183
|
+
const outcome = unknown('unsupported-load', plan.detail);
|
|
184
|
+
report(logger, input, outcome);
|
|
185
|
+
return outcome;
|
|
186
|
+
}
|
|
187
|
+
const residentBytes = await (options.residentModelBytes ?? defaultResidentModelBytes)();
|
|
188
|
+
const residentReserveMiB = Math.ceil(residentBytes / BYTES_PER_MIB);
|
|
189
|
+
const runFit = await resolveRunFit(options.runFit, mobile);
|
|
190
|
+
// Always sent, even with a zero reserve: relying on the addon default for
|
|
191
|
+
// the base margin would leave two sources of truth that match today and
|
|
192
|
+
// diverge silently if the addon default moves.
|
|
193
|
+
const result = await runFit(plan.loadKind, { ...plan.config, marginMiB: ADVISORY_FIT_BASE_MARGIN_MIB + residentReserveMiB }, {
|
|
194
|
+
timeoutMs: options.timeoutMs ?? ADVISORY_FIT_TIMEOUT_MS,
|
|
195
|
+
...(options.signal !== undefined && { signal: options.signal })
|
|
196
|
+
});
|
|
197
|
+
const outcome = result.status === 'completed'
|
|
198
|
+
? classify(result.result)
|
|
199
|
+
: unknown(result.reason, result.message);
|
|
200
|
+
report(logger, input, outcome);
|
|
201
|
+
return outcome;
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
const outcome = unknown('internal-error', error instanceof Error ? `${error.name}: ${error.message}` : String(error));
|
|
205
|
+
try {
|
|
206
|
+
if (logger !== undefined)
|
|
207
|
+
report(logger, input, outcome);
|
|
208
|
+
}
|
|
209
|
+
catch {
|
|
210
|
+
// A failing logger must not turn an advisory check into a load failure.
|
|
211
|
+
}
|
|
212
|
+
return outcome;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { FitLlamaProcessConfig, LlamaLoadKind } from '@qvac/model-fit/process';
|
|
2
|
+
import { type CanonicalModelType } from '../../../schemas/index.js';
|
|
3
|
+
export type LlamaFitRequestPlan = {
|
|
4
|
+
supported: true;
|
|
5
|
+
loadKind: LlamaLoadKind;
|
|
6
|
+
config: FitLlamaProcessConfig;
|
|
7
|
+
} | {
|
|
8
|
+
supported: false;
|
|
9
|
+
detail: string;
|
|
10
|
+
};
|
|
11
|
+
export interface CreateLlamaFitRequestParams {
|
|
12
|
+
modelType: CanonicalModelType;
|
|
13
|
+
modelPath: string;
|
|
14
|
+
modelConfig: unknown;
|
|
15
|
+
artifacts?: Record<string, string> | undefined;
|
|
16
|
+
isShardedModel: boolean;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Builds a protocol-v2 fit request from the same resolved model config the real
|
|
20
|
+
* load is about to use, or explains why this load cannot be answered for.
|
|
21
|
+
*
|
|
22
|
+
* Structural shapes the SDK owns are refused here so the fitter is not asked
|
|
23
|
+
* a question it cannot answer. Value-level policy (device names, symbolic GPU
|
|
24
|
+
* selection, context bounds) stays inside `@qvac/model-fit`.
|
|
25
|
+
*/
|
|
26
|
+
export declare function createLlamaFitRequest(params: CreateLlamaFitRequestParams): LlamaFitRequestPlan;
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { ModelType } from '../../../schemas/index.js';
|
|
2
|
+
import { transformLlmConfig } from '../../../plugins/builtin/llamacpp-completion/transform.js';
|
|
3
|
+
import { transformEmbedConfig } from '../../../plugins/builtin/llamacpp-embedding/transform.js';
|
|
4
|
+
/**
|
|
5
|
+
* Keys `@qvac/model-fit` reads as load evidence, in the spelling the SDK's own
|
|
6
|
+
* completion/embedding transforms emit. The package canonicalizes `_` to `-`,
|
|
7
|
+
* so `ctx_size` and `ctx-size` reach the same native setting.
|
|
8
|
+
*
|
|
9
|
+
* This mirrors `SUPPORTED_LOAD_KEYS` in the package's `LlamaLoadConfig.cpp`
|
|
10
|
+
* intersected with what the two transforms can produce. It is a duplicated
|
|
11
|
+
* policy for the duration of the experiment: a key added to a load config
|
|
12
|
+
* without being classified here must not silently change the question the
|
|
13
|
+
* fitter answers, so `partitionParams` refuses the request instead.
|
|
14
|
+
*/
|
|
15
|
+
const FIT_LOAD_KEYS = {
|
|
16
|
+
completion: [
|
|
17
|
+
'device',
|
|
18
|
+
'ctx_size',
|
|
19
|
+
'gpu_layers',
|
|
20
|
+
'load_mode',
|
|
21
|
+
'parallel',
|
|
22
|
+
'cache-type-k',
|
|
23
|
+
'cache-type-v',
|
|
24
|
+
// Alters KV/compute memory; `model-fit` reads it as evidence.
|
|
25
|
+
'flash-attn',
|
|
26
|
+
'main-gpu',
|
|
27
|
+
'split-mode',
|
|
28
|
+
'tensor-split'
|
|
29
|
+
],
|
|
30
|
+
embedding: [
|
|
31
|
+
'device',
|
|
32
|
+
'gpu_layers',
|
|
33
|
+
'batch_size',
|
|
34
|
+
'flash_attn',
|
|
35
|
+
'main-gpu',
|
|
36
|
+
'split-mode',
|
|
37
|
+
'tensor-split'
|
|
38
|
+
]
|
|
39
|
+
};
|
|
40
|
+
/**
|
|
41
|
+
* Keys that reach the addon but cannot move a load's memory footprint —
|
|
42
|
+
* sampling, generation, logging, and JS-side presentation settings. They are
|
|
43
|
+
* dropped rather than forwarded: `@qvac/model-fit` rejects everything outside
|
|
44
|
+
* its own allowlist, and a key it deliberately ignores must not turn a
|
|
45
|
+
* supported load into `unsupported-config`.
|
|
46
|
+
*/
|
|
47
|
+
const NON_FIT_KEYS = {
|
|
48
|
+
completion: [
|
|
49
|
+
'temp',
|
|
50
|
+
'top_p',
|
|
51
|
+
'top_k',
|
|
52
|
+
'seed',
|
|
53
|
+
'predict',
|
|
54
|
+
'presence_penalty',
|
|
55
|
+
'frequency_penalty',
|
|
56
|
+
'repeat_penalty',
|
|
57
|
+
'reverse_prompt',
|
|
58
|
+
'n_discarded',
|
|
59
|
+
'tools',
|
|
60
|
+
'verbosity',
|
|
61
|
+
'reasoning_budget',
|
|
62
|
+
'image_tile_mode',
|
|
63
|
+
'image_no_upscale',
|
|
64
|
+
'mmproj-use-gpu',
|
|
65
|
+
'openclCacheDir'
|
|
66
|
+
],
|
|
67
|
+
embedding: ['pooling', 'attention', 'embd_normalize', 'verbosity', 'openclCacheDir']
|
|
68
|
+
};
|
|
69
|
+
/**
|
|
70
|
+
* Load-config keys that describe a shape the fitter cannot answer for.
|
|
71
|
+
* `projection_model_src` reaches here when the config still carries the
|
|
72
|
+
* unresolved source, ahead of the resolved-artifact refusal below.
|
|
73
|
+
*/
|
|
74
|
+
const UNSUPPORTED_KEYS = ['lora', 'projection_model_src'];
|
|
75
|
+
/**
|
|
76
|
+
* A CPU load's weights stay file-backed and evictable, so the fitter projects
|
|
77
|
+
* `fits` for nearly any model the OS could page through. Measured on a 24 GiB
|
|
78
|
+
* M4 Pro against `@qvac/model-fit@0.8.0`: an 18.3 GiB model at 32k context
|
|
79
|
+
* still reports `fits` on `device: 'cpu'` (and it does decode — at 0.1 tok/s).
|
|
80
|
+
* That answer carries no admission information, so it is refused here rather
|
|
81
|
+
* than spending a fit call to produce it.
|
|
82
|
+
*/
|
|
83
|
+
function isCpuLoad(params) {
|
|
84
|
+
return params['device']?.toLowerCase() === 'cpu';
|
|
85
|
+
}
|
|
86
|
+
function loadKindFor(modelType) {
|
|
87
|
+
if (modelType === ModelType.llamacppCompletion)
|
|
88
|
+
return 'completion';
|
|
89
|
+
if (modelType === ModelType.llamacppEmbedding)
|
|
90
|
+
return 'embedding';
|
|
91
|
+
return undefined;
|
|
92
|
+
}
|
|
93
|
+
function unsupported(detail) {
|
|
94
|
+
return { supported: false, detail };
|
|
95
|
+
}
|
|
96
|
+
function partitionParams(loadKind, transformed) {
|
|
97
|
+
const forwarded = new Set(FIT_LOAD_KEYS[loadKind]);
|
|
98
|
+
const dropped = new Set(NON_FIT_KEYS[loadKind]);
|
|
99
|
+
const params = {};
|
|
100
|
+
for (const [key, value] of Object.entries(transformed)) {
|
|
101
|
+
if (UNSUPPORTED_KEYS.includes(key)) {
|
|
102
|
+
return { detail: `unsupported load setting: ${key}` };
|
|
103
|
+
}
|
|
104
|
+
if (forwarded.has(key)) {
|
|
105
|
+
params[key] = value;
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
if (dropped.has(key))
|
|
109
|
+
continue;
|
|
110
|
+
// Neither fit evidence nor a known non-memory setting: this load carries a
|
|
111
|
+
// setting the SDK cannot classify, so it must not be answered for.
|
|
112
|
+
return { detail: `unclassified load setting: ${key}` };
|
|
113
|
+
}
|
|
114
|
+
return { params };
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Pins the requested context so the fitter reports on the exact load the SDK
|
|
118
|
+
* is about to run. Left unset for an auto context (`0`, "use the model's
|
|
119
|
+
* trained context"), where the package's own floor applies and the fitter
|
|
120
|
+
* stays free to reduce.
|
|
121
|
+
*/
|
|
122
|
+
function contextFloor(params) {
|
|
123
|
+
const ctxSize = params['ctx_size'];
|
|
124
|
+
if (ctxSize === undefined)
|
|
125
|
+
return undefined;
|
|
126
|
+
const parsed = Number(ctxSize);
|
|
127
|
+
if (!Number.isSafeInteger(parsed) || parsed <= 0)
|
|
128
|
+
return undefined;
|
|
129
|
+
return parsed;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Builds a protocol-v2 fit request from the same resolved model config the real
|
|
133
|
+
* load is about to use, or explains why this load cannot be answered for.
|
|
134
|
+
*
|
|
135
|
+
* Structural shapes the SDK owns are refused here so the fitter is not asked
|
|
136
|
+
* a question it cannot answer. Value-level policy (device names, symbolic GPU
|
|
137
|
+
* selection, context bounds) stays inside `@qvac/model-fit`.
|
|
138
|
+
*/
|
|
139
|
+
export function createLlamaFitRequest(params) {
|
|
140
|
+
const loadKind = loadKindFor(params.modelType);
|
|
141
|
+
if (loadKind === undefined) {
|
|
142
|
+
return unsupported(`model type is not a llama.cpp load: ${params.modelType}`);
|
|
143
|
+
}
|
|
144
|
+
if (params.isShardedModel) {
|
|
145
|
+
return unsupported('sharded models are not representable');
|
|
146
|
+
}
|
|
147
|
+
if (loadKind === 'completion' && params.artifacts?.['projectionModelPath'] !== undefined) {
|
|
148
|
+
return unsupported('multimodal projection loads are not representable');
|
|
149
|
+
}
|
|
150
|
+
const modelConfig = (params.modelConfig ?? {});
|
|
151
|
+
const transformed = loadKind === 'completion'
|
|
152
|
+
? transformLlmConfig(modelConfig)
|
|
153
|
+
: transformEmbedConfig(modelConfig);
|
|
154
|
+
const partitioned = partitionParams(loadKind, transformed);
|
|
155
|
+
if ('detail' in partitioned)
|
|
156
|
+
return unsupported(partitioned.detail);
|
|
157
|
+
if (isCpuLoad(partitioned.params)) {
|
|
158
|
+
return unsupported('cpu loads carry no device-memory evidence');
|
|
159
|
+
}
|
|
160
|
+
const nCtxMin = contextFloor(partitioned.params);
|
|
161
|
+
return {
|
|
162
|
+
supported: true,
|
|
163
|
+
loadKind,
|
|
164
|
+
config: {
|
|
165
|
+
// Sharded loads are refused above, so the resolved path is the whole model.
|
|
166
|
+
modelPath: params.modelPath,
|
|
167
|
+
params: partitioned.params,
|
|
168
|
+
...(nCtxMin !== undefined && { nCtxMin })
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { FitConfig, FitResult } from '@qvac/model-fit';
|
|
2
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
3
|
+
import type { FitLlamaProcessConfig, LlamaLoadKind } from '@qvac/model-fit/process';
|
|
4
|
+
import type { IsolatedFitUnknownReason } from './run-isolated-fit.js';
|
|
5
|
+
export interface RunInProcessFitOptions {
|
|
6
|
+
signal?: AbortSignal;
|
|
7
|
+
timeoutMs?: number;
|
|
8
|
+
fit?: (config: FitConfig) => FitResult | Promise<FitResult>;
|
|
9
|
+
stateDir?: string;
|
|
10
|
+
}
|
|
11
|
+
export declare function toFitConfig(config: FitLlamaProcessConfig): {
|
|
12
|
+
splitMode?: number;
|
|
13
|
+
flashAttnType?: number;
|
|
14
|
+
typeV?: number;
|
|
15
|
+
typeK?: number;
|
|
16
|
+
mainGpu?: number;
|
|
17
|
+
nBatch?: number;
|
|
18
|
+
nGpuLayers?: number;
|
|
19
|
+
nCtx?: number;
|
|
20
|
+
nCtxMin?: number;
|
|
21
|
+
marginMiB?: number;
|
|
22
|
+
backendsDir?: string;
|
|
23
|
+
modelPath: string;
|
|
24
|
+
};
|
|
25
|
+
export declare function crashMarkerPath(stateDir: string, config: FitLlamaProcessConfig): string;
|
|
26
|
+
export declare function crashedMarkerPath(stateDir: string, config: FitLlamaProcessConfig): string;
|
|
27
|
+
/**
|
|
28
|
+
* In-process llama.cpp fit for hosts with no disposable child (Android/iOS).
|
|
29
|
+
*
|
|
30
|
+
* The fit runs on a model-fit worker thread, so the JS loop stays free, but a
|
|
31
|
+
* thread is not a process boundary: a leftover `.running` marker means a
|
|
32
|
+
* previous call aborted the process. That is converted to `.crashed` so later
|
|
33
|
+
* launches skip native for the same path+config instead of retrying the abort.
|
|
34
|
+
*/
|
|
35
|
+
export declare function runInProcessFit(_loadKind: LlamaLoadKind, config: FitLlamaProcessConfig, options?: RunInProcessFitOptions): Promise<{
|
|
36
|
+
status: string;
|
|
37
|
+
reason: IsolatedFitUnknownReason;
|
|
38
|
+
message: string;
|
|
39
|
+
} | {
|
|
40
|
+
status: "completed";
|
|
41
|
+
result: FitResult;
|
|
42
|
+
}>;
|