@qvac/inference 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +95 -39
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/delete-cache.js +3 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.js +169 -68
- package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
- package/dist/resources/model-fit/estimators/llm.js +3 -1
- package/dist/resources/model-fit/floor.d.ts +31 -0
- package/dist/resources/model-fit/floor.js +65 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/resources/native.js +5 -4
- package/dist/resources/types.d.ts +1 -3
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +84 -0
- package/dist/schemas/assess-model-fit.js +72 -3
- package/dist/schemas/audio-gen.d.ts +482 -7
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +10 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci.d.ts +1 -1
- package/dist/schemas/common.d.ts +782 -19
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +14 -1
- package/dist/schemas/completion-event.js +4 -0
- package/dist/schemas/completion-stream.d.ts +22 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2604 -180
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/model-ops.d.ts +3 -0
- package/dist/schemas/model-ops.js +6 -0
- package/dist/schemas/plugin.d.ts +21 -11
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +30 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +29 -26
package/dist/runtime/env.js
CHANGED
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import env from 'bare-env';
|
|
2
2
|
import { z } from 'zod';
|
|
3
3
|
const envSchema = z.object({
|
|
4
|
-
HOME_DIR: z.string()
|
|
4
|
+
HOME_DIR: z.string(),
|
|
5
|
+
/**
|
|
6
|
+
* Operator opt-out for the advisory llama.cpp fit check (on by default).
|
|
7
|
+
* Free-form rather than an enum so a typo cannot fail engine startup; only
|
|
8
|
+
* `0`/`false`/`off`/`no` disable it.
|
|
9
|
+
*/
|
|
10
|
+
QVAC_ADVISORY_MODEL_FIT: z.string().optional()
|
|
5
11
|
});
|
|
6
12
|
let validatedEnv = null;
|
|
7
13
|
/**
|
|
@@ -12,7 +18,8 @@ export function initEnv() {
|
|
|
12
18
|
// Snap's HOME can be revision-scoped; SNAP_USER_COMMON is stable.
|
|
13
19
|
env['SNAP_USER_COMMON'] ?? env['HOME'] ?? env['USERPROFILE'] ?? '/tmp';
|
|
14
20
|
let envConfig = {
|
|
15
|
-
HOME_DIR: defaultHomeDir
|
|
21
|
+
HOME_DIR: defaultHomeDir,
|
|
22
|
+
QVAC_ADVISORY_MODEL_FIT: env['QVAC_ADVISORY_MODEL_FIT']
|
|
16
23
|
};
|
|
17
24
|
const isBareKit = typeof globalThis.BareKit !== 'undefined';
|
|
18
25
|
if (isBareKit && Bare.argv[0]) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { initEnv } from './env.js';
|
|
2
2
|
import { closeAllRagInstances } from '../rag/index.js';
|
|
3
|
+
import { disposeAllVectorIndexes } from './vector-index-registry.js';
|
|
3
4
|
import { cleanupDownloads } from '../handlers/load-model/download-manager.js';
|
|
4
5
|
import { unloadAllModels } from './model-registry.js';
|
|
5
6
|
import { closeRegistryClient } from './registry-client.js';
|
|
@@ -57,6 +58,7 @@ async function runCleanup() {
|
|
|
57
58
|
cleanupRan = true;
|
|
58
59
|
destroyResourceCollector();
|
|
59
60
|
clearRegistries();
|
|
61
|
+
disposeAllVectorIndexes();
|
|
60
62
|
await Promise.allSettled([
|
|
61
63
|
closeAllRagInstances(),
|
|
62
64
|
cleanupDownloads(),
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CanonicalModelType } from '../schemas/index.js';
|
|
1
|
+
import type { CanonicalModelType, NativeProbeFit } from '../schemas/index.js';
|
|
2
2
|
interface AddonInterface {
|
|
3
3
|
cancel(jobId?: string): Promise<void>;
|
|
4
4
|
}
|
|
@@ -26,6 +26,7 @@ interface LocalOptions {
|
|
|
26
26
|
config: unknown;
|
|
27
27
|
modelType: CanonicalModelType;
|
|
28
28
|
name?: string | undefined;
|
|
29
|
+
fitProbe?: NativeProbeFit | undefined;
|
|
29
30
|
}
|
|
30
31
|
export type ModelEntry = {
|
|
31
32
|
id: string;
|
|
@@ -37,6 +38,7 @@ export declare function registerModel(id: string, options: {
|
|
|
37
38
|
config: unknown;
|
|
38
39
|
modelType: CanonicalModelType;
|
|
39
40
|
name?: string | undefined;
|
|
41
|
+
fitProbe?: NativeProbeFit | undefined;
|
|
40
42
|
}): void;
|
|
41
43
|
export declare function getModelEntry(id: string): ModelEntry | null;
|
|
42
44
|
export declare function getModel(id: string): AnyModel;
|
|
@@ -15,7 +15,8 @@ export function registerModel(id, options) {
|
|
|
15
15
|
loadedAt: new Date(),
|
|
16
16
|
config: options.config,
|
|
17
17
|
modelType: options.modelType,
|
|
18
|
-
name: options.name
|
|
18
|
+
name: options.name,
|
|
19
|
+
fitProbe: options.fitProbe
|
|
19
20
|
}
|
|
20
21
|
});
|
|
21
22
|
const nameStr = options.name ? ` (${options.name})` : '';
|
|
@@ -65,6 +65,23 @@ function installDefaultPolicies(r) {
|
|
|
65
65
|
onOverflow: 'queue',
|
|
66
66
|
maxQueueDepthPerModel: 64
|
|
67
67
|
});
|
|
68
|
+
// @qvac/tts-ggml is constructed with `exclusiveRun: true`, so it already
|
|
69
|
+
// serialises synthesis per model — but it queues internally, where the
|
|
70
|
+
// registry cannot see it. Mirroring the limit here keeps the registry
|
|
71
|
+
// authoritative, which is what model-scoped cancel needs: exactly one
|
|
72
|
+
// registered request owns the addon at a time, so cancelling a queued run
|
|
73
|
+
// aborts it before it starts instead of interrupting the run in flight.
|
|
74
|
+
// Deeper than the other lanes on purpose: the addon's own queue was
|
|
75
|
+
// unbounded, and a caller fanning a paragraph out sentence-by-sentence
|
|
76
|
+
// (`Promise.all` over a few dozen `textToSpeech` calls) is a realistic
|
|
77
|
+
// pattern that must keep working. Beyond this the call rejects with a typed
|
|
78
|
+
// RequestRejectedByPolicyError instead of piling up.
|
|
79
|
+
r.policy({
|
|
80
|
+
kind: 'tts',
|
|
81
|
+
maxConcurrentPerModel: 1,
|
|
82
|
+
onOverflow: 'queue',
|
|
83
|
+
maxQueueDepthPerModel: 256
|
|
84
|
+
});
|
|
68
85
|
// An ABot-World session runs one job at a time: the addon rejects a second
|
|
69
86
|
// step while a block is still streaming, and scene creation shares the same
|
|
70
87
|
// lane. Reject rather than queue, unlike every other policy here, because a
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { VectorIndexHit, VectorIndexStorage } from '../schemas/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* An open index as the vector index feature sees it. Row-major vectors and
|
|
4
|
+
* unsigned 64-bit ids go in; finished hit rows come out. Anything specific
|
|
5
|
+
* to a native engine (warm-up, result padding, how the storage mode is
|
|
6
|
+
* reported) is handled by the backend that implements this, not by callers.
|
|
7
|
+
*/
|
|
8
|
+
export interface OpenVectorIndex {
|
|
9
|
+
readonly dim: number;
|
|
10
|
+
readonly length: number;
|
|
11
|
+
readonly storage: VectorIndexStorage | undefined;
|
|
12
|
+
/** Adds `ids.length` vectors of `dim` components each, atomically. */
|
|
13
|
+
add(vectors: Float32Array, ids: BigUint64Array): void;
|
|
14
|
+
/** One hit list per query row, ordered by descending score; never padded. */
|
|
15
|
+
search(queries: Float32Array, k: number): VectorIndexHit[][];
|
|
16
|
+
contains(id: bigint): boolean;
|
|
17
|
+
/** Returns whether the id was present. */
|
|
18
|
+
remove(id: bigint): boolean;
|
|
19
|
+
write(path: string): void;
|
|
20
|
+
dispose(): void;
|
|
21
|
+
}
|
|
22
|
+
export interface VectorIndexBackend {
|
|
23
|
+
create(dim: number, storage: VectorIndexStorage): OpenVectorIndex;
|
|
24
|
+
load(path: string): OpenVectorIndex;
|
|
25
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { VectorIdWire, VectorIndexStorage } from '../schemas/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Snapshot paths are used as given when absolute. A relative path resolves
|
|
4
|
+
* against the QVAC data directory (the parent of the configured cache
|
|
5
|
+
* directory), the same root that holds the RAG workspace stores, so the
|
|
6
|
+
* same relative path works on every platform.
|
|
7
|
+
*/
|
|
8
|
+
export declare function resolveVectorIndexPath(snapshotPath: string): string;
|
|
9
|
+
export declare function createVectorIndex(params: {
|
|
10
|
+
dim: number;
|
|
11
|
+
storage: VectorIndexStorage;
|
|
12
|
+
}): {
|
|
13
|
+
indexId: string;
|
|
14
|
+
dim: number;
|
|
15
|
+
storage: VectorIndexStorage;
|
|
16
|
+
length: number;
|
|
17
|
+
};
|
|
18
|
+
export declare function loadVectorIndex(params: {
|
|
19
|
+
path: string;
|
|
20
|
+
}): {
|
|
21
|
+
storage?: VectorIndexStorage;
|
|
22
|
+
indexId: string;
|
|
23
|
+
dim: number;
|
|
24
|
+
length: number;
|
|
25
|
+
};
|
|
26
|
+
export declare function addVectors(params: {
|
|
27
|
+
indexId: string;
|
|
28
|
+
ids: VectorIdWire[];
|
|
29
|
+
vectors: number[][];
|
|
30
|
+
}): {
|
|
31
|
+
length: number;
|
|
32
|
+
};
|
|
33
|
+
export declare function searchVectors(params: {
|
|
34
|
+
indexId: string;
|
|
35
|
+
queries: number[][];
|
|
36
|
+
k: number;
|
|
37
|
+
}): {
|
|
38
|
+
results: {
|
|
39
|
+
id: string;
|
|
40
|
+
score: number;
|
|
41
|
+
}[][];
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* Removes ids one at a time, which is the only shape the backend offers. A
|
|
45
|
+
* backend throw here means the handle itself is unusable, so the batch stops
|
|
46
|
+
* rather than trying the rest — but ids before it are already gone, so the
|
|
47
|
+
* error names how many were applied. `length` is only reported on success,
|
|
48
|
+
* which is why the handle documents its own `length` as of the last call that
|
|
49
|
+
* reported one.
|
|
50
|
+
*/
|
|
51
|
+
export declare function removeVectors(params: {
|
|
52
|
+
indexId: string;
|
|
53
|
+
ids: VectorIdWire[];
|
|
54
|
+
}): {
|
|
55
|
+
removed: boolean[];
|
|
56
|
+
length: number;
|
|
57
|
+
};
|
|
58
|
+
export declare function containsVectors(params: {
|
|
59
|
+
indexId: string;
|
|
60
|
+
ids: VectorIdWire[];
|
|
61
|
+
}): {
|
|
62
|
+
present: boolean[];
|
|
63
|
+
};
|
|
64
|
+
export declare function writeVectorIndex(params: {
|
|
65
|
+
indexId: string;
|
|
66
|
+
path: string;
|
|
67
|
+
}): Promise<{
|
|
68
|
+
path: string;
|
|
69
|
+
}>;
|
|
70
|
+
export declare function disposeVectorIndex(params: {
|
|
71
|
+
indexId: string;
|
|
72
|
+
}): {
|
|
73
|
+
disposed: boolean;
|
|
74
|
+
};
|
|
75
|
+
export declare function getOpenVectorIndexCount(): number;
|
|
76
|
+
export declare function disposeAllVectorIndexes(): void;
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { promises as fsPromises } from 'bare-fs';
|
|
2
|
+
import path from 'bare-path';
|
|
3
|
+
import { QvacErrorBase } from '@qvac/error';
|
|
4
|
+
import { getTurboVecBackend } from '../plugins/turbovec-backend.js';
|
|
5
|
+
import { getConfiguredCacheDir } from './state.js';
|
|
6
|
+
import { generateRandomRequestId } from './request-id.js';
|
|
7
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
8
|
+
import { VectorIndexFailedError, VectorIndexInvalidVectorsError, VectorIndexNotFoundError, VectorIndexProviderUnavailableError } from '../errors/index.js';
|
|
9
|
+
// Worker-side bookkeeping for open indexes: ids, wire-to-native conversion,
|
|
10
|
+
// row validation, snapshot paths, and lifetime. Engine behaviour lives in
|
|
11
|
+
// the backend.
|
|
12
|
+
const indexes = new Map();
|
|
13
|
+
function requireBackend() {
|
|
14
|
+
const backend = getTurboVecBackend();
|
|
15
|
+
if (!backend)
|
|
16
|
+
throw new VectorIndexProviderUnavailableError();
|
|
17
|
+
return backend;
|
|
18
|
+
}
|
|
19
|
+
function getIndex(indexId) {
|
|
20
|
+
const index = indexes.get(indexId);
|
|
21
|
+
if (!index)
|
|
22
|
+
throw new VectorIndexNotFoundError(indexId);
|
|
23
|
+
return index;
|
|
24
|
+
}
|
|
25
|
+
// Backends throw plain TypeError/RangeError/Error values from the native
|
|
26
|
+
// layer; wrap them so the caller receives a coded error with the original
|
|
27
|
+
// message preserved as cause. `describe` adds caller context to the message
|
|
28
|
+
// and is evaluated at throw time, so it can report progress made so far.
|
|
29
|
+
function runBackend(action, describe) {
|
|
30
|
+
try {
|
|
31
|
+
return action();
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
if (error instanceof QvacErrorBase)
|
|
35
|
+
throw error;
|
|
36
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
37
|
+
throw new VectorIndexFailedError(describe ? describe(detail) : detail, error);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Snapshot paths are used as given when absolute. A relative path resolves
|
|
42
|
+
* against the QVAC data directory (the parent of the configured cache
|
|
43
|
+
* directory), the same root that holds the RAG workspace stores, so the
|
|
44
|
+
* same relative path works on every platform.
|
|
45
|
+
*/
|
|
46
|
+
export function resolveVectorIndexPath(snapshotPath) {
|
|
47
|
+
if (path.isAbsolute(snapshotPath))
|
|
48
|
+
return snapshotPath;
|
|
49
|
+
return path.join(path.dirname(getConfiguredCacheDir()), snapshotPath);
|
|
50
|
+
}
|
|
51
|
+
function flattenRows(rows, dim, label) {
|
|
52
|
+
const flat = new Float32Array(rows.length * dim);
|
|
53
|
+
rows.forEach((row, rowIndex) => {
|
|
54
|
+
if (row.length !== dim) {
|
|
55
|
+
throw new VectorIndexInvalidVectorsError(`${label}[${rowIndex}] has ${row.length} components, expected ${dim}`);
|
|
56
|
+
}
|
|
57
|
+
flat.set(row, rowIndex * dim);
|
|
58
|
+
});
|
|
59
|
+
return flat;
|
|
60
|
+
}
|
|
61
|
+
function toNativeIds(ids) {
|
|
62
|
+
return new BigUint64Array(ids.map((id) => BigInt(id)));
|
|
63
|
+
}
|
|
64
|
+
function track(index) {
|
|
65
|
+
const indexId = generateRandomRequestId();
|
|
66
|
+
indexes.set(indexId, index);
|
|
67
|
+
return indexId;
|
|
68
|
+
}
|
|
69
|
+
export function createVectorIndex(params) {
|
|
70
|
+
const backend = requireBackend();
|
|
71
|
+
const index = runBackend(() => backend.create(params.dim, params.storage));
|
|
72
|
+
const indexId = track(index);
|
|
73
|
+
return { indexId, dim: index.dim, storage: params.storage, length: index.length };
|
|
74
|
+
}
|
|
75
|
+
export function loadVectorIndex(params) {
|
|
76
|
+
const backend = requireBackend();
|
|
77
|
+
const index = runBackend(() => backend.load(resolveVectorIndexPath(params.path)));
|
|
78
|
+
const indexId = track(index);
|
|
79
|
+
return {
|
|
80
|
+
indexId,
|
|
81
|
+
dim: index.dim,
|
|
82
|
+
length: index.length,
|
|
83
|
+
...(index.storage !== undefined && { storage: index.storage })
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
export function addVectors(params) {
|
|
87
|
+
const index = getIndex(params.indexId);
|
|
88
|
+
if (params.ids.length !== params.vectors.length) {
|
|
89
|
+
throw new VectorIndexInvalidVectorsError(`ids has ${params.ids.length} entries but vectors has ${params.vectors.length} rows`);
|
|
90
|
+
}
|
|
91
|
+
const vectors = flattenRows(params.vectors, index.dim, 'vectors');
|
|
92
|
+
const ids = toNativeIds(params.ids);
|
|
93
|
+
runBackend(() => index.add(vectors, ids));
|
|
94
|
+
return { length: index.length };
|
|
95
|
+
}
|
|
96
|
+
export function searchVectors(params) {
|
|
97
|
+
const index = getIndex(params.indexId);
|
|
98
|
+
const queries = flattenRows(params.queries, index.dim, 'queries');
|
|
99
|
+
const results = runBackend(() => index.search(queries, params.k));
|
|
100
|
+
return { results };
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Removes ids one at a time, which is the only shape the backend offers. A
|
|
104
|
+
* backend throw here means the handle itself is unusable, so the batch stops
|
|
105
|
+
* rather than trying the rest — but ids before it are already gone, so the
|
|
106
|
+
* error names how many were applied. `length` is only reported on success,
|
|
107
|
+
* which is why the handle documents its own `length` as of the last call that
|
|
108
|
+
* reported one.
|
|
109
|
+
*/
|
|
110
|
+
export function removeVectors(params) {
|
|
111
|
+
const index = getIndex(params.indexId);
|
|
112
|
+
const removed = [];
|
|
113
|
+
for (const id of params.ids) {
|
|
114
|
+
removed.push(runBackend(() => index.remove(BigInt(id)), (detail) => `remove applied ${removed.length} of ${params.ids.length} ids before failing: ${detail}`));
|
|
115
|
+
}
|
|
116
|
+
return { removed, length: index.length };
|
|
117
|
+
}
|
|
118
|
+
export function containsVectors(params) {
|
|
119
|
+
const index = getIndex(params.indexId);
|
|
120
|
+
const present = params.ids.map((id) => runBackend(() => index.contains(BigInt(id))));
|
|
121
|
+
return { present };
|
|
122
|
+
}
|
|
123
|
+
export async function writeVectorIndex(params) {
|
|
124
|
+
const index = getIndex(params.indexId);
|
|
125
|
+
const snapshotPath = resolveVectorIndexPath(params.path);
|
|
126
|
+
try {
|
|
127
|
+
await fsPromises.mkdir(path.dirname(snapshotPath), { recursive: true });
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
throw new VectorIndexFailedError(`cannot create the snapshot directory for ${snapshotPath}: ${error instanceof Error ? error.message : String(error)}`, error);
|
|
131
|
+
}
|
|
132
|
+
runBackend(() => index.write(snapshotPath));
|
|
133
|
+
return { path: snapshotPath };
|
|
134
|
+
}
|
|
135
|
+
// Disposing an unknown id is not an error: the caller may retry after a
|
|
136
|
+
// worker restart, and the memory it refers to is already gone.
|
|
137
|
+
export function disposeVectorIndex(params) {
|
|
138
|
+
const index = indexes.get(params.indexId);
|
|
139
|
+
if (!index)
|
|
140
|
+
return { disposed: false };
|
|
141
|
+
// Drop the entry only once the backend confirms. Deleting first would strand
|
|
142
|
+
// a failed dispose: the retry would find nothing and report success, and
|
|
143
|
+
// `disposeAllVectorIndexes` could not reach it at shutdown either.
|
|
144
|
+
runBackend(() => index.dispose());
|
|
145
|
+
indexes.delete(params.indexId);
|
|
146
|
+
return { disposed: true };
|
|
147
|
+
}
|
|
148
|
+
export function getOpenVectorIndexCount() {
|
|
149
|
+
return indexes.size;
|
|
150
|
+
}
|
|
151
|
+
export function disposeAllVectorIndexes() {
|
|
152
|
+
for (const [indexId, index] of indexes) {
|
|
153
|
+
try {
|
|
154
|
+
index.dispose();
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
getEngineLogger().warn(`Failed to dispose vector index '${indexId}' during cleanup:`, error);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
indexes.clear();
|
|
161
|
+
}
|
|
@@ -120,6 +120,20 @@ export declare const modelFitBasisSchema: z.ZodEnum<{
|
|
|
120
120
|
"device-memory": "device-memory";
|
|
121
121
|
"device-budget": "device-budget";
|
|
122
122
|
}>;
|
|
123
|
+
/**
|
|
124
|
+
* What kind of evidence a verdict rests on.
|
|
125
|
+
*
|
|
126
|
+
* - `calibration`: a two-sided estimate from coefficients measured on this
|
|
127
|
+
* platform. The only evidence that can support `likely-fits`.
|
|
128
|
+
* - `computed-only`: a floor computed from catalog facts alone — artifact bytes,
|
|
129
|
+
* plus the KV cache for llama.cpp models. It omits every engine cost that
|
|
130
|
+
* only a real load can tell, so it can refuse a model but never confirm one.
|
|
131
|
+
* This is what an uncalibrated platform, including Android and iOS, reports.
|
|
132
|
+
*/
|
|
133
|
+
export declare const modelFitEvidenceSchema: z.ZodEnum<{
|
|
134
|
+
calibration: "calibration";
|
|
135
|
+
"computed-only": "computed-only";
|
|
136
|
+
}>;
|
|
123
137
|
export declare const modelFitBudgetSchema: z.ZodObject<{
|
|
124
138
|
totalBytes: z.ZodNumber;
|
|
125
139
|
usedBytes: z.ZodNumber;
|
|
@@ -134,10 +148,15 @@ export declare const modelFitModelResultSchema: z.ZodObject<{
|
|
|
134
148
|
"likely-fits": "likely-fits";
|
|
135
149
|
"likely-too-large": "likely-too-large";
|
|
136
150
|
}>;
|
|
151
|
+
evidence: z.ZodOptional<z.ZodEnum<{
|
|
152
|
+
calibration: "calibration";
|
|
153
|
+
"computed-only": "computed-only";
|
|
154
|
+
}>>;
|
|
137
155
|
estimate: z.ZodOptional<z.ZodObject<{
|
|
138
156
|
lowerBoundBytes: z.ZodNumber;
|
|
139
157
|
upperBoundBytes: z.ZodNumber;
|
|
140
158
|
}, z.core.$strip>>;
|
|
159
|
+
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
141
160
|
estimatorVersion: z.ZodOptional<z.ZodString>;
|
|
142
161
|
reasons: z.ZodArray<z.ZodString>;
|
|
143
162
|
}, z.core.$strip>;
|
|
@@ -157,6 +176,10 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
157
176
|
sequential: "sequential";
|
|
158
177
|
concurrent: "concurrent";
|
|
159
178
|
}>;
|
|
179
|
+
evidence: z.ZodOptional<z.ZodEnum<{
|
|
180
|
+
calibration: "calibration";
|
|
181
|
+
"computed-only": "computed-only";
|
|
182
|
+
}>>;
|
|
160
183
|
budget: z.ZodOptional<z.ZodObject<{
|
|
161
184
|
totalBytes: z.ZodNumber;
|
|
162
185
|
usedBytes: z.ZodNumber;
|
|
@@ -168,6 +191,7 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
168
191
|
lowerBoundBytes: z.ZodNumber;
|
|
169
192
|
upperBoundBytes: z.ZodNumber;
|
|
170
193
|
}, z.core.$strip>>;
|
|
194
|
+
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
171
195
|
models: z.ZodArray<z.ZodObject<{
|
|
172
196
|
name: z.ZodString;
|
|
173
197
|
verdict: z.ZodEnum<{
|
|
@@ -175,16 +199,62 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
175
199
|
"likely-fits": "likely-fits";
|
|
176
200
|
"likely-too-large": "likely-too-large";
|
|
177
201
|
}>;
|
|
202
|
+
evidence: z.ZodOptional<z.ZodEnum<{
|
|
203
|
+
calibration: "calibration";
|
|
204
|
+
"computed-only": "computed-only";
|
|
205
|
+
}>>;
|
|
178
206
|
estimate: z.ZodOptional<z.ZodObject<{
|
|
179
207
|
lowerBoundBytes: z.ZodNumber;
|
|
180
208
|
upperBoundBytes: z.ZodNumber;
|
|
181
209
|
}, z.core.$strip>>;
|
|
210
|
+
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
182
211
|
estimatorVersion: z.ZodOptional<z.ZodString>;
|
|
183
212
|
reasons: z.ZodArray<z.ZodString>;
|
|
184
213
|
}, z.core.$strip>>;
|
|
185
214
|
reasons: z.ZodArray<z.ZodString>;
|
|
186
215
|
assumptions: z.ZodArray<z.ZodString>;
|
|
187
216
|
}, z.core.$strip>;
|
|
217
|
+
/**
|
|
218
|
+
* The other fit question, kept in this file on purpose: `assessModelFit` above
|
|
219
|
+
* answers "should I download this?" from calibrated coefficients and a checksum,
|
|
220
|
+
* while the native probe answers "will the load I am about to run fit?" by
|
|
221
|
+
* reading the GGUF on disk in a disposable child. Two evidence classes, so two
|
|
222
|
+
* verdict vocabularies:
|
|
223
|
+
*
|
|
224
|
+
* - `likely-fits` / `likely-too-large` hedge because a formula is an estimate.
|
|
225
|
+
* - `fit` / `does-not-fit` do not, because the probe measured this build of this
|
|
226
|
+
* file against this device.
|
|
227
|
+
*
|
|
228
|
+
* Precedence when both have spoken about the same model: the probe wins, because
|
|
229
|
+
* it saw the artifact and the resolved load settings. `assessModelFit` is the
|
|
230
|
+
* answer available before the bytes are on disk. Neither denies a load today.
|
|
231
|
+
*/
|
|
232
|
+
export declare const nativeProbeVerdictSchema: z.ZodEnum<{
|
|
233
|
+
unknown: "unknown";
|
|
234
|
+
fit: "fit";
|
|
235
|
+
"does-not-fit": "does-not-fit";
|
|
236
|
+
}>;
|
|
237
|
+
export declare const nativeProbePlanSchema: z.ZodObject<{
|
|
238
|
+
nCtx: z.ZodNumber;
|
|
239
|
+
nGpuLayers: z.ZodNumber;
|
|
240
|
+
nGpuDevices: z.ZodNumber;
|
|
241
|
+
}, z.core.$strip>;
|
|
242
|
+
export declare const nativeProbeFitSchema: z.ZodObject<{
|
|
243
|
+
verdict: z.ZodEnum<{
|
|
244
|
+
unknown: "unknown";
|
|
245
|
+
fit: "fit";
|
|
246
|
+
"does-not-fit": "does-not-fit";
|
|
247
|
+
}>;
|
|
248
|
+
basis: z.ZodLiteral<"native-probe">;
|
|
249
|
+
estimatorVersion: z.ZodString;
|
|
250
|
+
reason: z.ZodString;
|
|
251
|
+
message: z.ZodOptional<z.ZodString>;
|
|
252
|
+
plan: z.ZodOptional<z.ZodObject<{
|
|
253
|
+
nCtx: z.ZodNumber;
|
|
254
|
+
nGpuLayers: z.ZodNumber;
|
|
255
|
+
nGpuDevices: z.ZodNumber;
|
|
256
|
+
}, z.core.$strip>>;
|
|
257
|
+
}, z.core.$strip>;
|
|
188
258
|
export declare const assessModelFitRequestSchema: z.ZodObject<{
|
|
189
259
|
models: z.ZodArray<z.ZodObject<{
|
|
190
260
|
model: z.ZodObject<{
|
|
@@ -228,6 +298,10 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
228
298
|
sequential: "sequential";
|
|
229
299
|
concurrent: "concurrent";
|
|
230
300
|
}>;
|
|
301
|
+
evidence: z.ZodOptional<z.ZodEnum<{
|
|
302
|
+
calibration: "calibration";
|
|
303
|
+
"computed-only": "computed-only";
|
|
304
|
+
}>>;
|
|
231
305
|
budget: z.ZodOptional<z.ZodObject<{
|
|
232
306
|
totalBytes: z.ZodNumber;
|
|
233
307
|
usedBytes: z.ZodNumber;
|
|
@@ -239,6 +313,7 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
239
313
|
lowerBoundBytes: z.ZodNumber;
|
|
240
314
|
upperBoundBytes: z.ZodNumber;
|
|
241
315
|
}, z.core.$strip>>;
|
|
316
|
+
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
242
317
|
models: z.ZodArray<z.ZodObject<{
|
|
243
318
|
name: z.ZodString;
|
|
244
319
|
verdict: z.ZodEnum<{
|
|
@@ -246,10 +321,15 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
246
321
|
"likely-fits": "likely-fits";
|
|
247
322
|
"likely-too-large": "likely-too-large";
|
|
248
323
|
}>;
|
|
324
|
+
evidence: z.ZodOptional<z.ZodEnum<{
|
|
325
|
+
calibration: "calibration";
|
|
326
|
+
"computed-only": "computed-only";
|
|
327
|
+
}>>;
|
|
249
328
|
estimate: z.ZodOptional<z.ZodObject<{
|
|
250
329
|
lowerBoundBytes: z.ZodNumber;
|
|
251
330
|
upperBoundBytes: z.ZodNumber;
|
|
252
331
|
}, z.core.$strip>>;
|
|
332
|
+
floorBytes: z.ZodOptional<z.ZodNumber>;
|
|
253
333
|
estimatorVersion: z.ZodOptional<z.ZodString>;
|
|
254
334
|
reasons: z.ZodArray<z.ZodString>;
|
|
255
335
|
}, z.core.$strip>>;
|
|
@@ -257,12 +337,16 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
257
337
|
assumptions: z.ZodArray<z.ZodString>;
|
|
258
338
|
type: z.ZodLiteral<"assessModelFit">;
|
|
259
339
|
}, z.core.$strip>;
|
|
340
|
+
export type NativeProbeVerdict = z.infer<typeof nativeProbeVerdictSchema>;
|
|
341
|
+
export type NativeProbePlan = z.infer<typeof nativeProbePlanSchema>;
|
|
342
|
+
export type NativeProbeFit = z.infer<typeof nativeProbeFitSchema>;
|
|
260
343
|
export type ModelFitVerdict = z.infer<typeof modelFitVerdictSchema>;
|
|
261
344
|
export type ModelFitModelRef = z.infer<typeof modelFitModelRefSchema>;
|
|
262
345
|
export type ModelFitWorkload = z.infer<typeof modelFitWorkloadSchema>;
|
|
263
346
|
export type ModelFitCandidate = z.infer<typeof modelFitCandidateSchema>;
|
|
264
347
|
export type ModelFitExecution = z.infer<typeof modelFitExecutionSchema>;
|
|
265
348
|
export type ModelFitBasis = z.infer<typeof modelFitBasisSchema>;
|
|
349
|
+
export type ModelFitEvidence = z.infer<typeof modelFitEvidenceSchema>;
|
|
266
350
|
export type ModelFitBudget = z.infer<typeof modelFitBudgetSchema>;
|
|
267
351
|
export type ModelFitModelResult = z.infer<typeof modelFitModelResultSchema>;
|
|
268
352
|
export type AssessModelFitInput = z.input<typeof assessModelFitInputSchema>;
|
|
@@ -108,6 +108,17 @@ export const modelFitBasisSchema = z.enum([
|
|
|
108
108
|
'device-memory',
|
|
109
109
|
'device-budget'
|
|
110
110
|
]);
|
|
111
|
+
/**
|
|
112
|
+
* What kind of evidence a verdict rests on.
|
|
113
|
+
*
|
|
114
|
+
* - `calibration`: a two-sided estimate from coefficients measured on this
|
|
115
|
+
* platform. The only evidence that can support `likely-fits`.
|
|
116
|
+
* - `computed-only`: a floor computed from catalog facts alone — artifact bytes,
|
|
117
|
+
* plus the KV cache for llama.cpp models. It omits every engine cost that
|
|
118
|
+
* only a real load can tell, so it can refuse a model but never confirm one.
|
|
119
|
+
* This is what an uncalibrated platform, including Android and iOS, reports.
|
|
120
|
+
*/
|
|
121
|
+
export const modelFitEvidenceSchema = z.enum(['calibration', 'computed-only']);
|
|
111
122
|
export const modelFitBudgetSchema = z.object({
|
|
112
123
|
totalBytes: z
|
|
113
124
|
.number()
|
|
@@ -126,25 +137,83 @@ export const modelFitBudgetSchema = z.object({
|
|
|
126
137
|
export const modelFitModelResultSchema = z.object({
|
|
127
138
|
name: z.string().describe('Catalog name of the assessed model.'),
|
|
128
139
|
verdict: modelFitVerdictSchema,
|
|
129
|
-
|
|
140
|
+
evidence: modelFitEvidenceSchema
|
|
141
|
+
.optional()
|
|
142
|
+
.describe('What the verdict rests on. Absent when nothing could be computed for this model, e.g. no catalog profile.'),
|
|
143
|
+
estimate: byteRangeSchema
|
|
144
|
+
.optional()
|
|
145
|
+
.describe('Two-sided bound from calibrated coefficients. Absent under computed-only evidence, or when this model assessed as `unknown` for want of any evidence.'),
|
|
146
|
+
floorBytes: z
|
|
147
|
+
.number()
|
|
148
|
+
.optional()
|
|
149
|
+
.describe('Under computed-only evidence: the smallest resident footprint the catalog facts prove — artifact bytes, plus the KV cache for llama.cpp. A floor only; the true cost is above it by an unmeasured amount.'),
|
|
130
150
|
estimatorVersion: z
|
|
131
151
|
.string()
|
|
132
152
|
.optional()
|
|
133
|
-
.describe('Estimator that produced the bounds, e.g. `llm-v1
|
|
153
|
+
.describe('Estimator that produced the bounds, e.g. `llm-v1`, or `floor-v1` for a computed floor.'),
|
|
134
154
|
reasons: z.array(z.string()).describe('Why this model got this verdict.')
|
|
135
155
|
});
|
|
136
156
|
export const assessModelFitResultSchema = z.object({
|
|
137
157
|
verdict: modelFitVerdictSchema.describe('Combined verdict across every candidate.'),
|
|
138
158
|
basis: modelFitBasisSchema.describe('The evidence the budget was derived from — system RAM, the per-process ceiling on iOS, a discrete GPU’s own memory, or on Windows the GPU memory budget the OS grants this process. The two device bases also require the system-memory budget to hold.'),
|
|
139
159
|
execution: modelFitExecutionSchema.describe('The declared execution mode this result assumed.'),
|
|
160
|
+
evidence: modelFitEvidenceSchema
|
|
161
|
+
.optional()
|
|
162
|
+
.describe('The weakest evidence among the candidates: `computed-only` as soon as one model has only a floor, since the combined verdict can then never be `likely-fits`. Absent whenever any candidate could not be assessed at all, so an `unknown` that carries `evidence` is a near-miss or an uncalibrated floor, never a missing model.'),
|
|
140
163
|
budget: modelFitBudgetSchema.optional().describe('Absent when memory evidence was unusable.'),
|
|
141
|
-
estimate: byteRangeSchema
|
|
164
|
+
estimate: byteRangeSchema
|
|
165
|
+
.optional()
|
|
166
|
+
.describe('Combined two-sided bound. Absent when the combined verdict is `unknown` for want of evidence, and under computed-only evidence, which has no upper bound.'),
|
|
167
|
+
floorBytes: z
|
|
168
|
+
.number()
|
|
169
|
+
.optional()
|
|
170
|
+
.describe('Under computed-only evidence: the combined floor across every candidate, aggregated under `execution`. Compared against the budget for `likely-too-large`; never enough for `likely-fits`.'),
|
|
142
171
|
models: z.array(modelFitModelResultSchema).describe('Per-candidate verdicts, in input order.'),
|
|
143
172
|
reasons: z.array(z.string()).describe('Why the combined verdict came out this way.'),
|
|
144
173
|
assumptions: z
|
|
145
174
|
.array(z.string())
|
|
146
175
|
.describe('Everything the result took for granted, including estimator defaults.')
|
|
147
176
|
});
|
|
177
|
+
// ============== Native probe (pre-load) ==============
|
|
178
|
+
/**
|
|
179
|
+
* The other fit question, kept in this file on purpose: `assessModelFit` above
|
|
180
|
+
* answers "should I download this?" from calibrated coefficients and a checksum,
|
|
181
|
+
* while the native probe answers "will the load I am about to run fit?" by
|
|
182
|
+
* reading the GGUF on disk in a disposable child. Two evidence classes, so two
|
|
183
|
+
* verdict vocabularies:
|
|
184
|
+
*
|
|
185
|
+
* - `likely-fits` / `likely-too-large` hedge because a formula is an estimate.
|
|
186
|
+
* - `fit` / `does-not-fit` do not, because the probe measured this build of this
|
|
187
|
+
* file against this device.
|
|
188
|
+
*
|
|
189
|
+
* Precedence when both have spoken about the same model: the probe wins, because
|
|
190
|
+
* it saw the artifact and the resolved load settings. `assessModelFit` is the
|
|
191
|
+
* answer available before the bytes are on disk. Neither denies a load today.
|
|
192
|
+
*/
|
|
193
|
+
export const nativeProbeVerdictSchema = z.enum(['fit', 'does-not-fit', 'unknown']);
|
|
194
|
+
export const nativeProbePlanSchema = z.object({
|
|
195
|
+
nCtx: z.number().int().describe('Context the probe resolved for this load, in tokens.'),
|
|
196
|
+
nGpuLayers: z.number().int().describe('Layers the probe would offload.'),
|
|
197
|
+
nGpuDevices: z.number().int().describe('GPU devices the offload would span.')
|
|
198
|
+
});
|
|
199
|
+
export const nativeProbeFitSchema = z
|
|
200
|
+
.object({
|
|
201
|
+
verdict: nativeProbeVerdictSchema.describe('Advisory outcome. `unknown` means no verdict was obtainable — the check was disabled, the load shape is unsupported, or the child produced no usable answer.'),
|
|
202
|
+
basis: z
|
|
203
|
+
.literal('native-probe')
|
|
204
|
+
.describe('Evidence class: a disposable llama.cpp child that read the model file and the resolved load settings.'),
|
|
205
|
+
estimatorVersion: z
|
|
206
|
+
.string()
|
|
207
|
+
.describe('Version of the probe integration that produced this outcome, covering the load-setting partitioning and the headroom policy — `native-probe-v1` withholds 1024 MiB plus the on-disk bytes of every model already resident in this worker.'),
|
|
208
|
+
reason: z
|
|
209
|
+
.string()
|
|
210
|
+
.describe('Machine-readable reason for the verdict. Never parsed out of log text.'),
|
|
211
|
+
message: z.string().optional().describe('Human-readable detail, when the reason has any.'),
|
|
212
|
+
plan: nativeProbePlanSchema
|
|
213
|
+
.optional()
|
|
214
|
+
.describe('Placement the probe projected. Present only on a `fit` verdict.')
|
|
215
|
+
})
|
|
216
|
+
.meta({ title: 'NativeProbeFit' });
|
|
148
217
|
export const assessModelFitRequestSchema = assessModelFitInputSchema.extend({
|
|
149
218
|
type: z.literal('assessModelFit')
|
|
150
219
|
});
|