@qvac/inference 0.19.1 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +57 -17
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/assess-model-fit.d.ts +30 -6
- package/dist/handlers/assess-model-fit.js +68 -6
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +69 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +10 -1
- package/dist/resources/model-fit/assess.js +43 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +83 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +133 -0
- package/dist/resources/model-fit/fit-stub/index.d.ts +2 -0
- package/dist/resources/model-fit/fit-stub/index.js +2 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +31 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +36 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +191 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +67 -0
- package/dist/schemas/assess-model-fit.js +56 -2
- package/dist/schemas/audio-gen.d.ts +479 -4
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +7 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/common.d.ts +816 -15
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +11 -1
- package/dist/schemas/completion-event.js +3 -0
- package/dist/schemas/completion-stream.d.ts +20 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +186 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +88 -8
- package/dist/schemas/llamacpp-config.js +108 -8
- package/dist/schemas/load-model.d.ts +2906 -206
- package/dist/schemas/model-config-schemas.d.ts +251 -5
- package/dist/schemas/plugin.d.ts +10 -0
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +29 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +29 -26
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CanonicalModelType } from '../schemas/index.js';
|
|
1
|
+
import type { CanonicalModelType, NativeProbeFit } from '../schemas/index.js';
|
|
2
2
|
interface AddonInterface {
|
|
3
3
|
cancel(jobId?: string): Promise<void>;
|
|
4
4
|
}
|
|
@@ -26,6 +26,7 @@ interface LocalOptions {
|
|
|
26
26
|
config: unknown;
|
|
27
27
|
modelType: CanonicalModelType;
|
|
28
28
|
name?: string | undefined;
|
|
29
|
+
fitProbe?: NativeProbeFit | undefined;
|
|
29
30
|
}
|
|
30
31
|
export type ModelEntry = {
|
|
31
32
|
id: string;
|
|
@@ -37,6 +38,7 @@ export declare function registerModel(id: string, options: {
|
|
|
37
38
|
config: unknown;
|
|
38
39
|
modelType: CanonicalModelType;
|
|
39
40
|
name?: string | undefined;
|
|
41
|
+
fitProbe?: NativeProbeFit | undefined;
|
|
40
42
|
}): void;
|
|
41
43
|
export declare function getModelEntry(id: string): ModelEntry | null;
|
|
42
44
|
export declare function getModel(id: string): AnyModel;
|
|
@@ -15,7 +15,8 @@ export function registerModel(id, options) {
|
|
|
15
15
|
loadedAt: new Date(),
|
|
16
16
|
config: options.config,
|
|
17
17
|
modelType: options.modelType,
|
|
18
|
-
name: options.name
|
|
18
|
+
name: options.name,
|
|
19
|
+
fitProbe: options.fitProbe
|
|
19
20
|
}
|
|
20
21
|
});
|
|
21
22
|
const nameStr = options.name ? ` (${options.name})` : '';
|
|
@@ -65,6 +65,23 @@ function installDefaultPolicies(r) {
|
|
|
65
65
|
onOverflow: 'queue',
|
|
66
66
|
maxQueueDepthPerModel: 64
|
|
67
67
|
});
|
|
68
|
+
// @qvac/tts-ggml is constructed with `exclusiveRun: true`, so it already
|
|
69
|
+
// serialises synthesis per model — but it queues internally, where the
|
|
70
|
+
// registry cannot see it. Mirroring the limit here keeps the registry
|
|
71
|
+
// authoritative, which is what model-scoped cancel needs: exactly one
|
|
72
|
+
// registered request owns the addon at a time, so cancelling a queued run
|
|
73
|
+
// aborts it before it starts instead of interrupting the run in flight.
|
|
74
|
+
// Deeper than the other lanes on purpose: the addon's own queue was
|
|
75
|
+
// unbounded, and a caller fanning a paragraph out sentence-by-sentence
|
|
76
|
+
// (`Promise.all` over a few dozen `textToSpeech` calls) is a realistic
|
|
77
|
+
// pattern that must keep working. Beyond this the call rejects with a typed
|
|
78
|
+
// RequestRejectedByPolicyError instead of piling up.
|
|
79
|
+
r.policy({
|
|
80
|
+
kind: 'tts',
|
|
81
|
+
maxConcurrentPerModel: 1,
|
|
82
|
+
onOverflow: 'queue',
|
|
83
|
+
maxQueueDepthPerModel: 256
|
|
84
|
+
});
|
|
68
85
|
// An ABot-World session runs one job at a time: the addon rejects a second
|
|
69
86
|
// step while a block is still streaming, and scene creation shares the same
|
|
70
87
|
// lane. Reject rather than queue, unlike every other policy here, because a
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { VectorIndexHit, VectorIndexStorage } from '../schemas/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* An open index as the vector index feature sees it. Row-major vectors and
|
|
4
|
+
* unsigned 64-bit ids go in; finished hit rows come out. Anything specific
|
|
5
|
+
* to a native engine (warm-up, result padding, how the storage mode is
|
|
6
|
+
* reported) is handled by the backend that implements this, not by callers.
|
|
7
|
+
*/
|
|
8
|
+
export interface OpenVectorIndex {
|
|
9
|
+
readonly dim: number;
|
|
10
|
+
readonly length: number;
|
|
11
|
+
readonly storage: VectorIndexStorage | undefined;
|
|
12
|
+
/** Adds `ids.length` vectors of `dim` components each, atomically. */
|
|
13
|
+
add(vectors: Float32Array, ids: BigUint64Array): void;
|
|
14
|
+
/** One hit list per query row, ordered by descending score; never padded. */
|
|
15
|
+
search(queries: Float32Array, k: number): VectorIndexHit[][];
|
|
16
|
+
contains(id: bigint): boolean;
|
|
17
|
+
/** Returns whether the id was present. */
|
|
18
|
+
remove(id: bigint): boolean;
|
|
19
|
+
write(path: string): void;
|
|
20
|
+
dispose(): void;
|
|
21
|
+
}
|
|
22
|
+
export interface VectorIndexBackend {
|
|
23
|
+
create(dim: number, storage: VectorIndexStorage): OpenVectorIndex;
|
|
24
|
+
load(path: string): OpenVectorIndex;
|
|
25
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import type { VectorIdWire, VectorIndexStorage } from '../schemas/index.js';
|
|
2
|
+
/**
|
|
3
|
+
* Snapshot paths are used as given when absolute. A relative path resolves
|
|
4
|
+
* against the QVAC data directory (the parent of the configured cache
|
|
5
|
+
* directory), the same root that holds the RAG workspace stores, so the
|
|
6
|
+
* same relative path works on every platform.
|
|
7
|
+
*/
|
|
8
|
+
export declare function resolveVectorIndexPath(snapshotPath: string): string;
|
|
9
|
+
export declare function createVectorIndex(params: {
|
|
10
|
+
dim: number;
|
|
11
|
+
storage: VectorIndexStorage;
|
|
12
|
+
}): {
|
|
13
|
+
indexId: string;
|
|
14
|
+
dim: number;
|
|
15
|
+
storage: VectorIndexStorage;
|
|
16
|
+
length: number;
|
|
17
|
+
};
|
|
18
|
+
export declare function loadVectorIndex(params: {
|
|
19
|
+
path: string;
|
|
20
|
+
}): {
|
|
21
|
+
storage?: VectorIndexStorage;
|
|
22
|
+
indexId: string;
|
|
23
|
+
dim: number;
|
|
24
|
+
length: number;
|
|
25
|
+
};
|
|
26
|
+
export declare function addVectors(params: {
|
|
27
|
+
indexId: string;
|
|
28
|
+
ids: VectorIdWire[];
|
|
29
|
+
vectors: number[][];
|
|
30
|
+
}): {
|
|
31
|
+
length: number;
|
|
32
|
+
};
|
|
33
|
+
export declare function searchVectors(params: {
|
|
34
|
+
indexId: string;
|
|
35
|
+
queries: number[][];
|
|
36
|
+
k: number;
|
|
37
|
+
}): {
|
|
38
|
+
results: {
|
|
39
|
+
id: string;
|
|
40
|
+
score: number;
|
|
41
|
+
}[][];
|
|
42
|
+
};
|
|
43
|
+
/**
|
|
44
|
+
* Removes ids one at a time, which is the only shape the backend offers. A
|
|
45
|
+
* backend throw here means the handle itself is unusable, so the batch stops
|
|
46
|
+
* rather than trying the rest — but ids before it are already gone, so the
|
|
47
|
+
* error names how many were applied. `length` is only reported on success,
|
|
48
|
+
* which is why the handle documents its own `length` as of the last call that
|
|
49
|
+
* reported one.
|
|
50
|
+
*/
|
|
51
|
+
export declare function removeVectors(params: {
|
|
52
|
+
indexId: string;
|
|
53
|
+
ids: VectorIdWire[];
|
|
54
|
+
}): {
|
|
55
|
+
removed: boolean[];
|
|
56
|
+
length: number;
|
|
57
|
+
};
|
|
58
|
+
export declare function containsVectors(params: {
|
|
59
|
+
indexId: string;
|
|
60
|
+
ids: VectorIdWire[];
|
|
61
|
+
}): {
|
|
62
|
+
present: boolean[];
|
|
63
|
+
};
|
|
64
|
+
export declare function writeVectorIndex(params: {
|
|
65
|
+
indexId: string;
|
|
66
|
+
path: string;
|
|
67
|
+
}): Promise<{
|
|
68
|
+
path: string;
|
|
69
|
+
}>;
|
|
70
|
+
export declare function disposeVectorIndex(params: {
|
|
71
|
+
indexId: string;
|
|
72
|
+
}): {
|
|
73
|
+
disposed: boolean;
|
|
74
|
+
};
|
|
75
|
+
export declare function getOpenVectorIndexCount(): number;
|
|
76
|
+
export declare function disposeAllVectorIndexes(): void;
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { promises as fsPromises } from 'bare-fs';
|
|
2
|
+
import path from 'bare-path';
|
|
3
|
+
import { QvacErrorBase } from '@qvac/error';
|
|
4
|
+
import { getTurboVecBackend } from '../plugins/turbovec-backend.js';
|
|
5
|
+
import { getConfiguredCacheDir } from './state.js';
|
|
6
|
+
import { generateRandomRequestId } from './request-id.js';
|
|
7
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
8
|
+
import { VectorIndexFailedError, VectorIndexInvalidVectorsError, VectorIndexNotFoundError, VectorIndexProviderUnavailableError } from '../errors/index.js';
|
|
9
|
+
// Worker-side bookkeeping for open indexes: ids, wire-to-native conversion,
|
|
10
|
+
// row validation, snapshot paths, and lifetime. Engine behaviour lives in
|
|
11
|
+
// the backend.
|
|
12
|
+
const indexes = new Map();
|
|
13
|
+
function requireBackend() {
|
|
14
|
+
const backend = getTurboVecBackend();
|
|
15
|
+
if (!backend)
|
|
16
|
+
throw new VectorIndexProviderUnavailableError();
|
|
17
|
+
return backend;
|
|
18
|
+
}
|
|
19
|
+
function getIndex(indexId) {
|
|
20
|
+
const index = indexes.get(indexId);
|
|
21
|
+
if (!index)
|
|
22
|
+
throw new VectorIndexNotFoundError(indexId);
|
|
23
|
+
return index;
|
|
24
|
+
}
|
|
25
|
+
// Backends throw plain TypeError/RangeError/Error values from the native
|
|
26
|
+
// layer; wrap them so the caller receives a coded error with the original
|
|
27
|
+
// message preserved as cause. `describe` adds caller context to the message
|
|
28
|
+
// and is evaluated at throw time, so it can report progress made so far.
|
|
29
|
+
function runBackend(action, describe) {
|
|
30
|
+
try {
|
|
31
|
+
return action();
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
if (error instanceof QvacErrorBase)
|
|
35
|
+
throw error;
|
|
36
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
37
|
+
throw new VectorIndexFailedError(describe ? describe(detail) : detail, error);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Snapshot paths are used as given when absolute. A relative path resolves
|
|
42
|
+
* against the QVAC data directory (the parent of the configured cache
|
|
43
|
+
* directory), the same root that holds the RAG workspace stores, so the
|
|
44
|
+
* same relative path works on every platform.
|
|
45
|
+
*/
|
|
46
|
+
export function resolveVectorIndexPath(snapshotPath) {
|
|
47
|
+
if (path.isAbsolute(snapshotPath))
|
|
48
|
+
return snapshotPath;
|
|
49
|
+
return path.join(path.dirname(getConfiguredCacheDir()), snapshotPath);
|
|
50
|
+
}
|
|
51
|
+
function flattenRows(rows, dim, label) {
|
|
52
|
+
const flat = new Float32Array(rows.length * dim);
|
|
53
|
+
rows.forEach((row, rowIndex) => {
|
|
54
|
+
if (row.length !== dim) {
|
|
55
|
+
throw new VectorIndexInvalidVectorsError(`${label}[${rowIndex}] has ${row.length} components, expected ${dim}`);
|
|
56
|
+
}
|
|
57
|
+
flat.set(row, rowIndex * dim);
|
|
58
|
+
});
|
|
59
|
+
return flat;
|
|
60
|
+
}
|
|
61
|
+
function toNativeIds(ids) {
|
|
62
|
+
return new BigUint64Array(ids.map((id) => BigInt(id)));
|
|
63
|
+
}
|
|
64
|
+
function track(index) {
|
|
65
|
+
const indexId = generateRandomRequestId();
|
|
66
|
+
indexes.set(indexId, index);
|
|
67
|
+
return indexId;
|
|
68
|
+
}
|
|
69
|
+
export function createVectorIndex(params) {
|
|
70
|
+
const backend = requireBackend();
|
|
71
|
+
const index = runBackend(() => backend.create(params.dim, params.storage));
|
|
72
|
+
const indexId = track(index);
|
|
73
|
+
return { indexId, dim: index.dim, storage: params.storage, length: index.length };
|
|
74
|
+
}
|
|
75
|
+
export function loadVectorIndex(params) {
|
|
76
|
+
const backend = requireBackend();
|
|
77
|
+
const index = runBackend(() => backend.load(resolveVectorIndexPath(params.path)));
|
|
78
|
+
const indexId = track(index);
|
|
79
|
+
return {
|
|
80
|
+
indexId,
|
|
81
|
+
dim: index.dim,
|
|
82
|
+
length: index.length,
|
|
83
|
+
...(index.storage !== undefined && { storage: index.storage })
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
export function addVectors(params) {
|
|
87
|
+
const index = getIndex(params.indexId);
|
|
88
|
+
if (params.ids.length !== params.vectors.length) {
|
|
89
|
+
throw new VectorIndexInvalidVectorsError(`ids has ${params.ids.length} entries but vectors has ${params.vectors.length} rows`);
|
|
90
|
+
}
|
|
91
|
+
const vectors = flattenRows(params.vectors, index.dim, 'vectors');
|
|
92
|
+
const ids = toNativeIds(params.ids);
|
|
93
|
+
runBackend(() => index.add(vectors, ids));
|
|
94
|
+
return { length: index.length };
|
|
95
|
+
}
|
|
96
|
+
export function searchVectors(params) {
|
|
97
|
+
const index = getIndex(params.indexId);
|
|
98
|
+
const queries = flattenRows(params.queries, index.dim, 'queries');
|
|
99
|
+
const results = runBackend(() => index.search(queries, params.k));
|
|
100
|
+
return { results };
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Removes ids one at a time, which is the only shape the backend offers. A
|
|
104
|
+
* backend throw here means the handle itself is unusable, so the batch stops
|
|
105
|
+
* rather than trying the rest — but ids before it are already gone, so the
|
|
106
|
+
* error names how many were applied. `length` is only reported on success,
|
|
107
|
+
* which is why the handle documents its own `length` as of the last call that
|
|
108
|
+
* reported one.
|
|
109
|
+
*/
|
|
110
|
+
export function removeVectors(params) {
|
|
111
|
+
const index = getIndex(params.indexId);
|
|
112
|
+
const removed = [];
|
|
113
|
+
for (const id of params.ids) {
|
|
114
|
+
removed.push(runBackend(() => index.remove(BigInt(id)), (detail) => `remove applied ${removed.length} of ${params.ids.length} ids before failing: ${detail}`));
|
|
115
|
+
}
|
|
116
|
+
return { removed, length: index.length };
|
|
117
|
+
}
|
|
118
|
+
export function containsVectors(params) {
|
|
119
|
+
const index = getIndex(params.indexId);
|
|
120
|
+
const present = params.ids.map((id) => runBackend(() => index.contains(BigInt(id))));
|
|
121
|
+
return { present };
|
|
122
|
+
}
|
|
123
|
+
export async function writeVectorIndex(params) {
|
|
124
|
+
const index = getIndex(params.indexId);
|
|
125
|
+
const snapshotPath = resolveVectorIndexPath(params.path);
|
|
126
|
+
try {
|
|
127
|
+
await fsPromises.mkdir(path.dirname(snapshotPath), { recursive: true });
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
throw new VectorIndexFailedError(`cannot create the snapshot directory for ${snapshotPath}: ${error instanceof Error ? error.message : String(error)}`, error);
|
|
131
|
+
}
|
|
132
|
+
runBackend(() => index.write(snapshotPath));
|
|
133
|
+
return { path: snapshotPath };
|
|
134
|
+
}
|
|
135
|
+
// Disposing an unknown id is not an error: the caller may retry after a
|
|
136
|
+
// worker restart, and the memory it refers to is already gone.
|
|
137
|
+
export function disposeVectorIndex(params) {
|
|
138
|
+
const index = indexes.get(params.indexId);
|
|
139
|
+
if (!index)
|
|
140
|
+
return { disposed: false };
|
|
141
|
+
// Drop the entry only once the backend confirms. Deleting first would strand
|
|
142
|
+
// a failed dispose: the retry would find nothing and report success, and
|
|
143
|
+
// `disposeAllVectorIndexes` could not reach it at shutdown either.
|
|
144
|
+
runBackend(() => index.dispose());
|
|
145
|
+
indexes.delete(params.indexId);
|
|
146
|
+
return { disposed: true };
|
|
147
|
+
}
|
|
148
|
+
export function getOpenVectorIndexCount() {
|
|
149
|
+
return indexes.size;
|
|
150
|
+
}
|
|
151
|
+
export function disposeAllVectorIndexes() {
|
|
152
|
+
for (const [indexId, index] of indexes) {
|
|
153
|
+
try {
|
|
154
|
+
index.dispose();
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
getEngineLogger().warn(`Failed to dispose vector index '${indexId}' during cleanup:`, error);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
indexes.clear();
|
|
161
|
+
}
|
|
@@ -38,6 +38,8 @@ export declare const modelFitWorkloadSchema: z.ZodDiscriminatedUnion<[z.ZodObjec
|
|
|
38
38
|
export declare const modelFitModelRefSchema: z.ZodObject<{
|
|
39
39
|
sha256Checksum: z.ZodString;
|
|
40
40
|
name: z.ZodString;
|
|
41
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
42
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
41
43
|
}, z.core.$strip>;
|
|
42
44
|
/**
|
|
43
45
|
* One model the caller is considering, with the workload it would run.
|
|
@@ -51,10 +53,14 @@ export declare const modelFitCandidateSchema: z.ZodObject<{
|
|
|
51
53
|
model: z.ZodObject<{
|
|
52
54
|
sha256Checksum: z.ZodString;
|
|
53
55
|
name: z.ZodString;
|
|
56
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
57
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
54
58
|
}, z.core.$strip>;
|
|
55
59
|
artifacts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
56
60
|
sha256Checksum: z.ZodString;
|
|
57
61
|
name: z.ZodString;
|
|
62
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
63
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
58
64
|
}, z.core.$strip>>>;
|
|
59
65
|
workload: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
60
66
|
kind: z.ZodLiteral<"llm">;
|
|
@@ -84,10 +90,14 @@ export declare const assessModelFitInputSchema: z.ZodObject<{
|
|
|
84
90
|
model: z.ZodObject<{
|
|
85
91
|
sha256Checksum: z.ZodString;
|
|
86
92
|
name: z.ZodString;
|
|
93
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
94
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
87
95
|
}, z.core.$strip>;
|
|
88
96
|
artifacts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
89
97
|
sha256Checksum: z.ZodString;
|
|
90
98
|
name: z.ZodString;
|
|
99
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
100
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
91
101
|
}, z.core.$strip>>>;
|
|
92
102
|
workload: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
93
103
|
kind: z.ZodLiteral<"llm">;
|
|
@@ -129,10 +139,14 @@ export declare const modelFitBasisSchema: z.ZodEnum<{
|
|
|
129
139
|
* plus the KV cache for llama.cpp models. It omits every engine cost that
|
|
130
140
|
* only a real load can tell, so it can refuse a model but never confirm one.
|
|
131
141
|
* This is what an uncalibrated platform, including Android and iOS, reports.
|
|
142
|
+
* - `native-fit`: the engine's own fitter, run against the registry's weightless
|
|
143
|
+
* description of the artifact. It is the answer the loader would give on this
|
|
144
|
+
* machine, not a model of it, so it outranks `calibration` where both exist.
|
|
132
145
|
*/
|
|
133
146
|
export declare const modelFitEvidenceSchema: z.ZodEnum<{
|
|
134
147
|
calibration: "calibration";
|
|
135
148
|
"computed-only": "computed-only";
|
|
149
|
+
"native-fit": "native-fit";
|
|
136
150
|
}>;
|
|
137
151
|
export declare const modelFitBudgetSchema: z.ZodObject<{
|
|
138
152
|
totalBytes: z.ZodNumber;
|
|
@@ -151,6 +165,7 @@ export declare const modelFitModelResultSchema: z.ZodObject<{
|
|
|
151
165
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
152
166
|
calibration: "calibration";
|
|
153
167
|
"computed-only": "computed-only";
|
|
168
|
+
"native-fit": "native-fit";
|
|
154
169
|
}>>;
|
|
155
170
|
estimate: z.ZodOptional<z.ZodObject<{
|
|
156
171
|
lowerBoundBytes: z.ZodNumber;
|
|
@@ -179,6 +194,7 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
179
194
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
180
195
|
calibration: "calibration";
|
|
181
196
|
"computed-only": "computed-only";
|
|
197
|
+
"native-fit": "native-fit";
|
|
182
198
|
}>>;
|
|
183
199
|
budget: z.ZodOptional<z.ZodObject<{
|
|
184
200
|
totalBytes: z.ZodNumber;
|
|
@@ -202,6 +218,7 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
202
218
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
203
219
|
calibration: "calibration";
|
|
204
220
|
"computed-only": "computed-only";
|
|
221
|
+
"native-fit": "native-fit";
|
|
205
222
|
}>>;
|
|
206
223
|
estimate: z.ZodOptional<z.ZodObject<{
|
|
207
224
|
lowerBoundBytes: z.ZodNumber;
|
|
@@ -214,15 +231,60 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
214
231
|
reasons: z.ZodArray<z.ZodString>;
|
|
215
232
|
assumptions: z.ZodArray<z.ZodString>;
|
|
216
233
|
}, z.core.$strip>;
|
|
234
|
+
/**
|
|
235
|
+
* The other fit question, kept in this file on purpose: `assessModelFit` above
|
|
236
|
+
* answers "should I download this?" from calibrated coefficients and a checksum,
|
|
237
|
+
* while the native probe answers "will the load I am about to run fit?" by
|
|
238
|
+
* reading the GGUF on disk in a disposable child. Two evidence classes, so two
|
|
239
|
+
* verdict vocabularies:
|
|
240
|
+
*
|
|
241
|
+
* - `likely-fits` / `likely-too-large` hedge because a formula is an estimate.
|
|
242
|
+
* - `fit` / `does-not-fit` do not, because the probe measured this build of this
|
|
243
|
+
* file against this device.
|
|
244
|
+
*
|
|
245
|
+
* Precedence when both have spoken about the same model: the probe wins, because
|
|
246
|
+
* it saw the artifact and the resolved load settings. `assessModelFit` is the
|
|
247
|
+
* answer available before the bytes are on disk. Neither denies a load today.
|
|
248
|
+
*/
|
|
249
|
+
export declare const nativeProbeVerdictSchema: z.ZodEnum<{
|
|
250
|
+
unknown: "unknown";
|
|
251
|
+
fit: "fit";
|
|
252
|
+
"does-not-fit": "does-not-fit";
|
|
253
|
+
}>;
|
|
254
|
+
export declare const nativeProbePlanSchema: z.ZodObject<{
|
|
255
|
+
nCtx: z.ZodNumber;
|
|
256
|
+
nGpuLayers: z.ZodNumber;
|
|
257
|
+
nGpuDevices: z.ZodNumber;
|
|
258
|
+
}, z.core.$strip>;
|
|
259
|
+
export declare const nativeProbeFitSchema: z.ZodObject<{
|
|
260
|
+
verdict: z.ZodEnum<{
|
|
261
|
+
unknown: "unknown";
|
|
262
|
+
fit: "fit";
|
|
263
|
+
"does-not-fit": "does-not-fit";
|
|
264
|
+
}>;
|
|
265
|
+
basis: z.ZodLiteral<"native-probe">;
|
|
266
|
+
estimatorVersion: z.ZodString;
|
|
267
|
+
reason: z.ZodString;
|
|
268
|
+
message: z.ZodOptional<z.ZodString>;
|
|
269
|
+
plan: z.ZodOptional<z.ZodObject<{
|
|
270
|
+
nCtx: z.ZodNumber;
|
|
271
|
+
nGpuLayers: z.ZodNumber;
|
|
272
|
+
nGpuDevices: z.ZodNumber;
|
|
273
|
+
}, z.core.$strip>>;
|
|
274
|
+
}, z.core.$strip>;
|
|
217
275
|
export declare const assessModelFitRequestSchema: z.ZodObject<{
|
|
218
276
|
models: z.ZodArray<z.ZodObject<{
|
|
219
277
|
model: z.ZodObject<{
|
|
220
278
|
sha256Checksum: z.ZodString;
|
|
221
279
|
name: z.ZodString;
|
|
280
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
281
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
222
282
|
}, z.core.$strip>;
|
|
223
283
|
artifacts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
224
284
|
sha256Checksum: z.ZodString;
|
|
225
285
|
name: z.ZodString;
|
|
286
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
287
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
226
288
|
}, z.core.$strip>>>;
|
|
227
289
|
workload: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
228
290
|
kind: z.ZodLiteral<"llm">;
|
|
@@ -260,6 +322,7 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
260
322
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
261
323
|
calibration: "calibration";
|
|
262
324
|
"computed-only": "computed-only";
|
|
325
|
+
"native-fit": "native-fit";
|
|
263
326
|
}>>;
|
|
264
327
|
budget: z.ZodOptional<z.ZodObject<{
|
|
265
328
|
totalBytes: z.ZodNumber;
|
|
@@ -283,6 +346,7 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
283
346
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
284
347
|
calibration: "calibration";
|
|
285
348
|
"computed-only": "computed-only";
|
|
349
|
+
"native-fit": "native-fit";
|
|
286
350
|
}>>;
|
|
287
351
|
estimate: z.ZodOptional<z.ZodObject<{
|
|
288
352
|
lowerBoundBytes: z.ZodNumber;
|
|
@@ -296,6 +360,9 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
296
360
|
assumptions: z.ZodArray<z.ZodString>;
|
|
297
361
|
type: z.ZodLiteral<"assessModelFit">;
|
|
298
362
|
}, z.core.$strip>;
|
|
363
|
+
export type NativeProbeVerdict = z.infer<typeof nativeProbeVerdictSchema>;
|
|
364
|
+
export type NativeProbePlan = z.infer<typeof nativeProbePlanSchema>;
|
|
365
|
+
export type NativeProbeFit = z.infer<typeof nativeProbeFitSchema>;
|
|
299
366
|
export type ModelFitVerdict = z.infer<typeof modelFitVerdictSchema>;
|
|
300
367
|
export type ModelFitModelRef = z.infer<typeof modelFitModelRefSchema>;
|
|
301
368
|
export type ModelFitWorkload = z.infer<typeof modelFitWorkloadSchema>;
|
|
@@ -46,9 +46,20 @@ export const modelFitWorkloadSchema = z.discriminatedUnion('kind', [
|
|
|
46
46
|
* are read; everything else — engine, byte totals, transformer facts — comes
|
|
47
47
|
* from the resolved resource profile, not from the caller.
|
|
48
48
|
*/
|
|
49
|
-
export const modelFitModelRefSchema = modelRegistryEntrySchema
|
|
49
|
+
export const modelFitModelRefSchema = modelRegistryEntrySchema
|
|
50
|
+
.pick({
|
|
50
51
|
name: true,
|
|
51
52
|
sha256Checksum: true
|
|
53
|
+
})
|
|
54
|
+
.extend({
|
|
55
|
+
registryPath: z
|
|
56
|
+
.string()
|
|
57
|
+
.optional()
|
|
58
|
+
.describe('Registry coordinates. Present on a catalog constant; without them no fit stub can be resolved and the assessment falls back to calibration.'),
|
|
59
|
+
registrySource: z
|
|
60
|
+
.string()
|
|
61
|
+
.optional()
|
|
62
|
+
.describe('Registry source identifier, e.g. `huggingface`.')
|
|
52
63
|
});
|
|
53
64
|
/**
|
|
54
65
|
* One model the caller is considering, with the workload it would run.
|
|
@@ -117,8 +128,11 @@ export const modelFitBasisSchema = z.enum([
|
|
|
117
128
|
* plus the KV cache for llama.cpp models. It omits every engine cost that
|
|
118
129
|
* only a real load can tell, so it can refuse a model but never confirm one.
|
|
119
130
|
* This is what an uncalibrated platform, including Android and iOS, reports.
|
|
131
|
+
* - `native-fit`: the engine's own fitter, run against the registry's weightless
|
|
132
|
+
* description of the artifact. It is the answer the loader would give on this
|
|
133
|
+
* machine, not a model of it, so it outranks `calibration` where both exist.
|
|
120
134
|
*/
|
|
121
|
-
export const modelFitEvidenceSchema = z.enum(['calibration', 'computed-only']);
|
|
135
|
+
export const modelFitEvidenceSchema = z.enum(['calibration', 'computed-only', 'native-fit']);
|
|
122
136
|
export const modelFitBudgetSchema = z.object({
|
|
123
137
|
totalBytes: z
|
|
124
138
|
.number()
|
|
@@ -174,6 +188,46 @@ export const assessModelFitResultSchema = z.object({
|
|
|
174
188
|
.array(z.string())
|
|
175
189
|
.describe('Everything the result took for granted, including estimator defaults.')
|
|
176
190
|
});
|
|
191
|
+
// ============== Native probe (pre-load) ==============
|
|
192
|
+
/**
|
|
193
|
+
* The other fit question, kept in this file on purpose: `assessModelFit` above
|
|
194
|
+
* answers "should I download this?" from calibrated coefficients and a checksum,
|
|
195
|
+
* while the native probe answers "will the load I am about to run fit?" by
|
|
196
|
+
* reading the GGUF on disk in a disposable child. Two evidence classes, so two
|
|
197
|
+
* verdict vocabularies:
|
|
198
|
+
*
|
|
199
|
+
* - `likely-fits` / `likely-too-large` hedge because a formula is an estimate.
|
|
200
|
+
* - `fit` / `does-not-fit` do not, because the probe measured this build of this
|
|
201
|
+
* file against this device.
|
|
202
|
+
*
|
|
203
|
+
* Precedence when both have spoken about the same model: the probe wins, because
|
|
204
|
+
* it saw the artifact and the resolved load settings. `assessModelFit` is the
|
|
205
|
+
* answer available before the bytes are on disk. Neither denies a load today.
|
|
206
|
+
*/
|
|
207
|
+
export const nativeProbeVerdictSchema = z.enum(['fit', 'does-not-fit', 'unknown']);
|
|
208
|
+
export const nativeProbePlanSchema = z.object({
|
|
209
|
+
nCtx: z.number().int().describe('Context the probe resolved for this load, in tokens.'),
|
|
210
|
+
nGpuLayers: z.number().int().describe('Layers the probe would offload.'),
|
|
211
|
+
nGpuDevices: z.number().int().describe('GPU devices the offload would span.')
|
|
212
|
+
});
|
|
213
|
+
export const nativeProbeFitSchema = z
|
|
214
|
+
.object({
|
|
215
|
+
verdict: nativeProbeVerdictSchema.describe('Advisory outcome. `unknown` means no verdict was obtainable — the check was disabled, the load shape is unsupported, or the child produced no usable answer.'),
|
|
216
|
+
basis: z
|
|
217
|
+
.literal('native-probe')
|
|
218
|
+
.describe('Evidence class: a disposable llama.cpp child that read the model file and the resolved load settings.'),
|
|
219
|
+
estimatorVersion: z
|
|
220
|
+
.string()
|
|
221
|
+
.describe('Version of the probe integration that produced this outcome, covering the load-setting partitioning and the headroom policy — `native-probe-v1` withholds 1024 MiB plus the on-disk bytes of every model already resident in this worker.'),
|
|
222
|
+
reason: z
|
|
223
|
+
.string()
|
|
224
|
+
.describe('Machine-readable reason for the verdict. Never parsed out of log text.'),
|
|
225
|
+
message: z.string().optional().describe('Human-readable detail, when the reason has any.'),
|
|
226
|
+
plan: nativeProbePlanSchema
|
|
227
|
+
.optional()
|
|
228
|
+
.describe('Placement the probe projected. Present only on a `fit` verdict.')
|
|
229
|
+
})
|
|
230
|
+
.meta({ title: 'NativeProbeFit' });
|
|
177
231
|
export const assessModelFitRequestSchema = assessModelFitInputSchema.extend({
|
|
178
232
|
type: z.literal('assessModelFit')
|
|
179
233
|
});
|