@qvac/inference 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +43 -11
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +8 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +44 -0
- package/dist/schemas/assess-model-fit.js +40 -0
- package/dist/schemas/audio-gen.d.ts +479 -4
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +7 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/common.d.ts +749 -15
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +11 -1
- package/dist/schemas/completion-event.js +3 -0
- package/dist/schemas/completion-stream.d.ts +20 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2503 -139
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/plugin.d.ts +10 -0
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +29 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +28 -25
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { promises as fsPromises } from 'bare-fs';
|
|
2
|
+
import path from 'bare-path';
|
|
3
|
+
import { QvacErrorBase } from '@qvac/error';
|
|
4
|
+
import { getTurboVecBackend } from '../plugins/turbovec-backend.js';
|
|
5
|
+
import { getConfiguredCacheDir } from './state.js';
|
|
6
|
+
import { generateRandomRequestId } from './request-id.js';
|
|
7
|
+
import { getEngineLogger } from '../logging/index.js';
|
|
8
|
+
import { VectorIndexFailedError, VectorIndexInvalidVectorsError, VectorIndexNotFoundError, VectorIndexProviderUnavailableError } from '../errors/index.js';
|
|
9
|
+
// Worker-side bookkeeping for open indexes: ids, wire-to-native conversion,
|
|
10
|
+
// row validation, snapshot paths, and lifetime. Engine behaviour lives in
|
|
11
|
+
// the backend.
|
|
12
|
+
const indexes = new Map();
|
|
13
|
+
function requireBackend() {
|
|
14
|
+
const backend = getTurboVecBackend();
|
|
15
|
+
if (!backend)
|
|
16
|
+
throw new VectorIndexProviderUnavailableError();
|
|
17
|
+
return backend;
|
|
18
|
+
}
|
|
19
|
+
function getIndex(indexId) {
|
|
20
|
+
const index = indexes.get(indexId);
|
|
21
|
+
if (!index)
|
|
22
|
+
throw new VectorIndexNotFoundError(indexId);
|
|
23
|
+
return index;
|
|
24
|
+
}
|
|
25
|
+
// Backends throw plain TypeError/RangeError/Error values from the native
|
|
26
|
+
// layer; wrap them so the caller receives a coded error with the original
|
|
27
|
+
// message preserved as cause. `describe` adds caller context to the message
|
|
28
|
+
// and is evaluated at throw time, so it can report progress made so far.
|
|
29
|
+
function runBackend(action, describe) {
|
|
30
|
+
try {
|
|
31
|
+
return action();
|
|
32
|
+
}
|
|
33
|
+
catch (error) {
|
|
34
|
+
if (error instanceof QvacErrorBase)
|
|
35
|
+
throw error;
|
|
36
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
37
|
+
throw new VectorIndexFailedError(describe ? describe(detail) : detail, error);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Snapshot paths are used as given when absolute. A relative path resolves
|
|
42
|
+
* against the QVAC data directory (the parent of the configured cache
|
|
43
|
+
* directory), the same root that holds the RAG workspace stores, so the
|
|
44
|
+
* same relative path works on every platform.
|
|
45
|
+
*/
|
|
46
|
+
export function resolveVectorIndexPath(snapshotPath) {
|
|
47
|
+
if (path.isAbsolute(snapshotPath))
|
|
48
|
+
return snapshotPath;
|
|
49
|
+
return path.join(path.dirname(getConfiguredCacheDir()), snapshotPath);
|
|
50
|
+
}
|
|
51
|
+
function flattenRows(rows, dim, label) {
|
|
52
|
+
const flat = new Float32Array(rows.length * dim);
|
|
53
|
+
rows.forEach((row, rowIndex) => {
|
|
54
|
+
if (row.length !== dim) {
|
|
55
|
+
throw new VectorIndexInvalidVectorsError(`${label}[${rowIndex}] has ${row.length} components, expected ${dim}`);
|
|
56
|
+
}
|
|
57
|
+
flat.set(row, rowIndex * dim);
|
|
58
|
+
});
|
|
59
|
+
return flat;
|
|
60
|
+
}
|
|
61
|
+
function toNativeIds(ids) {
|
|
62
|
+
return new BigUint64Array(ids.map((id) => BigInt(id)));
|
|
63
|
+
}
|
|
64
|
+
function track(index) {
|
|
65
|
+
const indexId = generateRandomRequestId();
|
|
66
|
+
indexes.set(indexId, index);
|
|
67
|
+
return indexId;
|
|
68
|
+
}
|
|
69
|
+
export function createVectorIndex(params) {
|
|
70
|
+
const backend = requireBackend();
|
|
71
|
+
const index = runBackend(() => backend.create(params.dim, params.storage));
|
|
72
|
+
const indexId = track(index);
|
|
73
|
+
return { indexId, dim: index.dim, storage: params.storage, length: index.length };
|
|
74
|
+
}
|
|
75
|
+
export function loadVectorIndex(params) {
|
|
76
|
+
const backend = requireBackend();
|
|
77
|
+
const index = runBackend(() => backend.load(resolveVectorIndexPath(params.path)));
|
|
78
|
+
const indexId = track(index);
|
|
79
|
+
return {
|
|
80
|
+
indexId,
|
|
81
|
+
dim: index.dim,
|
|
82
|
+
length: index.length,
|
|
83
|
+
...(index.storage !== undefined && { storage: index.storage })
|
|
84
|
+
};
|
|
85
|
+
}
|
|
86
|
+
export function addVectors(params) {
|
|
87
|
+
const index = getIndex(params.indexId);
|
|
88
|
+
if (params.ids.length !== params.vectors.length) {
|
|
89
|
+
throw new VectorIndexInvalidVectorsError(`ids has ${params.ids.length} entries but vectors has ${params.vectors.length} rows`);
|
|
90
|
+
}
|
|
91
|
+
const vectors = flattenRows(params.vectors, index.dim, 'vectors');
|
|
92
|
+
const ids = toNativeIds(params.ids);
|
|
93
|
+
runBackend(() => index.add(vectors, ids));
|
|
94
|
+
return { length: index.length };
|
|
95
|
+
}
|
|
96
|
+
export function searchVectors(params) {
|
|
97
|
+
const index = getIndex(params.indexId);
|
|
98
|
+
const queries = flattenRows(params.queries, index.dim, 'queries');
|
|
99
|
+
const results = runBackend(() => index.search(queries, params.k));
|
|
100
|
+
return { results };
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Removes ids one at a time, which is the only shape the backend offers. A
|
|
104
|
+
* backend throw here means the handle itself is unusable, so the batch stops
|
|
105
|
+
* rather than trying the rest — but ids before it are already gone, so the
|
|
106
|
+
* error names how many were applied. `length` is only reported on success,
|
|
107
|
+
* which is why the handle documents its own `length` as of the last call that
|
|
108
|
+
* reported one.
|
|
109
|
+
*/
|
|
110
|
+
export function removeVectors(params) {
|
|
111
|
+
const index = getIndex(params.indexId);
|
|
112
|
+
const removed = [];
|
|
113
|
+
for (const id of params.ids) {
|
|
114
|
+
removed.push(runBackend(() => index.remove(BigInt(id)), (detail) => `remove applied ${removed.length} of ${params.ids.length} ids before failing: ${detail}`));
|
|
115
|
+
}
|
|
116
|
+
return { removed, length: index.length };
|
|
117
|
+
}
|
|
118
|
+
export function containsVectors(params) {
|
|
119
|
+
const index = getIndex(params.indexId);
|
|
120
|
+
const present = params.ids.map((id) => runBackend(() => index.contains(BigInt(id))));
|
|
121
|
+
return { present };
|
|
122
|
+
}
|
|
123
|
+
export async function writeVectorIndex(params) {
|
|
124
|
+
const index = getIndex(params.indexId);
|
|
125
|
+
const snapshotPath = resolveVectorIndexPath(params.path);
|
|
126
|
+
try {
|
|
127
|
+
await fsPromises.mkdir(path.dirname(snapshotPath), { recursive: true });
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
throw new VectorIndexFailedError(`cannot create the snapshot directory for ${snapshotPath}: ${error instanceof Error ? error.message : String(error)}`, error);
|
|
131
|
+
}
|
|
132
|
+
runBackend(() => index.write(snapshotPath));
|
|
133
|
+
return { path: snapshotPath };
|
|
134
|
+
}
|
|
135
|
+
// Disposing an unknown id is not an error: the caller may retry after a
|
|
136
|
+
// worker restart, and the memory it refers to is already gone.
|
|
137
|
+
export function disposeVectorIndex(params) {
|
|
138
|
+
const index = indexes.get(params.indexId);
|
|
139
|
+
if (!index)
|
|
140
|
+
return { disposed: false };
|
|
141
|
+
// Drop the entry only once the backend confirms. Deleting first would strand
|
|
142
|
+
// a failed dispose: the retry would find nothing and report success, and
|
|
143
|
+
// `disposeAllVectorIndexes` could not reach it at shutdown either.
|
|
144
|
+
runBackend(() => index.dispose());
|
|
145
|
+
indexes.delete(params.indexId);
|
|
146
|
+
return { disposed: true };
|
|
147
|
+
}
|
|
148
|
+
export function getOpenVectorIndexCount() {
|
|
149
|
+
return indexes.size;
|
|
150
|
+
}
|
|
151
|
+
export function disposeAllVectorIndexes() {
|
|
152
|
+
for (const [indexId, index] of indexes) {
|
|
153
|
+
try {
|
|
154
|
+
index.dispose();
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
getEngineLogger().warn(`Failed to dispose vector index '${indexId}' during cleanup:`, error);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
indexes.clear();
|
|
161
|
+
}
|
|
@@ -214,6 +214,47 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
214
214
|
reasons: z.ZodArray<z.ZodString>;
|
|
215
215
|
assumptions: z.ZodArray<z.ZodString>;
|
|
216
216
|
}, z.core.$strip>;
|
|
217
|
+
/**
|
|
218
|
+
* The other fit question, kept in this file on purpose: `assessModelFit` above
|
|
219
|
+
* answers "should I download this?" from calibrated coefficients and a checksum,
|
|
220
|
+
* while the native probe answers "will the load I am about to run fit?" by
|
|
221
|
+
* reading the GGUF on disk in a disposable child. Two evidence classes, so two
|
|
222
|
+
* verdict vocabularies:
|
|
223
|
+
*
|
|
224
|
+
* - `likely-fits` / `likely-too-large` hedge because a formula is an estimate.
|
|
225
|
+
* - `fit` / `does-not-fit` do not, because the probe measured this build of this
|
|
226
|
+
* file against this device.
|
|
227
|
+
*
|
|
228
|
+
* Precedence when both have spoken about the same model: the probe wins, because
|
|
229
|
+
* it saw the artifact and the resolved load settings. `assessModelFit` is the
|
|
230
|
+
* answer available before the bytes are on disk. Neither denies a load today.
|
|
231
|
+
*/
|
|
232
|
+
export declare const nativeProbeVerdictSchema: z.ZodEnum<{
|
|
233
|
+
unknown: "unknown";
|
|
234
|
+
fit: "fit";
|
|
235
|
+
"does-not-fit": "does-not-fit";
|
|
236
|
+
}>;
|
|
237
|
+
export declare const nativeProbePlanSchema: z.ZodObject<{
|
|
238
|
+
nCtx: z.ZodNumber;
|
|
239
|
+
nGpuLayers: z.ZodNumber;
|
|
240
|
+
nGpuDevices: z.ZodNumber;
|
|
241
|
+
}, z.core.$strip>;
|
|
242
|
+
export declare const nativeProbeFitSchema: z.ZodObject<{
|
|
243
|
+
verdict: z.ZodEnum<{
|
|
244
|
+
unknown: "unknown";
|
|
245
|
+
fit: "fit";
|
|
246
|
+
"does-not-fit": "does-not-fit";
|
|
247
|
+
}>;
|
|
248
|
+
basis: z.ZodLiteral<"native-probe">;
|
|
249
|
+
estimatorVersion: z.ZodString;
|
|
250
|
+
reason: z.ZodString;
|
|
251
|
+
message: z.ZodOptional<z.ZodString>;
|
|
252
|
+
plan: z.ZodOptional<z.ZodObject<{
|
|
253
|
+
nCtx: z.ZodNumber;
|
|
254
|
+
nGpuLayers: z.ZodNumber;
|
|
255
|
+
nGpuDevices: z.ZodNumber;
|
|
256
|
+
}, z.core.$strip>>;
|
|
257
|
+
}, z.core.$strip>;
|
|
217
258
|
export declare const assessModelFitRequestSchema: z.ZodObject<{
|
|
218
259
|
models: z.ZodArray<z.ZodObject<{
|
|
219
260
|
model: z.ZodObject<{
|
|
@@ -296,6 +337,9 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
296
337
|
assumptions: z.ZodArray<z.ZodString>;
|
|
297
338
|
type: z.ZodLiteral<"assessModelFit">;
|
|
298
339
|
}, z.core.$strip>;
|
|
340
|
+
export type NativeProbeVerdict = z.infer<typeof nativeProbeVerdictSchema>;
|
|
341
|
+
export type NativeProbePlan = z.infer<typeof nativeProbePlanSchema>;
|
|
342
|
+
export type NativeProbeFit = z.infer<typeof nativeProbeFitSchema>;
|
|
299
343
|
export type ModelFitVerdict = z.infer<typeof modelFitVerdictSchema>;
|
|
300
344
|
export type ModelFitModelRef = z.infer<typeof modelFitModelRefSchema>;
|
|
301
345
|
export type ModelFitWorkload = z.infer<typeof modelFitWorkloadSchema>;
|
|
@@ -174,6 +174,46 @@ export const assessModelFitResultSchema = z.object({
|
|
|
174
174
|
.array(z.string())
|
|
175
175
|
.describe('Everything the result took for granted, including estimator defaults.')
|
|
176
176
|
});
|
|
177
|
+
// ============== Native probe (pre-load) ==============
|
|
178
|
+
/**
|
|
179
|
+
* The other fit question, kept in this file on purpose: `assessModelFit` above
|
|
180
|
+
* answers "should I download this?" from calibrated coefficients and a checksum,
|
|
181
|
+
* while the native probe answers "will the load I am about to run fit?" by
|
|
182
|
+
* reading the GGUF on disk in a disposable child. Two evidence classes, so two
|
|
183
|
+
* verdict vocabularies:
|
|
184
|
+
*
|
|
185
|
+
* - `likely-fits` / `likely-too-large` hedge because a formula is an estimate.
|
|
186
|
+
* - `fit` / `does-not-fit` do not, because the probe measured this build of this
|
|
187
|
+
* file against this device.
|
|
188
|
+
*
|
|
189
|
+
* Precedence when both have spoken about the same model: the probe wins, because
|
|
190
|
+
* it saw the artifact and the resolved load settings. `assessModelFit` is the
|
|
191
|
+
* answer available before the bytes are on disk. Neither denies a load today.
|
|
192
|
+
*/
|
|
193
|
+
export const nativeProbeVerdictSchema = z.enum(['fit', 'does-not-fit', 'unknown']);
|
|
194
|
+
export const nativeProbePlanSchema = z.object({
|
|
195
|
+
nCtx: z.number().int().describe('Context the probe resolved for this load, in tokens.'),
|
|
196
|
+
nGpuLayers: z.number().int().describe('Layers the probe would offload.'),
|
|
197
|
+
nGpuDevices: z.number().int().describe('GPU devices the offload would span.')
|
|
198
|
+
});
|
|
199
|
+
export const nativeProbeFitSchema = z
|
|
200
|
+
.object({
|
|
201
|
+
verdict: nativeProbeVerdictSchema.describe('Advisory outcome. `unknown` means no verdict was obtainable — the check was disabled, the load shape is unsupported, or the child produced no usable answer.'),
|
|
202
|
+
basis: z
|
|
203
|
+
.literal('native-probe')
|
|
204
|
+
.describe('Evidence class: a disposable llama.cpp child that read the model file and the resolved load settings.'),
|
|
205
|
+
estimatorVersion: z
|
|
206
|
+
.string()
|
|
207
|
+
.describe('Version of the probe integration that produced this outcome, covering the load-setting partitioning and the headroom policy — `native-probe-v1` withholds 1024 MiB plus the on-disk bytes of every model already resident in this worker.'),
|
|
208
|
+
reason: z
|
|
209
|
+
.string()
|
|
210
|
+
.describe('Machine-readable reason for the verdict. Never parsed out of log text.'),
|
|
211
|
+
message: z.string().optional().describe('Human-readable detail, when the reason has any.'),
|
|
212
|
+
plan: nativeProbePlanSchema
|
|
213
|
+
.optional()
|
|
214
|
+
.describe('Placement the probe projected. Present only on a `fit` verdict.')
|
|
215
|
+
})
|
|
216
|
+
.meta({ title: 'NativeProbeFit' });
|
|
177
217
|
export const assessModelFitRequestSchema = assessModelFitInputSchema.extend({
|
|
178
218
|
type: z.literal('assessModelFit')
|
|
179
219
|
});
|