@qvac/inference 0.20.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +19 -11
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/handlers/assess-model-fit.d.ts +30 -6
- package/dist/handlers/assess-model-fit.js +68 -6
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +61 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/resources/model-fit/assess.d.ts +10 -1
- package/dist/resources/model-fit/assess.js +43 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +83 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +133 -0
- package/dist/resources/model-fit/fit-stub/index.d.ts +2 -0
- package/dist/resources/model-fit/fit-stub/index.js +2 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +31 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +36 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +38 -18
- package/dist/schemas/assess-model-fit.d.ts +23 -0
- package/dist/schemas/assess-model-fit.js +16 -2
- package/dist/schemas/common.d.ts +67 -0
- package/dist/schemas/config.d.ts +138 -0
- package/dist/schemas/llamacpp-config.d.ts +84 -0
- package/dist/schemas/llamacpp-config.js +104 -4
- package/dist/schemas/load-model.d.ts +336 -0
- package/dist/schemas/model-config-schemas.d.ts +23 -0
- package/package.json +2 -2
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import fs, { promises as fsPromises } from 'bare-fs';
|
|
2
|
+
import path from 'bare-path';
|
|
3
|
+
import { AbortController } from 'bare-abort-controller';
|
|
4
|
+
import { getCacheDir } from '../../../utils/cache/paths.js';
|
|
5
|
+
import { calculateFileChecksum } from '../../../utils/checksum.js';
|
|
6
|
+
import { getEngineLogger } from '../../../logging/index.js';
|
|
7
|
+
/**
|
|
8
|
+
* Bounds the registry lookup and the blob fetch together. The client's own
|
|
9
|
+
* defaults — 30s per attempt, three attempts, a peer wait between them, and an
|
|
10
|
+
* uncapped wait for the registry view on first use — suit a weights download.
|
|
11
|
+
* Here an unreachable registry is the whole latency of the call, so it is
|
|
12
|
+
* capped. Work still in flight at expiry is abandoned: the download is
|
|
13
|
+
* aborted, and a lookup that cannot be cancelled settles on its own with
|
|
14
|
+
* nothing waiting for it.
|
|
15
|
+
*/
|
|
16
|
+
export const FIT_STUB_BUDGET_MS = 10_000;
|
|
17
|
+
async function defaultGetEntry(registryPath, registrySource) {
|
|
18
|
+
const { getRegistryClient } = await import('../../../runtime/registry-client.js');
|
|
19
|
+
const client = await getRegistryClient();
|
|
20
|
+
return client.getModel(registryPath, registrySource);
|
|
21
|
+
}
|
|
22
|
+
async function defaultDownloadBlob(binding, outputFile, signal) {
|
|
23
|
+
const { getRegistryClient } = await import('../../../runtime/registry-client.js');
|
|
24
|
+
const client = await getRegistryClient();
|
|
25
|
+
return client.downloadBlob(binding, {
|
|
26
|
+
outputFile,
|
|
27
|
+
maxRetries: 1,
|
|
28
|
+
...(signal !== undefined && { signal })
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
/** Removes a fetched stub and the directory it was staged in. Never throws. */
|
|
32
|
+
export async function removeStub(stubPath) {
|
|
33
|
+
await fsPromises.rm(path.dirname(stubPath), { recursive: true, force: true }).catch(() => { });
|
|
34
|
+
}
|
|
35
|
+
function unavailable(reason, message) {
|
|
36
|
+
return message === undefined
|
|
37
|
+
? { status: 'unavailable', reason }
|
|
38
|
+
: { status: 'unavailable', reason, message };
|
|
39
|
+
}
|
|
40
|
+
function describe(error) {
|
|
41
|
+
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Fetches the fit stub for one artifact into its own staging directory and
|
|
45
|
+
* returns its path. The stub is a per-call payload, not a cache: the caller
|
|
46
|
+
* removes it once the fitter has read it, so nothing accumulates under the
|
|
47
|
+
* QVAC root and no cleanup path has to know about it.
|
|
48
|
+
*
|
|
49
|
+
* Never throws: a missing entry, an entry without a description, a failed or
|
|
50
|
+
* corrupt download and an expired budget are all `unavailable`, because a
|
|
51
|
+
* pre-download assessment has to survive an offline caller and an older
|
|
52
|
+
* registry record.
|
|
53
|
+
*
|
|
54
|
+
* Only the artifact named by `ref` is fetched. A split model's shards each
|
|
55
|
+
* carry their own binding, and the fitter needs the whole set laid out under
|
|
56
|
+
* llama's shard naming to read any of it — assembling that is not done here.
|
|
57
|
+
*/
|
|
58
|
+
export async function fetchFitStub(ref, options = {}) {
|
|
59
|
+
const budgetMs = options.budgetMs ?? FIT_STUB_BUDGET_MS;
|
|
60
|
+
const controller = new AbortController();
|
|
61
|
+
const onAbort = () => controller.abort(new Error('fit stub fetch aborted by the caller'));
|
|
62
|
+
options.signal?.addEventListener('abort', onAbort, { once: true });
|
|
63
|
+
let timer;
|
|
64
|
+
const expiry = new Promise((resolve) => {
|
|
65
|
+
timer = setTimeout(() => {
|
|
66
|
+
controller.abort(new Error('fit stub budget expired'));
|
|
67
|
+
resolve(unavailable('timed-out', `no fit stub for ${ref.name} from the registry within ${budgetMs}ms`));
|
|
68
|
+
}, budgetMs);
|
|
69
|
+
});
|
|
70
|
+
try {
|
|
71
|
+
return await Promise.race([fetchWithin(ref, options, controller.signal), expiry]);
|
|
72
|
+
}
|
|
73
|
+
finally {
|
|
74
|
+
clearTimeout(timer);
|
|
75
|
+
options.signal?.removeEventListener('abort', onAbort);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
async function fetchWithin(ref, options, signal) {
|
|
79
|
+
const logger = options.logger ?? getEngineLogger();
|
|
80
|
+
try {
|
|
81
|
+
const { registryPath, registrySource } = ref;
|
|
82
|
+
if (!registryPath || !registrySource)
|
|
83
|
+
return unavailable('unresolvable-ref');
|
|
84
|
+
const getEntry = options.getEntry ?? defaultGetEntry;
|
|
85
|
+
const entry = await getEntry(registryPath, registrySource);
|
|
86
|
+
if (entry === null || entry === undefined)
|
|
87
|
+
return unavailable('not-in-registry');
|
|
88
|
+
const binding = entry.fitBlobBinding;
|
|
89
|
+
if (binding === undefined || binding === null)
|
|
90
|
+
return unavailable('no-fit-blob');
|
|
91
|
+
if (signal.aborted)
|
|
92
|
+
return unavailable('timed-out');
|
|
93
|
+
// One directory per fetch, so two assessments of the same model never share
|
|
94
|
+
// a file that one of them is about to remove.
|
|
95
|
+
const root = options.cacheDir ?? getCacheDir('fit-stubs');
|
|
96
|
+
await fsPromises.mkdir(root, { recursive: true });
|
|
97
|
+
const dir = await fsPromises.mkdtemp(path.join(root, 'stub-'));
|
|
98
|
+
const dest = path.join(dir, `${binding.sha256}.gguf`);
|
|
99
|
+
const downloadBlob = options.downloadBlob ?? defaultDownloadBlob;
|
|
100
|
+
let verified = false;
|
|
101
|
+
try {
|
|
102
|
+
await downloadBlob(binding, dest, signal);
|
|
103
|
+
const bytes = fs.statSync(dest).size;
|
|
104
|
+
if (bytes !== binding.byteLength) {
|
|
105
|
+
return unavailable('download-failed', `fit stub for ${ref.name} is ${bytes} bytes, the record says ${binding.byteLength}`);
|
|
106
|
+
}
|
|
107
|
+
// The record binds the description by digest; a stub that reads back
|
|
108
|
+
// differently is not the description the fitter should answer for.
|
|
109
|
+
const digest = await calculateFileChecksum(dest);
|
|
110
|
+
if (digest.toLowerCase() !== binding.sha256.toLowerCase()) {
|
|
111
|
+
return unavailable('download-failed', `fit stub for ${ref.name} hashes to ${digest}, the record says ${binding.sha256}`);
|
|
112
|
+
}
|
|
113
|
+
// Past the budget nobody is waiting for this result, so the stub must
|
|
114
|
+
// not be marked as kept.
|
|
115
|
+
if (signal.aborted)
|
|
116
|
+
return unavailable('timed-out');
|
|
117
|
+
verified = true;
|
|
118
|
+
return { status: 'ready', path: dest, bytes };
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
return unavailable('download-failed', describe(error));
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
// A short, corrupt, half-written or late stub must not outlive the call.
|
|
125
|
+
if (!verified)
|
|
126
|
+
await removeStub(dest);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
catch (error) {
|
|
130
|
+
logger.debug(`fit stub for ${ref.name} unavailable: ${describe(error)}`);
|
|
131
|
+
return unavailable('download-failed', describe(error));
|
|
132
|
+
}
|
|
133
|
+
}
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
export { fetchFitStub, type FitBlobBinding, type FitStubEntry, type FitStubOptions, type FitStubOutcome, type FitStubRef, type FitStubUnavailableReason } from './fetch-fit-stub.js';
|
|
2
|
+
export { projectFitFromStub, type StubFitInput, type StubFitOptions, type StubFitOutcome } from './project-fit-from-stub.js';
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { CanonicalModelType } from '../../../schemas/index.js';
|
|
2
|
+
import { type FitStubOptions, type FitStubRef, type FitStubUnavailableReason } from './fetch-fit-stub.js';
|
|
3
|
+
import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '../native-probe/advisory-fit.js';
|
|
4
|
+
export interface StubFitInput {
|
|
5
|
+
model: FitStubRef;
|
|
6
|
+
modelType: CanonicalModelType;
|
|
7
|
+
/** The load the caller intends, in the spelling the engine's config uses. */
|
|
8
|
+
modelConfig: unknown;
|
|
9
|
+
}
|
|
10
|
+
export type StubFitOutcome = {
|
|
11
|
+
status: 'projected';
|
|
12
|
+
fit: AdvisoryFitOutcome;
|
|
13
|
+
} | {
|
|
14
|
+
status: 'no-stub';
|
|
15
|
+
reason: FitStubUnavailableReason;
|
|
16
|
+
message?: string;
|
|
17
|
+
};
|
|
18
|
+
export interface StubFitOptions {
|
|
19
|
+
stub?: FitStubOptions;
|
|
20
|
+
fit?: AdvisoryFitOptions;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Projects a load from the registry's fit stub, before the weights exist
|
|
24
|
+
* locally.
|
|
25
|
+
*
|
|
26
|
+
* The stub is an ordinary GGUF as far as the fitter is concerned, so this is
|
|
27
|
+
* the same probe `loadModel` runs, pointed at a file that is tens of KB instead
|
|
28
|
+
* of gigabytes. It inherits that probe's fail-open contract: every failure is
|
|
29
|
+
* an `unknown` verdict or a `no-stub` outcome, never a throw.
|
|
30
|
+
*/
|
|
31
|
+
export declare function projectFitFromStub(input: StubFitInput, options?: StubFitOptions): Promise<StubFitOutcome>;
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { fetchFitStub, removeStub } from './fetch-fit-stub.js';
|
|
2
|
+
import { runAdvisoryFitCheck } from '../native-probe/advisory-fit.js';
|
|
3
|
+
/**
|
|
4
|
+
* Projects a load from the registry's fit stub, before the weights exist
|
|
5
|
+
* locally.
|
|
6
|
+
*
|
|
7
|
+
* The stub is an ordinary GGUF as far as the fitter is concerned, so this is
|
|
8
|
+
* the same probe `loadModel` runs, pointed at a file that is tens of KB instead
|
|
9
|
+
* of gigabytes. It inherits that probe's fail-open contract: every failure is
|
|
10
|
+
* an `unknown` verdict or a `no-stub` outcome, never a throw.
|
|
11
|
+
*/
|
|
12
|
+
export async function projectFitFromStub(input, options = {}) {
|
|
13
|
+
const stub = await fetchFitStub(input.model, options.stub);
|
|
14
|
+
if (stub.status !== 'ready') {
|
|
15
|
+
return stub.message === undefined
|
|
16
|
+
? { status: 'no-stub', reason: stub.reason }
|
|
17
|
+
: { status: 'no-stub', reason: stub.reason, message: stub.message };
|
|
18
|
+
}
|
|
19
|
+
try {
|
|
20
|
+
const fit = await runAdvisoryFitCheck({
|
|
21
|
+
modelId: input.model.name,
|
|
22
|
+
modelType: input.modelType,
|
|
23
|
+
modelPath: stub.path,
|
|
24
|
+
modelConfig: input.modelConfig,
|
|
25
|
+
// One stub is one artifact: `fetchFitStub` does not assemble a shard set,
|
|
26
|
+
// so nothing here may claim the fitter is looking at a split model.
|
|
27
|
+
isShardedModel: false
|
|
28
|
+
}, options.fit);
|
|
29
|
+
return { status: 'projected', fit };
|
|
30
|
+
}
|
|
31
|
+
finally {
|
|
32
|
+
// The stub is a per-call payload. Nothing else knows the path, so nothing
|
|
33
|
+
// else could remove it.
|
|
34
|
+
await removeStub(stub.path);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
@@ -2,16 +2,19 @@ import { ModelType } from '../../../schemas/index.js';
|
|
|
2
2
|
import { transformLlmConfig } from '../../../plugins/builtin/llamacpp-completion/transform.js';
|
|
3
3
|
import { transformEmbedConfig } from '../../../plugins/builtin/llamacpp-embedding/transform.js';
|
|
4
4
|
/**
|
|
5
|
-
* Keys
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* Keys forwarded to model-fit, in the spelling the SDK's completion and
|
|
6
|
+
* embedding transforms emit. model-fit canonicalizes `_` to `-`, so `ctx_size`
|
|
7
|
+
* and `ctx-size` reach the same native setting.
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
9
|
+
* A key belongs here when it could change a load's device-memory footprint.
|
|
10
|
+
*
|
|
11
|
+
* model-fit answers unsupported for a key it does not recognize, and the
|
|
12
|
+
* advisory then reports unknown for every load that sets it. A key added here
|
|
13
|
+
* stays in that state until model-fit learns it.
|
|
14
14
|
*/
|
|
15
|
+
// NOTE: adding a key here needs a follow-up in model-fit to recognize it and
|
|
16
|
+
// account for it in the projection. Until that lands, every load setting the
|
|
17
|
+
// key gets no fit verdict.
|
|
15
18
|
const FIT_LOAD_KEYS = {
|
|
16
19
|
completion: [
|
|
17
20
|
'device',
|
|
@@ -21,11 +24,26 @@ const FIT_LOAD_KEYS = {
|
|
|
21
24
|
'parallel',
|
|
22
25
|
'cache-type-k',
|
|
23
26
|
'cache-type-v',
|
|
24
|
-
// Alters KV/compute memory; `model-fit` reads it as evidence.
|
|
25
27
|
'flash-attn',
|
|
26
28
|
'main-gpu',
|
|
27
29
|
'split-mode',
|
|
28
|
-
'tensor-split'
|
|
30
|
+
'tensor-split',
|
|
31
|
+
'batch-size',
|
|
32
|
+
'ubatch-size',
|
|
33
|
+
'cpu-moe',
|
|
34
|
+
'n-cpu-moe',
|
|
35
|
+
'n-cpu-ffn',
|
|
36
|
+
'override-tensor',
|
|
37
|
+
'moe-cache-mib',
|
|
38
|
+
'kv-offload',
|
|
39
|
+
'no-kv-offload',
|
|
40
|
+
'prefetch-weights',
|
|
41
|
+
'tensor-read-lazy',
|
|
42
|
+
'fit',
|
|
43
|
+
'fit-target',
|
|
44
|
+
'fit-ctx',
|
|
45
|
+
'image-max-tokens',
|
|
46
|
+
'image-min-tokens'
|
|
29
47
|
],
|
|
30
48
|
embedding: [
|
|
31
49
|
'device',
|
|
@@ -38,11 +56,10 @@ const FIT_LOAD_KEYS = {
|
|
|
38
56
|
]
|
|
39
57
|
};
|
|
40
58
|
/**
|
|
41
|
-
* Keys
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
* supported load into `unsupported-config`.
|
|
59
|
+
* Keys withheld from model-fit — sampling, generation, logging, CPU scheduling
|
|
60
|
+
* and JS-side presentation settings. A key belongs here when it cannot change
|
|
61
|
+
* a load's device-memory footprint, so withholding it leaves the verdict
|
|
62
|
+
* intact.
|
|
46
63
|
*/
|
|
47
64
|
const NON_FIT_KEYS = {
|
|
48
65
|
completion: [
|
|
@@ -62,7 +79,11 @@ const NON_FIT_KEYS = {
|
|
|
62
79
|
'image_tile_mode',
|
|
63
80
|
'image_no_upscale',
|
|
64
81
|
'mmproj-use-gpu',
|
|
65
|
-
'openclCacheDir'
|
|
82
|
+
'openclCacheDir',
|
|
83
|
+
'threads',
|
|
84
|
+
'threads-batch',
|
|
85
|
+
'cpu-mask',
|
|
86
|
+
'cpu-mask-batch'
|
|
66
87
|
],
|
|
67
88
|
embedding: ['pooling', 'attention', 'embd_normalize', 'verbosity', 'openclCacheDir']
|
|
68
89
|
};
|
|
@@ -107,8 +128,7 @@ function partitionParams(loadKind, transformed) {
|
|
|
107
128
|
}
|
|
108
129
|
if (dropped.has(key))
|
|
109
130
|
continue;
|
|
110
|
-
//
|
|
111
|
-
// setting the SDK cannot classify, so it must not be answered for.
|
|
131
|
+
// In neither list, so its effect on the projection is unknown.
|
|
112
132
|
return { detail: `unclassified load setting: ${key}` };
|
|
113
133
|
}
|
|
114
134
|
return { params };
|
|
@@ -38,6 +38,8 @@ export declare const modelFitWorkloadSchema: z.ZodDiscriminatedUnion<[z.ZodObjec
|
|
|
38
38
|
export declare const modelFitModelRefSchema: z.ZodObject<{
|
|
39
39
|
sha256Checksum: z.ZodString;
|
|
40
40
|
name: z.ZodString;
|
|
41
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
42
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
41
43
|
}, z.core.$strip>;
|
|
42
44
|
/**
|
|
43
45
|
* One model the caller is considering, with the workload it would run.
|
|
@@ -51,10 +53,14 @@ export declare const modelFitCandidateSchema: z.ZodObject<{
|
|
|
51
53
|
model: z.ZodObject<{
|
|
52
54
|
sha256Checksum: z.ZodString;
|
|
53
55
|
name: z.ZodString;
|
|
56
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
57
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
54
58
|
}, z.core.$strip>;
|
|
55
59
|
artifacts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
56
60
|
sha256Checksum: z.ZodString;
|
|
57
61
|
name: z.ZodString;
|
|
62
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
63
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
58
64
|
}, z.core.$strip>>>;
|
|
59
65
|
workload: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
60
66
|
kind: z.ZodLiteral<"llm">;
|
|
@@ -84,10 +90,14 @@ export declare const assessModelFitInputSchema: z.ZodObject<{
|
|
|
84
90
|
model: z.ZodObject<{
|
|
85
91
|
sha256Checksum: z.ZodString;
|
|
86
92
|
name: z.ZodString;
|
|
93
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
94
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
87
95
|
}, z.core.$strip>;
|
|
88
96
|
artifacts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
89
97
|
sha256Checksum: z.ZodString;
|
|
90
98
|
name: z.ZodString;
|
|
99
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
100
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
91
101
|
}, z.core.$strip>>>;
|
|
92
102
|
workload: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
93
103
|
kind: z.ZodLiteral<"llm">;
|
|
@@ -129,10 +139,14 @@ export declare const modelFitBasisSchema: z.ZodEnum<{
|
|
|
129
139
|
* plus the KV cache for llama.cpp models. It omits every engine cost that
|
|
130
140
|
* only a real load can tell, so it can refuse a model but never confirm one.
|
|
131
141
|
* This is what an uncalibrated platform, including Android and iOS, reports.
|
|
142
|
+
* - `native-fit`: the engine's own fitter, run against the registry's weightless
|
|
143
|
+
* description of the artifact. It is the answer the loader would give on this
|
|
144
|
+
* machine, not a model of it, so it outranks `calibration` where both exist.
|
|
132
145
|
*/
|
|
133
146
|
export declare const modelFitEvidenceSchema: z.ZodEnum<{
|
|
134
147
|
calibration: "calibration";
|
|
135
148
|
"computed-only": "computed-only";
|
|
149
|
+
"native-fit": "native-fit";
|
|
136
150
|
}>;
|
|
137
151
|
export declare const modelFitBudgetSchema: z.ZodObject<{
|
|
138
152
|
totalBytes: z.ZodNumber;
|
|
@@ -151,6 +165,7 @@ export declare const modelFitModelResultSchema: z.ZodObject<{
|
|
|
151
165
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
152
166
|
calibration: "calibration";
|
|
153
167
|
"computed-only": "computed-only";
|
|
168
|
+
"native-fit": "native-fit";
|
|
154
169
|
}>>;
|
|
155
170
|
estimate: z.ZodOptional<z.ZodObject<{
|
|
156
171
|
lowerBoundBytes: z.ZodNumber;
|
|
@@ -179,6 +194,7 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
179
194
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
180
195
|
calibration: "calibration";
|
|
181
196
|
"computed-only": "computed-only";
|
|
197
|
+
"native-fit": "native-fit";
|
|
182
198
|
}>>;
|
|
183
199
|
budget: z.ZodOptional<z.ZodObject<{
|
|
184
200
|
totalBytes: z.ZodNumber;
|
|
@@ -202,6 +218,7 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
|
|
|
202
218
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
203
219
|
calibration: "calibration";
|
|
204
220
|
"computed-only": "computed-only";
|
|
221
|
+
"native-fit": "native-fit";
|
|
205
222
|
}>>;
|
|
206
223
|
estimate: z.ZodOptional<z.ZodObject<{
|
|
207
224
|
lowerBoundBytes: z.ZodNumber;
|
|
@@ -260,10 +277,14 @@ export declare const assessModelFitRequestSchema: z.ZodObject<{
|
|
|
260
277
|
model: z.ZodObject<{
|
|
261
278
|
sha256Checksum: z.ZodString;
|
|
262
279
|
name: z.ZodString;
|
|
280
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
281
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
263
282
|
}, z.core.$strip>;
|
|
264
283
|
artifacts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
265
284
|
sha256Checksum: z.ZodString;
|
|
266
285
|
name: z.ZodString;
|
|
286
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
287
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
267
288
|
}, z.core.$strip>>>;
|
|
268
289
|
workload: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
269
290
|
kind: z.ZodLiteral<"llm">;
|
|
@@ -301,6 +322,7 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
301
322
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
302
323
|
calibration: "calibration";
|
|
303
324
|
"computed-only": "computed-only";
|
|
325
|
+
"native-fit": "native-fit";
|
|
304
326
|
}>>;
|
|
305
327
|
budget: z.ZodOptional<z.ZodObject<{
|
|
306
328
|
totalBytes: z.ZodNumber;
|
|
@@ -324,6 +346,7 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
|
|
|
324
346
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
325
347
|
calibration: "calibration";
|
|
326
348
|
"computed-only": "computed-only";
|
|
349
|
+
"native-fit": "native-fit";
|
|
327
350
|
}>>;
|
|
328
351
|
estimate: z.ZodOptional<z.ZodObject<{
|
|
329
352
|
lowerBoundBytes: z.ZodNumber;
|
|
@@ -46,9 +46,20 @@ export const modelFitWorkloadSchema = z.discriminatedUnion('kind', [
|
|
|
46
46
|
* are read; everything else — engine, byte totals, transformer facts — comes
|
|
47
47
|
* from the resolved resource profile, not from the caller.
|
|
48
48
|
*/
|
|
49
|
-
export const modelFitModelRefSchema = modelRegistryEntrySchema
|
|
49
|
+
export const modelFitModelRefSchema = modelRegistryEntrySchema
|
|
50
|
+
.pick({
|
|
50
51
|
name: true,
|
|
51
52
|
sha256Checksum: true
|
|
53
|
+
})
|
|
54
|
+
.extend({
|
|
55
|
+
registryPath: z
|
|
56
|
+
.string()
|
|
57
|
+
.optional()
|
|
58
|
+
.describe('Registry coordinates. Present on a catalog constant; without them no fit stub can be resolved and the assessment falls back to calibration.'),
|
|
59
|
+
registrySource: z
|
|
60
|
+
.string()
|
|
61
|
+
.optional()
|
|
62
|
+
.describe('Registry source identifier, e.g. `huggingface`.')
|
|
52
63
|
});
|
|
53
64
|
/**
|
|
54
65
|
* One model the caller is considering, with the workload it would run.
|
|
@@ -117,8 +128,11 @@ export const modelFitBasisSchema = z.enum([
|
|
|
117
128
|
* plus the KV cache for llama.cpp models. It omits every engine cost that
|
|
118
129
|
* only a real load can tell, so it can refuse a model but never confirm one.
|
|
119
130
|
* This is what an uncalibrated platform, including Android and iOS, reports.
|
|
131
|
+
* - `native-fit`: the engine's own fitter, run against the registry's weightless
|
|
132
|
+
* description of the artifact. It is the answer the loader would give on this
|
|
133
|
+
* machine, not a model of it, so it outranks `calibration` where both exist.
|
|
120
134
|
*/
|
|
121
|
-
export const modelFitEvidenceSchema = z.enum(['calibration', 'computed-only']);
|
|
135
|
+
export const modelFitEvidenceSchema = z.enum(['calibration', 'computed-only', 'native-fit']);
|
|
122
136
|
export const modelFitBudgetSchema = z.object({
|
|
123
137
|
totalBytes: z
|
|
124
138
|
.number()
|
package/dist/schemas/common.d.ts
CHANGED
|
@@ -179,6 +179,29 @@ export declare const requestSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
179
179
|
auto: "auto";
|
|
180
180
|
}>>;
|
|
181
181
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
182
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
183
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
184
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
185
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
186
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
187
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
188
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
189
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
190
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
191
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
192
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
193
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
194
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
195
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
196
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
197
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
198
|
+
on: "on";
|
|
199
|
+
off: "off";
|
|
200
|
+
auto: "auto";
|
|
201
|
+
}>>;
|
|
202
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
203
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
204
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
182
205
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
183
206
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
184
207
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -2649,6 +2672,25 @@ export declare const requestSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
2649
2672
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
2650
2673
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
2651
2674
|
'tensor-split'?: string | undefined;
|
|
2675
|
+
'batch-size'?: number | undefined;
|
|
2676
|
+
'ubatch-size'?: number | undefined;
|
|
2677
|
+
'cpu-moe'?: boolean | undefined;
|
|
2678
|
+
'n-cpu-moe'?: number | undefined;
|
|
2679
|
+
'kv-offload'?: boolean | undefined;
|
|
2680
|
+
'image-max-tokens'?: number | undefined;
|
|
2681
|
+
'image-min-tokens'?: number | undefined;
|
|
2682
|
+
threads?: number | undefined;
|
|
2683
|
+
'threads-batch'?: number | undefined;
|
|
2684
|
+
'cpu-mask'?: string | undefined;
|
|
2685
|
+
'cpu-mask-batch'?: string | undefined;
|
|
2686
|
+
'override-tensor'?: string | undefined;
|
|
2687
|
+
'n-cpu-ffn'?: number | undefined;
|
|
2688
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
2689
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
2690
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
2691
|
+
fit?: boolean | undefined;
|
|
2692
|
+
'fit-target'?: string | number | undefined;
|
|
2693
|
+
'fit-ctx'?: number | undefined;
|
|
2652
2694
|
openclCacheDir?: string | undefined;
|
|
2653
2695
|
reasoning_budget?: number | undefined;
|
|
2654
2696
|
projectionModelSrc?: string | {
|
|
@@ -3612,6 +3654,25 @@ export declare const requestSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
3612
3654
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
3613
3655
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
3614
3656
|
'tensor-split'?: string | undefined;
|
|
3657
|
+
'batch-size'?: number | undefined;
|
|
3658
|
+
'ubatch-size'?: number | undefined;
|
|
3659
|
+
'cpu-moe'?: boolean | undefined;
|
|
3660
|
+
'n-cpu-moe'?: number | undefined;
|
|
3661
|
+
'kv-offload'?: boolean | undefined;
|
|
3662
|
+
'image-max-tokens'?: number | undefined;
|
|
3663
|
+
'image-min-tokens'?: number | undefined;
|
|
3664
|
+
threads?: number | undefined;
|
|
3665
|
+
'threads-batch'?: number | undefined;
|
|
3666
|
+
'cpu-mask'?: string | undefined;
|
|
3667
|
+
'cpu-mask-batch'?: string | undefined;
|
|
3668
|
+
'override-tensor'?: string | undefined;
|
|
3669
|
+
'n-cpu-ffn'?: number | undefined;
|
|
3670
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
3671
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
3672
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
3673
|
+
fit?: boolean | undefined;
|
|
3674
|
+
'fit-target'?: string | number | undefined;
|
|
3675
|
+
'fit-ctx'?: number | undefined;
|
|
3615
3676
|
openclCacheDir?: string | undefined;
|
|
3616
3677
|
reasoning_budget?: number | undefined;
|
|
3617
3678
|
projectionModelSrc?: string | {
|
|
@@ -5267,10 +5328,14 @@ export declare const requestSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
|
5267
5328
|
model: z.ZodObject<{
|
|
5268
5329
|
sha256Checksum: z.ZodString;
|
|
5269
5330
|
name: z.ZodString;
|
|
5331
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
5332
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
5270
5333
|
}, z.core.$strip>;
|
|
5271
5334
|
artifacts: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
5272
5335
|
sha256Checksum: z.ZodString;
|
|
5273
5336
|
name: z.ZodString;
|
|
5337
|
+
registryPath: z.ZodOptional<z.ZodString>;
|
|
5338
|
+
registrySource: z.ZodOptional<z.ZodString>;
|
|
5274
5339
|
}, z.core.$strip>>>;
|
|
5275
5340
|
workload: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
5276
5341
|
kind: z.ZodLiteral<"llm">;
|
|
@@ -7665,6 +7730,7 @@ export declare const responseSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
7665
7730
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
7666
7731
|
calibration: "calibration";
|
|
7667
7732
|
"computed-only": "computed-only";
|
|
7733
|
+
"native-fit": "native-fit";
|
|
7668
7734
|
}>>;
|
|
7669
7735
|
budget: z.ZodOptional<z.ZodObject<{
|
|
7670
7736
|
totalBytes: z.ZodNumber;
|
|
@@ -7688,6 +7754,7 @@ export declare const responseSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
|
7688
7754
|
evidence: z.ZodOptional<z.ZodEnum<{
|
|
7689
7755
|
calibration: "calibration";
|
|
7690
7756
|
"computed-only": "computed-only";
|
|
7757
|
+
"native-fit": "native-fit";
|
|
7691
7758
|
}>>;
|
|
7692
7759
|
estimate: z.ZodOptional<z.ZodObject<{
|
|
7693
7760
|
lowerBoundBytes: z.ZodNumber;
|