@qvac/inference 0.20.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +19 -11
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/handlers/assess-model-fit.d.ts +30 -6
- package/dist/handlers/assess-model-fit.js +68 -6
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +61 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/resources/model-fit/assess.d.ts +10 -1
- package/dist/resources/model-fit/assess.js +43 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +83 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +133 -0
- package/dist/resources/model-fit/fit-stub/index.d.ts +2 -0
- package/dist/resources/model-fit/fit-stub/index.js +2 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +31 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +36 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +38 -18
- package/dist/schemas/assess-model-fit.d.ts +23 -0
- package/dist/schemas/assess-model-fit.js +16 -2
- package/dist/schemas/common.d.ts +67 -0
- package/dist/schemas/config.d.ts +138 -0
- package/dist/schemas/llamacpp-config.d.ts +84 -0
- package/dist/schemas/llamacpp-config.js +104 -4
- package/dist/schemas/load-model.d.ts +336 -0
- package/dist/schemas/model-config-schemas.d.ts +23 -0
- package/package.json +2 -2
package/NOTICE
CHANGED
|
@@ -73,12 +73,16 @@ Third-Party Model Licenses
|
|
|
73
73
|
https://github.com/tetherto/qvac/releases/download/bci-test-assets-v0.1.0/bci-embedder.bin
|
|
74
74
|
cosyvoice3-campplus-f32
|
|
75
75
|
https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
|
76
|
+
cosyvoice3-flow-bf16
|
|
77
|
+
https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
|
76
78
|
cosyvoice3-flow-f32
|
|
77
79
|
https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
|
78
80
|
cosyvoice3-hift-f32
|
|
79
81
|
https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
|
80
82
|
cosyvoice3-llm-q8_0
|
|
81
83
|
https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
|
84
|
+
cosyvoice3-llm-q8_0-fused
|
|
85
|
+
https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
|
82
86
|
cosyvoice3-s3tok-f16
|
|
83
87
|
https://huggingface.co/FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
|
84
88
|
craft_mlt_25k
|
|
@@ -330,8 +334,12 @@ Third-Party Model Licenses
|
|
|
330
334
|
|
|
331
335
|
--- mit (MIT License) ---
|
|
332
336
|
|
|
337
|
+
acestep-5Hz-lm-0.6B-BF16
|
|
338
|
+
https://huggingface.co/ACE-Step/acestep-5Hz-lm-0.6B
|
|
333
339
|
acestep-5Hz-lm-0.6B-Q8_0
|
|
334
340
|
https://huggingface.co/ACE-Step/acestep-5Hz-lm-0.6B
|
|
341
|
+
acestep-v15-base-Q4_K_M
|
|
342
|
+
https://huggingface.co/ACE-Step/acestep-v15-base
|
|
335
343
|
acestep-v15-sft-Q8_0
|
|
336
344
|
https://huggingface.co/ACE-Step/acestep-v15-sft
|
|
337
345
|
acestep-v15-turbo-Q4_K_M
|
|
@@ -684,17 +692,17 @@ JavaScript Dependencies
|
|
|
684
692
|
@hyperswarm/secret-stream@6.9.2
|
|
685
693
|
https://github.com/holepunchto/hyperswarm-secret-stream
|
|
686
694
|
@qvac/error@0.1.1
|
|
687
|
-
@qvac/fabric@0.16.
|
|
695
|
+
@qvac/fabric@0.16.1
|
|
688
696
|
https://github.com/tetherto/qvac
|
|
689
697
|
@qvac/logging@0.1.1
|
|
690
698
|
https://github.com/tetherto/qvac
|
|
691
|
-
@qvac/model-fit@0.12.
|
|
699
|
+
@qvac/model-fit@0.12.2
|
|
692
700
|
https://github.com/tetherto/qvac
|
|
693
701
|
@qvac/rag@0.8.1
|
|
694
702
|
https://github.com/tetherto/qvac
|
|
695
|
-
@qvac/registry-client@0.
|
|
703
|
+
@qvac/registry-client@0.7.0
|
|
696
704
|
https://github.com/tetherto/qvac
|
|
697
|
-
@qvac/registry-schema@0.
|
|
705
|
+
@qvac/registry-schema@0.4.0
|
|
698
706
|
https://github.com/tetherto/qvac
|
|
699
707
|
adaptive-timeout@1.0.1
|
|
700
708
|
https://github.com/holepunchto/adaptive-timeout
|
|
@@ -772,9 +780,9 @@ JavaScript Dependencies
|
|
|
772
780
|
https://github.com/holepunchto/bare-stdio
|
|
773
781
|
bare-stream@2.13.4
|
|
774
782
|
https://github.com/holepunchto/bare-stream
|
|
775
|
-
bare-structured-clone@
|
|
783
|
+
bare-structured-clone@2.0.1
|
|
776
784
|
https://github.com/holepunchto/bare-structured-clone
|
|
777
|
-
bare-subprocess@6.
|
|
785
|
+
bare-subprocess@6.2.0
|
|
778
786
|
https://github.com/holepunchto/bare-subprocess
|
|
779
787
|
bare-tcp@2.6.1
|
|
780
788
|
https://github.com/holepunchto/bare-tcp
|
|
@@ -782,7 +790,7 @@ JavaScript Dependencies
|
|
|
782
790
|
https://github.com/holepunchto/bare-tls
|
|
783
791
|
bare-tty@5.2.1
|
|
784
792
|
https://github.com/holepunchto/bare-tty
|
|
785
|
-
bare-type@1.
|
|
793
|
+
bare-type@1.4.0
|
|
786
794
|
https://github.com/holepunchto/bare-type
|
|
787
795
|
bare-url@2.5.4
|
|
788
796
|
https://github.com/holepunchto/bare-url
|
|
@@ -822,7 +830,7 @@ JavaScript Dependencies
|
|
|
822
830
|
https://github.com/holepunchto/hyperdht-stats
|
|
823
831
|
hyperdispatch@1.6.0
|
|
824
832
|
https://github.com/holepunchto/hyperdispatch
|
|
825
|
-
hyperdrive@13.3.
|
|
833
|
+
hyperdrive@13.3.4
|
|
826
834
|
https://github.com/holepunchto/hyperdrive
|
|
827
835
|
hyperschema@1.26.2
|
|
828
836
|
https://github.com/holepunchto/hyperschema
|
|
@@ -854,7 +862,7 @@ JavaScript Dependencies
|
|
|
854
862
|
https://github.com/holepunchto/require-asset
|
|
855
863
|
resource-on-exit@1.0.0
|
|
856
864
|
https://github.com/holepunchto/bare-teardown
|
|
857
|
-
rocksdb-native@3.18.
|
|
865
|
+
rocksdb-native@3.18.1
|
|
858
866
|
https://github.com/holepunchto/rocksdb-native
|
|
859
867
|
scope-lock@1.2.4
|
|
860
868
|
https://github.com/holepunchto/scope-lock
|
|
@@ -864,7 +872,7 @@ JavaScript Dependencies
|
|
|
864
872
|
https://github.com/holepunchto/sub-encoder
|
|
865
873
|
text-decoder@1.2.7
|
|
866
874
|
https://github.com/holepunchto/text-decoder
|
|
867
|
-
udx-native@1.21.
|
|
875
|
+
udx-native@1.21.3
|
|
868
876
|
https://github.com/holepunchto/udx-native
|
|
869
877
|
unslab@1.3.0
|
|
870
878
|
https://github.com/holepunchto/unslab
|
|
@@ -920,7 +928,7 @@ JavaScript Dependencies
|
|
|
920
928
|
https://github.com/mafintosh/hypercore-crypto
|
|
921
929
|
hyperdht@6.34.0
|
|
922
930
|
https://github.com/holepunchto/hyperdht
|
|
923
|
-
hyperswarm@4.17.
|
|
931
|
+
hyperswarm@4.17.2
|
|
924
932
|
https://github.com/holepunchto/hyperswarm
|
|
925
933
|
is-options@1.0.2
|
|
926
934
|
https://github.com/mafintosh/is-options
|
|
@@ -3,10 +3,15 @@ import { type AssessModelFitInput, type AssessModelFitResult } from '../schemas/
|
|
|
3
3
|
* Assesses, before anything is downloaded, whether the given models are likely
|
|
4
4
|
* to fit in this device's memory.
|
|
5
5
|
*
|
|
6
|
-
* Advisory only: it does not download weights,
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
6
|
+
* Advisory only: it does not download weights, block `loadModel`, reserve
|
|
7
|
+
* memory, or make a performance claim. `unknown` is a real answer — it means
|
|
8
|
+
* the available evidence does not support a verdict either way, and callers
|
|
9
|
+
* should treat it as "cannot say", not "no".
|
|
10
|
+
*
|
|
11
|
+
* For a single candidate it fetches the registry's weightless description of
|
|
12
|
+
* the artifact — tens of KB, never the weights — and runs the engine's own
|
|
13
|
+
* fitter against it, reported as `native-fit` evidence. Where that is
|
|
14
|
+
* unavailable, and for a set of candidates, the calibrated estimate stands.
|
|
10
15
|
*
|
|
11
16
|
* @param input - Candidates with their intended workloads, the declared
|
|
12
17
|
* execution mode, and the headroom policy.
|
|
@@ -5,10 +5,15 @@ import { InvalidResponseError } from '../errors/index.js';
|
|
|
5
5
|
* Assesses, before anything is downloaded, whether the given models are likely
|
|
6
6
|
* to fit in this device's memory.
|
|
7
7
|
*
|
|
8
|
-
* Advisory only: it does not download weights,
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* Advisory only: it does not download weights, block `loadModel`, reserve
|
|
9
|
+
* memory, or make a performance claim. `unknown` is a real answer — it means
|
|
10
|
+
* the available evidence does not support a verdict either way, and callers
|
|
11
|
+
* should treat it as "cannot say", not "no".
|
|
12
|
+
*
|
|
13
|
+
* For a single candidate it fetches the registry's weightless description of
|
|
14
|
+
* the artifact — tens of KB, never the weights — and runs the engine's own
|
|
15
|
+
* fitter against it, reported as `native-fit` evidence. Where that is
|
|
16
|
+
* unavailable, and for a set of candidates, the calibrated estimate stands.
|
|
12
17
|
*
|
|
13
18
|
* @param input - Candidates with their intended workloads, the declared
|
|
14
19
|
* execution mode, and the headroom policy.
|
|
@@ -1,10 +1,34 @@
|
|
|
1
|
-
import type { AssessModelFitRequest, AssessModelFitResponse } from '../schemas/assess-model-fit.js';
|
|
1
|
+
import type { AssessModelFitRequest, AssessModelFitResponse, ModelFitCandidate } from '../schemas/assess-model-fit.js';
|
|
2
|
+
import { type CanonicalModelType } from '../schemas/index.js';
|
|
3
|
+
import { type ProfileResolver } from '../resources/model-fit/assess.js';
|
|
2
4
|
/**
|
|
3
5
|
* Runs a pre-download fit assessment worker-side.
|
|
4
6
|
*
|
|
5
|
-
* This lives on the worker because that is where the
|
|
6
|
-
* already are: the resource collector for a fresh memory sample,
|
|
7
|
-
*
|
|
8
|
-
* loaded
|
|
7
|
+
* This lives on the worker because that is where the three things it needs
|
|
8
|
+
* already are: the resource collector for a fresh memory sample, the runtime's
|
|
9
|
+
* own platform/arch pair, and the registry client. No weights are read and
|
|
10
|
+
* nothing is loaded — a single candidate additionally has the registry's
|
|
11
|
+
* weightless description fetched, tens of KB, so the engine's own fitter can
|
|
12
|
+
* answer instead of the coefficients modelling it.
|
|
9
13
|
*/
|
|
10
|
-
export declare function handleAssessModelFit(request: AssessModelFitRequest): AssessModelFitResponse
|
|
14
|
+
export declare function handleAssessModelFit(request: AssessModelFitRequest): Promise<AssessModelFitResponse>;
|
|
15
|
+
export interface FitLoad {
|
|
16
|
+
modelType: CanonicalModelType;
|
|
17
|
+
modelConfig: Record<string, unknown>;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* The load `loadModel` would run for this candidate, or `undefined` where no
|
|
21
|
+
* fitter covers it.
|
|
22
|
+
*
|
|
23
|
+
* The engine comes from the catalog profile, not from the workload: `llm` names
|
|
24
|
+
* a token-context workload and covers embedding models too, which the
|
|
25
|
+
* completion fitter refuses for asking more context than they declare. The
|
|
26
|
+
* config carries the same device defaults a real load resolves — the fitter
|
|
27
|
+
* answers `unsupported-config` for a load with no `device`. An embedding load
|
|
28
|
+
* has no context setting; the fitter reads the window the model declares.
|
|
29
|
+
*
|
|
30
|
+
* A candidate with companion `artifacts` gets no fitter: the estimator counts
|
|
31
|
+
* their bytes, the fitter reads one file, and a verdict for the model alone
|
|
32
|
+
* would stand in for the set.
|
|
33
|
+
*/
|
|
34
|
+
export declare function fitLoad(candidate: ModelFitCandidate, resolveProfile?: ProfileResolver): FitLoad | undefined;
|
|
@@ -1,26 +1,88 @@
|
|
|
1
1
|
import os from 'bare-os';
|
|
2
|
+
import { ModelType } from '../schemas/index.js';
|
|
3
|
+
import { projectFitFromStub } from '../resources/model-fit/fit-stub/project-fit-from-stub.js';
|
|
4
|
+
import { resolveModelConfig } from '../runtime/state.js';
|
|
5
|
+
import { getModelResourceProfile } from '../models/registry/resource-profiles.js';
|
|
2
6
|
import { getResourceCollector } from '../resources/instance.js';
|
|
3
7
|
import { assessModelFitFromResources } from '../resources/model-fit/assess.js';
|
|
4
8
|
import { getPlatformCalibration } from '../resources/model-fit/calibration/index.js';
|
|
5
9
|
/**
|
|
6
10
|
* Runs a pre-download fit assessment worker-side.
|
|
7
11
|
*
|
|
8
|
-
* This lives on the worker because that is where the
|
|
9
|
-
* already are: the resource collector for a fresh memory sample,
|
|
10
|
-
*
|
|
11
|
-
* loaded
|
|
12
|
+
* This lives on the worker because that is where the three things it needs
|
|
13
|
+
* already are: the resource collector for a fresh memory sample, the runtime's
|
|
14
|
+
* own platform/arch pair, and the registry client. No weights are read and
|
|
15
|
+
* nothing is loaded — a single candidate additionally has the registry's
|
|
16
|
+
* weightless description fetched, tens of KB, so the engine's own fitter can
|
|
17
|
+
* answer instead of the coefficients modelling it.
|
|
12
18
|
*/
|
|
13
|
-
export function handleAssessModelFit(request) {
|
|
19
|
+
export async function handleAssessModelFit(request) {
|
|
14
20
|
const platform = detectPlatform();
|
|
15
21
|
const result = assessModelFitFromResources({
|
|
16
22
|
models: request.models,
|
|
17
23
|
execution: request.execution,
|
|
18
24
|
resources: readResources(),
|
|
19
25
|
platform,
|
|
20
|
-
calibration: platform ? getPlatformCalibration(platform) : undefined
|
|
26
|
+
calibration: platform ? getPlatformCalibration(platform) : undefined,
|
|
27
|
+
nativeFit: await resolveNativeFit(request)
|
|
21
28
|
});
|
|
22
29
|
return { type: 'assessModelFit', ...result };
|
|
23
30
|
}
|
|
31
|
+
/**
|
|
32
|
+
* The engine fitter's verdict for a single candidate, read from the registry's
|
|
33
|
+
* fit stub.
|
|
34
|
+
*
|
|
35
|
+
* Only for a one-candidate request: the probe measures one model against the
|
|
36
|
+
* whole machine, which cannot be aggregated across a set. Everything else — no
|
|
37
|
+
* stub published, an offline caller, an engine with no fit path — resolves to
|
|
38
|
+
* `undefined` and the modelled assessment stands.
|
|
39
|
+
*/
|
|
40
|
+
async function resolveNativeFit(request) {
|
|
41
|
+
if (request.models.length !== 1)
|
|
42
|
+
return undefined;
|
|
43
|
+
const candidate = request.models[0];
|
|
44
|
+
if (!candidate)
|
|
45
|
+
return undefined;
|
|
46
|
+
const load = fitLoad(candidate);
|
|
47
|
+
if (!load)
|
|
48
|
+
return undefined;
|
|
49
|
+
const outcome = await projectFitFromStub({ model: candidate.model, ...load });
|
|
50
|
+
return outcome.status === 'projected' ? outcome.fit : undefined;
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* The load `loadModel` would run for this candidate, or `undefined` where no
|
|
54
|
+
* fitter covers it.
|
|
55
|
+
*
|
|
56
|
+
* The engine comes from the catalog profile, not from the workload: `llm` names
|
|
57
|
+
* a token-context workload and covers embedding models too, which the
|
|
58
|
+
* completion fitter refuses for asking more context than they declare. The
|
|
59
|
+
* config carries the same device defaults a real load resolves — the fitter
|
|
60
|
+
* answers `unsupported-config` for a load with no `device`. An embedding load
|
|
61
|
+
* has no context setting; the fitter reads the window the model declares.
|
|
62
|
+
*
|
|
63
|
+
* A candidate with companion `artifacts` gets no fitter: the estimator counts
|
|
64
|
+
* their bytes, the fitter reads one file, and a verdict for the model alone
|
|
65
|
+
* would stand in for the set.
|
|
66
|
+
*/
|
|
67
|
+
export function fitLoad(candidate, resolveProfile = getModelResourceProfile) {
|
|
68
|
+
if (candidate.workload.kind !== 'llm')
|
|
69
|
+
return undefined;
|
|
70
|
+
if (candidate.artifacts?.length)
|
|
71
|
+
return undefined;
|
|
72
|
+
// A model outside the catalog is taken as a completion model, the common
|
|
73
|
+
// case for an `llm` workload.
|
|
74
|
+
const engine = resolveProfile(candidate.model.sha256Checksum)?.engine ?? ModelType.llamacppCompletion;
|
|
75
|
+
if (engine === ModelType.llamacppEmbedding) {
|
|
76
|
+
return { modelType: engine, modelConfig: resolveModelConfig(engine, {}) };
|
|
77
|
+
}
|
|
78
|
+
if (engine === ModelType.llamacppCompletion) {
|
|
79
|
+
return {
|
|
80
|
+
modelType: engine,
|
|
81
|
+
modelConfig: resolveModelConfig(engine, { ctx_size: candidate.workload.contextTokens })
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
return undefined;
|
|
85
|
+
}
|
|
24
86
|
function readResources() {
|
|
25
87
|
const collector = getResourceCollector();
|
|
26
88
|
if (!collector) {
|
|
@@ -14,7 +14,8 @@ export function buildBlobBinding(meta) {
|
|
|
14
14
|
blockOffset: meta.blobBlockOffset,
|
|
15
15
|
blockLength: meta.blobBlockLength,
|
|
16
16
|
byteOffset: meta.blobByteOffset,
|
|
17
|
-
byteLength: meta.expectedSize
|
|
17
|
+
byteLength: meta.expectedSize,
|
|
18
|
+
sha256: meta.sha256Checksum
|
|
18
19
|
};
|
|
19
20
|
}
|
|
20
21
|
/**
|
|
@@ -50,6 +50,29 @@ export declare const llmPlugin: {
|
|
|
50
50
|
auto: "auto";
|
|
51
51
|
}>>;
|
|
52
52
|
'tensor-split': import("zod").ZodOptional<import("zod").ZodString>;
|
|
53
|
+
'batch-size': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
54
|
+
'ubatch-size': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
55
|
+
'cpu-moe': import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
56
|
+
'n-cpu-moe': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
57
|
+
'kv-offload': import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
58
|
+
'image-max-tokens': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
59
|
+
'image-min-tokens': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
60
|
+
threads: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
61
|
+
'threads-batch': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
62
|
+
'cpu-mask': import("zod").ZodOptional<import("zod").ZodString>;
|
|
63
|
+
'cpu-mask-batch': import("zod").ZodOptional<import("zod").ZodString>;
|
|
64
|
+
'override-tensor': import("zod").ZodOptional<import("zod").ZodString>;
|
|
65
|
+
'n-cpu-ffn': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
66
|
+
'moe-cache-mib': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodLiteral<"auto">]>>;
|
|
67
|
+
'prefetch-weights': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodBoolean, import("zod").ZodLiteral<"auto">]>>;
|
|
68
|
+
'tensor-read-lazy': import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
69
|
+
on: "on";
|
|
70
|
+
off: "off";
|
|
71
|
+
auto: "auto";
|
|
72
|
+
}>>;
|
|
73
|
+
fit: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
74
|
+
'fit-target': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodString]>>;
|
|
75
|
+
'fit-ctx': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
53
76
|
openclCacheDir: import("zod").ZodOptional<import("zod").ZodString>;
|
|
54
77
|
reasoning_budget: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
55
78
|
projectionModelSrc: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodObject<{
|
|
@@ -128,6 +151,25 @@ export declare const llmPlugin: {
|
|
|
128
151
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
129
152
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
130
153
|
'tensor-split'?: string | undefined;
|
|
154
|
+
'batch-size'?: number | undefined;
|
|
155
|
+
'ubatch-size'?: number | undefined;
|
|
156
|
+
'cpu-moe'?: boolean | undefined;
|
|
157
|
+
'n-cpu-moe'?: number | undefined;
|
|
158
|
+
'kv-offload'?: boolean | undefined;
|
|
159
|
+
'image-max-tokens'?: number | undefined;
|
|
160
|
+
'image-min-tokens'?: number | undefined;
|
|
161
|
+
threads?: number | undefined;
|
|
162
|
+
'threads-batch'?: number | undefined;
|
|
163
|
+
'cpu-mask'?: string | undefined;
|
|
164
|
+
'cpu-mask-batch'?: string | undefined;
|
|
165
|
+
'override-tensor'?: string | undefined;
|
|
166
|
+
'n-cpu-ffn'?: number | undefined;
|
|
167
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
168
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
169
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
170
|
+
fit?: boolean | undefined;
|
|
171
|
+
'fit-target'?: string | number | undefined;
|
|
172
|
+
'fit-ctx'?: number | undefined;
|
|
131
173
|
openclCacheDir?: string | undefined;
|
|
132
174
|
reasoning_budget?: number | undefined;
|
|
133
175
|
image_tile_mode?: "disabled" | "batched" | "sequential" | undefined;
|
|
@@ -161,6 +203,25 @@ export declare const llmPlugin: {
|
|
|
161
203
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
162
204
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
163
205
|
'tensor-split'?: string | undefined;
|
|
206
|
+
'batch-size'?: number | undefined;
|
|
207
|
+
'ubatch-size'?: number | undefined;
|
|
208
|
+
'cpu-moe'?: boolean | undefined;
|
|
209
|
+
'n-cpu-moe'?: number | undefined;
|
|
210
|
+
'kv-offload'?: boolean | undefined;
|
|
211
|
+
'image-max-tokens'?: number | undefined;
|
|
212
|
+
'image-min-tokens'?: number | undefined;
|
|
213
|
+
threads?: number | undefined;
|
|
214
|
+
'threads-batch'?: number | undefined;
|
|
215
|
+
'cpu-mask'?: string | undefined;
|
|
216
|
+
'cpu-mask-batch'?: string | undefined;
|
|
217
|
+
'override-tensor'?: string | undefined;
|
|
218
|
+
'n-cpu-ffn'?: number | undefined;
|
|
219
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
220
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
221
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
222
|
+
fit?: boolean | undefined;
|
|
223
|
+
'fit-target'?: string | number | undefined;
|
|
224
|
+
'fit-ctx'?: number | undefined;
|
|
164
225
|
openclCacheDir?: string | undefined;
|
|
165
226
|
reasoning_budget?: number | undefined;
|
|
166
227
|
image_tile_mode?: "disabled" | "batched" | "sequential" | undefined;
|
|
@@ -23,5 +23,24 @@ export function transformLlmConfig(llmConfig) {
|
|
|
23
23
|
transformed['openclCacheDir'] = transformed['opencl_cache_dir'];
|
|
24
24
|
delete transformed['opencl_cache_dir'];
|
|
25
25
|
}
|
|
26
|
+
if ('cpu-moe' in transformed) {
|
|
27
|
+
if (transformed['cpu-moe'] === 'true') {
|
|
28
|
+
transformed['cpu-moe'] = '';
|
|
29
|
+
}
|
|
30
|
+
else {
|
|
31
|
+
delete transformed['cpu-moe'];
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
if ('kv-offload' in transformed) {
|
|
35
|
+
const enabled = transformed['kv-offload'] === 'true';
|
|
36
|
+
delete transformed['kv-offload'];
|
|
37
|
+
transformed[enabled ? 'kv-offload' : 'no-kv-offload'] = '';
|
|
38
|
+
}
|
|
39
|
+
if (transformed['prefetch-weights'] === 'true') {
|
|
40
|
+
transformed['prefetch-weights'] = '1';
|
|
41
|
+
}
|
|
42
|
+
else if (transformed['prefetch-weights'] === 'false') {
|
|
43
|
+
transformed['prefetch-weights'] = '0';
|
|
44
|
+
}
|
|
26
45
|
return transformed;
|
|
27
46
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { AssessModelFitResult, ModelFitCandidate, ModelFitExecution } from '../../schemas/assess-model-fit.js';
|
|
1
|
+
import type { AssessModelFitResult, ModelFitCandidate, ModelFitExecution, NativeProbeFit } from '../../schemas/assess-model-fit.js';
|
|
2
2
|
import type { SystemResources } from '../../schemas/system-resources.js';
|
|
3
3
|
import type { ModelResourceProfile } from '../../schemas/model-resource-profile.js';
|
|
4
4
|
import type { ModelFitPlatform, PlatformCalibration } from './types.js';
|
|
@@ -24,6 +24,15 @@ export interface AssessModelFitOptions {
|
|
|
24
24
|
resolveSharedGpuCalibration?: (platform: ModelFitPlatform, backend: string) => PlatformCalibration | undefined;
|
|
25
25
|
/** Defaults to the generated catalog table; injected in tests. */
|
|
26
26
|
resolveProfile?: ProfileResolver;
|
|
27
|
+
/**
|
|
28
|
+
* The engine's own verdict for the single candidate, from the registry's fit
|
|
29
|
+
* stub. Resolved in the handler, which is where the network is.
|
|
30
|
+
*
|
|
31
|
+
* Only ever set for a one-candidate assessment: the fitter answers about one
|
|
32
|
+
* model against the whole machine, and two such answers cannot be summed into
|
|
33
|
+
* a combined budget the way two byte estimates can.
|
|
34
|
+
*/
|
|
35
|
+
nativeFit?: NativeProbeFit | undefined;
|
|
27
36
|
}
|
|
28
37
|
/**
|
|
29
38
|
* Turns catalog profiles plus a fresh system-memory sample into an advisory
|
|
@@ -168,6 +168,23 @@ export function assessModelFitFromResources(options) {
|
|
|
168
168
|
if (anyFloor) {
|
|
169
169
|
reasons.push('at least one model is bounded only by its computed floor (weights, plus the KV cache for llama.cpp), which can refuse the set but never confirm a fit');
|
|
170
170
|
}
|
|
171
|
+
// The engine's own fitter, where it reached a verdict, replaces the modelled
|
|
172
|
+
// one: it read this model's tensor list and measured this machine, which is
|
|
173
|
+
// what the coefficients above approximate. The estimator's reasons and
|
|
174
|
+
// assumptions describe the path not taken, so none of them are reported.
|
|
175
|
+
const native = nativeModelResult(options.nativeFit, modelResults);
|
|
176
|
+
if (native) {
|
|
177
|
+
return {
|
|
178
|
+
verdict: native.verdict,
|
|
179
|
+
basis,
|
|
180
|
+
execution,
|
|
181
|
+
evidence: 'native-fit',
|
|
182
|
+
...(budget && { budget }),
|
|
183
|
+
models: [native],
|
|
184
|
+
reasons: ['the engine fitter read the registry description of this model'],
|
|
185
|
+
assumptions: []
|
|
186
|
+
};
|
|
187
|
+
}
|
|
171
188
|
return {
|
|
172
189
|
verdict,
|
|
173
190
|
basis,
|
|
@@ -183,6 +200,32 @@ export function assessModelFitFromResources(options) {
|
|
|
183
200
|
assumptions
|
|
184
201
|
};
|
|
185
202
|
}
|
|
203
|
+
/**
|
|
204
|
+
* The per-model result a native fit produces, or `undefined` when the modelled
|
|
205
|
+
* one stands.
|
|
206
|
+
*
|
|
207
|
+
* The probe answers about one model against the whole machine. Two such answers
|
|
208
|
+
* carry no byte demand that could be summed under one budget, so a multi-model
|
|
209
|
+
* assessment keeps the estimator that can aggregate. `unknown` — the probe
|
|
210
|
+
* disabled, the load shape unsupported, the child unusable — is not a verdict
|
|
211
|
+
* and falls through the same way.
|
|
212
|
+
*/
|
|
213
|
+
function nativeModelResult(nativeFit, modelResults) {
|
|
214
|
+
if (!nativeFit || modelResults.length !== 1)
|
|
215
|
+
return undefined;
|
|
216
|
+
if (nativeFit.verdict === 'unknown')
|
|
217
|
+
return undefined;
|
|
218
|
+
const modelled = modelResults[0];
|
|
219
|
+
if (!modelled)
|
|
220
|
+
return undefined;
|
|
221
|
+
return {
|
|
222
|
+
name: modelled.name,
|
|
223
|
+
verdict: nativeFit.verdict === 'fit' ? 'likely-fits' : 'likely-too-large',
|
|
224
|
+
evidence: 'native-fit',
|
|
225
|
+
estimatorVersion: nativeFit.estimatorVersion,
|
|
226
|
+
reasons: nativeFit.message === undefined ? [nativeFit.reason] : [nativeFit.reason, nativeFit.message]
|
|
227
|
+
};
|
|
228
|
+
}
|
|
186
229
|
/**
|
|
187
230
|
* Whether every allocation a load makes comes out of the memory the system (or
|
|
188
231
|
* process) budget measures — the precondition for the computed floor to be
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { type AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { QVACBlobBinding } from '@qvac/registry-client';
|
|
3
|
+
import type { Logger } from '../../../logging/types.js';
|
|
4
|
+
/**
|
|
5
|
+
* A weightless description of an artifact, served by the registry beside the
|
|
6
|
+
* weights: for a GGUF, the tensor list and the settings with the tokenizer
|
|
7
|
+
* tables stripped and no data section — tens of KB against gigabytes. The
|
|
8
|
+
* fitter reads it exactly as it reads the artifact, so a projection can be made
|
|
9
|
+
* before anything is downloaded.
|
|
10
|
+
*/
|
|
11
|
+
export interface FitStubRef {
|
|
12
|
+
name: string;
|
|
13
|
+
sha256Checksum: string;
|
|
14
|
+
/** Registry coordinates. A constant that carries neither cannot be resolved. */
|
|
15
|
+
registryPath?: string | undefined;
|
|
16
|
+
registrySource?: string | undefined;
|
|
17
|
+
}
|
|
18
|
+
export type FitStubUnavailableReason =
|
|
19
|
+
/** The ref carries no registry coordinates to look up. */
|
|
20
|
+
'unresolvable-ref'
|
|
21
|
+
/** The registry has no entry at those coordinates. */
|
|
22
|
+
| 'not-in-registry'
|
|
23
|
+
/** The entry predates fit blobs, or its description could not be built. */
|
|
24
|
+
| 'no-fit-blob' | 'download-failed'
|
|
25
|
+
/** The lookup and the fetch together did not finish within the budget. */
|
|
26
|
+
| 'timed-out';
|
|
27
|
+
export type FitStubOutcome =
|
|
28
|
+
/** `path` is the caller's to remove once the fitter has read it. */
|
|
29
|
+
{
|
|
30
|
+
status: 'ready';
|
|
31
|
+
path: string;
|
|
32
|
+
bytes: number;
|
|
33
|
+
} | {
|
|
34
|
+
status: 'unavailable';
|
|
35
|
+
reason: FitStubUnavailableReason;
|
|
36
|
+
message?: string;
|
|
37
|
+
};
|
|
38
|
+
export type FitBlobBinding = QVACBlobBinding;
|
|
39
|
+
export interface FitStubEntry {
|
|
40
|
+
fitBlobBinding?: FitBlobBinding | undefined;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Injection seams. Every field defaults to the real runtime dependency; tests
|
|
44
|
+
* substitute them rather than mocking modules.
|
|
45
|
+
*/
|
|
46
|
+
export interface FitStubOptions {
|
|
47
|
+
signal?: AbortSignal;
|
|
48
|
+
/** Where stubs are staged. Defaults to `fit-stubs` under the QVAC cache root. */
|
|
49
|
+
cacheDir?: string;
|
|
50
|
+
/** Overall budget for the lookup and the fetch. Defaults to `FIT_STUB_BUDGET_MS`. */
|
|
51
|
+
budgetMs?: number;
|
|
52
|
+
logger?: Logger;
|
|
53
|
+
getEntry?: (registryPath: string, registrySource: string) => Promise<FitStubEntry | null>;
|
|
54
|
+
downloadBlob?: (binding: FitBlobBinding, outputFile: string, signal?: AbortSignal) => Promise<unknown>;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Bounds the registry lookup and the blob fetch together. The client's own
|
|
58
|
+
* defaults — 30s per attempt, three attempts, a peer wait between them, and an
|
|
59
|
+
* uncapped wait for the registry view on first use — suit a weights download.
|
|
60
|
+
* Here an unreachable registry is the whole latency of the call, so it is
|
|
61
|
+
* capped. Work still in flight at expiry is abandoned: the download is
|
|
62
|
+
* aborted, and a lookup that cannot be cancelled settles on its own with
|
|
63
|
+
* nothing waiting for it.
|
|
64
|
+
*/
|
|
65
|
+
export declare const FIT_STUB_BUDGET_MS = 10000;
|
|
66
|
+
/** Removes a fetched stub and the directory it was staged in. Never throws. */
|
|
67
|
+
export declare function removeStub(stubPath: string): Promise<void>;
|
|
68
|
+
/**
|
|
69
|
+
* Fetches the fit stub for one artifact into its own staging directory and
|
|
70
|
+
* returns its path. The stub is a per-call payload, not a cache: the caller
|
|
71
|
+
* removes it once the fitter has read it, so nothing accumulates under the
|
|
72
|
+
* QVAC root and no cleanup path has to know about it.
|
|
73
|
+
*
|
|
74
|
+
* Never throws: a missing entry, an entry without a description, a failed or
|
|
75
|
+
* corrupt download and an expired budget are all `unavailable`, because a
|
|
76
|
+
* pre-download assessment has to survive an offline caller and an older
|
|
77
|
+
* registry record.
|
|
78
|
+
*
|
|
79
|
+
* Only the artifact named by `ref` is fetched. A split model's shards each
|
|
80
|
+
* carry their own binding, and the fitter needs the whole set laid out under
|
|
81
|
+
* llama's shard naming to read any of it — assembling that is not done here.
|
|
82
|
+
*/
|
|
83
|
+
export declare function fetchFitStub(ref: FitStubRef, options?: FitStubOptions): Promise<FitStubOutcome>;
|