@qvac/inference 0.20.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +19 -11
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/handlers/assess-model-fit.d.ts +30 -6
- package/dist/handlers/assess-model-fit.js +68 -6
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +61 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/resources/model-fit/assess.d.ts +10 -1
- package/dist/resources/model-fit/assess.js +43 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +83 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +133 -0
- package/dist/resources/model-fit/fit-stub/index.d.ts +2 -0
- package/dist/resources/model-fit/fit-stub/index.js +2 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +31 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +36 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +38 -18
- package/dist/schemas/assess-model-fit.d.ts +23 -0
- package/dist/schemas/assess-model-fit.js +16 -2
- package/dist/schemas/common.d.ts +67 -0
- package/dist/schemas/config.d.ts +138 -0
- package/dist/schemas/llamacpp-config.d.ts +84 -0
- package/dist/schemas/llamacpp-config.js +104 -4
- package/dist/schemas/load-model.d.ts +336 -0
- package/dist/schemas/model-config-schemas.d.ts +23 -0
- package/package.json +2 -2
package/dist/schemas/config.d.ts
CHANGED
|
@@ -69,6 +69,29 @@ export declare const deviceConfigDefaultsSchema: z.ZodObject<{
|
|
|
69
69
|
auto: "auto";
|
|
70
70
|
}>>;
|
|
71
71
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
72
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
73
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
74
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
75
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
76
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
77
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
78
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
79
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
80
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
81
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
82
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
83
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
84
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
85
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
86
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
87
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
88
|
+
on: "on";
|
|
89
|
+
off: "off";
|
|
90
|
+
auto: "auto";
|
|
91
|
+
}>>;
|
|
92
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
93
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
94
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
72
95
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
73
96
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
74
97
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -1003,6 +1026,29 @@ export declare const deviceConfigDefaultsSchema: z.ZodObject<{
|
|
|
1003
1026
|
auto: "auto";
|
|
1004
1027
|
}>>;
|
|
1005
1028
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
1029
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
1030
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
1031
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
1032
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
1033
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
1034
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
1035
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
1036
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
1037
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
1038
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
1039
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
1040
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
1041
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
1042
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
1043
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
1044
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
1045
|
+
on: "on";
|
|
1046
|
+
off: "off";
|
|
1047
|
+
auto: "auto";
|
|
1048
|
+
}>>;
|
|
1049
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
1050
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
1051
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
1006
1052
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
1007
1053
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
1008
1054
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -1957,6 +2003,29 @@ export declare const devicePatternSchema: z.ZodObject<{
|
|
|
1957
2003
|
auto: "auto";
|
|
1958
2004
|
}>>;
|
|
1959
2005
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
2006
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
2007
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
2008
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
2009
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
2010
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
2011
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
2012
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
2013
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
2014
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
2015
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
2016
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
2017
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
2018
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
2019
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
2020
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
2021
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
2022
|
+
on: "on";
|
|
2023
|
+
off: "off";
|
|
2024
|
+
auto: "auto";
|
|
2025
|
+
}>>;
|
|
2026
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
2027
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
2028
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
1960
2029
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
1961
2030
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
1962
2031
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -2891,6 +2960,29 @@ export declare const devicePatternSchema: z.ZodObject<{
|
|
|
2891
2960
|
auto: "auto";
|
|
2892
2961
|
}>>;
|
|
2893
2962
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
2963
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
2964
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
2965
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
2966
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
2967
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
2968
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
2969
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
2970
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
2971
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
2972
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
2973
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
2974
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
2975
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
2976
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
2977
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
2978
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
2979
|
+
on: "on";
|
|
2980
|
+
off: "off";
|
|
2981
|
+
auto: "auto";
|
|
2982
|
+
}>>;
|
|
2983
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
2984
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
2985
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
2894
2986
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
2895
2987
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
2896
2988
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -3868,6 +3960,29 @@ export declare const qvacConfigSchema: z.ZodObject<{
|
|
|
3868
3960
|
auto: "auto";
|
|
3869
3961
|
}>>;
|
|
3870
3962
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
3963
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
3964
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
3965
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
3966
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
3967
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
3968
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
3969
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
3970
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
3971
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
3972
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
3973
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
3974
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
3975
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
3976
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
3977
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
3978
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
3979
|
+
on: "on";
|
|
3980
|
+
off: "off";
|
|
3981
|
+
auto: "auto";
|
|
3982
|
+
}>>;
|
|
3983
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
3984
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
3985
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
3871
3986
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
3872
3987
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
3873
3988
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -4802,6 +4917,29 @@ export declare const qvacConfigSchema: z.ZodObject<{
|
|
|
4802
4917
|
auto: "auto";
|
|
4803
4918
|
}>>;
|
|
4804
4919
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
4920
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
4921
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
4922
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
4923
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
4924
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
4925
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
4926
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
4927
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
4928
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
4929
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
4930
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
4931
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
4932
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
4933
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
4934
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
4935
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
4936
|
+
on: "on";
|
|
4937
|
+
off: "off";
|
|
4938
|
+
auto: "auto";
|
|
4939
|
+
}>>;
|
|
4940
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
4941
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
4942
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
4805
4943
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
4806
4944
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
4807
4945
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -58,6 +58,29 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
|
|
|
58
58
|
auto: "auto";
|
|
59
59
|
}>>;
|
|
60
60
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
61
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
62
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
63
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
64
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
65
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
66
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
67
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
68
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
69
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
70
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
71
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
72
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
73
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
74
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
75
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
76
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
77
|
+
on: "on";
|
|
78
|
+
off: "off";
|
|
79
|
+
auto: "auto";
|
|
80
|
+
}>>;
|
|
81
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
82
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
83
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
61
84
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
62
85
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
63
86
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -165,6 +188,29 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
165
188
|
auto: "auto";
|
|
166
189
|
}>>;
|
|
167
190
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
191
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
192
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
193
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
194
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
195
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
196
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
197
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
198
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
199
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
200
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
201
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
202
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
203
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
204
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
205
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
206
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
207
|
+
on: "on";
|
|
208
|
+
off: "off";
|
|
209
|
+
auto: "auto";
|
|
210
|
+
}>>;
|
|
211
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
212
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
213
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
168
214
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
169
215
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
170
216
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -241,6 +287,25 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
241
287
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
242
288
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
243
289
|
'tensor-split'?: string | undefined;
|
|
290
|
+
'batch-size'?: number | undefined;
|
|
291
|
+
'ubatch-size'?: number | undefined;
|
|
292
|
+
'cpu-moe'?: boolean | undefined;
|
|
293
|
+
'n-cpu-moe'?: number | undefined;
|
|
294
|
+
'kv-offload'?: boolean | undefined;
|
|
295
|
+
'image-max-tokens'?: number | undefined;
|
|
296
|
+
'image-min-tokens'?: number | undefined;
|
|
297
|
+
threads?: number | undefined;
|
|
298
|
+
'threads-batch'?: number | undefined;
|
|
299
|
+
'cpu-mask'?: string | undefined;
|
|
300
|
+
'cpu-mask-batch'?: string | undefined;
|
|
301
|
+
'override-tensor'?: string | undefined;
|
|
302
|
+
'n-cpu-ffn'?: number | undefined;
|
|
303
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
304
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
305
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
306
|
+
fit?: boolean | undefined;
|
|
307
|
+
'fit-target'?: string | number | undefined;
|
|
308
|
+
'fit-ctx'?: number | undefined;
|
|
244
309
|
openclCacheDir?: string | undefined;
|
|
245
310
|
reasoning_budget?: number | undefined;
|
|
246
311
|
projectionModelSrc?: string | {
|
|
@@ -284,6 +349,25 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
284
349
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
285
350
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
286
351
|
'tensor-split'?: string | undefined;
|
|
352
|
+
'batch-size'?: number | undefined;
|
|
353
|
+
'ubatch-size'?: number | undefined;
|
|
354
|
+
'cpu-moe'?: boolean | undefined;
|
|
355
|
+
'n-cpu-moe'?: number | undefined;
|
|
356
|
+
'kv-offload'?: boolean | undefined;
|
|
357
|
+
'image-max-tokens'?: number | undefined;
|
|
358
|
+
'image-min-tokens'?: number | undefined;
|
|
359
|
+
threads?: number | undefined;
|
|
360
|
+
'threads-batch'?: number | undefined;
|
|
361
|
+
'cpu-mask'?: string | undefined;
|
|
362
|
+
'cpu-mask-batch'?: string | undefined;
|
|
363
|
+
'override-tensor'?: string | undefined;
|
|
364
|
+
'n-cpu-ffn'?: number | undefined;
|
|
365
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
366
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
367
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
368
|
+
fit?: boolean | undefined;
|
|
369
|
+
'fit-target'?: string | number | undefined;
|
|
370
|
+
'fit-ctx'?: number | undefined;
|
|
287
371
|
openclCacheDir?: string | undefined;
|
|
288
372
|
reasoning_budget?: number | undefined;
|
|
289
373
|
projectionModelSrc?: string | {
|
|
@@ -115,6 +115,98 @@ export const llmConfigBaseSchema = z.object({
|
|
|
115
115
|
.string()
|
|
116
116
|
.optional()
|
|
117
117
|
.describe("Proportions for distributing layers/rows across GPUs, e.g. `'1,1'` (equal) or `'3,1'` (75/25)."),
|
|
118
|
+
'batch-size': z
|
|
119
|
+
.number()
|
|
120
|
+
.int()
|
|
121
|
+
.min(1)
|
|
122
|
+
.optional()
|
|
123
|
+
.describe('Logical maximum batch size, the tokens submitted per decode call. Default 2048.'),
|
|
124
|
+
'ubatch-size': z
|
|
125
|
+
.number()
|
|
126
|
+
.int()
|
|
127
|
+
.min(1)
|
|
128
|
+
.optional()
|
|
129
|
+
.describe('Physical maximum batch size, the tokens computed in one pass. Default 512.'),
|
|
130
|
+
'cpu-moe': z
|
|
131
|
+
.boolean()
|
|
132
|
+
.optional()
|
|
133
|
+
.describe('Keep all Mixture-of-Experts weights on the CPU, freeing accelerator memory for the rest of the model. Mixture-of-Experts models only. Default false.'),
|
|
134
|
+
'n-cpu-moe': z
|
|
135
|
+
.number()
|
|
136
|
+
.int()
|
|
137
|
+
.min(0)
|
|
138
|
+
.optional()
|
|
139
|
+
.describe('Keep the Mixture-of-Experts weights of the first N layers on the CPU. Mixture-of-Experts models only, and a finer-grained alternative to `cpu-moe`.'),
|
|
140
|
+
'kv-offload': z
|
|
141
|
+
.boolean()
|
|
142
|
+
.optional()
|
|
143
|
+
.describe('Whether to hold the KV cache in accelerator memory. Default true. Set false to keep it in system memory, which frees accelerator memory at the cost of decode speed.'),
|
|
144
|
+
'image-max-tokens': z
|
|
145
|
+
.number()
|
|
146
|
+
.int()
|
|
147
|
+
.min(1)
|
|
148
|
+
.optional()
|
|
149
|
+
.describe('Upper bound on the tokens one image may occupy. Vision models only. Unset uses the model default.'),
|
|
150
|
+
'image-min-tokens': z
|
|
151
|
+
.number()
|
|
152
|
+
.int()
|
|
153
|
+
.min(1)
|
|
154
|
+
.optional()
|
|
155
|
+
.describe('Lower bound on the tokens one image may occupy. Vision models only. Unset uses the model default.'),
|
|
156
|
+
threads: z
|
|
157
|
+
.number()
|
|
158
|
+
.int()
|
|
159
|
+
.optional()
|
|
160
|
+
.describe('CPU threads to use during generation. Unset uses one thread per physical core; a value of 0 or below uses every logical CPU, which is a different and usually larger count.'),
|
|
161
|
+
'threads-batch': z
|
|
162
|
+
.number()
|
|
163
|
+
.int()
|
|
164
|
+
.optional()
|
|
165
|
+
.describe('CPU threads to use during batch and prompt processing. Setting it governs generation too and discards `threads` and `cpu-mask`; leave it unset for `threads` to apply to both.'),
|
|
166
|
+
'cpu-mask': z
|
|
167
|
+
.string()
|
|
168
|
+
.optional()
|
|
169
|
+
.describe("CPU affinity mask as an arbitrarily long hex string, e.g. `'ff'`. Unset applies no affinity. Ignored when `threads-batch` is set."),
|
|
170
|
+
'cpu-mask-batch': z
|
|
171
|
+
.string()
|
|
172
|
+
.optional()
|
|
173
|
+
.describe('CPU affinity mask for batch and prompt processing, in the same form as `cpu-mask`. Honoured only alongside `threads-batch`; without it `cpu-mask` takes over.'),
|
|
174
|
+
'override-tensor': z
|
|
175
|
+
.string()
|
|
176
|
+
.optional()
|
|
177
|
+
.describe("Buffer type overrides for tensors matching a pattern, as comma-separated `<tensor name pattern>=<buffer type>` entries, e.g. `'blk\\.(1[0-9])\\.ffn_(up|down|gate)_exps=CPU'` to keep those experts in system memory."),
|
|
178
|
+
'n-cpu-ffn': z
|
|
179
|
+
.number()
|
|
180
|
+
.int()
|
|
181
|
+
.min(0)
|
|
182
|
+
.optional()
|
|
183
|
+
.describe('Keep the dense FFN weights of the first N layers on the CPU.'),
|
|
184
|
+
'moe-cache-mib': z
|
|
185
|
+
.union([z.number().int().min(0), z.literal('auto')])
|
|
186
|
+
.optional()
|
|
187
|
+
.describe("Persistent accelerator cache for Mixture-of-Experts weights, in MiB. Default `'auto'`, which sizes it during `fit`."),
|
|
188
|
+
'prefetch-weights': z
|
|
189
|
+
.union([z.boolean(), z.literal('auto')])
|
|
190
|
+
.optional()
|
|
191
|
+
.describe("Overlap host-to-accelerator weight copies with compute. Default `'auto'`, which enables it for fitted dense models."),
|
|
192
|
+
'tensor-read-lazy': z
|
|
193
|
+
.enum(['on', 'auto', 'off'])
|
|
194
|
+
.optional()
|
|
195
|
+
.describe("On-demand reading of large tensors such as per-layer embeddings, which requires mmap: `'on'` reads their rows from disk instead of keeping them resident, `'off'` always keeps them resident, `'auto'` (default) applies `'on'` above 4 GiB."),
|
|
196
|
+
fit: z
|
|
197
|
+
.boolean()
|
|
198
|
+
.optional()
|
|
199
|
+
.describe('Adjust the arguments left unset so the model fits device memory. Default true. It chooses layer offload, tensor split and tensor buffer overrides, and the context size only when `ctx_size` is 0. Setting this field releases the `gpu_layers` default so fit can choose it, which offloads every layer rather than 99 if fit then gives up; setting `gpu_layers` alongside it pins the value and makes fit abort, leaving every argument unchanged. Incompatible with `split-mode: tensor`, which disables it.'),
|
|
200
|
+
'fit-target': z
|
|
201
|
+
.union([z.number().int().min(0), z.string().regex(/^\d+(,\d+)*$/)])
|
|
202
|
+
.optional()
|
|
203
|
+
.describe("Free margin `fit` aims to leave per device, in MiB. A single value is broadcast across devices; a comma-separated list such as `'1024,512'` sets them individually. Default 1024."),
|
|
204
|
+
'fit-ctx': z
|
|
205
|
+
.number()
|
|
206
|
+
.int()
|
|
207
|
+
.min(0)
|
|
208
|
+
.optional()
|
|
209
|
+
.describe('Smallest context size `fit` may settle on. Default 4096.'),
|
|
118
210
|
openclCacheDir: z
|
|
119
211
|
.string()
|
|
120
212
|
.optional()
|
|
@@ -150,11 +242,19 @@ export const LLM_CONFIG_DEFAULTS = {
|
|
|
150
242
|
system_prompt: 'You are a helpful assistant.',
|
|
151
243
|
image_tile_mode: 'sequential'
|
|
152
244
|
};
|
|
245
|
+
/**
|
|
246
|
+
* `llama_model_default_params().n_gpu_layers`. Fabric's fit aborts and rolls
|
|
247
|
+
* back every argument it had computed when `n_gpu_layers` holds any other
|
|
248
|
+
* value, so a caller opting into `fit` gets this in place of the default above.
|
|
249
|
+
*/
|
|
250
|
+
const LLAMA_GPU_LAYERS_DEFAULT = -1;
|
|
153
251
|
// Full schema - applies defaults via transform (no duplication)
|
|
154
|
-
export const llmConfigSchema = llmConfigBaseSchema.transform((data) =>
|
|
155
|
-
...LLM_CONFIG_DEFAULTS,
|
|
156
|
-
|
|
157
|
-
}
|
|
252
|
+
export const llmConfigSchema = llmConfigBaseSchema.transform((data) => {
|
|
253
|
+
const merged = { ...LLM_CONFIG_DEFAULTS, ...data };
|
|
254
|
+
return data.fit !== undefined && data.gpu_layers === undefined
|
|
255
|
+
? { ...merged, gpu_layers: LLAMA_GPU_LAYERS_DEFAULT }
|
|
256
|
+
: merged;
|
|
257
|
+
});
|
|
158
258
|
// Base schema - validates types, all fields optional (for input validation)
|
|
159
259
|
export const embedConfigBaseSchema = z.object({
|
|
160
260
|
gpuLayers: z
|