@qvac/inference 0.20.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +19 -11
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/handlers/assess-model-fit.d.ts +30 -6
- package/dist/handlers/assess-model-fit.js +68 -6
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +61 -0
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/resources/model-fit/assess.d.ts +10 -1
- package/dist/resources/model-fit/assess.js +43 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +83 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +133 -0
- package/dist/resources/model-fit/fit-stub/index.d.ts +2 -0
- package/dist/resources/model-fit/fit-stub/index.js +2 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +31 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +36 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +38 -18
- package/dist/schemas/assess-model-fit.d.ts +23 -0
- package/dist/schemas/assess-model-fit.js +16 -2
- package/dist/schemas/common.d.ts +67 -0
- package/dist/schemas/config.d.ts +138 -0
- package/dist/schemas/llamacpp-config.d.ts +84 -0
- package/dist/schemas/llamacpp-config.js +104 -4
- package/dist/schemas/load-model.d.ts +336 -0
- package/dist/schemas/model-config-schemas.d.ts +23 -0
- package/package.json +2 -2
|
@@ -65,6 +65,29 @@ export declare const loadBuiltinModelOptionsBaseSchema: z.ZodUnion<readonly [z.Z
|
|
|
65
65
|
auto: "auto";
|
|
66
66
|
}>>;
|
|
67
67
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
68
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
69
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
70
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
71
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
72
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
73
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
74
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
75
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
76
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
77
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
78
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
79
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
80
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
81
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
82
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
83
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
84
|
+
on: "on";
|
|
85
|
+
off: "off";
|
|
86
|
+
auto: "auto";
|
|
87
|
+
}>>;
|
|
88
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
89
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
90
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
68
91
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
69
92
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
70
93
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -3043,6 +3066,29 @@ export declare const loadModelOptionsBaseSchema: z.ZodUnion<readonly [z.ZodUnion
|
|
|
3043
3066
|
auto: "auto";
|
|
3044
3067
|
}>>;
|
|
3045
3068
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
3069
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
3070
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
3071
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
3072
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
3073
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
3074
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
3075
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
3076
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
3077
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
3078
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
3079
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
3080
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
3081
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
3082
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
3083
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
3084
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
3085
|
+
on: "on";
|
|
3086
|
+
off: "off";
|
|
3087
|
+
auto: "auto";
|
|
3088
|
+
}>>;
|
|
3089
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
3090
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
3091
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
3046
3092
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
3047
3093
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
3048
3094
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -6020,6 +6066,29 @@ export declare const loadModelOptionsSchema: z.ZodPipe<z.ZodUnion<readonly [z.Zo
|
|
|
6020
6066
|
auto: "auto";
|
|
6021
6067
|
}>>;
|
|
6022
6068
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
6069
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
6070
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
6071
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
6072
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
6073
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
6074
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
6075
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
6076
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
6077
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
6078
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
6079
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
6080
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
6081
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
6082
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
6083
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
6084
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
6085
|
+
on: "on";
|
|
6086
|
+
off: "off";
|
|
6087
|
+
auto: "auto";
|
|
6088
|
+
}>>;
|
|
6089
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
6090
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
6091
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
6023
6092
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
6024
6093
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
6025
6094
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -8985,6 +9054,25 @@ export declare const loadModelOptionsSchema: z.ZodPipe<z.ZodUnion<readonly [z.Zo
|
|
|
8985
9054
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
8986
9055
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
8987
9056
|
'tensor-split'?: string | undefined;
|
|
9057
|
+
'batch-size'?: number | undefined;
|
|
9058
|
+
'ubatch-size'?: number | undefined;
|
|
9059
|
+
'cpu-moe'?: boolean | undefined;
|
|
9060
|
+
'n-cpu-moe'?: number | undefined;
|
|
9061
|
+
'kv-offload'?: boolean | undefined;
|
|
9062
|
+
'image-max-tokens'?: number | undefined;
|
|
9063
|
+
'image-min-tokens'?: number | undefined;
|
|
9064
|
+
threads?: number | undefined;
|
|
9065
|
+
'threads-batch'?: number | undefined;
|
|
9066
|
+
'cpu-mask'?: string | undefined;
|
|
9067
|
+
'cpu-mask-batch'?: string | undefined;
|
|
9068
|
+
'override-tensor'?: string | undefined;
|
|
9069
|
+
'n-cpu-ffn'?: number | undefined;
|
|
9070
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
9071
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
9072
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
9073
|
+
fit?: boolean | undefined;
|
|
9074
|
+
'fit-target'?: string | number | undefined;
|
|
9075
|
+
'fit-ctx'?: number | undefined;
|
|
8988
9076
|
openclCacheDir?: string | undefined;
|
|
8989
9077
|
reasoning_budget?: number | undefined;
|
|
8990
9078
|
projectionModelSrc?: string | {
|
|
@@ -10052,6 +10140,25 @@ export declare const loadModelOptionsSchema: z.ZodPipe<z.ZodUnion<readonly [z.Zo
|
|
|
10052
10140
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
10053
10141
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
10054
10142
|
'tensor-split'?: string | undefined;
|
|
10143
|
+
'batch-size'?: number | undefined;
|
|
10144
|
+
'ubatch-size'?: number | undefined;
|
|
10145
|
+
'cpu-moe'?: boolean | undefined;
|
|
10146
|
+
'n-cpu-moe'?: number | undefined;
|
|
10147
|
+
'kv-offload'?: boolean | undefined;
|
|
10148
|
+
'image-max-tokens'?: number | undefined;
|
|
10149
|
+
'image-min-tokens'?: number | undefined;
|
|
10150
|
+
threads?: number | undefined;
|
|
10151
|
+
'threads-batch'?: number | undefined;
|
|
10152
|
+
'cpu-mask'?: string | undefined;
|
|
10153
|
+
'cpu-mask-batch'?: string | undefined;
|
|
10154
|
+
'override-tensor'?: string | undefined;
|
|
10155
|
+
'n-cpu-ffn'?: number | undefined;
|
|
10156
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
10157
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
10158
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
10159
|
+
fit?: boolean | undefined;
|
|
10160
|
+
'fit-target'?: string | number | undefined;
|
|
10161
|
+
'fit-ctx'?: number | undefined;
|
|
10055
10162
|
openclCacheDir?: string | undefined;
|
|
10056
10163
|
reasoning_budget?: number | undefined;
|
|
10057
10164
|
projectionModelSrc?: string | {
|
|
@@ -11133,6 +11240,29 @@ export declare const loadBuiltinToRequestSchema: z.ZodDiscriminatedUnion<[z.ZodP
|
|
|
11133
11240
|
auto: "auto";
|
|
11134
11241
|
}>>;
|
|
11135
11242
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
11243
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
11244
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
11245
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
11246
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
11247
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
11248
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
11249
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
11250
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
11251
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
11252
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
11253
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
11254
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
11255
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
11256
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
11257
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
11258
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
11259
|
+
on: "on";
|
|
11260
|
+
off: "off";
|
|
11261
|
+
auto: "auto";
|
|
11262
|
+
}>>;
|
|
11263
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
11264
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
11265
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
11136
11266
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
11137
11267
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
11138
11268
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -11284,6 +11414,25 @@ export declare const loadBuiltinToRequestSchema: z.ZodDiscriminatedUnion<[z.ZodP
|
|
|
11284
11414
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
11285
11415
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
11286
11416
|
'tensor-split'?: string | undefined;
|
|
11417
|
+
'batch-size'?: number | undefined;
|
|
11418
|
+
'ubatch-size'?: number | undefined;
|
|
11419
|
+
'cpu-moe'?: boolean | undefined;
|
|
11420
|
+
'n-cpu-moe'?: number | undefined;
|
|
11421
|
+
'kv-offload'?: boolean | undefined;
|
|
11422
|
+
'image-max-tokens'?: number | undefined;
|
|
11423
|
+
'image-min-tokens'?: number | undefined;
|
|
11424
|
+
threads?: number | undefined;
|
|
11425
|
+
'threads-batch'?: number | undefined;
|
|
11426
|
+
'cpu-mask'?: string | undefined;
|
|
11427
|
+
'cpu-mask-batch'?: string | undefined;
|
|
11428
|
+
'override-tensor'?: string | undefined;
|
|
11429
|
+
'n-cpu-ffn'?: number | undefined;
|
|
11430
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
11431
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
11432
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
11433
|
+
fit?: boolean | undefined;
|
|
11434
|
+
'fit-target'?: string | number | undefined;
|
|
11435
|
+
'fit-ctx'?: number | undefined;
|
|
11287
11436
|
openclCacheDir?: string | undefined;
|
|
11288
11437
|
reasoning_budget?: number | undefined;
|
|
11289
11438
|
projectionModelSrc?: string | {
|
|
@@ -15980,6 +16129,29 @@ export declare const loadModelOptionsToRequestSchema: z.ZodPipe<z.ZodUnknown, z.
|
|
|
15980
16129
|
auto: "auto";
|
|
15981
16130
|
}>>;
|
|
15982
16131
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
16132
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
16133
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
16134
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
16135
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
16136
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
16137
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
16138
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
16139
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
16140
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
16141
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
16142
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
16143
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
16144
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
16145
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
16146
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
16147
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
16148
|
+
on: "on";
|
|
16149
|
+
off: "off";
|
|
16150
|
+
auto: "auto";
|
|
16151
|
+
}>>;
|
|
16152
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
16153
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
16154
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
15983
16155
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
15984
16156
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
15985
16157
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -16131,6 +16303,25 @@ export declare const loadModelOptionsToRequestSchema: z.ZodPipe<z.ZodUnknown, z.
|
|
|
16131
16303
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
16132
16304
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
16133
16305
|
'tensor-split'?: string | undefined;
|
|
16306
|
+
'batch-size'?: number | undefined;
|
|
16307
|
+
'ubatch-size'?: number | undefined;
|
|
16308
|
+
'cpu-moe'?: boolean | undefined;
|
|
16309
|
+
'n-cpu-moe'?: number | undefined;
|
|
16310
|
+
'kv-offload'?: boolean | undefined;
|
|
16311
|
+
'image-max-tokens'?: number | undefined;
|
|
16312
|
+
'image-min-tokens'?: number | undefined;
|
|
16313
|
+
threads?: number | undefined;
|
|
16314
|
+
'threads-batch'?: number | undefined;
|
|
16315
|
+
'cpu-mask'?: string | undefined;
|
|
16316
|
+
'cpu-mask-batch'?: string | undefined;
|
|
16317
|
+
'override-tensor'?: string | undefined;
|
|
16318
|
+
'n-cpu-ffn'?: number | undefined;
|
|
16319
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
16320
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
16321
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
16322
|
+
fit?: boolean | undefined;
|
|
16323
|
+
'fit-target'?: string | number | undefined;
|
|
16324
|
+
'fit-ctx'?: number | undefined;
|
|
16134
16325
|
openclCacheDir?: string | undefined;
|
|
16135
16326
|
reasoning_budget?: number | undefined;
|
|
16136
16327
|
projectionModelSrc?: string | {
|
|
@@ -20832,6 +21023,29 @@ export declare const loadLlmModelRequestSchema: z.ZodObject<{
|
|
|
20832
21023
|
auto: "auto";
|
|
20833
21024
|
}>>;
|
|
20834
21025
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
21026
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
21027
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
21028
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
21029
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
21030
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
21031
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
21032
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
21033
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
21034
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
21035
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
21036
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
21037
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
21038
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
21039
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
21040
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
21041
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
21042
|
+
on: "on";
|
|
21043
|
+
off: "off";
|
|
21044
|
+
auto: "auto";
|
|
21045
|
+
}>>;
|
|
21046
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
21047
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
21048
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
20835
21049
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
20836
21050
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
20837
21051
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -23343,6 +23557,29 @@ export declare const loadModelSrcRequestSchema: z.ZodPipe<z.ZodUnion<readonly [z
|
|
|
23343
23557
|
auto: "auto";
|
|
23344
23558
|
}>>;
|
|
23345
23559
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
23560
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
23561
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
23562
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
23563
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
23564
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
23565
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
23566
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
23567
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
23568
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
23569
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
23570
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
23571
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
23572
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
23573
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
23574
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
23575
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
23576
|
+
on: "on";
|
|
23577
|
+
off: "off";
|
|
23578
|
+
auto: "auto";
|
|
23579
|
+
}>>;
|
|
23580
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
23581
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
23582
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
23346
23583
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
23347
23584
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
23348
23585
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -25813,6 +26050,25 @@ export declare const loadModelSrcRequestSchema: z.ZodPipe<z.ZodUnion<readonly [z
|
|
|
25813
26050
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
25814
26051
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
25815
26052
|
'tensor-split'?: string | undefined;
|
|
26053
|
+
'batch-size'?: number | undefined;
|
|
26054
|
+
'ubatch-size'?: number | undefined;
|
|
26055
|
+
'cpu-moe'?: boolean | undefined;
|
|
26056
|
+
'n-cpu-moe'?: number | undefined;
|
|
26057
|
+
'kv-offload'?: boolean | undefined;
|
|
26058
|
+
'image-max-tokens'?: number | undefined;
|
|
26059
|
+
'image-min-tokens'?: number | undefined;
|
|
26060
|
+
threads?: number | undefined;
|
|
26061
|
+
'threads-batch'?: number | undefined;
|
|
26062
|
+
'cpu-mask'?: string | undefined;
|
|
26063
|
+
'cpu-mask-batch'?: string | undefined;
|
|
26064
|
+
'override-tensor'?: string | undefined;
|
|
26065
|
+
'n-cpu-ffn'?: number | undefined;
|
|
26066
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
26067
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
26068
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
26069
|
+
fit?: boolean | undefined;
|
|
26070
|
+
'fit-target'?: string | number | undefined;
|
|
26071
|
+
'fit-ctx'?: number | undefined;
|
|
25816
26072
|
openclCacheDir?: string | undefined;
|
|
25817
26073
|
reasoning_budget?: number | undefined;
|
|
25818
26074
|
projectionModelSrc?: string | {
|
|
@@ -26776,6 +27032,25 @@ export declare const loadModelSrcRequestSchema: z.ZodPipe<z.ZodUnion<readonly [z
|
|
|
26776
27032
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
26777
27033
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
26778
27034
|
'tensor-split'?: string | undefined;
|
|
27035
|
+
'batch-size'?: number | undefined;
|
|
27036
|
+
'ubatch-size'?: number | undefined;
|
|
27037
|
+
'cpu-moe'?: boolean | undefined;
|
|
27038
|
+
'n-cpu-moe'?: number | undefined;
|
|
27039
|
+
'kv-offload'?: boolean | undefined;
|
|
27040
|
+
'image-max-tokens'?: number | undefined;
|
|
27041
|
+
'image-min-tokens'?: number | undefined;
|
|
27042
|
+
threads?: number | undefined;
|
|
27043
|
+
'threads-batch'?: number | undefined;
|
|
27044
|
+
'cpu-mask'?: string | undefined;
|
|
27045
|
+
'cpu-mask-batch'?: string | undefined;
|
|
27046
|
+
'override-tensor'?: string | undefined;
|
|
27047
|
+
'n-cpu-ffn'?: number | undefined;
|
|
27048
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
27049
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
27050
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
27051
|
+
fit?: boolean | undefined;
|
|
27052
|
+
'fit-target'?: string | number | undefined;
|
|
27053
|
+
'fit-ctx'?: number | undefined;
|
|
26779
27054
|
openclCacheDir?: string | undefined;
|
|
26780
27055
|
reasoning_budget?: number | undefined;
|
|
26781
27056
|
projectionModelSrc?: string | {
|
|
@@ -27770,6 +28045,29 @@ export declare const loadModelRequestSchema: z.ZodUnion<readonly [z.ZodPipe<z.Zo
|
|
|
27770
28045
|
auto: "auto";
|
|
27771
28046
|
}>>;
|
|
27772
28047
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
28048
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
28049
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
28050
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
28051
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
28052
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
28053
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
28054
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
28055
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
28056
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
28057
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
28058
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
28059
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
28060
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
28061
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
28062
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
28063
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
28064
|
+
on: "on";
|
|
28065
|
+
off: "off";
|
|
28066
|
+
auto: "auto";
|
|
28067
|
+
}>>;
|
|
28068
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
28069
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
28070
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
27773
28071
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
27774
28072
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
27775
28073
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -30240,6 +30538,25 @@ export declare const loadModelRequestSchema: z.ZodUnion<readonly [z.ZodPipe<z.Zo
|
|
|
30240
30538
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
30241
30539
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
30242
30540
|
'tensor-split'?: string | undefined;
|
|
30541
|
+
'batch-size'?: number | undefined;
|
|
30542
|
+
'ubatch-size'?: number | undefined;
|
|
30543
|
+
'cpu-moe'?: boolean | undefined;
|
|
30544
|
+
'n-cpu-moe'?: number | undefined;
|
|
30545
|
+
'kv-offload'?: boolean | undefined;
|
|
30546
|
+
'image-max-tokens'?: number | undefined;
|
|
30547
|
+
'image-min-tokens'?: number | undefined;
|
|
30548
|
+
threads?: number | undefined;
|
|
30549
|
+
'threads-batch'?: number | undefined;
|
|
30550
|
+
'cpu-mask'?: string | undefined;
|
|
30551
|
+
'cpu-mask-batch'?: string | undefined;
|
|
30552
|
+
'override-tensor'?: string | undefined;
|
|
30553
|
+
'n-cpu-ffn'?: number | undefined;
|
|
30554
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
30555
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
30556
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
30557
|
+
fit?: boolean | undefined;
|
|
30558
|
+
'fit-target'?: string | number | undefined;
|
|
30559
|
+
'fit-ctx'?: number | undefined;
|
|
30243
30560
|
openclCacheDir?: string | undefined;
|
|
30244
30561
|
reasoning_budget?: number | undefined;
|
|
30245
30562
|
projectionModelSrc?: string | {
|
|
@@ -31203,6 +31520,25 @@ export declare const loadModelRequestSchema: z.ZodUnion<readonly [z.ZodPipe<z.Zo
|
|
|
31203
31520
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
31204
31521
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
31205
31522
|
'tensor-split'?: string | undefined;
|
|
31523
|
+
'batch-size'?: number | undefined;
|
|
31524
|
+
'ubatch-size'?: number | undefined;
|
|
31525
|
+
'cpu-moe'?: boolean | undefined;
|
|
31526
|
+
'n-cpu-moe'?: number | undefined;
|
|
31527
|
+
'kv-offload'?: boolean | undefined;
|
|
31528
|
+
'image-max-tokens'?: number | undefined;
|
|
31529
|
+
'image-min-tokens'?: number | undefined;
|
|
31530
|
+
threads?: number | undefined;
|
|
31531
|
+
'threads-batch'?: number | undefined;
|
|
31532
|
+
'cpu-mask'?: string | undefined;
|
|
31533
|
+
'cpu-mask-batch'?: string | undefined;
|
|
31534
|
+
'override-tensor'?: string | undefined;
|
|
31535
|
+
'n-cpu-ffn'?: number | undefined;
|
|
31536
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
31537
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
31538
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
31539
|
+
fit?: boolean | undefined;
|
|
31540
|
+
'fit-target'?: string | number | undefined;
|
|
31541
|
+
'fit-ctx'?: number | undefined;
|
|
31206
31542
|
openclCacheDir?: string | undefined;
|
|
31207
31543
|
reasoning_budget?: number | undefined;
|
|
31208
31544
|
projectionModelSrc?: string | {
|
|
@@ -47,6 +47,29 @@ export declare const MODEL_CONFIG_SCHEMA_BY_TYPE: {
|
|
|
47
47
|
auto: "auto";
|
|
48
48
|
}>>;
|
|
49
49
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
50
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
51
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
52
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
53
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
54
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
55
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
56
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
57
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
58
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
59
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
60
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
61
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
62
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
63
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
64
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
65
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
66
|
+
on: "on";
|
|
67
|
+
off: "off";
|
|
68
|
+
auto: "auto";
|
|
69
|
+
}>>;
|
|
70
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
71
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
72
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
50
73
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
51
74
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
52
75
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qvac/inference",
|
|
3
|
-
"version": "0.20.
|
|
3
|
+
"version": "0.20.1",
|
|
4
4
|
"description": "Bare-only in-process core of the QVAC SDK. Runs inference directly on the Bare runtime with explicit plugin assembly and no RPC, worker, or subprocess layer.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -165,7 +165,7 @@
|
|
|
165
165
|
"@qvac/logging": "^0.1.1",
|
|
166
166
|
"@qvac/model-fit": "^0.12.0",
|
|
167
167
|
"@qvac/rag": "^0.8.1",
|
|
168
|
-
"@qvac/registry-client": "^0.
|
|
168
|
+
"@qvac/registry-client": "^0.7.0",
|
|
169
169
|
"bare-abort-controller": "^1.1.2",
|
|
170
170
|
"bare-buffer": "^3.6.2",
|
|
171
171
|
"bare-cpu-info": "0.1.1",
|