@qvac/inference 0.19.1 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +57 -17
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/assess-model-fit.d.ts +30 -6
- package/dist/handlers/assess-model-fit.js +68 -6
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +69 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +10 -1
- package/dist/resources/model-fit/assess.js +43 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +83 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +133 -0
- package/dist/resources/model-fit/fit-stub/index.d.ts +2 -0
- package/dist/resources/model-fit/fit-stub/index.js +2 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +31 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +36 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +191 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +67 -0
- package/dist/schemas/assess-model-fit.js +56 -2
- package/dist/schemas/audio-gen.d.ts +479 -4
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +7 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/common.d.ts +816 -15
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +11 -1
- package/dist/schemas/completion-event.js +3 -0
- package/dist/schemas/completion-stream.d.ts +20 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +186 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +88 -8
- package/dist/schemas/llamacpp-config.js +108 -8
- package/dist/schemas/load-model.d.ts +2906 -206
- package/dist/schemas/model-config-schemas.d.ts +251 -5
- package/dist/schemas/plugin.d.ts +10 -0
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +29 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +29 -26
|
@@ -50,7 +50,6 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
|
|
|
50
50
|
'split-mode': z.ZodOptional<z.ZodEnum<{
|
|
51
51
|
none: "none";
|
|
52
52
|
layer: "layer";
|
|
53
|
-
row: "row";
|
|
54
53
|
tensor: "tensor";
|
|
55
54
|
}>>;
|
|
56
55
|
'flash-attn': z.ZodOptional<z.ZodEnum<{
|
|
@@ -59,6 +58,29 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
|
|
|
59
58
|
auto: "auto";
|
|
60
59
|
}>>;
|
|
61
60
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
61
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
62
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
63
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
64
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
65
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
66
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
67
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
68
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
69
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
70
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
71
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
72
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
73
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
74
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
75
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
76
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
77
|
+
on: "on";
|
|
78
|
+
off: "off";
|
|
79
|
+
auto: "auto";
|
|
80
|
+
}>>;
|
|
81
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
82
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
83
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
62
84
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
63
85
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
64
86
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -158,7 +180,6 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
158
180
|
'split-mode': z.ZodOptional<z.ZodEnum<{
|
|
159
181
|
none: "none";
|
|
160
182
|
layer: "layer";
|
|
161
|
-
row: "row";
|
|
162
183
|
tensor: "tensor";
|
|
163
184
|
}>>;
|
|
164
185
|
'flash-attn': z.ZodOptional<z.ZodEnum<{
|
|
@@ -167,6 +188,29 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
167
188
|
auto: "auto";
|
|
168
189
|
}>>;
|
|
169
190
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
191
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
192
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
193
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
194
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
195
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
196
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
197
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
198
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
199
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
200
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
201
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
202
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
203
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
204
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
205
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
206
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
207
|
+
on: "on";
|
|
208
|
+
off: "off";
|
|
209
|
+
auto: "auto";
|
|
210
|
+
}>>;
|
|
211
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
212
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
213
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
170
214
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
171
215
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
172
216
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -240,9 +284,28 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
240
284
|
'cache-type-k'?: string | undefined;
|
|
241
285
|
'cache-type-v'?: string | undefined;
|
|
242
286
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
243
|
-
'split-mode'?: "none" | "layer" | "
|
|
287
|
+
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
244
288
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
245
289
|
'tensor-split'?: string | undefined;
|
|
290
|
+
'batch-size'?: number | undefined;
|
|
291
|
+
'ubatch-size'?: number | undefined;
|
|
292
|
+
'cpu-moe'?: boolean | undefined;
|
|
293
|
+
'n-cpu-moe'?: number | undefined;
|
|
294
|
+
'kv-offload'?: boolean | undefined;
|
|
295
|
+
'image-max-tokens'?: number | undefined;
|
|
296
|
+
'image-min-tokens'?: number | undefined;
|
|
297
|
+
threads?: number | undefined;
|
|
298
|
+
'threads-batch'?: number | undefined;
|
|
299
|
+
'cpu-mask'?: string | undefined;
|
|
300
|
+
'cpu-mask-batch'?: string | undefined;
|
|
301
|
+
'override-tensor'?: string | undefined;
|
|
302
|
+
'n-cpu-ffn'?: number | undefined;
|
|
303
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
304
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
305
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
306
|
+
fit?: boolean | undefined;
|
|
307
|
+
'fit-target'?: string | number | undefined;
|
|
308
|
+
'fit-ctx'?: number | undefined;
|
|
246
309
|
openclCacheDir?: string | undefined;
|
|
247
310
|
reasoning_budget?: number | undefined;
|
|
248
311
|
projectionModelSrc?: string | {
|
|
@@ -283,9 +346,28 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
283
346
|
'cache-type-k'?: string | undefined;
|
|
284
347
|
'cache-type-v'?: string | undefined;
|
|
285
348
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
286
|
-
'split-mode'?: "none" | "layer" | "
|
|
349
|
+
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
287
350
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
288
351
|
'tensor-split'?: string | undefined;
|
|
352
|
+
'batch-size'?: number | undefined;
|
|
353
|
+
'ubatch-size'?: number | undefined;
|
|
354
|
+
'cpu-moe'?: boolean | undefined;
|
|
355
|
+
'n-cpu-moe'?: number | undefined;
|
|
356
|
+
'kv-offload'?: boolean | undefined;
|
|
357
|
+
'image-max-tokens'?: number | undefined;
|
|
358
|
+
'image-min-tokens'?: number | undefined;
|
|
359
|
+
threads?: number | undefined;
|
|
360
|
+
'threads-batch'?: number | undefined;
|
|
361
|
+
'cpu-mask'?: string | undefined;
|
|
362
|
+
'cpu-mask-batch'?: string | undefined;
|
|
363
|
+
'override-tensor'?: string | undefined;
|
|
364
|
+
'n-cpu-ffn'?: number | undefined;
|
|
365
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
366
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
367
|
+
'tensor-read-lazy'?: "on" | "off" | "auto" | undefined;
|
|
368
|
+
fit?: boolean | undefined;
|
|
369
|
+
'fit-target'?: string | number | undefined;
|
|
370
|
+
'fit-ctx'?: number | undefined;
|
|
289
371
|
openclCacheDir?: string | undefined;
|
|
290
372
|
reasoning_budget?: number | undefined;
|
|
291
373
|
projectionModelSrc?: string | {
|
|
@@ -337,7 +419,6 @@ export declare const embedConfigBaseSchema: z.ZodObject<{
|
|
|
337
419
|
splitMode: z.ZodOptional<z.ZodEnum<{
|
|
338
420
|
none: "none";
|
|
339
421
|
layer: "layer";
|
|
340
|
-
row: "row";
|
|
341
422
|
}>>;
|
|
342
423
|
tensorSplit: z.ZodOptional<z.ZodString>;
|
|
343
424
|
verbosity: z.ZodOptional<z.ZodEnum<{
|
|
@@ -385,7 +466,6 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
385
466
|
splitMode: z.ZodOptional<z.ZodEnum<{
|
|
386
467
|
none: "none";
|
|
387
468
|
layer: "layer";
|
|
388
|
-
row: "row";
|
|
389
469
|
}>>;
|
|
390
470
|
tensorSplit: z.ZodOptional<z.ZodString>;
|
|
391
471
|
verbosity: z.ZodOptional<z.ZodEnum<{
|
|
@@ -404,7 +484,7 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
404
484
|
embdNormalize?: number | undefined;
|
|
405
485
|
flashAttention?: "on" | "off" | "auto" | undefined;
|
|
406
486
|
mainGpu?: number | "integrated" | "dedicated" | undefined;
|
|
407
|
-
splitMode?: "none" | "layer" |
|
|
487
|
+
splitMode?: "none" | "layer" | undefined;
|
|
408
488
|
tensorSplit?: string | undefined;
|
|
409
489
|
verbosity?: 0 | 3 | 2 | 1 | undefined;
|
|
410
490
|
openclCacheDir?: string | undefined;
|
|
@@ -417,7 +497,7 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
417
497
|
embdNormalize?: number | undefined;
|
|
418
498
|
flashAttention?: "on" | "off" | "auto" | undefined;
|
|
419
499
|
mainGpu?: number | "integrated" | "dedicated" | undefined;
|
|
420
|
-
splitMode?: "none" | "layer" |
|
|
500
|
+
splitMode?: "none" | "layer" | undefined;
|
|
421
501
|
tensorSplit?: string | undefined;
|
|
422
502
|
verbosity?: 0 | 3 | 2 | 1 | undefined;
|
|
423
503
|
openclCacheDir?: string | undefined;
|
|
@@ -104,9 +104,9 @@ export const llmConfigBaseSchema = z.object({
|
|
|
104
104
|
.optional()
|
|
105
105
|
.describe("GPU to use on multi-GPU systems: a device index, or `'integrated'`/`'dedicated'` to restrict selection to that class."),
|
|
106
106
|
'split-mode': z
|
|
107
|
-
.enum(['none', 'layer', '
|
|
107
|
+
.enum(['none', 'layer', 'tensor'])
|
|
108
108
|
.optional()
|
|
109
|
-
.describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism),
|
|
109
|
+
.describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'tensor'` (EXPERIMENTAL tensor parallelism across all visible GPUs; desktop-only, requires flash attention, and disables auto-fit, so set `ctx_size` explicitly)."),
|
|
110
110
|
'flash-attn': z
|
|
111
111
|
.enum(['on', 'off', 'auto'])
|
|
112
112
|
.optional()
|
|
@@ -115,6 +115,98 @@ export const llmConfigBaseSchema = z.object({
|
|
|
115
115
|
.string()
|
|
116
116
|
.optional()
|
|
117
117
|
.describe("Proportions for distributing layers/rows across GPUs, e.g. `'1,1'` (equal) or `'3,1'` (75/25)."),
|
|
118
|
+
'batch-size': z
|
|
119
|
+
.number()
|
|
120
|
+
.int()
|
|
121
|
+
.min(1)
|
|
122
|
+
.optional()
|
|
123
|
+
.describe('Logical maximum batch size, the tokens submitted per decode call. Default 2048.'),
|
|
124
|
+
'ubatch-size': z
|
|
125
|
+
.number()
|
|
126
|
+
.int()
|
|
127
|
+
.min(1)
|
|
128
|
+
.optional()
|
|
129
|
+
.describe('Physical maximum batch size, the tokens computed in one pass. Default 512.'),
|
|
130
|
+
'cpu-moe': z
|
|
131
|
+
.boolean()
|
|
132
|
+
.optional()
|
|
133
|
+
.describe('Keep all Mixture-of-Experts weights on the CPU, freeing accelerator memory for the rest of the model. Mixture-of-Experts models only. Default false.'),
|
|
134
|
+
'n-cpu-moe': z
|
|
135
|
+
.number()
|
|
136
|
+
.int()
|
|
137
|
+
.min(0)
|
|
138
|
+
.optional()
|
|
139
|
+
.describe('Keep the Mixture-of-Experts weights of the first N layers on the CPU. Mixture-of-Experts models only, and a finer-grained alternative to `cpu-moe`.'),
|
|
140
|
+
'kv-offload': z
|
|
141
|
+
.boolean()
|
|
142
|
+
.optional()
|
|
143
|
+
.describe('Whether to hold the KV cache in accelerator memory. Default true. Set false to keep it in system memory, which frees accelerator memory at the cost of decode speed.'),
|
|
144
|
+
'image-max-tokens': z
|
|
145
|
+
.number()
|
|
146
|
+
.int()
|
|
147
|
+
.min(1)
|
|
148
|
+
.optional()
|
|
149
|
+
.describe('Upper bound on the tokens one image may occupy. Vision models only. Unset uses the model default.'),
|
|
150
|
+
'image-min-tokens': z
|
|
151
|
+
.number()
|
|
152
|
+
.int()
|
|
153
|
+
.min(1)
|
|
154
|
+
.optional()
|
|
155
|
+
.describe('Lower bound on the tokens one image may occupy. Vision models only. Unset uses the model default.'),
|
|
156
|
+
threads: z
|
|
157
|
+
.number()
|
|
158
|
+
.int()
|
|
159
|
+
.optional()
|
|
160
|
+
.describe('CPU threads to use during generation. Unset uses one thread per physical core; a value of 0 or below uses every logical CPU, which is a different and usually larger count.'),
|
|
161
|
+
'threads-batch': z
|
|
162
|
+
.number()
|
|
163
|
+
.int()
|
|
164
|
+
.optional()
|
|
165
|
+
.describe('CPU threads to use during batch and prompt processing. Setting it governs generation too and discards `threads` and `cpu-mask`; leave it unset for `threads` to apply to both.'),
|
|
166
|
+
'cpu-mask': z
|
|
167
|
+
.string()
|
|
168
|
+
.optional()
|
|
169
|
+
.describe("CPU affinity mask as an arbitrarily long hex string, e.g. `'ff'`. Unset applies no affinity. Ignored when `threads-batch` is set."),
|
|
170
|
+
'cpu-mask-batch': z
|
|
171
|
+
.string()
|
|
172
|
+
.optional()
|
|
173
|
+
.describe('CPU affinity mask for batch and prompt processing, in the same form as `cpu-mask`. Honoured only alongside `threads-batch`; without it `cpu-mask` takes over.'),
|
|
174
|
+
'override-tensor': z
|
|
175
|
+
.string()
|
|
176
|
+
.optional()
|
|
177
|
+
.describe("Buffer type overrides for tensors matching a pattern, as comma-separated `<tensor name pattern>=<buffer type>` entries, e.g. `'blk\\.(1[0-9])\\.ffn_(up|down|gate)_exps=CPU'` to keep those experts in system memory."),
|
|
178
|
+
'n-cpu-ffn': z
|
|
179
|
+
.number()
|
|
180
|
+
.int()
|
|
181
|
+
.min(0)
|
|
182
|
+
.optional()
|
|
183
|
+
.describe('Keep the dense FFN weights of the first N layers on the CPU.'),
|
|
184
|
+
'moe-cache-mib': z
|
|
185
|
+
.union([z.number().int().min(0), z.literal('auto')])
|
|
186
|
+
.optional()
|
|
187
|
+
.describe("Persistent accelerator cache for Mixture-of-Experts weights, in MiB. Default `'auto'`, which sizes it during `fit`."),
|
|
188
|
+
'prefetch-weights': z
|
|
189
|
+
.union([z.boolean(), z.literal('auto')])
|
|
190
|
+
.optional()
|
|
191
|
+
.describe("Overlap host-to-accelerator weight copies with compute. Default `'auto'`, which enables it for fitted dense models."),
|
|
192
|
+
'tensor-read-lazy': z
|
|
193
|
+
.enum(['on', 'auto', 'off'])
|
|
194
|
+
.optional()
|
|
195
|
+
.describe("On-demand reading of large tensors such as per-layer embeddings, which requires mmap: `'on'` reads their rows from disk instead of keeping them resident, `'off'` always keeps them resident, `'auto'` (default) applies `'on'` above 4 GiB."),
|
|
196
|
+
fit: z
|
|
197
|
+
.boolean()
|
|
198
|
+
.optional()
|
|
199
|
+
.describe('Adjust the arguments left unset so the model fits device memory. Default true. It chooses layer offload, tensor split and tensor buffer overrides, and the context size only when `ctx_size` is 0. Setting this field releases the `gpu_layers` default so fit can choose it, which offloads every layer rather than 99 if fit then gives up; setting `gpu_layers` alongside it pins the value and makes fit abort, leaving every argument unchanged. Incompatible with `split-mode: tensor`, which disables it.'),
|
|
200
|
+
'fit-target': z
|
|
201
|
+
.union([z.number().int().min(0), z.string().regex(/^\d+(,\d+)*$/)])
|
|
202
|
+
.optional()
|
|
203
|
+
.describe("Free margin `fit` aims to leave per device, in MiB. A single value is broadcast across devices; a comma-separated list such as `'1024,512'` sets them individually. Default 1024."),
|
|
204
|
+
'fit-ctx': z
|
|
205
|
+
.number()
|
|
206
|
+
.int()
|
|
207
|
+
.min(0)
|
|
208
|
+
.optional()
|
|
209
|
+
.describe('Smallest context size `fit` may settle on. Default 4096.'),
|
|
118
210
|
openclCacheDir: z
|
|
119
211
|
.string()
|
|
120
212
|
.optional()
|
|
@@ -150,11 +242,19 @@ export const LLM_CONFIG_DEFAULTS = {
|
|
|
150
242
|
system_prompt: 'You are a helpful assistant.',
|
|
151
243
|
image_tile_mode: 'sequential'
|
|
152
244
|
};
|
|
245
|
+
/**
|
|
246
|
+
* `llama_model_default_params().n_gpu_layers`. Fabric's fit aborts and rolls
|
|
247
|
+
* back every argument it had computed when `n_gpu_layers` holds any other
|
|
248
|
+
* value, so a caller opting into `fit` gets this in place of the default above.
|
|
249
|
+
*/
|
|
250
|
+
const LLAMA_GPU_LAYERS_DEFAULT = -1;
|
|
153
251
|
// Full schema - applies defaults via transform (no duplication)
|
|
154
|
-
export const llmConfigSchema = llmConfigBaseSchema.transform((data) =>
|
|
155
|
-
...LLM_CONFIG_DEFAULTS,
|
|
156
|
-
|
|
157
|
-
}
|
|
252
|
+
export const llmConfigSchema = llmConfigBaseSchema.transform((data) => {
|
|
253
|
+
const merged = { ...LLM_CONFIG_DEFAULTS, ...data };
|
|
254
|
+
return data.fit !== undefined && data.gpu_layers === undefined
|
|
255
|
+
? { ...merged, gpu_layers: LLAMA_GPU_LAYERS_DEFAULT }
|
|
256
|
+
: merged;
|
|
257
|
+
});
|
|
158
258
|
// Base schema - validates types, all fields optional (for input validation)
|
|
159
259
|
export const embedConfigBaseSchema = z.object({
|
|
160
260
|
gpuLayers: z
|
|
@@ -194,9 +294,9 @@ export const embedConfigBaseSchema = z.object({
|
|
|
194
294
|
.optional()
|
|
195
295
|
.describe("GPU to use on multi-GPU systems: a device index, or `'integrated'`/`'dedicated'` to restrict selection to that class."),
|
|
196
296
|
splitMode: z
|
|
197
|
-
.enum(['none', 'layer'
|
|
297
|
+
.enum(['none', 'layer'])
|
|
198
298
|
.optional()
|
|
199
|
-
.describe("How to split the model across GPUs: `'none'` (default, single GPU)
|
|
299
|
+
.describe("How to split the model across GPUs: `'none'` (default, single GPU) or `'layer'` (pipeline parallelism)."),
|
|
200
300
|
tensorSplit: z
|
|
201
301
|
.string()
|
|
202
302
|
.optional()
|