@qvac/sdk 0.17.1 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +124 -64
- package/README.md +19 -0
- package/dist/client/api/audio-gen.d.ts +17 -0
- package/dist/client/api/audio-gen.js +17 -0
- package/dist/client/api/finetune.d.ts +2 -0
- package/dist/client/api/finetune.js +7 -1
- package/dist/client/api/index.d.ts +1 -1
- package/dist/client/api/index.js +1 -1
- package/dist/client/api/load-model.d.ts +8 -0
- package/dist/client/api/transcribe.d.ts +19 -2
- package/dist/client/api/transcribe.js +66 -27
- package/dist/client/api/translate.d.ts +1 -0
- package/dist/client/api/translate.js +40 -19
- package/dist/client/api/vla.d.ts +42 -1
- package/dist/client/api/vla.js +50 -1
- package/dist/commands/bundle/entry-gen.d.ts +4 -0
- package/dist/commands/bundle/entry-gen.js +6 -0
- package/dist/commands/bundle/index.js +19 -12
- package/dist/examples/asr/parakeet-indic-conformer-filesystem.d.ts +2 -0
- package/dist/examples/asr/parakeet-indic-conformer-filesystem.js +52 -0
- package/dist/examples/audiogen/generate-cover.d.ts +2 -0
- package/dist/examples/audiogen/generate-cover.js +100 -0
- package/dist/examples/llamacpp-fallback.d.ts +2 -0
- package/dist/examples/llamacpp-fallback.js +34 -0
- package/dist/examples/multi-job-completion.d.ts +28 -0
- package/dist/examples/multi-job-completion.js +128 -0
- package/dist/examples/profiling/basic.js +32 -1
- package/dist/examples/system-resources.d.ts +2 -0
- package/dist/examples/system-resources.js +86 -0
- package/dist/examples/tts/audio8.d.ts +2 -0
- package/dist/examples/tts/audio8.js +54 -0
- package/dist/examples/tts/cosyvoice3.d.ts +2 -0
- package/dist/examples/tts/cosyvoice3.js +52 -0
- package/dist/examples/vla-groot.js +92 -49
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/models/registry/models.d.ts +782 -6
- package/dist/models/registry/models.js +2664 -1859
- package/dist/schemas/audio-gen.d.ts +103 -0
- package/dist/schemas/audio-gen.js +135 -4
- package/dist/schemas/common.d.ts +550 -11
- package/dist/schemas/constants-registry.d.ts +18 -4
- package/dist/schemas/constants-registry.js +16 -3
- package/dist/schemas/index.d.ts +1 -1
- package/dist/schemas/index.js +1 -1
- package/dist/schemas/llamacpp-config.d.ts +10 -10
- package/dist/schemas/llamacpp-config.js +168 -82
- package/dist/schemas/load-model.d.ts +3678 -323
- package/dist/schemas/load-model.js +21 -3
- package/dist/schemas/public.d.ts +3 -0
- package/dist/schemas/public.js +6 -0
- package/dist/schemas/sdk-config.d.ts +60 -24
- package/dist/schemas/sdk-errors-server.d.ts +1 -0
- package/dist/schemas/sdk-errors-server.js +5 -0
- package/dist/schemas/text-to-speech.d.ts +1086 -110
- package/dist/schemas/text-to-speech.js +194 -4
- package/dist/schemas/tools.d.ts +0 -16
- package/dist/schemas/tools.js +0 -15
- package/dist/schemas/transcription-config.d.ts +2 -0
- package/dist/schemas/transcription-config.js +10 -4
- package/dist/schemas/transcription.d.ts +3 -0
- package/dist/schemas/vla.d.ts +83 -0
- package/dist/schemas/vla.js +105 -2
- package/dist/scripts/link-workspace-inference.d.ts +2 -0
- package/dist/scripts/link-workspace-inference.js +31 -0
- package/dist/server/bare/ops/cancel.d.ts +1 -1
- package/dist/server/bare/ops/cancel.js +1 -1
- package/dist/server/bare/ops/kv-cache-retention.js +11 -0
- package/dist/server/bare/ops/kv-cache-utils.d.ts +1 -1
- package/dist/server/bare/ops/kv-cache-utils.js +40 -12
- package/dist/server/bare/ops/transcribe.d.ts +2 -2
- package/dist/server/bare/ops/transcribe.js +54 -48
- package/dist/server/bare/ops/translate-stats.d.ts +5 -0
- package/dist/server/bare/ops/translate-stats.js +11 -0
- package/dist/server/bare/ops/translate.js +66 -38
- package/dist/server/bare/ops/unload-model.js +30 -24
- package/dist/server/bare/plugins/audiogen-ggml/ops/audio-gen-input.d.ts +20 -0
- package/dist/server/bare/plugins/audiogen-ggml/ops/audio-gen-input.js +153 -0
- package/dist/server/bare/plugins/audiogen-ggml/ops/audio-gen-stream.js +52 -1
- package/dist/server/bare/plugins/audiogen-ggml/plugin.d.ts +28 -0
- package/dist/server/bare/plugins/ggml-ocr/plugin.js +3 -1
- package/dist/server/bare/plugins/ggml-vla/ops/vla-set-embodiment.d.ts +3 -0
- package/dist/server/bare/plugins/ggml-vla/ops/vla-set-embodiment.js +9 -0
- package/dist/server/bare/plugins/ggml-vla/plugin.d.ts +37 -0
- package/dist/server/bare/plugins/ggml-vla/plugin.js +21 -4
- package/dist/server/bare/plugins/llamacpp-completion/ops/batch-completion-stream.js +16 -15
- package/dist/server/bare/plugins/llamacpp-completion/ops/cache-logger.d.ts +1 -1
- package/dist/server/bare/plugins/llamacpp-completion/ops/cache-logger.js +2 -2
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stats.d.ts +6 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stats.js +10 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.d.ts +1 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.js +131 -123
- package/dist/server/bare/plugins/llamacpp-completion/ops/finetune.js +110 -20
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-session.d.ts +49 -23
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-session.js +308 -143
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.d.ts +1 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.js +2 -2
- package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts +6 -6
- package/dist/server/bare/plugins/llamacpp-completion/plugin.js +55 -36
- package/dist/server/bare/plugins/llamacpp-completion/transform.js +1 -7
- package/dist/server/bare/plugins/llamacpp-embedding/plugin.js +2 -3
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.js +6 -5
- package/dist/server/bare/plugins/parakeet-transcription/plugin.js +27 -16
- package/dist/server/bare/plugins/tts-ggml/ops/parler-options.js +23 -3
- package/dist/server/bare/plugins/tts-ggml/plugin.d.ts +302 -4
- package/dist/server/bare/plugins/tts-ggml/plugin.js +131 -0
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.js +48 -39
- package/dist/server/bare/registry/model-registry.d.ts +1 -0
- package/dist/server/bare/runtime/request-registry-singleton.js +29 -15
- package/dist/server/bare/runtime/request-registry.d.ts +27 -0
- package/dist/server/bare/runtime/request-registry.js +246 -76
- package/dist/server/bare/types/addon-responses.d.ts +4 -12
- package/dist/server/rpc/handlers/load-model/handler.js +2 -2
- package/dist/server/rpc/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/server/rpc/handlers/load-model/resolve-session.js +3 -3
- package/dist/server/rpc/handlers/load-model/resolve.d.ts +12 -2
- package/dist/server/rpc/handlers/load-model/resolve.js +90 -12
- package/dist/server/utils/audio/decoder.d.ts +37 -1
- package/dist/server/utils/audio/decoder.js +81 -51
- package/dist/server/utils/config-transform.d.ts +3 -0
- package/dist/server/utils/config-transform.js +6 -0
- package/dist/server/utils/index.d.ts +0 -1
- package/dist/server/utils/index.js +0 -1
- package/dist/server/utils/shard-utils.d.ts +1 -0
- package/dist/server/utils/shard-utils.js +14 -0
- package/dist/server/utils/tool-integration.d.ts +3 -10
- package/dist/server/utils/tool-integration.js +3 -12
- package/dist/utils/errors-server.d.ts +3 -0
- package/dist/utils/errors-server.js +5 -0
- package/package.json +17 -14
- package/dist/examples/llamacpp-dynamic-tools.d.ts +0 -2
- package/dist/examples/llamacpp-dynamic-tools.js +0 -152
- package/dist/models/update-models/codegen.d.ts +0 -3
- package/dist/models/update-models/codegen.js +0 -196
- package/dist/models/update-models/companions.d.ts +0 -30
- package/dist/models/update-models/companions.js +0 -296
- package/dist/models/update-models/history.d.ts +0 -24
- package/dist/models/update-models/history.js +0 -107
- package/dist/models/update-models/index.d.ts +0 -2
- package/dist/models/update-models/index.js +0 -123
- package/dist/models/update-models/naming.d.ts +0 -4
- package/dist/models/update-models/naming.js +0 -396
- package/dist/models/update-models/processing.d.ts +0 -8
- package/dist/models/update-models/processing.js +0 -65
- package/dist/models/update-models/registry.d.ts +0 -5
- package/dist/models/update-models/registry.js +0 -87
- package/dist/models/update-models/shards.d.ts +0 -4
- package/dist/models/update-models/shards.js +0 -57
- package/dist/models/update-models/types.d.ts +0 -88
- package/dist/models/update-models/types.js +0 -1
- package/dist/models/update-models/utils.d.ts +0 -3
- package/dist/models/update-models/utils.js +0 -21
- package/dist/server/bare/plugins/llamacpp-completion/ops/batch-cancelled.d.ts +0 -23
- package/dist/server/bare/plugins/llamacpp-completion/ops/batch-cancelled.js +0 -30
- package/dist/server/utils/expand-gguf-shards.d.ts +0 -3
- package/dist/server/utils/expand-gguf-shards.js +0 -22
|
@@ -39,10 +39,6 @@ export declare const constantsRegistry: {
|
|
|
39
39
|
readonly ggmlVla: "ggml-vla";
|
|
40
40
|
readonly ggmlClassification: "ggml-classification";
|
|
41
41
|
}>;
|
|
42
|
-
readonly ToolsMode: z.ZodEnum<{
|
|
43
|
-
readonly static: "static";
|
|
44
|
-
readonly dynamic: "dynamic";
|
|
45
|
-
}>;
|
|
46
42
|
readonly Verbosity: z.ZodEnum<{
|
|
47
43
|
readonly ERROR: 0;
|
|
48
44
|
readonly WARN: 1;
|
|
@@ -65,5 +61,23 @@ export declare const constantsRegistry: {
|
|
|
65
61
|
readonly SupportedAudioFormat: z.ZodEnum<{
|
|
66
62
|
[k: string]: string;
|
|
67
63
|
}>;
|
|
64
|
+
readonly TtsPace: z.ZodEnum<{
|
|
65
|
+
[k: string]: string;
|
|
66
|
+
}>;
|
|
67
|
+
readonly TtsCosyvoice3Emotion: z.ZodEnum<{
|
|
68
|
+
[k: string]: string;
|
|
69
|
+
}>;
|
|
70
|
+
readonly TtsCosyvoice3InstructDialect: z.ZodEnum<{
|
|
71
|
+
[k: string]: string;
|
|
72
|
+
}>;
|
|
73
|
+
readonly TtsCosyvoice3InstructVolume: z.ZodEnum<{
|
|
74
|
+
[k: string]: string;
|
|
75
|
+
}>;
|
|
76
|
+
readonly TtsCosyvoice3InstructStyle: z.ZodEnum<{
|
|
77
|
+
[k: string]: string;
|
|
78
|
+
}>;
|
|
79
|
+
readonly AudioGenTaskType: z.ZodEnum<{
|
|
80
|
+
[k: string]: string;
|
|
81
|
+
}>;
|
|
68
82
|
};
|
|
69
83
|
//# sourceMappingURL=constants-registry.d.ts.map
|
|
@@ -1,9 +1,17 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { ModelType } from './model-types.js';
|
|
3
|
-
import { TOOLS_MODE } from './tools.js';
|
|
4
3
|
import { VERBOSITY } from './llamacpp-config.js';
|
|
5
4
|
import { PLUGIN_LLM, PLUGIN_EMBEDDING, PLUGIN_WHISPER, PLUGIN_BCI, PLUGIN_NMT, PLUGIN_TTS, PLUGIN_OCR, PLUGIN_DIFFUSION, PLUGIN_AUDIOGEN, PLUGIN_VLA, PLUGIN_CLASSIFICATION } from './plugin.js';
|
|
6
5
|
import { SUPPORTED_AUDIO_FORMATS } from '../constants/audio.js';
|
|
6
|
+
import { AUDIOGEN_TASK_TYPES } from './audio-gen.js';
|
|
7
|
+
import { TTS_PACES, TTS_COSYVOICE3_EMOTIONS, TTS_COSYVOICE3_INSTRUCT_DIALECTS, TTS_COSYVOICE3_INSTRUCT_VOLUMES, TTS_COSYVOICE3_INSTRUCT_STYLES } from './text-to-speech.js';
|
|
8
|
+
// Uppercases each vocabulary value into an identifier-safe varname (e.g.
|
|
9
|
+
// 'slow' -> 'SLOW', 'cover-nofsq' -> 'COVER_NOFSQ'), mirroring the
|
|
10
|
+
// SupportedAudioFormat derivation so names can't drift from the single
|
|
11
|
+
// source-of-truth array.
|
|
12
|
+
function enumFromVocabulary(values) {
|
|
13
|
+
return z.enum(Object.fromEntries(values.map((value) => [value.toUpperCase().replace(/-/g, '_'), value])));
|
|
14
|
+
}
|
|
7
15
|
/**
|
|
8
16
|
* Every public constant `enum` from index.ts that downstream (non-JS)
|
|
9
17
|
* client generators should get as a real, named type instead of a
|
|
@@ -30,7 +38,6 @@ import { SUPPORTED_AUDIO_FORMATS } from '../constants/audio.js';
|
|
|
30
38
|
*/
|
|
31
39
|
export const constantsRegistry = {
|
|
32
40
|
ModelType: z.enum(ModelType),
|
|
33
|
-
ToolsMode: z.enum(TOOLS_MODE),
|
|
34
41
|
Verbosity: z.enum(VERBOSITY),
|
|
35
42
|
PluginId: z.enum({
|
|
36
43
|
LLM: PLUGIN_LLM,
|
|
@@ -48,5 +55,11 @@ export const constantsRegistry = {
|
|
|
48
55
|
// Derived from SUPPORTED_AUDIO_FORMATS itself (e.g. '.mp3' -> 'MP3') rather
|
|
49
56
|
// than a hand-written record, so the varnames can't drift from the one
|
|
50
57
|
// source of truth every other consumer of SUPPORTED_AUDIO_FORMATS shares.
|
|
51
|
-
SupportedAudioFormat: z.enum(Object.fromEntries(SUPPORTED_AUDIO_FORMATS.map((ext) => [ext.slice(1).toUpperCase(), ext])))
|
|
58
|
+
SupportedAudioFormat: z.enum(Object.fromEntries(SUPPORTED_AUDIO_FORMATS.map((ext) => [ext.slice(1).toUpperCase(), ext]))),
|
|
59
|
+
TtsPace: enumFromVocabulary(TTS_PACES),
|
|
60
|
+
TtsCosyvoice3Emotion: enumFromVocabulary(TTS_COSYVOICE3_EMOTIONS),
|
|
61
|
+
TtsCosyvoice3InstructDialect: enumFromVocabulary(TTS_COSYVOICE3_INSTRUCT_DIALECTS),
|
|
62
|
+
TtsCosyvoice3InstructVolume: enumFromVocabulary(TTS_COSYVOICE3_INSTRUCT_VOLUMES),
|
|
63
|
+
TtsCosyvoice3InstructStyle: enumFromVocabulary(TTS_COSYVOICE3_INSTRUCT_STYLES),
|
|
64
|
+
AudioGenTaskType: enumFromVocabulary(AUDIOGEN_TASK_TYPES)
|
|
52
65
|
};
|
package/dist/schemas/index.d.ts
CHANGED
|
@@ -3,7 +3,7 @@ export * from './cancel.js';
|
|
|
3
3
|
export * from './batch-completion-stream.js';
|
|
4
4
|
export * from './completion-stream.js';
|
|
5
5
|
export * from './completion-event.js';
|
|
6
|
-
export { toolSchema, toolCallSchema, toolCallErrorSchema,
|
|
6
|
+
export { toolSchema, toolCallSchema, toolCallErrorSchema, type Tool, type ToolCall, type ToolCallError, type ToolCallWithCall } from './tools.js';
|
|
7
7
|
export * from './delegate.js';
|
|
8
8
|
export * from './delete-cache.js';
|
|
9
9
|
export * from './download-asset.js';
|
package/dist/schemas/index.js
CHANGED
|
@@ -4,7 +4,7 @@ export * from './cancel.js';
|
|
|
4
4
|
export * from './batch-completion-stream.js';
|
|
5
5
|
export * from './completion-stream.js';
|
|
6
6
|
export * from './completion-event.js';
|
|
7
|
-
export { toolSchema, toolCallSchema, toolCallErrorSchema
|
|
7
|
+
export { toolSchema, toolCallSchema, toolCallErrorSchema } from './tools.js';
|
|
8
8
|
export * from './delegate.js';
|
|
9
9
|
export * from './delete-cache.js';
|
|
10
10
|
export * from './download-asset.js';
|
|
@@ -36,10 +36,6 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
|
|
|
36
36
|
n_discarded: z.ZodOptional<z.ZodNumber>;
|
|
37
37
|
parallel: z.ZodOptional<z.ZodNumber>;
|
|
38
38
|
tools: z.ZodOptional<z.ZodBoolean>;
|
|
39
|
-
toolsMode: z.ZodOptional<z.ZodEnum<{
|
|
40
|
-
static: "static";
|
|
41
|
-
dynamic: "dynamic";
|
|
42
|
-
}>>;
|
|
43
39
|
'cache-type-k': z.ZodOptional<z.ZodString>;
|
|
44
40
|
'cache-type-v': z.ZodOptional<z.ZodString>;
|
|
45
41
|
'main-gpu': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodEnum<{
|
|
@@ -98,6 +94,10 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
|
|
|
98
94
|
batched: "batched";
|
|
99
95
|
sequential: "sequential";
|
|
100
96
|
}>>;
|
|
97
|
+
image_no_upscale: z.ZodOptional<z.ZodEnum<{
|
|
98
|
+
on: "on";
|
|
99
|
+
off: "off";
|
|
100
|
+
}>>;
|
|
101
101
|
'mmproj-use-gpu': z.ZodOptional<z.ZodBoolean>;
|
|
102
102
|
}, z.core.$strip>;
|
|
103
103
|
export type LlmConfigInput = z.infer<typeof llmConfigBaseSchema>;
|
|
@@ -133,10 +133,6 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
133
133
|
n_discarded: z.ZodOptional<z.ZodNumber>;
|
|
134
134
|
parallel: z.ZodOptional<z.ZodNumber>;
|
|
135
135
|
tools: z.ZodOptional<z.ZodBoolean>;
|
|
136
|
-
toolsMode: z.ZodOptional<z.ZodEnum<{
|
|
137
|
-
static: "static";
|
|
138
|
-
dynamic: "dynamic";
|
|
139
|
-
}>>;
|
|
140
136
|
'cache-type-k': z.ZodOptional<z.ZodString>;
|
|
141
137
|
'cache-type-v': z.ZodOptional<z.ZodString>;
|
|
142
138
|
'main-gpu': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodEnum<{
|
|
@@ -195,6 +191,10 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
195
191
|
batched: "batched";
|
|
196
192
|
sequential: "sequential";
|
|
197
193
|
}>>;
|
|
194
|
+
image_no_upscale: z.ZodOptional<z.ZodEnum<{
|
|
195
|
+
on: "on";
|
|
196
|
+
off: "off";
|
|
197
|
+
}>>;
|
|
198
198
|
'mmproj-use-gpu': z.ZodOptional<z.ZodBoolean>;
|
|
199
199
|
}, z.core.$strip>, z.ZodTransform<{
|
|
200
200
|
ctx_size: number | undefined;
|
|
@@ -216,7 +216,6 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
216
216
|
n_discarded?: number | undefined;
|
|
217
217
|
parallel?: number | undefined;
|
|
218
218
|
tools?: boolean | undefined;
|
|
219
|
-
toolsMode?: "static" | "dynamic" | undefined;
|
|
220
219
|
'cache-type-k'?: string | undefined;
|
|
221
220
|
'cache-type-v'?: string | undefined;
|
|
222
221
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
@@ -238,6 +237,7 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
238
237
|
addon?: "embeddings" | "diffusion" | "audiogen" | "tts" | "ocr" | "vla" | "llamacpp-completion" | "whispercpp-transcription" | "bci-whispercpp-transcription" | "llamacpp-embedding" | "nmtcpp-translation" | "onnx-tts" | "tts-ggml" | "parakeet-transcription" | "ggml-ocr" | "sdcpp-generation" | "audiogen-ggml" | "ggml-vla" | "ggml-classification" | "llm" | "whisper" | "bci" | "nmt" | "parakeet" | "classification" | "vad" | undefined;
|
|
239
238
|
} | undefined;
|
|
240
239
|
image_tile_mode: "disabled" | "batched" | "sequential" | undefined;
|
|
240
|
+
image_no_upscale?: "on" | "off" | undefined;
|
|
241
241
|
'mmproj-use-gpu'?: boolean | undefined;
|
|
242
242
|
}, {
|
|
243
243
|
ctx_size?: number | undefined;
|
|
@@ -259,7 +259,6 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
259
259
|
n_discarded?: number | undefined;
|
|
260
260
|
parallel?: number | undefined;
|
|
261
261
|
tools?: boolean | undefined;
|
|
262
|
-
toolsMode?: "static" | "dynamic" | undefined;
|
|
263
262
|
'cache-type-k'?: string | undefined;
|
|
264
263
|
'cache-type-v'?: string | undefined;
|
|
265
264
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
@@ -281,6 +280,7 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
281
280
|
addon?: "embeddings" | "diffusion" | "audiogen" | "tts" | "ocr" | "vla" | "llamacpp-completion" | "whispercpp-transcription" | "bci-whispercpp-transcription" | "llamacpp-embedding" | "nmtcpp-translation" | "onnx-tts" | "tts-ggml" | "parakeet-transcription" | "ggml-ocr" | "sdcpp-generation" | "audiogen-ggml" | "ggml-vla" | "ggml-classification" | "llm" | "whisper" | "bci" | "nmt" | "parakeet" | "classification" | "vad" | undefined;
|
|
282
281
|
} | undefined;
|
|
283
282
|
image_tile_mode?: "disabled" | "batched" | "sequential" | undefined;
|
|
283
|
+
image_no_upscale?: "on" | "off" | undefined;
|
|
284
284
|
'mmproj-use-gpu'?: boolean | undefined;
|
|
285
285
|
}>>;
|
|
286
286
|
export type LlmConfig = z.infer<typeof llmConfigSchema>;
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { modelSrcInputSchema } from './model-src-utils.js';
|
|
3
|
-
import { TOOLS_MODE } from './tools.js';
|
|
4
3
|
/**
|
|
5
4
|
* Upper bound for `reasoning_budget`. Mirrors the llm-llamacpp addon, which
|
|
6
5
|
* stores the budget as a 32-bit `int` and rejects values above
|
|
@@ -16,77 +15,129 @@ export const VERBOSITY = {
|
|
|
16
15
|
const verbositySchema = z.enum(VERBOSITY);
|
|
17
16
|
// Base schema - validates types, all fields optional (for client-side validation)
|
|
18
17
|
export const llmConfigBaseSchema = z.object({
|
|
19
|
-
ctx_size: z
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
18
|
+
ctx_size: z
|
|
19
|
+
.number()
|
|
20
|
+
.optional()
|
|
21
|
+
.describe("Context window size in tokens; `0` uses the model's trained context length. Default 1024."),
|
|
22
|
+
temp: z.number().min(0).max(2).optional().describe('Sampling temperature (0–2). Default 0.8.'),
|
|
23
|
+
top_p: z
|
|
24
|
+
.number()
|
|
25
|
+
.min(0)
|
|
26
|
+
.max(1)
|
|
27
|
+
.optional()
|
|
28
|
+
.describe('Top-p (nucleus) sampling cutoff (0–1). Default 0.9.'),
|
|
29
|
+
top_k: z
|
|
30
|
+
.number()
|
|
31
|
+
.int()
|
|
32
|
+
.min(0)
|
|
33
|
+
.max(128)
|
|
34
|
+
.optional()
|
|
35
|
+
.describe('Top-k sampling — keep only the top K tokens (0–128). Default 40.'),
|
|
36
|
+
seed: z.number().optional().describe('Sampling RNG seed; `-1` (default) picks a random seed.'),
|
|
37
|
+
gpu_layers: z
|
|
38
|
+
.number()
|
|
39
|
+
.optional()
|
|
40
|
+
.describe('Number of model layers to offload to the GPU. Default 99 (offload all).'),
|
|
41
|
+
lora: z
|
|
42
|
+
.string()
|
|
43
|
+
.optional()
|
|
44
|
+
.describe('Path to a LoRA adapter file to apply on top of the base model.'),
|
|
45
|
+
device: z
|
|
46
|
+
.string()
|
|
47
|
+
.optional()
|
|
48
|
+
.describe("Device to run inference on: `'gpu'` or `'cpu'`. Default `'gpu'`."),
|
|
27
49
|
predict: z
|
|
28
50
|
.union([
|
|
29
51
|
z.literal(-1), // special: until stop token
|
|
30
52
|
z.literal(-2), // special: until context filled
|
|
31
53
|
z.number().int().min(1) // positive integer: fixed token count
|
|
32
54
|
])
|
|
33
|
-
.optional()
|
|
34
|
-
|
|
35
|
-
system_prompt: z
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
.
|
|
53
|
-
.optional()
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
'
|
|
55
|
+
.optional()
|
|
56
|
+
.describe('Max tokens to predict. `-1` = until stop token, `-2` = until context filled.'),
|
|
57
|
+
system_prompt: z
|
|
58
|
+
.string()
|
|
59
|
+
.optional()
|
|
60
|
+
.describe("Seeds conversation history on the JS side only; never forwarded to the addon. Default `'You are a helpful assistant.'`"),
|
|
61
|
+
no_mmap: z.boolean().optional().describe('Disable memory-mapped model loading. Default false.'),
|
|
62
|
+
verbosity: verbositySchema
|
|
63
|
+
.optional()
|
|
64
|
+
.describe('Native log verbosity: `0`=ERROR, `1`=WARN, `2`=INFO, `3`=DEBUG. Default 0.'),
|
|
65
|
+
presence_penalty: z
|
|
66
|
+
.number()
|
|
67
|
+
.optional()
|
|
68
|
+
.describe('Presence penalty applied to tokens that have already appeared. Default 0.'),
|
|
69
|
+
frequency_penalty: z
|
|
70
|
+
.number()
|
|
71
|
+
.optional()
|
|
72
|
+
.describe('Frequency penalty applied to tokens by their frequency so far. Default 0.'),
|
|
73
|
+
repeat_penalty: z
|
|
74
|
+
.number()
|
|
75
|
+
.optional()
|
|
76
|
+
.describe('Repetition penalty applied to repeated tokens. Default 1.1.'),
|
|
77
|
+
stop_sequences: z
|
|
78
|
+
.array(z.string())
|
|
79
|
+
.optional()
|
|
80
|
+
.describe('Strings that stop generation when produced (forwarded to the addon as `reverse_prompt`).'),
|
|
81
|
+
n_discarded: z
|
|
82
|
+
.number()
|
|
83
|
+
.optional()
|
|
84
|
+
.describe('Tokens to discard from the front of the context when it fills (sliding window); `0` (default) disables sliding. In batch mode clamped to the per-slot window (`ctx_size / parallel`).'),
|
|
85
|
+
parallel: z
|
|
86
|
+
.number()
|
|
87
|
+
.int()
|
|
88
|
+
.min(1)
|
|
89
|
+
.optional()
|
|
90
|
+
.describe('Concurrent sequence slots for continuous batching (1–256). Default 1 (sequential, batching off); `>= 2` enables batched decoding and splits the KV cache evenly across slots.'),
|
|
91
|
+
tools: z
|
|
92
|
+
.boolean()
|
|
93
|
+
.optional()
|
|
94
|
+
.describe("Enable tool calling via the model's jinja chat template. Default false."),
|
|
95
|
+
'cache-type-k': z
|
|
96
|
+
.string()
|
|
97
|
+
.optional()
|
|
98
|
+
.describe('KV-cache key quantization type (`f16`, `f32`, `bf16`, `q8_0`, `q4_0`, …). Unset selects a backend-safe default.'),
|
|
99
|
+
'cache-type-v': z
|
|
100
|
+
.string()
|
|
101
|
+
.optional()
|
|
102
|
+
.describe('KV-cache value quantization type; quantizing the value cache requires flash attention. Unset selects a backend-safe default.'),
|
|
103
|
+
'main-gpu': z
|
|
104
|
+
.union([z.number().int().min(0), z.enum(['integrated', 'dedicated'])])
|
|
105
|
+
.optional()
|
|
106
|
+
.describe("GPU to use on multi-GPU systems: a device index, or `'integrated'`/`'dedicated'` to restrict selection to that class."),
|
|
107
|
+
'split-mode': z
|
|
108
|
+
.enum(['none', 'layer', 'row'])
|
|
109
|
+
.optional()
|
|
110
|
+
.describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'row'` (tensor parallelism)."),
|
|
111
|
+
'tensor-split': z
|
|
112
|
+
.string()
|
|
113
|
+
.optional()
|
|
114
|
+
.describe("Proportions for distributing layers/rows across GPUs, e.g. `'1,1'` (equal) or `'3,1'` (75/25)."),
|
|
115
|
+
openclCacheDir: z
|
|
116
|
+
.string()
|
|
117
|
+
.optional()
|
|
118
|
+
.describe('Writable directory for the OpenCL kernel binary cache; required on Android for fast GPU startup.'),
|
|
119
|
+
reasoning_budget: z
|
|
120
|
+
.number()
|
|
121
|
+
.int()
|
|
122
|
+
.min(-1)
|
|
123
|
+
.max(REASONING_BUDGET_MAX)
|
|
124
|
+
.optional()
|
|
125
|
+
.describe('Reasoning-channel token budget. `-1` (default) unrestricted, `0` disables it, any positive integer caps the reasoning channel at that many tokens (the closing think tag is force-emitted once the budget is spent).'),
|
|
126
|
+
projectionModelSrc: modelSrcInputSchema
|
|
127
|
+
.optional()
|
|
128
|
+
.describe('Multimodal projection (mmproj / vision encoder) model source; multimodal models only.'),
|
|
129
|
+
image_tile_mode: z
|
|
130
|
+
.enum(['disabled', 'batched', 'sequential'])
|
|
131
|
+
.optional()
|
|
132
|
+
.describe("Qwen3.5-VL multi-tile image encoding mode (multimodal models only): `'sequential'` (default) encodes image tiles one at a time, `'batched'` encodes all tiles in a single batched pass, `'disabled'` does no multi-tile encoding (single tile). Ignored by text-only models."),
|
|
133
|
+
image_no_upscale: z
|
|
134
|
+
.enum(['on', 'off'])
|
|
135
|
+
.optional()
|
|
136
|
+
.describe("idefics3-style image preprocessing rule (multimodal models only): `'on'` rounds the image's long side up to a whole number of slices and caps it, so an image smaller than the cap keeps its own resolution and becomes far fewer slices; `'off'` always stretches the long side to the cap. When unset, the model's own GGUF value is used; ignored with a warning by models that do not use idefics3-style preprocessing. Changes the number of image tokens (and therefore both accuracy and encode time), so a checkpoint whose GGUF omits the key needs this set to preprocess correctly."),
|
|
137
|
+
'mmproj-use-gpu': z
|
|
138
|
+
.boolean()
|
|
139
|
+
.optional()
|
|
140
|
+
.describe('Run the multimodal projector (mmproj / vision encoder) on the GPU (multimodal models only). `true` forces GPU, `false` forces CPU. When unset, the backend is auto-selected per device class: GPU on desktop/iOS and Android Adreno 800+; CPU on every other Android GPU (Arm Mali, Adreno <800, and undetectable Adreno tiers), with the LLM layers still on the GPU. Only honoured when the model itself runs on a GPU backend — ignored with a warning on CPU.')
|
|
90
141
|
});
|
|
91
142
|
// Default values - typed as partial of the config
|
|
92
143
|
export const LLM_CONFIG_DEFAULTS = {
|
|
@@ -103,22 +154,57 @@ export const llmConfigSchema = llmConfigBaseSchema.transform((data) => ({
|
|
|
103
154
|
}));
|
|
104
155
|
// Base schema - validates types, all fields optional (for client-side validation)
|
|
105
156
|
export const embedConfigBaseSchema = z.object({
|
|
106
|
-
gpuLayers: z
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
157
|
+
gpuLayers: z
|
|
158
|
+
.number()
|
|
159
|
+
.int()
|
|
160
|
+
.optional()
|
|
161
|
+
.describe('Number of model layers to offload to the GPU. Default 99 (offload all).'),
|
|
162
|
+
device: z
|
|
163
|
+
.enum(['gpu', 'cpu'])
|
|
164
|
+
.optional()
|
|
165
|
+
.describe("Device to run inference on: `'gpu'` or `'cpu'`. Default `'gpu'`."),
|
|
166
|
+
batchSize: z
|
|
167
|
+
.number()
|
|
168
|
+
.int()
|
|
169
|
+
.min(1)
|
|
170
|
+
.optional()
|
|
171
|
+
.describe('Tokens processed per batch (input throughput). Default 1024.'),
|
|
172
|
+
pooling: z
|
|
173
|
+
.enum(['none', 'mean', 'cls', 'last', 'rank'])
|
|
174
|
+
.optional()
|
|
175
|
+
.describe("Pooling strategy collapsing token embeddings into one sequence vector: `'none'`, `'mean'`, `'cls'`, `'last'`, or `'rank'`. Unset uses the model default."),
|
|
176
|
+
attention: z
|
|
177
|
+
.enum(['causal', 'non-causal'])
|
|
178
|
+
.optional()
|
|
179
|
+
.describe("Attention type: `'causal'` or `'non-causal'`. Unset uses the model default."),
|
|
180
|
+
embdNormalize: z
|
|
181
|
+
.number()
|
|
182
|
+
.int()
|
|
183
|
+
.optional()
|
|
184
|
+
.describe('Embedding normalization: `-1` none, `0` max-abs int16, `1` taxicab, `2` euclidean, `>2` p-norm. Default 2 (euclidean).'),
|
|
185
|
+
flashAttention: z
|
|
186
|
+
.enum(['on', 'off', 'auto'])
|
|
187
|
+
.optional()
|
|
188
|
+
.describe("Flash attention: `'on'`, `'off'`, or `'auto'`. Default `'auto'`."),
|
|
189
|
+
mainGpu: z
|
|
190
|
+
.union([z.number().int().min(0), z.enum(['integrated', 'dedicated'])])
|
|
191
|
+
.optional()
|
|
192
|
+
.describe("GPU to use on multi-GPU systems: a device index, or `'integrated'`/`'dedicated'` to restrict selection to that class."),
|
|
193
|
+
splitMode: z
|
|
194
|
+
.enum(['none', 'layer', 'row'])
|
|
195
|
+
.optional()
|
|
196
|
+
.describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'row'` (tensor parallelism)."),
|
|
197
|
+
tensorSplit: z
|
|
198
|
+
.string()
|
|
199
|
+
.optional()
|
|
200
|
+
.describe("Proportions for distributing layers/rows across GPUs, e.g. `'1,1'` (equal) or `'3,1'` (75/25)."),
|
|
201
|
+
verbosity: verbositySchema
|
|
202
|
+
.optional()
|
|
203
|
+
.describe('Native log verbosity: `0`=ERROR, `1`=WARN, `2`=INFO, `3`=DEBUG. Default 0.'),
|
|
204
|
+
openclCacheDir: z
|
|
205
|
+
.string()
|
|
206
|
+
.optional()
|
|
207
|
+
.describe('Writable directory for the OpenCL kernel binary cache; required on Android for fast GPU startup.')
|
|
122
208
|
});
|
|
123
209
|
// Default values - typed as partial of the config
|
|
124
210
|
export const EMBED_CONFIG_DEFAULTS = {
|