@qvac/inference 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +107 -33
- package/README.md +8 -5
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +18 -6
- package/dist/handlers/assess-model-fit.js +215 -35
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +80 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +46 -2
- package/dist/resources/model-fit/assess.js +67 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +102 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +212 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +284 -77
- package/dist/schemas/assess-model-fit.js +93 -55
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +765 -56
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +236 -30
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +105 -14
- package/dist/schemas/llamacpp-config.js +113 -3
- package/dist/schemas/load-model.d.ts +3102 -627
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +205 -5
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +44 -30
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -171
package/dist/schemas/config.js
CHANGED
|
@@ -166,6 +166,14 @@ export const qvacConfigSchema = z.object({
|
|
|
166
166
|
* Defaults to 60000 (60 seconds).
|
|
167
167
|
*/
|
|
168
168
|
registryStreamTimeoutMs: z.number().int().positive().optional(),
|
|
169
|
+
/**
|
|
170
|
+
* Budget in milliseconds for the registry fetch of a model's weightless
|
|
171
|
+
* description during `assessModelFit`. One budget covers the lookup and the
|
|
172
|
+
* whole companion set; past it the assessment falls back to the computed
|
|
173
|
+
* floor. Raise this on slow or high-latency connections.
|
|
174
|
+
* Defaults to 40000 (40 seconds).
|
|
175
|
+
*/
|
|
176
|
+
fitStubBudgetMs: z.number().int().positive().optional(),
|
|
169
177
|
/**
|
|
170
178
|
* Device-specific config defaults.
|
|
171
179
|
* Use this to override model config defaults for specific devices.
|
|
@@ -203,6 +211,8 @@ export const qvacConfigSchema = z.object({
|
|
|
203
211
|
* ending in `/plugin`.
|
|
204
212
|
*/
|
|
205
213
|
plugins: z.array(z.string()).optional(),
|
|
214
|
+
/** Optional provider module selected at SDK bundle time. Direct Bare callers register in code. */
|
|
215
|
+
rpcServerProvider: z.string().min(1).optional(),
|
|
206
216
|
/**
|
|
207
217
|
* Inert: we have no bundling or ABI-verification step, so this field is
|
|
208
218
|
* unused. Carried over from the bundle-time config schema.
|
package/dist/schemas/embed.d.ts
CHANGED
|
@@ -14,8 +14,8 @@ export declare const embedStatsSchema: z.ZodObject<{
|
|
|
14
14
|
tokensPerSecond: z.ZodOptional<z.ZodNumber>;
|
|
15
15
|
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
16
16
|
backendDevice: z.ZodOptional<z.ZodEnum<{
|
|
17
|
-
gpu: "gpu";
|
|
18
17
|
cpu: "cpu";
|
|
18
|
+
gpu: "gpu";
|
|
19
19
|
}>>;
|
|
20
20
|
contextSize: z.ZodOptional<z.ZodNumber>;
|
|
21
21
|
}, z.core.$strip>;
|
|
@@ -28,8 +28,8 @@ export declare const embedResponseSchema: z.ZodObject<{
|
|
|
28
28
|
tokensPerSecond: z.ZodOptional<z.ZodNumber>;
|
|
29
29
|
totalTokens: z.ZodOptional<z.ZodNumber>;
|
|
30
30
|
backendDevice: z.ZodOptional<z.ZodEnum<{
|
|
31
|
-
gpu: "gpu";
|
|
32
31
|
cpu: "cpu";
|
|
32
|
+
gpu: "gpu";
|
|
33
33
|
}>>;
|
|
34
34
|
contextSize: z.ZodOptional<z.ZodNumber>;
|
|
35
35
|
}, z.core.$strip>>;
|
package/dist/schemas/error.js
CHANGED
|
@@ -34,6 +34,17 @@ function hasTypedFields(error) {
|
|
|
34
34
|
function isQvacError(error) {
|
|
35
35
|
return error instanceof QvacErrorBase;
|
|
36
36
|
}
|
|
37
|
+
function serializeCause(cause, depth = 0) {
|
|
38
|
+
if (!(cause instanceof Error))
|
|
39
|
+
return typeof cause === 'string' ? cause : String(cause);
|
|
40
|
+
return {
|
|
41
|
+
name: cause.name,
|
|
42
|
+
message: cause.message,
|
|
43
|
+
stack: cause.stack,
|
|
44
|
+
...('code' in cause && typeof cause.code === 'number' && { code: cause.code }),
|
|
45
|
+
...(cause.cause !== undefined && depth < 5 && { cause: serializeCause(cause.cause, depth + 1) })
|
|
46
|
+
};
|
|
47
|
+
}
|
|
37
48
|
export function createErrorResponse(error) {
|
|
38
49
|
if (isQvacError(error)) {
|
|
39
50
|
const qvacData = error.toJSON();
|
|
@@ -45,6 +56,8 @@ export function createErrorResponse(error) {
|
|
|
45
56
|
stack: qvacData.stack,
|
|
46
57
|
timestamp: new Date().toISOString()
|
|
47
58
|
};
|
|
59
|
+
if (qvacData.cause !== undefined)
|
|
60
|
+
response.cause = serializeCause(qvacData.cause);
|
|
48
61
|
if (hasTypedFields(error)) {
|
|
49
62
|
response.typedFields = error.toErrorResponseFields();
|
|
50
63
|
}
|
package/dist/schemas/errors.d.ts
CHANGED
|
@@ -28,6 +28,7 @@ export declare const ERROR_CODES: {
|
|
|
28
28
|
readonly TTS_REFERENCE_AUDIO_REQUIRED: 52209;
|
|
29
29
|
readonly LEGACY_PARAKEET_MODEL_DEPRECATED: 52210;
|
|
30
30
|
readonly LEGACY_TTS_MODEL_DEPRECATED: 52211;
|
|
31
|
+
readonly RPC_SERVER_OPERATION_FAILED: 52423;
|
|
31
32
|
readonly MODEL_UNLOAD_FAILED: 52400;
|
|
32
33
|
readonly EMBED_FAILED: 52401;
|
|
33
34
|
readonly EMBED_NO_EMBEDDINGS: 52402;
|
package/dist/schemas/errors.js
CHANGED
|
@@ -37,6 +37,7 @@ export const ERROR_CODES = {
|
|
|
37
37
|
LEGACY_PARAKEET_MODEL_DEPRECATED: 52210,
|
|
38
38
|
LEGACY_TTS_MODEL_DEPRECATED: 52211,
|
|
39
39
|
// Model operations (52,400-52,799)
|
|
40
|
+
RPC_SERVER_OPERATION_FAILED: 52423,
|
|
40
41
|
MODEL_UNLOAD_FAILED: 52400,
|
|
41
42
|
EMBED_FAILED: 52401,
|
|
42
43
|
EMBED_NO_EMBEDDINGS: 52402,
|
|
@@ -209,6 +210,10 @@ const errorDefinitions = {
|
|
|
209
210
|
name: 'UNKNOWN_MODEL_TYPE',
|
|
210
211
|
message: (modelType) => `Unknown model type: ${modelType}. Register the plugin for "${modelType}" in code with \`registerPlugin\` / \`plugins([...])\` before this call.`
|
|
211
212
|
},
|
|
213
|
+
[ERROR_CODES.RPC_SERVER_OPERATION_FAILED]: {
|
|
214
|
+
name: 'RPC_SERVER_OPERATION_FAILED',
|
|
215
|
+
message: (operation, details) => `${operation}: ${details}`
|
|
216
|
+
},
|
|
212
217
|
// Model loading
|
|
213
218
|
[ERROR_CODES.MODEL_LOAD_FAILED]: {
|
|
214
219
|
name: 'MODEL_LOAD_FAILED',
|
package/dist/schemas/index.d.ts
CHANGED
package/dist/schemas/index.js
CHANGED
|
@@ -47,17 +47,42 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
|
|
|
47
47
|
integrated: "integrated";
|
|
48
48
|
dedicated: "dedicated";
|
|
49
49
|
}>]>>;
|
|
50
|
+
'rpc-servers': z.ZodOptional<z.ZodString>;
|
|
51
|
+
devices: z.ZodOptional<z.ZodString>;
|
|
50
52
|
'split-mode': z.ZodOptional<z.ZodEnum<{
|
|
51
53
|
none: "none";
|
|
52
54
|
layer: "layer";
|
|
53
55
|
tensor: "tensor";
|
|
54
56
|
}>>;
|
|
55
57
|
'flash-attn': z.ZodOptional<z.ZodEnum<{
|
|
58
|
+
auto: "auto";
|
|
56
59
|
on: "on";
|
|
57
60
|
off: "off";
|
|
58
|
-
auto: "auto";
|
|
59
61
|
}>>;
|
|
60
62
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
63
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
64
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
65
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
66
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
67
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
68
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
69
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
70
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
71
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
72
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
73
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
74
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
75
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
76
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
77
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
78
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
79
|
+
auto: "auto";
|
|
80
|
+
on: "on";
|
|
81
|
+
off: "off";
|
|
82
|
+
}>>;
|
|
83
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
84
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
85
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
61
86
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
62
87
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
63
88
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -113,7 +138,6 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
|
|
|
113
138
|
export type LlmConfigInput = z.infer<typeof llmConfigBaseSchema>;
|
|
114
139
|
export declare const LLM_CONFIG_DEFAULTS: {
|
|
115
140
|
readonly ctx_size: 1024;
|
|
116
|
-
readonly gpu_layers: 99;
|
|
117
141
|
readonly device: "gpu";
|
|
118
142
|
readonly system_prompt: "You are a helpful assistant.";
|
|
119
143
|
readonly image_tile_mode: "sequential";
|
|
@@ -154,17 +178,42 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
154
178
|
integrated: "integrated";
|
|
155
179
|
dedicated: "dedicated";
|
|
156
180
|
}>]>>;
|
|
181
|
+
'rpc-servers': z.ZodOptional<z.ZodString>;
|
|
182
|
+
devices: z.ZodOptional<z.ZodString>;
|
|
157
183
|
'split-mode': z.ZodOptional<z.ZodEnum<{
|
|
158
184
|
none: "none";
|
|
159
185
|
layer: "layer";
|
|
160
186
|
tensor: "tensor";
|
|
161
187
|
}>>;
|
|
162
188
|
'flash-attn': z.ZodOptional<z.ZodEnum<{
|
|
189
|
+
auto: "auto";
|
|
163
190
|
on: "on";
|
|
164
191
|
off: "off";
|
|
165
|
-
auto: "auto";
|
|
166
192
|
}>>;
|
|
167
193
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
194
|
+
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
195
|
+
'ubatch-size': z.ZodOptional<z.ZodNumber>;
|
|
196
|
+
'cpu-moe': z.ZodOptional<z.ZodBoolean>;
|
|
197
|
+
'n-cpu-moe': z.ZodOptional<z.ZodNumber>;
|
|
198
|
+
'kv-offload': z.ZodOptional<z.ZodBoolean>;
|
|
199
|
+
'image-max-tokens': z.ZodOptional<z.ZodNumber>;
|
|
200
|
+
'image-min-tokens': z.ZodOptional<z.ZodNumber>;
|
|
201
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
202
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
203
|
+
'cpu-mask': z.ZodOptional<z.ZodString>;
|
|
204
|
+
'cpu-mask-batch': z.ZodOptional<z.ZodString>;
|
|
205
|
+
'override-tensor': z.ZodOptional<z.ZodString>;
|
|
206
|
+
'n-cpu-ffn': z.ZodOptional<z.ZodNumber>;
|
|
207
|
+
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
208
|
+
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
209
|
+
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
210
|
+
auto: "auto";
|
|
211
|
+
on: "on";
|
|
212
|
+
off: "off";
|
|
213
|
+
}>>;
|
|
214
|
+
fit: z.ZodOptional<z.ZodBoolean>;
|
|
215
|
+
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
216
|
+
'fit-ctx': z.ZodOptional<z.ZodNumber>;
|
|
168
217
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
169
218
|
reasoning_budget: z.ZodOptional<z.ZodNumber>;
|
|
170
219
|
projectionModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
@@ -222,7 +271,7 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
222
271
|
top_p?: number | undefined;
|
|
223
272
|
top_k?: number | undefined;
|
|
224
273
|
seed?: number | undefined;
|
|
225
|
-
gpu_layers
|
|
274
|
+
gpu_layers?: number | undefined;
|
|
226
275
|
lora?: string | undefined;
|
|
227
276
|
device: string | undefined;
|
|
228
277
|
predict?: number | undefined;
|
|
@@ -238,9 +287,30 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
238
287
|
'cache-type-k'?: string | undefined;
|
|
239
288
|
'cache-type-v'?: string | undefined;
|
|
240
289
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
290
|
+
'rpc-servers'?: string | undefined;
|
|
291
|
+
devices?: string | undefined;
|
|
241
292
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
242
|
-
'flash-attn'?: "
|
|
293
|
+
'flash-attn'?: "auto" | "on" | "off" | undefined;
|
|
243
294
|
'tensor-split'?: string | undefined;
|
|
295
|
+
'batch-size'?: number | undefined;
|
|
296
|
+
'ubatch-size'?: number | undefined;
|
|
297
|
+
'cpu-moe'?: boolean | undefined;
|
|
298
|
+
'n-cpu-moe'?: number | undefined;
|
|
299
|
+
'kv-offload'?: boolean | undefined;
|
|
300
|
+
'image-max-tokens'?: number | undefined;
|
|
301
|
+
'image-min-tokens'?: number | undefined;
|
|
302
|
+
threads?: number | undefined;
|
|
303
|
+
'threads-batch'?: number | undefined;
|
|
304
|
+
'cpu-mask'?: string | undefined;
|
|
305
|
+
'cpu-mask-batch'?: string | undefined;
|
|
306
|
+
'override-tensor'?: string | undefined;
|
|
307
|
+
'n-cpu-ffn'?: number | undefined;
|
|
308
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
309
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
310
|
+
'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
|
|
311
|
+
fit?: boolean | undefined;
|
|
312
|
+
'fit-target'?: string | number | undefined;
|
|
313
|
+
'fit-ctx'?: number | undefined;
|
|
244
314
|
openclCacheDir?: string | undefined;
|
|
245
315
|
reasoning_budget?: number | undefined;
|
|
246
316
|
projectionModelSrc?: string | {
|
|
@@ -281,9 +351,30 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
281
351
|
'cache-type-k'?: string | undefined;
|
|
282
352
|
'cache-type-v'?: string | undefined;
|
|
283
353
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
354
|
+
'rpc-servers'?: string | undefined;
|
|
355
|
+
devices?: string | undefined;
|
|
284
356
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
285
|
-
'flash-attn'?: "
|
|
357
|
+
'flash-attn'?: "auto" | "on" | "off" | undefined;
|
|
286
358
|
'tensor-split'?: string | undefined;
|
|
359
|
+
'batch-size'?: number | undefined;
|
|
360
|
+
'ubatch-size'?: number | undefined;
|
|
361
|
+
'cpu-moe'?: boolean | undefined;
|
|
362
|
+
'n-cpu-moe'?: number | undefined;
|
|
363
|
+
'kv-offload'?: boolean | undefined;
|
|
364
|
+
'image-max-tokens'?: number | undefined;
|
|
365
|
+
'image-min-tokens'?: number | undefined;
|
|
366
|
+
threads?: number | undefined;
|
|
367
|
+
'threads-batch'?: number | undefined;
|
|
368
|
+
'cpu-mask'?: string | undefined;
|
|
369
|
+
'cpu-mask-batch'?: string | undefined;
|
|
370
|
+
'override-tensor'?: string | undefined;
|
|
371
|
+
'n-cpu-ffn'?: number | undefined;
|
|
372
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
373
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
374
|
+
'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
|
|
375
|
+
fit?: boolean | undefined;
|
|
376
|
+
'fit-target'?: string | number | undefined;
|
|
377
|
+
'fit-ctx'?: number | undefined;
|
|
287
378
|
openclCacheDir?: string | undefined;
|
|
288
379
|
reasoning_budget?: number | undefined;
|
|
289
380
|
projectionModelSrc?: string | {
|
|
@@ -307,8 +398,8 @@ export type LlmConfig = z.infer<typeof llmConfigSchema>;
|
|
|
307
398
|
export declare const embedConfigBaseSchema: z.ZodObject<{
|
|
308
399
|
gpuLayers: z.ZodOptional<z.ZodNumber>;
|
|
309
400
|
device: z.ZodOptional<z.ZodEnum<{
|
|
310
|
-
gpu: "gpu";
|
|
311
401
|
cpu: "cpu";
|
|
402
|
+
gpu: "gpu";
|
|
312
403
|
}>>;
|
|
313
404
|
batchSize: z.ZodOptional<z.ZodNumber>;
|
|
314
405
|
pooling: z.ZodOptional<z.ZodEnum<{
|
|
@@ -324,9 +415,9 @@ export declare const embedConfigBaseSchema: z.ZodObject<{
|
|
|
324
415
|
}>>;
|
|
325
416
|
embdNormalize: z.ZodOptional<z.ZodNumber>;
|
|
326
417
|
flashAttention: z.ZodOptional<z.ZodEnum<{
|
|
418
|
+
auto: "auto";
|
|
327
419
|
on: "on";
|
|
328
420
|
off: "off";
|
|
329
|
-
auto: "auto";
|
|
330
421
|
}>>;
|
|
331
422
|
mainGpu: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodEnum<{
|
|
332
423
|
integrated: "integrated";
|
|
@@ -354,8 +445,8 @@ export declare const EMBED_CONFIG_DEFAULTS: {
|
|
|
354
445
|
export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
355
446
|
gpuLayers: z.ZodOptional<z.ZodNumber>;
|
|
356
447
|
device: z.ZodOptional<z.ZodEnum<{
|
|
357
|
-
gpu: "gpu";
|
|
358
448
|
cpu: "cpu";
|
|
449
|
+
gpu: "gpu";
|
|
359
450
|
}>>;
|
|
360
451
|
batchSize: z.ZodOptional<z.ZodNumber>;
|
|
361
452
|
pooling: z.ZodOptional<z.ZodEnum<{
|
|
@@ -371,9 +462,9 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
371
462
|
}>>;
|
|
372
463
|
embdNormalize: z.ZodOptional<z.ZodNumber>;
|
|
373
464
|
flashAttention: z.ZodOptional<z.ZodEnum<{
|
|
465
|
+
auto: "auto";
|
|
374
466
|
on: "on";
|
|
375
467
|
off: "off";
|
|
376
|
-
auto: "auto";
|
|
377
468
|
}>>;
|
|
378
469
|
mainGpu: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodEnum<{
|
|
379
470
|
integrated: "integrated";
|
|
@@ -393,12 +484,12 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
393
484
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
394
485
|
}, z.core.$strip>, z.ZodTransform<{
|
|
395
486
|
gpuLayers: number | undefined;
|
|
396
|
-
device: "
|
|
487
|
+
device: "cpu" | "gpu" | undefined;
|
|
397
488
|
batchSize: number | undefined;
|
|
398
489
|
pooling?: "none" | "mean" | "cls" | "last" | "rank" | undefined;
|
|
399
490
|
attention?: "causal" | "non-causal" | undefined;
|
|
400
491
|
embdNormalize?: number | undefined;
|
|
401
|
-
flashAttention?: "
|
|
492
|
+
flashAttention?: "auto" | "on" | "off" | undefined;
|
|
402
493
|
mainGpu?: number | "integrated" | "dedicated" | undefined;
|
|
403
494
|
splitMode?: "none" | "layer" | undefined;
|
|
404
495
|
tensorSplit?: string | undefined;
|
|
@@ -406,12 +497,12 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
406
497
|
openclCacheDir?: string | undefined;
|
|
407
498
|
}, {
|
|
408
499
|
gpuLayers?: number | undefined;
|
|
409
|
-
device?: "
|
|
500
|
+
device?: "cpu" | "gpu" | undefined;
|
|
410
501
|
batchSize?: number | undefined;
|
|
411
502
|
pooling?: "none" | "mean" | "cls" | "last" | "rank" | undefined;
|
|
412
503
|
attention?: "causal" | "non-causal" | undefined;
|
|
413
504
|
embdNormalize?: number | undefined;
|
|
414
|
-
flashAttention?: "
|
|
505
|
+
flashAttention?: "auto" | "on" | "off" | undefined;
|
|
415
506
|
mainGpu?: number | "integrated" | "dedicated" | undefined;
|
|
416
507
|
splitMode?: "none" | "layer" | undefined;
|
|
417
508
|
tensorSplit?: string | undefined;
|
|
@@ -37,7 +37,7 @@ export const llmConfigBaseSchema = z.object({
|
|
|
37
37
|
gpu_layers: z
|
|
38
38
|
.number()
|
|
39
39
|
.optional()
|
|
40
|
-
.describe('Number of model layers to offload to the GPU.
|
|
40
|
+
.describe('Number of model layers to offload to the GPU. Unset by default, which lets the runtime fit the placement to free device memory (offloading every layer when it fits). Setting it pins the layer count and disables that fit.'),
|
|
41
41
|
lora: z
|
|
42
42
|
.string()
|
|
43
43
|
.optional()
|
|
@@ -103,10 +103,24 @@ export const llmConfigBaseSchema = z.object({
|
|
|
103
103
|
.union([z.number().int().min(0), z.enum(['integrated', 'dedicated'])])
|
|
104
104
|
.optional()
|
|
105
105
|
.describe("GPU to use on multi-GPU systems: a device index, or `'integrated'`/`'dedicated'` to restrict selection to that class."),
|
|
106
|
+
'rpc-servers': z
|
|
107
|
+
.string()
|
|
108
|
+
.min(1)
|
|
109
|
+
.refine((value) => {
|
|
110
|
+
const endpoints = value.split(',').map((endpoint) => endpoint.trim());
|
|
111
|
+
return new Set(endpoints).size === endpoints.length;
|
|
112
|
+
}, 'RPC server endpoints must be unique')
|
|
113
|
+
.optional()
|
|
114
|
+
.describe('Comma-separated unique native RPC host:port endpoints in device registration order for this load. Requires compatible RPC addon builds and a trusted private network; native traffic is unencrypted and unauthenticated.'),
|
|
115
|
+
devices: z
|
|
116
|
+
.string()
|
|
117
|
+
.min(1)
|
|
118
|
+
.optional()
|
|
119
|
+
.describe("Explicit ordered native device names, for example RPC0,RPC1. RPC indices enumerate every device in this load's rpc-servers endpoint order, starting at RPC0 for each load. Pass exactly those endpoints in that order to getRpcDeviceMap, including on reused workers. tensor-split weights follow the selected devices order."),
|
|
106
120
|
'split-mode': z
|
|
107
121
|
.enum(['none', 'layer', 'tensor'])
|
|
108
122
|
.optional()
|
|
109
|
-
.describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'tensor'` (EXPERIMENTAL tensor parallelism across
|
|
123
|
+
.describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'tensor'` (EXPERIMENTAL tensor parallelism across selected local or RPC GPUs; requires flash attention, and disables auto-fit, so set `ctx_size` explicitly)."),
|
|
110
124
|
'flash-attn': z
|
|
111
125
|
.enum(['on', 'off', 'auto'])
|
|
112
126
|
.optional()
|
|
@@ -115,6 +129,98 @@ export const llmConfigBaseSchema = z.object({
|
|
|
115
129
|
.string()
|
|
116
130
|
.optional()
|
|
117
131
|
.describe("Proportions for distributing layers/rows across GPUs, e.g. `'1,1'` (equal) or `'3,1'` (75/25)."),
|
|
132
|
+
'batch-size': z
|
|
133
|
+
.number()
|
|
134
|
+
.int()
|
|
135
|
+
.min(1)
|
|
136
|
+
.optional()
|
|
137
|
+
.describe('Logical maximum batch size, the tokens submitted per decode call. Default 2048.'),
|
|
138
|
+
'ubatch-size': z
|
|
139
|
+
.number()
|
|
140
|
+
.int()
|
|
141
|
+
.min(1)
|
|
142
|
+
.optional()
|
|
143
|
+
.describe('Physical maximum batch size, the tokens computed in one pass. Default 512.'),
|
|
144
|
+
'cpu-moe': z
|
|
145
|
+
.boolean()
|
|
146
|
+
.optional()
|
|
147
|
+
.describe('Keep all Mixture-of-Experts weights on the CPU, freeing accelerator memory for the rest of the model. Mixture-of-Experts models only. Default false.'),
|
|
148
|
+
'n-cpu-moe': z
|
|
149
|
+
.number()
|
|
150
|
+
.int()
|
|
151
|
+
.min(0)
|
|
152
|
+
.optional()
|
|
153
|
+
.describe('Keep the Mixture-of-Experts weights of the first N layers on the CPU. Mixture-of-Experts models only, and a finer-grained alternative to `cpu-moe`.'),
|
|
154
|
+
'kv-offload': z
|
|
155
|
+
.boolean()
|
|
156
|
+
.optional()
|
|
157
|
+
.describe('Whether to hold the KV cache in accelerator memory. Default true. Set false to keep it in system memory, which frees accelerator memory at the cost of decode speed.'),
|
|
158
|
+
'image-max-tokens': z
|
|
159
|
+
.number()
|
|
160
|
+
.int()
|
|
161
|
+
.min(1)
|
|
162
|
+
.optional()
|
|
163
|
+
.describe('Upper bound on the tokens one image may occupy. Vision models only. Unset uses the model default.'),
|
|
164
|
+
'image-min-tokens': z
|
|
165
|
+
.number()
|
|
166
|
+
.int()
|
|
167
|
+
.min(1)
|
|
168
|
+
.optional()
|
|
169
|
+
.describe('Lower bound on the tokens one image may occupy. Vision models only. Unset uses the model default.'),
|
|
170
|
+
threads: z
|
|
171
|
+
.number()
|
|
172
|
+
.int()
|
|
173
|
+
.optional()
|
|
174
|
+
.describe('CPU threads to use during generation. Unset uses one thread per physical core; a value of 0 or below uses every logical CPU, which is a different and usually larger count.'),
|
|
175
|
+
'threads-batch': z
|
|
176
|
+
.number()
|
|
177
|
+
.int()
|
|
178
|
+
.optional()
|
|
179
|
+
.describe('CPU threads to use during batch and prompt processing. Setting it governs generation too and discards `threads` and `cpu-mask`; leave it unset for `threads` to apply to both.'),
|
|
180
|
+
'cpu-mask': z
|
|
181
|
+
.string()
|
|
182
|
+
.optional()
|
|
183
|
+
.describe("CPU affinity mask as an arbitrarily long hex string, e.g. `'ff'`. Unset applies no affinity. Ignored when `threads-batch` is set."),
|
|
184
|
+
'cpu-mask-batch': z
|
|
185
|
+
.string()
|
|
186
|
+
.optional()
|
|
187
|
+
.describe('CPU affinity mask for batch and prompt processing, in the same form as `cpu-mask`. Honoured only alongside `threads-batch`; without it `cpu-mask` takes over.'),
|
|
188
|
+
'override-tensor': z
|
|
189
|
+
.string()
|
|
190
|
+
.optional()
|
|
191
|
+
.describe("Buffer type overrides for tensors matching a pattern, as comma-separated `<tensor name pattern>=<buffer type>` entries, e.g. `'blk\\.(1[0-9])\\.ffn_(up|down|gate)_exps=CPU'` to keep those experts in system memory."),
|
|
192
|
+
'n-cpu-ffn': z
|
|
193
|
+
.number()
|
|
194
|
+
.int()
|
|
195
|
+
.min(0)
|
|
196
|
+
.optional()
|
|
197
|
+
.describe('Keep the dense FFN weights of the first N layers on the CPU.'),
|
|
198
|
+
'moe-cache-mib': z
|
|
199
|
+
.union([z.number().int().min(0), z.literal('auto')])
|
|
200
|
+
.optional()
|
|
201
|
+
.describe("Persistent accelerator cache for Mixture-of-Experts weights, in MiB. Default `'auto'`, which sizes it during `fit`."),
|
|
202
|
+
'prefetch-weights': z
|
|
203
|
+
.union([z.boolean(), z.literal('auto')])
|
|
204
|
+
.optional()
|
|
205
|
+
.describe("Overlap host-to-accelerator weight copies with compute. Default `'auto'`, which enables it for fitted dense models."),
|
|
206
|
+
'tensor-read-lazy': z
|
|
207
|
+
.enum(['on', 'auto', 'off'])
|
|
208
|
+
.optional()
|
|
209
|
+
.describe("On-demand reading of large tensors such as per-layer embeddings, which requires mmap: `'on'` reads their rows from disk instead of keeping them resident, `'off'` always keeps them resident, `'auto'` (default) applies `'on'` above 4 GiB."),
|
|
210
|
+
fit: z
|
|
211
|
+
.boolean()
|
|
212
|
+
.optional()
|
|
213
|
+
.describe('Adjust the arguments left unset so the model fits device memory. Default true. It chooses layer offload, tensor split and tensor buffer overrides, and the context size only when `ctx_size` is 0. Setting `gpu_layers` pins the layer count and makes fit abort, leaving every argument unchanged. Incompatible with `split-mode: tensor`, which disables it.'),
|
|
214
|
+
'fit-target': z
|
|
215
|
+
.union([z.number().int().min(0), z.string().regex(/^\d+(,\d+)*$/)])
|
|
216
|
+
.optional()
|
|
217
|
+
.describe("Free margin `fit` aims to leave per device, in MiB. A single value is broadcast across devices; a comma-separated list such as `'1024,512'` sets them individually. Default 1024."),
|
|
218
|
+
'fit-ctx': z
|
|
219
|
+
.number()
|
|
220
|
+
.int()
|
|
221
|
+
.min(0)
|
|
222
|
+
.optional()
|
|
223
|
+
.describe('Smallest context size `fit` may settle on. Default 4096.'),
|
|
118
224
|
openclCacheDir: z
|
|
119
225
|
.string()
|
|
120
226
|
.optional()
|
|
@@ -143,9 +249,13 @@ export const llmConfigBaseSchema = z.object({
|
|
|
143
249
|
.describe('Run the multimodal projector (mmproj / vision encoder) on the GPU (multimodal models only). `true` forces GPU, `false` forces CPU. When unset, the backend is auto-selected per device class: GPU on desktop/iOS and Android Adreno 800+; CPU on every other Android GPU (Arm Mali, Adreno <800, and undetectable Adreno tiers), with the LLM layers still on the GPU. Only honoured when the model itself runs on a GPU backend — ignored with a warning on CPU.')
|
|
144
250
|
});
|
|
145
251
|
// Default values - typed as partial of the config
|
|
252
|
+
//
|
|
253
|
+
// gpu_layers carries no default: any value reads as user intent to qvac-fabric,
|
|
254
|
+
// which then aborts its fit instead of placing layers to free device memory.
|
|
255
|
+
// Absent emits no `--gpu-layers`, so the addon keeps fabric's own -1 — already
|
|
256
|
+
// "every layer" whenever the fit does not run.
|
|
146
257
|
export const LLM_CONFIG_DEFAULTS = {
|
|
147
258
|
ctx_size: 1024,
|
|
148
|
-
gpu_layers: 99,
|
|
149
259
|
device: 'gpu',
|
|
150
260
|
system_prompt: 'You are a helpful assistant.',
|
|
151
261
|
image_tile_mode: 'sequential'
|