@qvac/inference 0.20.1 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +102 -40
- package/README.md +8 -5
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +8 -20
- package/dist/handlers/assess-model-fit.js +195 -77
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -11
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +37 -2
- package/dist/resources/model-fit/assess.js +24 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +20 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +126 -47
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +269 -85
- package/dist/schemas/assess-model-fit.js +78 -54
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +708 -66
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +104 -36
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +25 -18
- package/dist/schemas/llamacpp-config.js +26 -16
- package/dist/schemas/load-model.d.ts +2608 -469
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +183 -6
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +43 -29
- package/dist/resources/model-fit/fit-stub/index.d.ts +0 -2
- package/dist/resources/model-fit/fit-stub/index.js +0 -2
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +0 -31
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +0 -36
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -191
package/dist/schemas/error.js
CHANGED
|
@@ -34,6 +34,17 @@ function hasTypedFields(error) {
|
|
|
34
34
|
function isQvacError(error) {
|
|
35
35
|
return error instanceof QvacErrorBase;
|
|
36
36
|
}
|
|
37
|
+
function serializeCause(cause, depth = 0) {
|
|
38
|
+
if (!(cause instanceof Error))
|
|
39
|
+
return typeof cause === 'string' ? cause : String(cause);
|
|
40
|
+
return {
|
|
41
|
+
name: cause.name,
|
|
42
|
+
message: cause.message,
|
|
43
|
+
stack: cause.stack,
|
|
44
|
+
...('code' in cause && typeof cause.code === 'number' && { code: cause.code }),
|
|
45
|
+
...(cause.cause !== undefined && depth < 5 && { cause: serializeCause(cause.cause, depth + 1) })
|
|
46
|
+
};
|
|
47
|
+
}
|
|
37
48
|
export function createErrorResponse(error) {
|
|
38
49
|
if (isQvacError(error)) {
|
|
39
50
|
const qvacData = error.toJSON();
|
|
@@ -45,6 +56,8 @@ export function createErrorResponse(error) {
|
|
|
45
56
|
stack: qvacData.stack,
|
|
46
57
|
timestamp: new Date().toISOString()
|
|
47
58
|
};
|
|
59
|
+
if (qvacData.cause !== undefined)
|
|
60
|
+
response.cause = serializeCause(qvacData.cause);
|
|
48
61
|
if (hasTypedFields(error)) {
|
|
49
62
|
response.typedFields = error.toErrorResponseFields();
|
|
50
63
|
}
|
package/dist/schemas/errors.d.ts
CHANGED
|
@@ -28,6 +28,7 @@ export declare const ERROR_CODES: {
|
|
|
28
28
|
readonly TTS_REFERENCE_AUDIO_REQUIRED: 52209;
|
|
29
29
|
readonly LEGACY_PARAKEET_MODEL_DEPRECATED: 52210;
|
|
30
30
|
readonly LEGACY_TTS_MODEL_DEPRECATED: 52211;
|
|
31
|
+
readonly RPC_SERVER_OPERATION_FAILED: 52423;
|
|
31
32
|
readonly MODEL_UNLOAD_FAILED: 52400;
|
|
32
33
|
readonly EMBED_FAILED: 52401;
|
|
33
34
|
readonly EMBED_NO_EMBEDDINGS: 52402;
|
package/dist/schemas/errors.js
CHANGED
|
@@ -37,6 +37,7 @@ export const ERROR_CODES = {
|
|
|
37
37
|
LEGACY_PARAKEET_MODEL_DEPRECATED: 52210,
|
|
38
38
|
LEGACY_TTS_MODEL_DEPRECATED: 52211,
|
|
39
39
|
// Model operations (52,400-52,799)
|
|
40
|
+
RPC_SERVER_OPERATION_FAILED: 52423,
|
|
40
41
|
MODEL_UNLOAD_FAILED: 52400,
|
|
41
42
|
EMBED_FAILED: 52401,
|
|
42
43
|
EMBED_NO_EMBEDDINGS: 52402,
|
|
@@ -209,6 +210,10 @@ const errorDefinitions = {
|
|
|
209
210
|
name: 'UNKNOWN_MODEL_TYPE',
|
|
210
211
|
message: (modelType) => `Unknown model type: ${modelType}. Register the plugin for "${modelType}" in code with \`registerPlugin\` / \`plugins([...])\` before this call.`
|
|
211
212
|
},
|
|
213
|
+
[ERROR_CODES.RPC_SERVER_OPERATION_FAILED]: {
|
|
214
|
+
name: 'RPC_SERVER_OPERATION_FAILED',
|
|
215
|
+
message: (operation, details) => `${operation}: ${details}`
|
|
216
|
+
},
|
|
212
217
|
// Model loading
|
|
213
218
|
[ERROR_CODES.MODEL_LOAD_FAILED]: {
|
|
214
219
|
name: 'MODEL_LOAD_FAILED',
|
package/dist/schemas/index.d.ts
CHANGED
package/dist/schemas/index.js
CHANGED
|
@@ -47,15 +47,17 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
|
|
|
47
47
|
integrated: "integrated";
|
|
48
48
|
dedicated: "dedicated";
|
|
49
49
|
}>]>>;
|
|
50
|
+
'rpc-servers': z.ZodOptional<z.ZodString>;
|
|
51
|
+
devices: z.ZodOptional<z.ZodString>;
|
|
50
52
|
'split-mode': z.ZodOptional<z.ZodEnum<{
|
|
51
53
|
none: "none";
|
|
52
54
|
layer: "layer";
|
|
53
55
|
tensor: "tensor";
|
|
54
56
|
}>>;
|
|
55
57
|
'flash-attn': z.ZodOptional<z.ZodEnum<{
|
|
58
|
+
auto: "auto";
|
|
56
59
|
on: "on";
|
|
57
60
|
off: "off";
|
|
58
|
-
auto: "auto";
|
|
59
61
|
}>>;
|
|
60
62
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
61
63
|
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
@@ -74,9 +76,9 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
|
|
|
74
76
|
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
75
77
|
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
76
78
|
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
79
|
+
auto: "auto";
|
|
77
80
|
on: "on";
|
|
78
81
|
off: "off";
|
|
79
|
-
auto: "auto";
|
|
80
82
|
}>>;
|
|
81
83
|
fit: z.ZodOptional<z.ZodBoolean>;
|
|
82
84
|
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
@@ -136,7 +138,6 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
|
|
|
136
138
|
export type LlmConfigInput = z.infer<typeof llmConfigBaseSchema>;
|
|
137
139
|
export declare const LLM_CONFIG_DEFAULTS: {
|
|
138
140
|
readonly ctx_size: 1024;
|
|
139
|
-
readonly gpu_layers: 99;
|
|
140
141
|
readonly device: "gpu";
|
|
141
142
|
readonly system_prompt: "You are a helpful assistant.";
|
|
142
143
|
readonly image_tile_mode: "sequential";
|
|
@@ -177,15 +178,17 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
177
178
|
integrated: "integrated";
|
|
178
179
|
dedicated: "dedicated";
|
|
179
180
|
}>]>>;
|
|
181
|
+
'rpc-servers': z.ZodOptional<z.ZodString>;
|
|
182
|
+
devices: z.ZodOptional<z.ZodString>;
|
|
180
183
|
'split-mode': z.ZodOptional<z.ZodEnum<{
|
|
181
184
|
none: "none";
|
|
182
185
|
layer: "layer";
|
|
183
186
|
tensor: "tensor";
|
|
184
187
|
}>>;
|
|
185
188
|
'flash-attn': z.ZodOptional<z.ZodEnum<{
|
|
189
|
+
auto: "auto";
|
|
186
190
|
on: "on";
|
|
187
191
|
off: "off";
|
|
188
|
-
auto: "auto";
|
|
189
192
|
}>>;
|
|
190
193
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
191
194
|
'batch-size': z.ZodOptional<z.ZodNumber>;
|
|
@@ -204,9 +207,9 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
204
207
|
'moe-cache-mib': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodLiteral<"auto">]>>;
|
|
205
208
|
'prefetch-weights': z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodLiteral<"auto">]>>;
|
|
206
209
|
'tensor-read-lazy': z.ZodOptional<z.ZodEnum<{
|
|
210
|
+
auto: "auto";
|
|
207
211
|
on: "on";
|
|
208
212
|
off: "off";
|
|
209
|
-
auto: "auto";
|
|
210
213
|
}>>;
|
|
211
214
|
fit: z.ZodOptional<z.ZodBoolean>;
|
|
212
215
|
'fit-target': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
|
|
@@ -268,7 +271,7 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
268
271
|
top_p?: number | undefined;
|
|
269
272
|
top_k?: number | undefined;
|
|
270
273
|
seed?: number | undefined;
|
|
271
|
-
gpu_layers
|
|
274
|
+
gpu_layers?: number | undefined;
|
|
272
275
|
lora?: string | undefined;
|
|
273
276
|
device: string | undefined;
|
|
274
277
|
predict?: number | undefined;
|
|
@@ -284,8 +287,10 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
284
287
|
'cache-type-k'?: string | undefined;
|
|
285
288
|
'cache-type-v'?: string | undefined;
|
|
286
289
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
290
|
+
'rpc-servers'?: string | undefined;
|
|
291
|
+
devices?: string | undefined;
|
|
287
292
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
288
|
-
'flash-attn'?: "
|
|
293
|
+
'flash-attn'?: "auto" | "on" | "off" | undefined;
|
|
289
294
|
'tensor-split'?: string | undefined;
|
|
290
295
|
'batch-size'?: number | undefined;
|
|
291
296
|
'ubatch-size'?: number | undefined;
|
|
@@ -302,7 +307,7 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
302
307
|
'n-cpu-ffn'?: number | undefined;
|
|
303
308
|
'moe-cache-mib'?: number | "auto" | undefined;
|
|
304
309
|
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
305
|
-
'tensor-read-lazy'?: "
|
|
310
|
+
'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
|
|
306
311
|
fit?: boolean | undefined;
|
|
307
312
|
'fit-target'?: string | number | undefined;
|
|
308
313
|
'fit-ctx'?: number | undefined;
|
|
@@ -346,8 +351,10 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
346
351
|
'cache-type-k'?: string | undefined;
|
|
347
352
|
'cache-type-v'?: string | undefined;
|
|
348
353
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
354
|
+
'rpc-servers'?: string | undefined;
|
|
355
|
+
devices?: string | undefined;
|
|
349
356
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
350
|
-
'flash-attn'?: "
|
|
357
|
+
'flash-attn'?: "auto" | "on" | "off" | undefined;
|
|
351
358
|
'tensor-split'?: string | undefined;
|
|
352
359
|
'batch-size'?: number | undefined;
|
|
353
360
|
'ubatch-size'?: number | undefined;
|
|
@@ -364,7 +371,7 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
364
371
|
'n-cpu-ffn'?: number | undefined;
|
|
365
372
|
'moe-cache-mib'?: number | "auto" | undefined;
|
|
366
373
|
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
367
|
-
'tensor-read-lazy'?: "
|
|
374
|
+
'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
|
|
368
375
|
fit?: boolean | undefined;
|
|
369
376
|
'fit-target'?: string | number | undefined;
|
|
370
377
|
'fit-ctx'?: number | undefined;
|
|
@@ -391,8 +398,8 @@ export type LlmConfig = z.infer<typeof llmConfigSchema>;
|
|
|
391
398
|
export declare const embedConfigBaseSchema: z.ZodObject<{
|
|
392
399
|
gpuLayers: z.ZodOptional<z.ZodNumber>;
|
|
393
400
|
device: z.ZodOptional<z.ZodEnum<{
|
|
394
|
-
gpu: "gpu";
|
|
395
401
|
cpu: "cpu";
|
|
402
|
+
gpu: "gpu";
|
|
396
403
|
}>>;
|
|
397
404
|
batchSize: z.ZodOptional<z.ZodNumber>;
|
|
398
405
|
pooling: z.ZodOptional<z.ZodEnum<{
|
|
@@ -408,9 +415,9 @@ export declare const embedConfigBaseSchema: z.ZodObject<{
|
|
|
408
415
|
}>>;
|
|
409
416
|
embdNormalize: z.ZodOptional<z.ZodNumber>;
|
|
410
417
|
flashAttention: z.ZodOptional<z.ZodEnum<{
|
|
418
|
+
auto: "auto";
|
|
411
419
|
on: "on";
|
|
412
420
|
off: "off";
|
|
413
|
-
auto: "auto";
|
|
414
421
|
}>>;
|
|
415
422
|
mainGpu: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodEnum<{
|
|
416
423
|
integrated: "integrated";
|
|
@@ -438,8 +445,8 @@ export declare const EMBED_CONFIG_DEFAULTS: {
|
|
|
438
445
|
export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
439
446
|
gpuLayers: z.ZodOptional<z.ZodNumber>;
|
|
440
447
|
device: z.ZodOptional<z.ZodEnum<{
|
|
441
|
-
gpu: "gpu";
|
|
442
448
|
cpu: "cpu";
|
|
449
|
+
gpu: "gpu";
|
|
443
450
|
}>>;
|
|
444
451
|
batchSize: z.ZodOptional<z.ZodNumber>;
|
|
445
452
|
pooling: z.ZodOptional<z.ZodEnum<{
|
|
@@ -455,9 +462,9 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
455
462
|
}>>;
|
|
456
463
|
embdNormalize: z.ZodOptional<z.ZodNumber>;
|
|
457
464
|
flashAttention: z.ZodOptional<z.ZodEnum<{
|
|
465
|
+
auto: "auto";
|
|
458
466
|
on: "on";
|
|
459
467
|
off: "off";
|
|
460
|
-
auto: "auto";
|
|
461
468
|
}>>;
|
|
462
469
|
mainGpu: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodEnum<{
|
|
463
470
|
integrated: "integrated";
|
|
@@ -477,12 +484,12 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
477
484
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
478
485
|
}, z.core.$strip>, z.ZodTransform<{
|
|
479
486
|
gpuLayers: number | undefined;
|
|
480
|
-
device: "
|
|
487
|
+
device: "cpu" | "gpu" | undefined;
|
|
481
488
|
batchSize: number | undefined;
|
|
482
489
|
pooling?: "none" | "mean" | "cls" | "last" | "rank" | undefined;
|
|
483
490
|
attention?: "causal" | "non-causal" | undefined;
|
|
484
491
|
embdNormalize?: number | undefined;
|
|
485
|
-
flashAttention?: "
|
|
492
|
+
flashAttention?: "auto" | "on" | "off" | undefined;
|
|
486
493
|
mainGpu?: number | "integrated" | "dedicated" | undefined;
|
|
487
494
|
splitMode?: "none" | "layer" | undefined;
|
|
488
495
|
tensorSplit?: string | undefined;
|
|
@@ -490,12 +497,12 @@ export declare const embedConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
490
497
|
openclCacheDir?: string | undefined;
|
|
491
498
|
}, {
|
|
492
499
|
gpuLayers?: number | undefined;
|
|
493
|
-
device?: "
|
|
500
|
+
device?: "cpu" | "gpu" | undefined;
|
|
494
501
|
batchSize?: number | undefined;
|
|
495
502
|
pooling?: "none" | "mean" | "cls" | "last" | "rank" | undefined;
|
|
496
503
|
attention?: "causal" | "non-causal" | undefined;
|
|
497
504
|
embdNormalize?: number | undefined;
|
|
498
|
-
flashAttention?: "
|
|
505
|
+
flashAttention?: "auto" | "on" | "off" | undefined;
|
|
499
506
|
mainGpu?: number | "integrated" | "dedicated" | undefined;
|
|
500
507
|
splitMode?: "none" | "layer" | undefined;
|
|
501
508
|
tensorSplit?: string | undefined;
|
|
@@ -37,7 +37,7 @@ export const llmConfigBaseSchema = z.object({
|
|
|
37
37
|
gpu_layers: z
|
|
38
38
|
.number()
|
|
39
39
|
.optional()
|
|
40
|
-
.describe('Number of model layers to offload to the GPU.
|
|
40
|
+
.describe('Number of model layers to offload to the GPU. Unset by default, which lets the runtime fit the placement to free device memory (offloading every layer when it fits). Setting it pins the layer count and disables that fit.'),
|
|
41
41
|
lora: z
|
|
42
42
|
.string()
|
|
43
43
|
.optional()
|
|
@@ -103,10 +103,24 @@ export const llmConfigBaseSchema = z.object({
|
|
|
103
103
|
.union([z.number().int().min(0), z.enum(['integrated', 'dedicated'])])
|
|
104
104
|
.optional()
|
|
105
105
|
.describe("GPU to use on multi-GPU systems: a device index, or `'integrated'`/`'dedicated'` to restrict selection to that class."),
|
|
106
|
+
'rpc-servers': z
|
|
107
|
+
.string()
|
|
108
|
+
.min(1)
|
|
109
|
+
.refine((value) => {
|
|
110
|
+
const endpoints = value.split(',').map((endpoint) => endpoint.trim());
|
|
111
|
+
return new Set(endpoints).size === endpoints.length;
|
|
112
|
+
}, 'RPC server endpoints must be unique')
|
|
113
|
+
.optional()
|
|
114
|
+
.describe('Comma-separated unique native RPC host:port endpoints in device registration order for this load. Requires compatible RPC addon builds and a trusted private network; native traffic is unencrypted and unauthenticated.'),
|
|
115
|
+
devices: z
|
|
116
|
+
.string()
|
|
117
|
+
.min(1)
|
|
118
|
+
.optional()
|
|
119
|
+
.describe("Explicit ordered native device names, for example RPC0,RPC1. RPC indices enumerate every device in this load's rpc-servers endpoint order, starting at RPC0 for each load. Pass exactly those endpoints in that order to getRpcDeviceMap, including on reused workers. tensor-split weights follow the selected devices order."),
|
|
106
120
|
'split-mode': z
|
|
107
121
|
.enum(['none', 'layer', 'tensor'])
|
|
108
122
|
.optional()
|
|
109
|
-
.describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'tensor'` (EXPERIMENTAL tensor parallelism across
|
|
123
|
+
.describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'tensor'` (EXPERIMENTAL tensor parallelism across selected local or RPC GPUs; requires flash attention, and disables auto-fit, so set `ctx_size` explicitly)."),
|
|
110
124
|
'flash-attn': z
|
|
111
125
|
.enum(['on', 'off', 'auto'])
|
|
112
126
|
.optional()
|
|
@@ -196,7 +210,7 @@ export const llmConfigBaseSchema = z.object({
|
|
|
196
210
|
fit: z
|
|
197
211
|
.boolean()
|
|
198
212
|
.optional()
|
|
199
|
-
.describe('Adjust the arguments left unset so the model fits device memory. Default true. It chooses layer offload, tensor split and tensor buffer overrides, and the context size only when `ctx_size` is 0. Setting
|
|
213
|
+
.describe('Adjust the arguments left unset so the model fits device memory. Default true. It chooses layer offload, tensor split and tensor buffer overrides, and the context size only when `ctx_size` is 0. Setting `gpu_layers` pins the layer count and makes fit abort, leaving every argument unchanged. Incompatible with `split-mode: tensor`, which disables it.'),
|
|
200
214
|
'fit-target': z
|
|
201
215
|
.union([z.number().int().min(0), z.string().regex(/^\d+(,\d+)*$/)])
|
|
202
216
|
.optional()
|
|
@@ -235,26 +249,22 @@ export const llmConfigBaseSchema = z.object({
|
|
|
235
249
|
.describe('Run the multimodal projector (mmproj / vision encoder) on the GPU (multimodal models only). `true` forces GPU, `false` forces CPU. When unset, the backend is auto-selected per device class: GPU on desktop/iOS and Android Adreno 800+; CPU on every other Android GPU (Arm Mali, Adreno <800, and undetectable Adreno tiers), with the LLM layers still on the GPU. Only honoured when the model itself runs on a GPU backend — ignored with a warning on CPU.')
|
|
236
250
|
});
|
|
237
251
|
// Default values - typed as partial of the config
|
|
252
|
+
//
|
|
253
|
+
// gpu_layers carries no default: any value reads as user intent to qvac-fabric,
|
|
254
|
+
// which then aborts its fit instead of placing layers to free device memory.
|
|
255
|
+
// Absent emits no `--gpu-layers`, so the addon keeps fabric's own -1 — already
|
|
256
|
+
// "every layer" whenever the fit does not run.
|
|
238
257
|
export const LLM_CONFIG_DEFAULTS = {
|
|
239
258
|
ctx_size: 1024,
|
|
240
|
-
gpu_layers: 99,
|
|
241
259
|
device: 'gpu',
|
|
242
260
|
system_prompt: 'You are a helpful assistant.',
|
|
243
261
|
image_tile_mode: 'sequential'
|
|
244
262
|
};
|
|
245
|
-
/**
|
|
246
|
-
* `llama_model_default_params().n_gpu_layers`. Fabric's fit aborts and rolls
|
|
247
|
-
* back every argument it had computed when `n_gpu_layers` holds any other
|
|
248
|
-
* value, so a caller opting into `fit` gets this in place of the default above.
|
|
249
|
-
*/
|
|
250
|
-
const LLAMA_GPU_LAYERS_DEFAULT = -1;
|
|
251
263
|
// Full schema - applies defaults via transform (no duplication)
|
|
252
|
-
export const llmConfigSchema = llmConfigBaseSchema.transform((data) => {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
: merged;
|
|
257
|
-
});
|
|
264
|
+
export const llmConfigSchema = llmConfigBaseSchema.transform((data) => ({
|
|
265
|
+
...LLM_CONFIG_DEFAULTS,
|
|
266
|
+
...data
|
|
267
|
+
}));
|
|
258
268
|
// Base schema - validates types, all fields optional (for input validation)
|
|
259
269
|
export const embedConfigBaseSchema = z.object({
|
|
260
270
|
gpuLayers: z
|