@qvac/inference 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +107 -33
- package/README.md +8 -5
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +18 -6
- package/dist/handlers/assess-model-fit.js +215 -35
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +80 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +46 -2
- package/dist/resources/model-fit/assess.js +67 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +102 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +212 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +284 -77
- package/dist/schemas/assess-model-fit.js +93 -55
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +765 -56
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +236 -30
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +105 -14
- package/dist/schemas/llamacpp-config.js +113 -3
- package/dist/schemas/load-model.d.ts +3102 -627
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +205 -5
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +44 -30
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -171
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { assessFit as audiogenAssessFit } from '@qvac/audiogen-ggml';
|
|
1
2
|
import { type CreateModelParams, type PluginModelResult } from '../../../schemas/index.js';
|
|
2
3
|
import { resolveAudioGenConfig } from './config.js';
|
|
3
4
|
export declare const audioGenPlugin: {
|
|
@@ -254,6 +255,7 @@ export declare const audioGenPlugin: {
|
|
|
254
255
|
}>, import("zod").ZodLiteral<"vad">]>>;
|
|
255
256
|
}, import("zod/v4/core").$strip>]>;
|
|
256
257
|
}, import("zod/v4/core").$strict>], "engine">;
|
|
258
|
+
assessFit: typeof audiogenAssessFit;
|
|
257
259
|
skipPrimaryModelPathValidation: true;
|
|
258
260
|
resolveConfig: typeof resolveAudioGenConfig;
|
|
259
261
|
createModel(params: CreateModelParams): PluginModelResult;
|
|
@@ -361,8 +363,8 @@ export declare const audioGenPlugin: {
|
|
|
361
363
|
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
362
364
|
selectedBackend: import("zod").ZodString;
|
|
363
365
|
selectedDevice: import("zod").ZodEnum<{
|
|
364
|
-
gpu: "gpu";
|
|
365
366
|
cpu: "cpu";
|
|
367
|
+
gpu: "gpu";
|
|
366
368
|
}>;
|
|
367
369
|
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
368
370
|
vulkan: "vulkan";
|
|
@@ -384,8 +386,8 @@ export declare const audioGenPlugin: {
|
|
|
384
386
|
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
385
387
|
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
386
388
|
requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
387
|
-
gpu: "gpu";
|
|
388
389
|
cpu: "cpu";
|
|
390
|
+
gpu: "gpu";
|
|
389
391
|
}>>;
|
|
390
392
|
reason: import("zod").ZodString;
|
|
391
393
|
}, import("zod/v4/core").$strip>>;
|
|
@@ -476,8 +478,8 @@ export declare const audioGenPlugin: {
|
|
|
476
478
|
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
477
479
|
selectedBackend: import("zod").ZodString;
|
|
478
480
|
selectedDevice: import("zod").ZodEnum<{
|
|
479
|
-
gpu: "gpu";
|
|
480
481
|
cpu: "cpu";
|
|
482
|
+
gpu: "gpu";
|
|
481
483
|
}>;
|
|
482
484
|
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
483
485
|
vulkan: "vulkan";
|
|
@@ -499,8 +501,8 @@ export declare const audioGenPlugin: {
|
|
|
499
501
|
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
500
502
|
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
501
503
|
requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
502
|
-
gpu: "gpu";
|
|
503
504
|
cpu: "cpu";
|
|
505
|
+
gpu: "gpu";
|
|
504
506
|
}>>;
|
|
505
507
|
reason: import("zod").ZodString;
|
|
506
508
|
}, import("zod/v4/core").$strip>>;
|
|
@@ -575,8 +577,8 @@ export declare const audioGenPlugin: {
|
|
|
575
577
|
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
576
578
|
selectedBackend: import("zod").ZodString;
|
|
577
579
|
selectedDevice: import("zod").ZodEnum<{
|
|
578
|
-
gpu: "gpu";
|
|
579
580
|
cpu: "cpu";
|
|
581
|
+
gpu: "gpu";
|
|
580
582
|
}>;
|
|
581
583
|
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
582
584
|
vulkan: "vulkan";
|
|
@@ -598,8 +600,8 @@ export declare const audioGenPlugin: {
|
|
|
598
600
|
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
599
601
|
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
600
602
|
requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
601
|
-
gpu: "gpu";
|
|
602
603
|
cpu: "cpu";
|
|
604
|
+
gpu: "gpu";
|
|
603
605
|
}>>;
|
|
604
606
|
reason: import("zod").ZodString;
|
|
605
607
|
}, import("zod/v4/core").$strip>>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AudioGen, ENGINE_ACESTEP, ENGINE_MINIMAX } from '@qvac/audiogen-ggml';
|
|
1
|
+
import { AudioGen, ENGINE_ACESTEP, ENGINE_MINIMAX, assessFit as audiogenAssessFit } from '@qvac/audiogen-ggml';
|
|
2
2
|
import { ADDON_AUDIOGEN, ModelType, audioEditStreamRequestSchema, audioEditStreamResponseSchema, audioGenConfigSchema, audioGenStreamRequestSchema, audioGenStreamResponseSchema, audioUnderstandRequestSchema, audioUnderstandResponseSchema, defineHandler, definePlugin } from '../../../schemas/index.js';
|
|
3
3
|
import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
|
|
4
4
|
import { resolveAudioGenConfig } from './config.js';
|
|
@@ -11,6 +11,7 @@ export const audioGenPlugin = definePlugin({
|
|
|
11
11
|
displayName: 'Audio Generation (GGML / ACE-Step and MiniMax)',
|
|
12
12
|
addonPackage: ADDON_AUDIOGEN,
|
|
13
13
|
loadConfigSchema: audioGenConfigSchema,
|
|
14
|
+
assessFit: audiogenAssessFit,
|
|
14
15
|
// AudioGen's primary `modelSrc` is intentionally empty: all required
|
|
15
16
|
// weights are config-owned artifacts (ACE-Step: four sources; MiniMax: LM + synth).
|
|
16
17
|
skipPrimaryModelPathValidation: true,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { assessFit as bciAssessFit } from '@qvac/bci-whispercpp';
|
|
1
2
|
import { type CreateModelParams, type PluginModelResult } from '../../../schemas/index.js';
|
|
2
3
|
import { resolveBciConfig } from './resolve-config.js';
|
|
3
4
|
export declare const bciPlugin: {
|
|
@@ -76,6 +77,7 @@ export declare const bciPlugin: {
|
|
|
76
77
|
}>, import("zod").ZodLiteral<"vad">]>>;
|
|
77
78
|
}, import("zod/v4/core").$strip>]>>;
|
|
78
79
|
}, import("zod/v4/core").$strip>;
|
|
80
|
+
assessFit: typeof bciAssessFit;
|
|
79
81
|
resolveConfig: typeof resolveBciConfig;
|
|
80
82
|
createModel(params: CreateModelParams): PluginModelResult;
|
|
81
83
|
handlers: {
|
|
@@ -110,6 +112,18 @@ export declare const bciPlugin: {
|
|
|
110
112
|
gpuUnsupported: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
111
113
|
gpuMemTotalMb: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
112
114
|
gpuMemFreeMb: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
115
|
+
totalTime: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
116
|
+
totalWallMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
117
|
+
totalSamples: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
118
|
+
processCalls: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
119
|
+
whisperSampleMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
120
|
+
whisperBatchdMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
121
|
+
whisperPromptMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
122
|
+
totalTranscriptions: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
123
|
+
modelLoadMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
124
|
+
totalEncodedFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
125
|
+
encoderOnCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
126
|
+
encoderUsedCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
113
127
|
}, import("zod/v4/core").$strip>>;
|
|
114
128
|
error: import("zod").ZodOptional<import("zod").ZodString>;
|
|
115
129
|
segment: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
@@ -118,8 +132,51 @@ export declare const bciPlugin: {
|
|
|
118
132
|
endMs: import("zod").ZodNumber;
|
|
119
133
|
append: import("zod").ZodBoolean;
|
|
120
134
|
id: import("zod").ZodNumber;
|
|
135
|
+
isEndOfTurn: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
136
|
+
startsWord: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
121
137
|
}, import("zod/v4/core").$strip>>;
|
|
122
138
|
type: import("zod").ZodLiteral<"bciTranscribe">;
|
|
139
|
+
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
140
|
+
selectedBackend: import("zod").ZodString;
|
|
141
|
+
selectedDevice: import("zod").ZodEnum<{
|
|
142
|
+
cpu: "cpu";
|
|
143
|
+
gpu: "gpu";
|
|
144
|
+
}>;
|
|
145
|
+
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
146
|
+
vulkan: "vulkan";
|
|
147
|
+
metal: "metal";
|
|
148
|
+
opencl: "opencl";
|
|
149
|
+
cuda: "cuda";
|
|
150
|
+
opengl: "opengl";
|
|
151
|
+
webgpu: "webgpu";
|
|
152
|
+
direct3d11: "direct3d11";
|
|
153
|
+
direct3d12: "direct3d12";
|
|
154
|
+
levelZero: "levelZero";
|
|
155
|
+
rocm: "rocm";
|
|
156
|
+
}>>;
|
|
157
|
+
driver: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
158
|
+
name: import("zod").ZodString;
|
|
159
|
+
version: import("zod").ZodOptional<import("zod").ZodString>;
|
|
160
|
+
}, import("zod/v4/core").$strip>>;
|
|
161
|
+
gpuId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
162
|
+
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
163
|
+
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
164
|
+
requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
165
|
+
cpu: "cpu";
|
|
166
|
+
gpu: "gpu";
|
|
167
|
+
}>>;
|
|
168
|
+
reason: import("zod").ZodString;
|
|
169
|
+
}, import("zod/v4/core").$strip>>;
|
|
170
|
+
probe: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
171
|
+
status: import("zod").ZodEnum<{
|
|
172
|
+
unknown: "unknown";
|
|
173
|
+
compatible: "compatible";
|
|
174
|
+
incompatible: "incompatible";
|
|
175
|
+
}>;
|
|
176
|
+
backend: import("zod").ZodString;
|
|
177
|
+
reason: import("zod").ZodOptional<import("zod").ZodString>;
|
|
178
|
+
}, import("zod/v4/core").$strip>>;
|
|
179
|
+
}, import("zod/v4/core").$strip>>;
|
|
123
180
|
}, import("zod/v4/core").$strip>>;
|
|
124
181
|
bciTranscribeStream: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
|
|
125
182
|
modelId: import("zod").ZodString;
|
|
@@ -153,16 +210,31 @@ export declare const bciPlugin: {
|
|
|
153
210
|
gpuUnsupported: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
154
211
|
gpuMemTotalMb: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
155
212
|
gpuMemFreeMb: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
213
|
+
totalTime: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
214
|
+
totalWallMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
215
|
+
totalSamples: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
216
|
+
processCalls: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
217
|
+
whisperSampleMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
218
|
+
whisperBatchdMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
219
|
+
whisperPromptMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
220
|
+
totalTranscriptions: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
221
|
+
modelLoadMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
222
|
+
totalEncodedFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
223
|
+
encoderOnCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
224
|
+
encoderUsedCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
156
225
|
}, import("zod/v4/core").$strip>>;
|
|
157
226
|
error: import("zod").ZodOptional<import("zod").ZodString>;
|
|
227
|
+
type: import("zod").ZodLiteral<"bciTranscribeStream">;
|
|
158
228
|
segment: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
159
229
|
text: import("zod").ZodString;
|
|
160
230
|
startMs: import("zod").ZodNumber;
|
|
161
231
|
endMs: import("zod").ZodNumber;
|
|
162
232
|
append: import("zod").ZodBoolean;
|
|
163
233
|
id: import("zod").ZodNumber;
|
|
234
|
+
isEndOfTurn: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
235
|
+
startsWord: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
236
|
+
windowStartTimestep: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
164
237
|
}, import("zod/v4/core").$strip>>;
|
|
165
|
-
type: import("zod").ZodLiteral<"bciTranscribeStream">;
|
|
166
238
|
}, import("zod/v4/core").$strip>>;
|
|
167
239
|
};
|
|
168
240
|
logging: {
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import BCIWhispercpp from '@qvac/bci-whispercpp';
|
|
1
|
+
import BCIWhispercpp, { assessFit as bciAssessFit } from '@qvac/bci-whispercpp';
|
|
2
2
|
import { definePlugin, defineHandler, defineDuplexHandler, bciTranscribeRequestSchema, bciTranscribeResponseSchema, bciTranscribeStreamRequestSchema, bciTranscribeStreamResponseSchema, ModelType, bciConfigSchema, ADDON_BCI } from '../../../schemas/index.js';
|
|
3
3
|
import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
|
|
4
4
|
import { bciTranscribe, bciTranscribeStream } from '../../ops/bci-transcribe.js';
|
|
5
5
|
import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
|
|
6
|
+
import { attachBackendDiagnostics } from '../../../profiling/backend-diagnostics.js';
|
|
6
7
|
import { buildBciWhispercppArgs } from './args.js';
|
|
7
8
|
import { resolveBciConfig } from './resolve-config.js';
|
|
8
9
|
function createBciModel(modelId, modelPath, bciConfig, embedderPath) {
|
|
@@ -17,6 +18,7 @@ export const bciPlugin = definePlugin({
|
|
|
17
18
|
displayName: 'BCI (whisper.cpp)',
|
|
18
19
|
addonPackage: ADDON_BCI,
|
|
19
20
|
loadConfigSchema: bciConfigSchema,
|
|
21
|
+
assessFit: bciAssessFit,
|
|
20
22
|
resolveConfig: resolveBciConfig,
|
|
21
23
|
createModel(params) {
|
|
22
24
|
const bciConfig = (params.modelConfig ?? {});
|
|
@@ -57,13 +59,17 @@ export const bciPlugin = definePlugin({
|
|
|
57
59
|
};
|
|
58
60
|
result = await stream.next();
|
|
59
61
|
}
|
|
60
|
-
const { modelExecutionMs, stats } = result.value;
|
|
61
|
-
|
|
62
|
+
const { modelExecutionMs, stats, diagnostics } = result.value;
|
|
63
|
+
// The field is what reaches an RPC client; the symbol is what the
|
|
64
|
+
// profiling layer reads to set `event.backend`, as audiogen does.
|
|
65
|
+
const terminal = attachModelExecutionMs({
|
|
62
66
|
type: 'bciTranscribe',
|
|
63
67
|
text: '',
|
|
64
68
|
done: true,
|
|
65
|
-
...(stats && { stats })
|
|
69
|
+
...(stats && { stats }),
|
|
70
|
+
...(diagnostics && { diagnostics })
|
|
66
71
|
}, modelExecutionMs);
|
|
72
|
+
yield diagnostics ? attachBackendDiagnostics(terminal, diagnostics) : terminal;
|
|
67
73
|
}
|
|
68
74
|
finally {
|
|
69
75
|
await stream.return?.(undefined);
|
|
@@ -84,22 +90,34 @@ export const bciPlugin = definePlugin({
|
|
|
84
90
|
const iterator = metadata
|
|
85
91
|
? bciTranscribeStream(request.modelId, inputStream, true, request.streamOpts, request.requestId)
|
|
86
92
|
: bciTranscribeStream(request.modelId, inputStream, false, request.streamOpts, request.requestId);
|
|
87
|
-
for await
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
93
|
+
// Iterated by hand rather than with `for await` so the generator's
|
|
94
|
+
// return value survives: it carries the execution time the profiling
|
|
95
|
+
// layer reads off the terminal frame.
|
|
96
|
+
try {
|
|
97
|
+
let result = await iterator.next();
|
|
98
|
+
while (!result.done) {
|
|
99
|
+
yield metadata
|
|
100
|
+
? {
|
|
101
|
+
type: 'bciTranscribeStream',
|
|
102
|
+
segment: result.value
|
|
103
|
+
}
|
|
104
|
+
: {
|
|
105
|
+
type: 'bciTranscribeStream',
|
|
106
|
+
text: result.value
|
|
107
|
+
};
|
|
108
|
+
result = await iterator.next();
|
|
109
|
+
}
|
|
110
|
+
// The addon reports no stats for a stream, so there is no backend
|
|
111
|
+
// verdict to attach here — only the timing.
|
|
112
|
+
yield attachModelExecutionMs({
|
|
113
|
+
type: 'bciTranscribeStream',
|
|
114
|
+
text: '',
|
|
115
|
+
done: true
|
|
116
|
+
}, result.value.modelExecutionMs);
|
|
117
|
+
}
|
|
118
|
+
finally {
|
|
119
|
+
await iterator.return?.(undefined);
|
|
97
120
|
}
|
|
98
|
-
yield {
|
|
99
|
-
type: 'bciTranscribeStream',
|
|
100
|
-
text: '',
|
|
101
|
-
done: true
|
|
102
|
-
};
|
|
103
121
|
}
|
|
104
122
|
})
|
|
105
123
|
},
|
|
@@ -5,8 +5,8 @@ export declare const vlaPlugin: {
|
|
|
5
5
|
addonPackage: "@qvac/vla-ggml";
|
|
6
6
|
loadConfigSchema: import("zod").ZodObject<{
|
|
7
7
|
backend: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
8
|
-
auto: "auto";
|
|
9
8
|
cpu: "cpu";
|
|
9
|
+
auto: "auto";
|
|
10
10
|
}>>;
|
|
11
11
|
verbosity: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
12
12
|
embodiment: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodNumber, import("zod").ZodObject<{
|
|
@@ -6,6 +6,7 @@ import { getResponseFormatJsonSchema } from '../../../../utils/response-format.j
|
|
|
6
6
|
import { seedConfiguredSystemPrompt, transformMessages } from './completion-stream.js';
|
|
7
7
|
import { normalizeCompletionStats } from './completion-stats.js';
|
|
8
8
|
import { prependToolsToHistory } from '../../../../utils/tool-integration.js';
|
|
9
|
+
import { resolveDeferredTools, toWireTool, withDeferredToolChoice } from '../../../../utils/tools/defer.js';
|
|
9
10
|
const logger = getEngineLogger();
|
|
10
11
|
function runBatchModel(model, prompts) {
|
|
11
12
|
const run = model.run.bind(model);
|
|
@@ -22,9 +23,10 @@ function mergeGenerationParams(generationParams, responseFormat) {
|
|
|
22
23
|
json_schema: jsonSchema
|
|
23
24
|
};
|
|
24
25
|
}
|
|
25
|
-
function renderPromptHistory(prompt, options) {
|
|
26
|
-
const tools = options.toolsEnabled && prompt.tools && prompt.tools.length > 0
|
|
27
|
-
|
|
26
|
+
function renderPromptHistory(prompt, history, deferred, options) {
|
|
27
|
+
const tools = options.toolsEnabled && prompt.tools && prompt.tools.length > 0
|
|
28
|
+
? (deferred?.toolsToRender ?? prompt.tools.map(toWireTool))
|
|
29
|
+
: undefined;
|
|
28
30
|
let historyWithTools = history;
|
|
29
31
|
if (tools) {
|
|
30
32
|
historyWithTools = prependToolsToHistory(history, tools);
|
|
@@ -34,10 +36,12 @@ function renderPromptHistory(prompt, options) {
|
|
|
34
36
|
return transformMessages(historyWithTools);
|
|
35
37
|
}
|
|
36
38
|
function buildBatchPrompt(prompt, options) {
|
|
37
|
-
const
|
|
39
|
+
const history = seedConfiguredSystemPrompt(prompt.history, options.modelConfig);
|
|
40
|
+
const deferred = resolveDeferredTools(prompt.tools, history);
|
|
41
|
+
const mergedGenerationParams = withDeferredToolChoice(mergeGenerationParams(prompt.generationParams, prompt.responseFormat), options.toolsEnabled ? deferred : null);
|
|
38
42
|
return {
|
|
39
43
|
...(prompt.id !== undefined && { id: prompt.id }),
|
|
40
|
-
prompt: renderPromptHistory(prompt, options),
|
|
44
|
+
prompt: renderPromptHistory(prompt, history, deferred, options),
|
|
41
45
|
...(mergedGenerationParams && {
|
|
42
46
|
runOptions: { generationParams: mergedGenerationParams }
|
|
43
47
|
})
|
|
@@ -10,7 +10,7 @@ export declare function normalizeCompletionStats(stats: LlmStats | undefined): {
|
|
|
10
10
|
emittedTokens?: number | undefined;
|
|
11
11
|
avgConcurrentSeq?: number | undefined;
|
|
12
12
|
toolDefinitionsDropped?: number | undefined;
|
|
13
|
-
backendDevice?: "
|
|
13
|
+
backendDevice?: "cpu" | "gpu" | undefined;
|
|
14
14
|
} | undefined;
|
|
15
15
|
/**
|
|
16
16
|
* Attach the count of non-empty pieces the addon streamed without
|
|
@@ -4,6 +4,7 @@ import { getModel, getModelConfig } from '../../../../runtime/model-registry.js'
|
|
|
4
4
|
import { decideCachedHistorySlice, shouldCommitCachedTurn } from './kv-cache-state.js';
|
|
5
5
|
import { createKvCacheSession, generateConfigHash } from './kv-cache-session.js';
|
|
6
6
|
import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
|
|
7
|
+
import { resolveDeferredTools, toWireTool, withDeferredToolChoice } from '../../../../utils/tools/defer.js';
|
|
7
8
|
import { parseToolCalls } from '../../../../utils/tools/index.js';
|
|
8
9
|
import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
|
|
9
10
|
import { toolChoiceDemandsCall } from '../../../../schemas/completion-stream.js';
|
|
@@ -209,7 +210,14 @@ export async function* completion(params, opts) {
|
|
|
209
210
|
const modelConfig = getModelConfig(modelId);
|
|
210
211
|
const history = seedConfiguredSystemPrompt(requestHistory, modelConfig);
|
|
211
212
|
const toolsEnabled = modelConfig.tools === true;
|
|
212
|
-
|
|
213
|
+
// With deferred tools the prompt and the parser see different lists: the
|
|
214
|
+
// prompt carries the always-loaded ones plus `tool_search` and its catalog,
|
|
215
|
+
// while the parser also accepts whatever earlier searches appended to the
|
|
216
|
+
// history. `null` when nothing defers, which leaves the existing path alone.
|
|
217
|
+
const deferred = resolveDeferredTools(tools, history);
|
|
218
|
+
const toolsToRender = deferred?.toolsToRender ?? tools?.map(toWireTool);
|
|
219
|
+
const callableTools = deferred?.callableTools ?? toolsToRender;
|
|
220
|
+
const toolsActive = !!toolsToRender?.length && toolsEnabled;
|
|
213
221
|
const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
|
|
214
222
|
// `responseFormat` is forwarded to the addon as a per-request
|
|
215
223
|
// `generationParams.json_schema`, which the addon converts to GBNF and
|
|
@@ -228,6 +236,7 @@ export async function* completion(params, opts) {
|
|
|
228
236
|
};
|
|
229
237
|
}
|
|
230
238
|
}
|
|
239
|
+
mergedGenerationParams = withDeferredToolChoice(mergedGenerationParams, toolsActive ? deferred : null);
|
|
231
240
|
const model = getModel(modelId);
|
|
232
241
|
// Per-request hard cancel: under continuous batching the model runs several
|
|
233
242
|
// jobs at once, so an abort must cancel only THIS request's job, not the
|
|
@@ -261,13 +270,13 @@ export async function* completion(params, opts) {
|
|
|
261
270
|
if (!kvCache) {
|
|
262
271
|
// KV-cache disabled — straight passthrough, no session involvement.
|
|
263
272
|
let historyWithTools = history;
|
|
264
|
-
if (toolsActive &&
|
|
265
|
-
historyWithTools = prependToolsToHistory(history,
|
|
273
|
+
if (toolsActive && toolsToRender) {
|
|
274
|
+
historyWithTools = prependToolsToHistory(history, toolsToRender);
|
|
266
275
|
}
|
|
267
276
|
const transformedHistory = transformMessages(historyWithTools);
|
|
268
277
|
logCacheDisabled();
|
|
269
278
|
logMessagesToAddon(transformedHistory, 'NO_CACHE');
|
|
270
|
-
return yield* processModelResponse(model, transformedHistory,
|
|
279
|
+
return yield* processModelResponse(model, transformedHistory, callableTools, mergedGenerationParams, undefined, dialect, setActiveResponse);
|
|
271
280
|
}
|
|
272
281
|
// ---- KV-cache path. The session owns every bookkeeping layer; the handler
|
|
273
282
|
// registers one deferred unwind that `commitTurn` short-circuits on the happy
|
|
@@ -281,7 +290,7 @@ export async function* completion(params, opts) {
|
|
|
281
290
|
// The tool block is baked into the cache on the turn that first sends it and
|
|
282
291
|
// never trimmed, so a late or changed tool set has to land on a fresh cache
|
|
283
292
|
// rather than a warm prefix holding the old block.
|
|
284
|
-
const configHash = generateConfigHash(systemPromptFromHistory, toolsActive ?
|
|
293
|
+
const configHash = generateConfigHash(systemPromptFromHistory, toolsActive ? toolsToRender : undefined);
|
|
285
294
|
let turn;
|
|
286
295
|
if (typeof kvCache === 'string') {
|
|
287
296
|
turn = await session.beginTurn({
|
|
@@ -315,7 +324,7 @@ export async function* completion(params, opts) {
|
|
|
315
324
|
scope.defer(() => (preserveCacheOnUnwind ? session.releaseTurn(turn) : session.rollback(turn)));
|
|
316
325
|
let payload;
|
|
317
326
|
try {
|
|
318
|
-
payload = await prepareMessagesForCache(session, turn, history, toolsActive ?
|
|
327
|
+
payload = await prepareMessagesForCache(session, turn, history, toolsActive ? toolsToRender : undefined, mergedGenerationParams?.tool_choice);
|
|
319
328
|
}
|
|
320
329
|
catch (error) {
|
|
321
330
|
// A missing attachment is caller input rejected before the addon runs,
|
|
@@ -328,7 +337,7 @@ export async function* completion(params, opts) {
|
|
|
328
337
|
let result;
|
|
329
338
|
let addonRunSettled = false;
|
|
330
339
|
try {
|
|
331
|
-
result = yield* processModelResponse(model, messagesToSend,
|
|
340
|
+
result = yield* processModelResponse(model, messagesToSend, callableTools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse, () => {
|
|
332
341
|
addonRunSettled = true;
|
|
333
342
|
});
|
|
334
343
|
}
|
|
@@ -39,17 +39,42 @@ export declare const llmPlugin: {
|
|
|
39
39
|
integrated: "integrated";
|
|
40
40
|
dedicated: "dedicated";
|
|
41
41
|
}>]>>;
|
|
42
|
+
'rpc-servers': import("zod").ZodOptional<import("zod").ZodString>;
|
|
43
|
+
devices: import("zod").ZodOptional<import("zod").ZodString>;
|
|
42
44
|
'split-mode': import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
43
45
|
none: "none";
|
|
44
46
|
layer: "layer";
|
|
45
47
|
tensor: "tensor";
|
|
46
48
|
}>>;
|
|
47
49
|
'flash-attn': import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
50
|
+
auto: "auto";
|
|
48
51
|
on: "on";
|
|
49
52
|
off: "off";
|
|
50
|
-
auto: "auto";
|
|
51
53
|
}>>;
|
|
52
54
|
'tensor-split': import("zod").ZodOptional<import("zod").ZodString>;
|
|
55
|
+
'batch-size': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
56
|
+
'ubatch-size': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
57
|
+
'cpu-moe': import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
58
|
+
'n-cpu-moe': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
59
|
+
'kv-offload': import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
60
|
+
'image-max-tokens': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
61
|
+
'image-min-tokens': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
62
|
+
threads: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
63
|
+
'threads-batch': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
64
|
+
'cpu-mask': import("zod").ZodOptional<import("zod").ZodString>;
|
|
65
|
+
'cpu-mask-batch': import("zod").ZodOptional<import("zod").ZodString>;
|
|
66
|
+
'override-tensor': import("zod").ZodOptional<import("zod").ZodString>;
|
|
67
|
+
'n-cpu-ffn': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
68
|
+
'moe-cache-mib': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodLiteral<"auto">]>>;
|
|
69
|
+
'prefetch-weights': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodBoolean, import("zod").ZodLiteral<"auto">]>>;
|
|
70
|
+
'tensor-read-lazy': import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
71
|
+
auto: "auto";
|
|
72
|
+
on: "on";
|
|
73
|
+
off: "off";
|
|
74
|
+
}>>;
|
|
75
|
+
fit: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
76
|
+
'fit-target': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodString]>>;
|
|
77
|
+
'fit-ctx': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
53
78
|
openclCacheDir: import("zod").ZodOptional<import("zod").ZodString>;
|
|
54
79
|
reasoning_budget: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
55
80
|
projectionModelSrc: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodObject<{
|
|
@@ -102,6 +127,7 @@ export declare const llmPlugin: {
|
|
|
102
127
|
}>>;
|
|
103
128
|
'mmproj-use-gpu': import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
104
129
|
}, import("zod/v4/core").$strip>;
|
|
130
|
+
assessFit: typeof import("node_modules/@qvac/llm-llamacpp/fit").assessFit;
|
|
105
131
|
resolveConfig(cfg: LlmConfigInput, ctx: ResolveContext): Promise<{
|
|
106
132
|
config: {
|
|
107
133
|
ctx_size?: number | undefined;
|
|
@@ -125,9 +151,30 @@ export declare const llmPlugin: {
|
|
|
125
151
|
'cache-type-k'?: string | undefined;
|
|
126
152
|
'cache-type-v'?: string | undefined;
|
|
127
153
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
154
|
+
'rpc-servers'?: string | undefined;
|
|
155
|
+
devices?: string | undefined;
|
|
128
156
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
129
|
-
'flash-attn'?: "
|
|
157
|
+
'flash-attn'?: "auto" | "on" | "off" | undefined;
|
|
130
158
|
'tensor-split'?: string | undefined;
|
|
159
|
+
'batch-size'?: number | undefined;
|
|
160
|
+
'ubatch-size'?: number | undefined;
|
|
161
|
+
'cpu-moe'?: boolean | undefined;
|
|
162
|
+
'n-cpu-moe'?: number | undefined;
|
|
163
|
+
'kv-offload'?: boolean | undefined;
|
|
164
|
+
'image-max-tokens'?: number | undefined;
|
|
165
|
+
'image-min-tokens'?: number | undefined;
|
|
166
|
+
threads?: number | undefined;
|
|
167
|
+
'threads-batch'?: number | undefined;
|
|
168
|
+
'cpu-mask'?: string | undefined;
|
|
169
|
+
'cpu-mask-batch'?: string | undefined;
|
|
170
|
+
'override-tensor'?: string | undefined;
|
|
171
|
+
'n-cpu-ffn'?: number | undefined;
|
|
172
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
173
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
174
|
+
'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
|
|
175
|
+
fit?: boolean | undefined;
|
|
176
|
+
'fit-target'?: string | number | undefined;
|
|
177
|
+
'fit-ctx'?: number | undefined;
|
|
131
178
|
openclCacheDir?: string | undefined;
|
|
132
179
|
reasoning_budget?: number | undefined;
|
|
133
180
|
image_tile_mode?: "disabled" | "batched" | "sequential" | undefined;
|
|
@@ -158,9 +205,30 @@ export declare const llmPlugin: {
|
|
|
158
205
|
'cache-type-k'?: string | undefined;
|
|
159
206
|
'cache-type-v'?: string | undefined;
|
|
160
207
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
208
|
+
'rpc-servers'?: string | undefined;
|
|
209
|
+
devices?: string | undefined;
|
|
161
210
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
162
|
-
'flash-attn'?: "
|
|
211
|
+
'flash-attn'?: "auto" | "on" | "off" | undefined;
|
|
163
212
|
'tensor-split'?: string | undefined;
|
|
213
|
+
'batch-size'?: number | undefined;
|
|
214
|
+
'ubatch-size'?: number | undefined;
|
|
215
|
+
'cpu-moe'?: boolean | undefined;
|
|
216
|
+
'n-cpu-moe'?: number | undefined;
|
|
217
|
+
'kv-offload'?: boolean | undefined;
|
|
218
|
+
'image-max-tokens'?: number | undefined;
|
|
219
|
+
'image-min-tokens'?: number | undefined;
|
|
220
|
+
threads?: number | undefined;
|
|
221
|
+
'threads-batch'?: number | undefined;
|
|
222
|
+
'cpu-mask'?: string | undefined;
|
|
223
|
+
'cpu-mask-batch'?: string | undefined;
|
|
224
|
+
'override-tensor'?: string | undefined;
|
|
225
|
+
'n-cpu-ffn'?: number | undefined;
|
|
226
|
+
'moe-cache-mib'?: number | "auto" | undefined;
|
|
227
|
+
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
228
|
+
'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
|
|
229
|
+
fit?: boolean | undefined;
|
|
230
|
+
'fit-target'?: string | number | undefined;
|
|
231
|
+
'fit-ctx'?: number | undefined;
|
|
164
232
|
openclCacheDir?: string | undefined;
|
|
165
233
|
reasoning_budget?: number | undefined;
|
|
166
234
|
image_tile_mode?: "disabled" | "batched" | "sequential" | undefined;
|
|
@@ -227,6 +295,8 @@ export declare const llmPlugin: {
|
|
|
227
295
|
type: import("zod").ZodLiteral<"function">;
|
|
228
296
|
name: import("zod").ZodString;
|
|
229
297
|
description: import("zod").ZodString;
|
|
298
|
+
deferLoading: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
299
|
+
group: import("zod").ZodOptional<import("zod").ZodString>;
|
|
230
300
|
parameters: import("zod").ZodObject<{
|
|
231
301
|
type: import("zod").ZodLiteral<"object">;
|
|
232
302
|
properties: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodObject<{
|
|
@@ -299,8 +369,8 @@ export declare const llmPlugin: {
|
|
|
299
369
|
avgConcurrentSeq: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
300
370
|
toolDefinitionsDropped: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
301
371
|
backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
302
|
-
gpu: "gpu";
|
|
303
372
|
cpu: "cpu";
|
|
373
|
+
gpu: "gpu";
|
|
304
374
|
}>>;
|
|
305
375
|
}, import("zod/v4/core").$strip>;
|
|
306
376
|
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
@@ -318,8 +388,8 @@ export declare const llmPlugin: {
|
|
|
318
388
|
seq: import("zod").ZodNumber;
|
|
319
389
|
stopReason: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
320
390
|
cancelled: "cancelled";
|
|
321
|
-
eos: "eos";
|
|
322
391
|
length: "length";
|
|
392
|
+
eos: "eos";
|
|
323
393
|
stopSequence: "stopSequence";
|
|
324
394
|
}>>;
|
|
325
395
|
raw: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
@@ -338,8 +408,8 @@ export declare const llmPlugin: {
|
|
|
338
408
|
avgConcurrentSeq: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
339
409
|
toolDefinitionsDropped: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
340
410
|
backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
341
|
-
gpu: "gpu";
|
|
342
411
|
cpu: "cpu";
|
|
412
|
+
gpu: "gpu";
|
|
343
413
|
}>>;
|
|
344
414
|
}, import("zod/v4/core").$strip>>;
|
|
345
415
|
}, import("zod/v4/core").$strict>>;
|
|
@@ -356,6 +426,8 @@ export declare const llmPlugin: {
|
|
|
356
426
|
type: import("zod").ZodLiteral<"function">;
|
|
357
427
|
name: import("zod").ZodString;
|
|
358
428
|
description: import("zod").ZodString;
|
|
429
|
+
deferLoading: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
430
|
+
group: import("zod").ZodOptional<import("zod").ZodString>;
|
|
359
431
|
parameters: import("zod").ZodObject<{
|
|
360
432
|
type: import("zod").ZodLiteral<"object">;
|
|
361
433
|
properties: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodObject<{
|
|
@@ -464,8 +536,8 @@ export declare const llmPlugin: {
|
|
|
464
536
|
avgConcurrentSeq: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
465
537
|
toolDefinitionsDropped: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
466
538
|
backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
467
|
-
gpu: "gpu";
|
|
468
539
|
cpu: "cpu";
|
|
540
|
+
gpu: "gpu";
|
|
469
541
|
}>>;
|
|
470
542
|
}, import("zod/v4/core").$strip>;
|
|
471
543
|
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
@@ -483,8 +555,8 @@ export declare const llmPlugin: {
|
|
|
483
555
|
seq: import("zod").ZodNumber;
|
|
484
556
|
stopReason: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
485
557
|
cancelled: "cancelled";
|
|
486
|
-
eos: "eos";
|
|
487
558
|
length: "length";
|
|
559
|
+
eos: "eos";
|
|
488
560
|
stopSequence: "stopSequence";
|
|
489
561
|
}>>;
|
|
490
562
|
raw: import("zod").ZodOptional<import("zod").ZodObject<{
|