@qvac/inference 0.20.1 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +90 -24
- package/README.md +8 -5
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +8 -20
- package/dist/handlers/assess-model-fit.js +195 -77
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -11
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +37 -2
- package/dist/resources/model-fit/assess.js +24 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +20 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +126 -47
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +269 -85
- package/dist/schemas/assess-model-fit.js +78 -54
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +708 -66
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +104 -36
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +25 -18
- package/dist/schemas/llamacpp-config.js +26 -16
- package/dist/schemas/load-model.d.ts +2608 -469
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +183 -6
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +43 -29
- package/dist/resources/model-fit/fit-stub/index.d.ts +0 -2
- package/dist/resources/model-fit/fit-stub/index.js +0 -2
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +0 -31
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +0 -36
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -191
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { assessFit as audiogenAssessFit } from '@qvac/audiogen-ggml';
|
|
1
2
|
import { type CreateModelParams, type PluginModelResult } from '../../../schemas/index.js';
|
|
2
3
|
import { resolveAudioGenConfig } from './config.js';
|
|
3
4
|
export declare const audioGenPlugin: {
|
|
@@ -254,6 +255,7 @@ export declare const audioGenPlugin: {
|
|
|
254
255
|
}>, import("zod").ZodLiteral<"vad">]>>;
|
|
255
256
|
}, import("zod/v4/core").$strip>]>;
|
|
256
257
|
}, import("zod/v4/core").$strict>], "engine">;
|
|
258
|
+
assessFit: typeof audiogenAssessFit;
|
|
257
259
|
skipPrimaryModelPathValidation: true;
|
|
258
260
|
resolveConfig: typeof resolveAudioGenConfig;
|
|
259
261
|
createModel(params: CreateModelParams): PluginModelResult;
|
|
@@ -361,8 +363,8 @@ export declare const audioGenPlugin: {
|
|
|
361
363
|
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
362
364
|
selectedBackend: import("zod").ZodString;
|
|
363
365
|
selectedDevice: import("zod").ZodEnum<{
|
|
364
|
-
gpu: "gpu";
|
|
365
366
|
cpu: "cpu";
|
|
367
|
+
gpu: "gpu";
|
|
366
368
|
}>;
|
|
367
369
|
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
368
370
|
vulkan: "vulkan";
|
|
@@ -384,8 +386,8 @@ export declare const audioGenPlugin: {
|
|
|
384
386
|
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
385
387
|
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
386
388
|
requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
387
|
-
gpu: "gpu";
|
|
388
389
|
cpu: "cpu";
|
|
390
|
+
gpu: "gpu";
|
|
389
391
|
}>>;
|
|
390
392
|
reason: import("zod").ZodString;
|
|
391
393
|
}, import("zod/v4/core").$strip>>;
|
|
@@ -476,8 +478,8 @@ export declare const audioGenPlugin: {
|
|
|
476
478
|
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
477
479
|
selectedBackend: import("zod").ZodString;
|
|
478
480
|
selectedDevice: import("zod").ZodEnum<{
|
|
479
|
-
gpu: "gpu";
|
|
480
481
|
cpu: "cpu";
|
|
482
|
+
gpu: "gpu";
|
|
481
483
|
}>;
|
|
482
484
|
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
483
485
|
vulkan: "vulkan";
|
|
@@ -499,8 +501,8 @@ export declare const audioGenPlugin: {
|
|
|
499
501
|
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
500
502
|
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
501
503
|
requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
502
|
-
gpu: "gpu";
|
|
503
504
|
cpu: "cpu";
|
|
505
|
+
gpu: "gpu";
|
|
504
506
|
}>>;
|
|
505
507
|
reason: import("zod").ZodString;
|
|
506
508
|
}, import("zod/v4/core").$strip>>;
|
|
@@ -575,8 +577,8 @@ export declare const audioGenPlugin: {
|
|
|
575
577
|
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
576
578
|
selectedBackend: import("zod").ZodString;
|
|
577
579
|
selectedDevice: import("zod").ZodEnum<{
|
|
578
|
-
gpu: "gpu";
|
|
579
580
|
cpu: "cpu";
|
|
581
|
+
gpu: "gpu";
|
|
580
582
|
}>;
|
|
581
583
|
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
582
584
|
vulkan: "vulkan";
|
|
@@ -598,8 +600,8 @@ export declare const audioGenPlugin: {
|
|
|
598
600
|
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
599
601
|
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
600
602
|
requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
601
|
-
gpu: "gpu";
|
|
602
603
|
cpu: "cpu";
|
|
604
|
+
gpu: "gpu";
|
|
603
605
|
}>>;
|
|
604
606
|
reason: import("zod").ZodString;
|
|
605
607
|
}, import("zod/v4/core").$strip>>;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { AudioGen, ENGINE_ACESTEP, ENGINE_MINIMAX } from '@qvac/audiogen-ggml';
|
|
1
|
+
import { AudioGen, ENGINE_ACESTEP, ENGINE_MINIMAX, assessFit as audiogenAssessFit } from '@qvac/audiogen-ggml';
|
|
2
2
|
import { ADDON_AUDIOGEN, ModelType, audioEditStreamRequestSchema, audioEditStreamResponseSchema, audioGenConfigSchema, audioGenStreamRequestSchema, audioGenStreamResponseSchema, audioUnderstandRequestSchema, audioUnderstandResponseSchema, defineHandler, definePlugin } from '../../../schemas/index.js';
|
|
3
3
|
import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
|
|
4
4
|
import { resolveAudioGenConfig } from './config.js';
|
|
@@ -11,6 +11,7 @@ export const audioGenPlugin = definePlugin({
|
|
|
11
11
|
displayName: 'Audio Generation (GGML / ACE-Step and MiniMax)',
|
|
12
12
|
addonPackage: ADDON_AUDIOGEN,
|
|
13
13
|
loadConfigSchema: audioGenConfigSchema,
|
|
14
|
+
assessFit: audiogenAssessFit,
|
|
14
15
|
// AudioGen's primary `modelSrc` is intentionally empty: all required
|
|
15
16
|
// weights are config-owned artifacts (ACE-Step: four sources; MiniMax: LM + synth).
|
|
16
17
|
skipPrimaryModelPathValidation: true,
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { assessFit as bciAssessFit } from '@qvac/bci-whispercpp';
|
|
1
2
|
import { type CreateModelParams, type PluginModelResult } from '../../../schemas/index.js';
|
|
2
3
|
import { resolveBciConfig } from './resolve-config.js';
|
|
3
4
|
export declare const bciPlugin: {
|
|
@@ -76,6 +77,7 @@ export declare const bciPlugin: {
|
|
|
76
77
|
}>, import("zod").ZodLiteral<"vad">]>>;
|
|
77
78
|
}, import("zod/v4/core").$strip>]>>;
|
|
78
79
|
}, import("zod/v4/core").$strip>;
|
|
80
|
+
assessFit: typeof bciAssessFit;
|
|
79
81
|
resolveConfig: typeof resolveBciConfig;
|
|
80
82
|
createModel(params: CreateModelParams): PluginModelResult;
|
|
81
83
|
handlers: {
|
|
@@ -110,6 +112,18 @@ export declare const bciPlugin: {
|
|
|
110
112
|
gpuUnsupported: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
111
113
|
gpuMemTotalMb: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
112
114
|
gpuMemFreeMb: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
115
|
+
totalTime: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
116
|
+
totalWallMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
117
|
+
totalSamples: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
118
|
+
processCalls: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
119
|
+
whisperSampleMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
120
|
+
whisperBatchdMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
121
|
+
whisperPromptMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
122
|
+
totalTranscriptions: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
123
|
+
modelLoadMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
124
|
+
totalEncodedFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
125
|
+
encoderOnCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
126
|
+
encoderUsedCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
113
127
|
}, import("zod/v4/core").$strip>>;
|
|
114
128
|
error: import("zod").ZodOptional<import("zod").ZodString>;
|
|
115
129
|
segment: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
@@ -118,8 +132,51 @@ export declare const bciPlugin: {
|
|
|
118
132
|
endMs: import("zod").ZodNumber;
|
|
119
133
|
append: import("zod").ZodBoolean;
|
|
120
134
|
id: import("zod").ZodNumber;
|
|
135
|
+
isEndOfTurn: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
136
|
+
startsWord: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
121
137
|
}, import("zod/v4/core").$strip>>;
|
|
122
138
|
type: import("zod").ZodLiteral<"bciTranscribe">;
|
|
139
|
+
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
140
|
+
selectedBackend: import("zod").ZodString;
|
|
141
|
+
selectedDevice: import("zod").ZodEnum<{
|
|
142
|
+
cpu: "cpu";
|
|
143
|
+
gpu: "gpu";
|
|
144
|
+
}>;
|
|
145
|
+
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
146
|
+
vulkan: "vulkan";
|
|
147
|
+
metal: "metal";
|
|
148
|
+
opencl: "opencl";
|
|
149
|
+
cuda: "cuda";
|
|
150
|
+
opengl: "opengl";
|
|
151
|
+
webgpu: "webgpu";
|
|
152
|
+
direct3d11: "direct3d11";
|
|
153
|
+
direct3d12: "direct3d12";
|
|
154
|
+
levelZero: "levelZero";
|
|
155
|
+
rocm: "rocm";
|
|
156
|
+
}>>;
|
|
157
|
+
driver: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
158
|
+
name: import("zod").ZodString;
|
|
159
|
+
version: import("zod").ZodOptional<import("zod").ZodString>;
|
|
160
|
+
}, import("zod/v4/core").$strip>>;
|
|
161
|
+
gpuId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
162
|
+
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
163
|
+
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
164
|
+
requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
165
|
+
cpu: "cpu";
|
|
166
|
+
gpu: "gpu";
|
|
167
|
+
}>>;
|
|
168
|
+
reason: import("zod").ZodString;
|
|
169
|
+
}, import("zod/v4/core").$strip>>;
|
|
170
|
+
probe: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
171
|
+
status: import("zod").ZodEnum<{
|
|
172
|
+
unknown: "unknown";
|
|
173
|
+
compatible: "compatible";
|
|
174
|
+
incompatible: "incompatible";
|
|
175
|
+
}>;
|
|
176
|
+
backend: import("zod").ZodString;
|
|
177
|
+
reason: import("zod").ZodOptional<import("zod").ZodString>;
|
|
178
|
+
}, import("zod/v4/core").$strip>>;
|
|
179
|
+
}, import("zod/v4/core").$strip>>;
|
|
123
180
|
}, import("zod/v4/core").$strip>>;
|
|
124
181
|
bciTranscribeStream: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
|
|
125
182
|
modelId: import("zod").ZodString;
|
|
@@ -153,16 +210,31 @@ export declare const bciPlugin: {
|
|
|
153
210
|
gpuUnsupported: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
154
211
|
gpuMemTotalMb: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
155
212
|
gpuMemFreeMb: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
213
|
+
totalTime: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
214
|
+
totalWallMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
215
|
+
totalSamples: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
216
|
+
processCalls: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
217
|
+
whisperSampleMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
218
|
+
whisperBatchdMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
219
|
+
whisperPromptMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
220
|
+
totalTranscriptions: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
221
|
+
modelLoadMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
222
|
+
totalEncodedFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
223
|
+
encoderOnCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
224
|
+
encoderUsedCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
156
225
|
}, import("zod/v4/core").$strip>>;
|
|
157
226
|
error: import("zod").ZodOptional<import("zod").ZodString>;
|
|
227
|
+
type: import("zod").ZodLiteral<"bciTranscribeStream">;
|
|
158
228
|
segment: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
159
229
|
text: import("zod").ZodString;
|
|
160
230
|
startMs: import("zod").ZodNumber;
|
|
161
231
|
endMs: import("zod").ZodNumber;
|
|
162
232
|
append: import("zod").ZodBoolean;
|
|
163
233
|
id: import("zod").ZodNumber;
|
|
234
|
+
isEndOfTurn: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
235
|
+
startsWord: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
236
|
+
windowStartTimestep: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
164
237
|
}, import("zod/v4/core").$strip>>;
|
|
165
|
-
type: import("zod").ZodLiteral<"bciTranscribeStream">;
|
|
166
238
|
}, import("zod/v4/core").$strip>>;
|
|
167
239
|
};
|
|
168
240
|
logging: {
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
-
import BCIWhispercpp from '@qvac/bci-whispercpp';
|
|
1
|
+
import BCIWhispercpp, { assessFit as bciAssessFit } from '@qvac/bci-whispercpp';
|
|
2
2
|
import { definePlugin, defineHandler, defineDuplexHandler, bciTranscribeRequestSchema, bciTranscribeResponseSchema, bciTranscribeStreamRequestSchema, bciTranscribeStreamResponseSchema, ModelType, bciConfigSchema, ADDON_BCI } from '../../../schemas/index.js';
|
|
3
3
|
import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
|
|
4
4
|
import { bciTranscribe, bciTranscribeStream } from '../../ops/bci-transcribe.js';
|
|
5
5
|
import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
|
|
6
|
+
import { attachBackendDiagnostics } from '../../../profiling/backend-diagnostics.js';
|
|
6
7
|
import { buildBciWhispercppArgs } from './args.js';
|
|
7
8
|
import { resolveBciConfig } from './resolve-config.js';
|
|
8
9
|
function createBciModel(modelId, modelPath, bciConfig, embedderPath) {
|
|
@@ -17,6 +18,7 @@ export const bciPlugin = definePlugin({
|
|
|
17
18
|
displayName: 'BCI (whisper.cpp)',
|
|
18
19
|
addonPackage: ADDON_BCI,
|
|
19
20
|
loadConfigSchema: bciConfigSchema,
|
|
21
|
+
assessFit: bciAssessFit,
|
|
20
22
|
resolveConfig: resolveBciConfig,
|
|
21
23
|
createModel(params) {
|
|
22
24
|
const bciConfig = (params.modelConfig ?? {});
|
|
@@ -57,13 +59,17 @@ export const bciPlugin = definePlugin({
|
|
|
57
59
|
};
|
|
58
60
|
result = await stream.next();
|
|
59
61
|
}
|
|
60
|
-
const { modelExecutionMs, stats } = result.value;
|
|
61
|
-
|
|
62
|
+
const { modelExecutionMs, stats, diagnostics } = result.value;
|
|
63
|
+
// The field is what reaches an RPC client; the symbol is what the
|
|
64
|
+
// profiling layer reads to set `event.backend`, as audiogen does.
|
|
65
|
+
const terminal = attachModelExecutionMs({
|
|
62
66
|
type: 'bciTranscribe',
|
|
63
67
|
text: '',
|
|
64
68
|
done: true,
|
|
65
|
-
...(stats && { stats })
|
|
69
|
+
...(stats && { stats }),
|
|
70
|
+
...(diagnostics && { diagnostics })
|
|
66
71
|
}, modelExecutionMs);
|
|
72
|
+
yield diagnostics ? attachBackendDiagnostics(terminal, diagnostics) : terminal;
|
|
67
73
|
}
|
|
68
74
|
finally {
|
|
69
75
|
await stream.return?.(undefined);
|
|
@@ -84,22 +90,34 @@ export const bciPlugin = definePlugin({
|
|
|
84
90
|
const iterator = metadata
|
|
85
91
|
? bciTranscribeStream(request.modelId, inputStream, true, request.streamOpts, request.requestId)
|
|
86
92
|
: bciTranscribeStream(request.modelId, inputStream, false, request.streamOpts, request.requestId);
|
|
87
|
-
for await
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
93
|
+
// Iterated by hand rather than with `for await` so the generator's
|
|
94
|
+
// return value survives: it carries the execution time the profiling
|
|
95
|
+
// layer reads off the terminal frame.
|
|
96
|
+
try {
|
|
97
|
+
let result = await iterator.next();
|
|
98
|
+
while (!result.done) {
|
|
99
|
+
yield metadata
|
|
100
|
+
? {
|
|
101
|
+
type: 'bciTranscribeStream',
|
|
102
|
+
segment: result.value
|
|
103
|
+
}
|
|
104
|
+
: {
|
|
105
|
+
type: 'bciTranscribeStream',
|
|
106
|
+
text: result.value
|
|
107
|
+
};
|
|
108
|
+
result = await iterator.next();
|
|
109
|
+
}
|
|
110
|
+
// The addon reports no stats for a stream, so there is no backend
|
|
111
|
+
// verdict to attach here — only the timing.
|
|
112
|
+
yield attachModelExecutionMs({
|
|
113
|
+
type: 'bciTranscribeStream',
|
|
114
|
+
text: '',
|
|
115
|
+
done: true
|
|
116
|
+
}, result.value.modelExecutionMs);
|
|
117
|
+
}
|
|
118
|
+
finally {
|
|
119
|
+
await iterator.return?.(undefined);
|
|
97
120
|
}
|
|
98
|
-
yield {
|
|
99
|
-
type: 'bciTranscribeStream',
|
|
100
|
-
text: '',
|
|
101
|
-
done: true
|
|
102
|
-
};
|
|
103
121
|
}
|
|
104
122
|
})
|
|
105
123
|
},
|
|
@@ -5,8 +5,8 @@ export declare const vlaPlugin: {
|
|
|
5
5
|
addonPackage: "@qvac/vla-ggml";
|
|
6
6
|
loadConfigSchema: import("zod").ZodObject<{
|
|
7
7
|
backend: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
8
|
-
auto: "auto";
|
|
9
8
|
cpu: "cpu";
|
|
9
|
+
auto: "auto";
|
|
10
10
|
}>>;
|
|
11
11
|
verbosity: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
12
12
|
embodiment: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodNumber, import("zod").ZodObject<{
|
|
@@ -6,6 +6,7 @@ import { getResponseFormatJsonSchema } from '../../../../utils/response-format.j
|
|
|
6
6
|
import { seedConfiguredSystemPrompt, transformMessages } from './completion-stream.js';
|
|
7
7
|
import { normalizeCompletionStats } from './completion-stats.js';
|
|
8
8
|
import { prependToolsToHistory } from '../../../../utils/tool-integration.js';
|
|
9
|
+
import { resolveDeferredTools, toWireTool, withDeferredToolChoice } from '../../../../utils/tools/defer.js';
|
|
9
10
|
const logger = getEngineLogger();
|
|
10
11
|
function runBatchModel(model, prompts) {
|
|
11
12
|
const run = model.run.bind(model);
|
|
@@ -22,9 +23,10 @@ function mergeGenerationParams(generationParams, responseFormat) {
|
|
|
22
23
|
json_schema: jsonSchema
|
|
23
24
|
};
|
|
24
25
|
}
|
|
25
|
-
function renderPromptHistory(prompt, options) {
|
|
26
|
-
const tools = options.toolsEnabled && prompt.tools && prompt.tools.length > 0
|
|
27
|
-
|
|
26
|
+
function renderPromptHistory(prompt, history, deferred, options) {
|
|
27
|
+
const tools = options.toolsEnabled && prompt.tools && prompt.tools.length > 0
|
|
28
|
+
? (deferred?.toolsToRender ?? prompt.tools.map(toWireTool))
|
|
29
|
+
: undefined;
|
|
28
30
|
let historyWithTools = history;
|
|
29
31
|
if (tools) {
|
|
30
32
|
historyWithTools = prependToolsToHistory(history, tools);
|
|
@@ -34,10 +36,12 @@ function renderPromptHistory(prompt, options) {
|
|
|
34
36
|
return transformMessages(historyWithTools);
|
|
35
37
|
}
|
|
36
38
|
function buildBatchPrompt(prompt, options) {
|
|
37
|
-
const
|
|
39
|
+
const history = seedConfiguredSystemPrompt(prompt.history, options.modelConfig);
|
|
40
|
+
const deferred = resolveDeferredTools(prompt.tools, history);
|
|
41
|
+
const mergedGenerationParams = withDeferredToolChoice(mergeGenerationParams(prompt.generationParams, prompt.responseFormat), options.toolsEnabled ? deferred : null);
|
|
38
42
|
return {
|
|
39
43
|
...(prompt.id !== undefined && { id: prompt.id }),
|
|
40
|
-
prompt: renderPromptHistory(prompt, options),
|
|
44
|
+
prompt: renderPromptHistory(prompt, history, deferred, options),
|
|
41
45
|
...(mergedGenerationParams && {
|
|
42
46
|
runOptions: { generationParams: mergedGenerationParams }
|
|
43
47
|
})
|
|
@@ -10,7 +10,7 @@ export declare function normalizeCompletionStats(stats: LlmStats | undefined): {
|
|
|
10
10
|
emittedTokens?: number | undefined;
|
|
11
11
|
avgConcurrentSeq?: number | undefined;
|
|
12
12
|
toolDefinitionsDropped?: number | undefined;
|
|
13
|
-
backendDevice?: "
|
|
13
|
+
backendDevice?: "cpu" | "gpu" | undefined;
|
|
14
14
|
} | undefined;
|
|
15
15
|
/**
|
|
16
16
|
* Attach the count of non-empty pieces the addon streamed without
|
|
@@ -4,6 +4,7 @@ import { getModel, getModelConfig } from '../../../../runtime/model-registry.js'
|
|
|
4
4
|
import { decideCachedHistorySlice, shouldCommitCachedTurn } from './kv-cache-state.js';
|
|
5
5
|
import { createKvCacheSession, generateConfigHash } from './kv-cache-session.js';
|
|
6
6
|
import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
|
|
7
|
+
import { resolveDeferredTools, toWireTool, withDeferredToolChoice } from '../../../../utils/tools/defer.js';
|
|
7
8
|
import { parseToolCalls } from '../../../../utils/tools/index.js';
|
|
8
9
|
import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
|
|
9
10
|
import { toolChoiceDemandsCall } from '../../../../schemas/completion-stream.js';
|
|
@@ -209,7 +210,14 @@ export async function* completion(params, opts) {
|
|
|
209
210
|
const modelConfig = getModelConfig(modelId);
|
|
210
211
|
const history = seedConfiguredSystemPrompt(requestHistory, modelConfig);
|
|
211
212
|
const toolsEnabled = modelConfig.tools === true;
|
|
212
|
-
|
|
213
|
+
// With deferred tools the prompt and the parser see different lists: the
|
|
214
|
+
// prompt carries the always-loaded ones plus `tool_search` and its catalog,
|
|
215
|
+
// while the parser also accepts whatever earlier searches appended to the
|
|
216
|
+
// history. `null` when nothing defers, which leaves the existing path alone.
|
|
217
|
+
const deferred = resolveDeferredTools(tools, history);
|
|
218
|
+
const toolsToRender = deferred?.toolsToRender ?? tools?.map(toWireTool);
|
|
219
|
+
const callableTools = deferred?.callableTools ?? toolsToRender;
|
|
220
|
+
const toolsActive = !!toolsToRender?.length && toolsEnabled;
|
|
213
221
|
const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
|
|
214
222
|
// `responseFormat` is forwarded to the addon as a per-request
|
|
215
223
|
// `generationParams.json_schema`, which the addon converts to GBNF and
|
|
@@ -228,6 +236,7 @@ export async function* completion(params, opts) {
|
|
|
228
236
|
};
|
|
229
237
|
}
|
|
230
238
|
}
|
|
239
|
+
mergedGenerationParams = withDeferredToolChoice(mergedGenerationParams, toolsActive ? deferred : null);
|
|
231
240
|
const model = getModel(modelId);
|
|
232
241
|
// Per-request hard cancel: under continuous batching the model runs several
|
|
233
242
|
// jobs at once, so an abort must cancel only THIS request's job, not the
|
|
@@ -261,13 +270,13 @@ export async function* completion(params, opts) {
|
|
|
261
270
|
if (!kvCache) {
|
|
262
271
|
// KV-cache disabled — straight passthrough, no session involvement.
|
|
263
272
|
let historyWithTools = history;
|
|
264
|
-
if (toolsActive &&
|
|
265
|
-
historyWithTools = prependToolsToHistory(history,
|
|
273
|
+
if (toolsActive && toolsToRender) {
|
|
274
|
+
historyWithTools = prependToolsToHistory(history, toolsToRender);
|
|
266
275
|
}
|
|
267
276
|
const transformedHistory = transformMessages(historyWithTools);
|
|
268
277
|
logCacheDisabled();
|
|
269
278
|
logMessagesToAddon(transformedHistory, 'NO_CACHE');
|
|
270
|
-
return yield* processModelResponse(model, transformedHistory,
|
|
279
|
+
return yield* processModelResponse(model, transformedHistory, callableTools, mergedGenerationParams, undefined, dialect, setActiveResponse);
|
|
271
280
|
}
|
|
272
281
|
// ---- KV-cache path. The session owns every bookkeeping layer; the handler
|
|
273
282
|
// registers one deferred unwind that `commitTurn` short-circuits on the happy
|
|
@@ -281,7 +290,7 @@ export async function* completion(params, opts) {
|
|
|
281
290
|
// The tool block is baked into the cache on the turn that first sends it and
|
|
282
291
|
// never trimmed, so a late or changed tool set has to land on a fresh cache
|
|
283
292
|
// rather than a warm prefix holding the old block.
|
|
284
|
-
const configHash = generateConfigHash(systemPromptFromHistory, toolsActive ?
|
|
293
|
+
const configHash = generateConfigHash(systemPromptFromHistory, toolsActive ? toolsToRender : undefined);
|
|
285
294
|
let turn;
|
|
286
295
|
if (typeof kvCache === 'string') {
|
|
287
296
|
turn = await session.beginTurn({
|
|
@@ -315,7 +324,7 @@ export async function* completion(params, opts) {
|
|
|
315
324
|
scope.defer(() => (preserveCacheOnUnwind ? session.releaseTurn(turn) : session.rollback(turn)));
|
|
316
325
|
let payload;
|
|
317
326
|
try {
|
|
318
|
-
payload = await prepareMessagesForCache(session, turn, history, toolsActive ?
|
|
327
|
+
payload = await prepareMessagesForCache(session, turn, history, toolsActive ? toolsToRender : undefined, mergedGenerationParams?.tool_choice);
|
|
319
328
|
}
|
|
320
329
|
catch (error) {
|
|
321
330
|
// A missing attachment is caller input rejected before the addon runs,
|
|
@@ -328,7 +337,7 @@ export async function* completion(params, opts) {
|
|
|
328
337
|
let result;
|
|
329
338
|
let addonRunSettled = false;
|
|
330
339
|
try {
|
|
331
|
-
result = yield* processModelResponse(model, messagesToSend,
|
|
340
|
+
result = yield* processModelResponse(model, messagesToSend, callableTools, mergedGenerationParams, { cacheKey: turn.cachePath, saveCacheToDisk: true }, dialect, setActiveResponse, () => {
|
|
332
341
|
addonRunSettled = true;
|
|
333
342
|
});
|
|
334
343
|
}
|
|
@@ -39,15 +39,17 @@ export declare const llmPlugin: {
|
|
|
39
39
|
integrated: "integrated";
|
|
40
40
|
dedicated: "dedicated";
|
|
41
41
|
}>]>>;
|
|
42
|
+
'rpc-servers': import("zod").ZodOptional<import("zod").ZodString>;
|
|
43
|
+
devices: import("zod").ZodOptional<import("zod").ZodString>;
|
|
42
44
|
'split-mode': import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
43
45
|
none: "none";
|
|
44
46
|
layer: "layer";
|
|
45
47
|
tensor: "tensor";
|
|
46
48
|
}>>;
|
|
47
49
|
'flash-attn': import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
50
|
+
auto: "auto";
|
|
48
51
|
on: "on";
|
|
49
52
|
off: "off";
|
|
50
|
-
auto: "auto";
|
|
51
53
|
}>>;
|
|
52
54
|
'tensor-split': import("zod").ZodOptional<import("zod").ZodString>;
|
|
53
55
|
'batch-size': import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
@@ -66,9 +68,9 @@ export declare const llmPlugin: {
|
|
|
66
68
|
'moe-cache-mib': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodLiteral<"auto">]>>;
|
|
67
69
|
'prefetch-weights': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodBoolean, import("zod").ZodLiteral<"auto">]>>;
|
|
68
70
|
'tensor-read-lazy': import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
71
|
+
auto: "auto";
|
|
69
72
|
on: "on";
|
|
70
73
|
off: "off";
|
|
71
|
-
auto: "auto";
|
|
72
74
|
}>>;
|
|
73
75
|
fit: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
74
76
|
'fit-target': import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodString]>>;
|
|
@@ -125,6 +127,7 @@ export declare const llmPlugin: {
|
|
|
125
127
|
}>>;
|
|
126
128
|
'mmproj-use-gpu': import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
127
129
|
}, import("zod/v4/core").$strip>;
|
|
130
|
+
assessFit: typeof import("node_modules/@qvac/llm-llamacpp/fit").assessFit;
|
|
128
131
|
resolveConfig(cfg: LlmConfigInput, ctx: ResolveContext): Promise<{
|
|
129
132
|
config: {
|
|
130
133
|
ctx_size?: number | undefined;
|
|
@@ -148,8 +151,10 @@ export declare const llmPlugin: {
|
|
|
148
151
|
'cache-type-k'?: string | undefined;
|
|
149
152
|
'cache-type-v'?: string | undefined;
|
|
150
153
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
154
|
+
'rpc-servers'?: string | undefined;
|
|
155
|
+
devices?: string | undefined;
|
|
151
156
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
152
|
-
'flash-attn'?: "
|
|
157
|
+
'flash-attn'?: "auto" | "on" | "off" | undefined;
|
|
153
158
|
'tensor-split'?: string | undefined;
|
|
154
159
|
'batch-size'?: number | undefined;
|
|
155
160
|
'ubatch-size'?: number | undefined;
|
|
@@ -166,7 +171,7 @@ export declare const llmPlugin: {
|
|
|
166
171
|
'n-cpu-ffn'?: number | undefined;
|
|
167
172
|
'moe-cache-mib'?: number | "auto" | undefined;
|
|
168
173
|
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
169
|
-
'tensor-read-lazy'?: "
|
|
174
|
+
'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
|
|
170
175
|
fit?: boolean | undefined;
|
|
171
176
|
'fit-target'?: string | number | undefined;
|
|
172
177
|
'fit-ctx'?: number | undefined;
|
|
@@ -200,8 +205,10 @@ export declare const llmPlugin: {
|
|
|
200
205
|
'cache-type-k'?: string | undefined;
|
|
201
206
|
'cache-type-v'?: string | undefined;
|
|
202
207
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
208
|
+
'rpc-servers'?: string | undefined;
|
|
209
|
+
devices?: string | undefined;
|
|
203
210
|
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
204
|
-
'flash-attn'?: "
|
|
211
|
+
'flash-attn'?: "auto" | "on" | "off" | undefined;
|
|
205
212
|
'tensor-split'?: string | undefined;
|
|
206
213
|
'batch-size'?: number | undefined;
|
|
207
214
|
'ubatch-size'?: number | undefined;
|
|
@@ -218,7 +225,7 @@ export declare const llmPlugin: {
|
|
|
218
225
|
'n-cpu-ffn'?: number | undefined;
|
|
219
226
|
'moe-cache-mib'?: number | "auto" | undefined;
|
|
220
227
|
'prefetch-weights'?: boolean | "auto" | undefined;
|
|
221
|
-
'tensor-read-lazy'?: "
|
|
228
|
+
'tensor-read-lazy'?: "auto" | "on" | "off" | undefined;
|
|
222
229
|
fit?: boolean | undefined;
|
|
223
230
|
'fit-target'?: string | number | undefined;
|
|
224
231
|
'fit-ctx'?: number | undefined;
|
|
@@ -288,6 +295,8 @@ export declare const llmPlugin: {
|
|
|
288
295
|
type: import("zod").ZodLiteral<"function">;
|
|
289
296
|
name: import("zod").ZodString;
|
|
290
297
|
description: import("zod").ZodString;
|
|
298
|
+
deferLoading: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
299
|
+
group: import("zod").ZodOptional<import("zod").ZodString>;
|
|
291
300
|
parameters: import("zod").ZodObject<{
|
|
292
301
|
type: import("zod").ZodLiteral<"object">;
|
|
293
302
|
properties: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodObject<{
|
|
@@ -360,8 +369,8 @@ export declare const llmPlugin: {
|
|
|
360
369
|
avgConcurrentSeq: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
361
370
|
toolDefinitionsDropped: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
362
371
|
backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
363
|
-
gpu: "gpu";
|
|
364
372
|
cpu: "cpu";
|
|
373
|
+
gpu: "gpu";
|
|
365
374
|
}>>;
|
|
366
375
|
}, import("zod/v4/core").$strip>;
|
|
367
376
|
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
@@ -379,8 +388,8 @@ export declare const llmPlugin: {
|
|
|
379
388
|
seq: import("zod").ZodNumber;
|
|
380
389
|
stopReason: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
381
390
|
cancelled: "cancelled";
|
|
382
|
-
eos: "eos";
|
|
383
391
|
length: "length";
|
|
392
|
+
eos: "eos";
|
|
384
393
|
stopSequence: "stopSequence";
|
|
385
394
|
}>>;
|
|
386
395
|
raw: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
@@ -399,8 +408,8 @@ export declare const llmPlugin: {
|
|
|
399
408
|
avgConcurrentSeq: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
400
409
|
toolDefinitionsDropped: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
401
410
|
backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
402
|
-
gpu: "gpu";
|
|
403
411
|
cpu: "cpu";
|
|
412
|
+
gpu: "gpu";
|
|
404
413
|
}>>;
|
|
405
414
|
}, import("zod/v4/core").$strip>>;
|
|
406
415
|
}, import("zod/v4/core").$strict>>;
|
|
@@ -417,6 +426,8 @@ export declare const llmPlugin: {
|
|
|
417
426
|
type: import("zod").ZodLiteral<"function">;
|
|
418
427
|
name: import("zod").ZodString;
|
|
419
428
|
description: import("zod").ZodString;
|
|
429
|
+
deferLoading: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
430
|
+
group: import("zod").ZodOptional<import("zod").ZodString>;
|
|
420
431
|
parameters: import("zod").ZodObject<{
|
|
421
432
|
type: import("zod").ZodLiteral<"object">;
|
|
422
433
|
properties: import("zod").ZodRecord<import("zod").ZodString, import("zod").ZodObject<{
|
|
@@ -525,8 +536,8 @@ export declare const llmPlugin: {
|
|
|
525
536
|
avgConcurrentSeq: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
526
537
|
toolDefinitionsDropped: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
527
538
|
backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
528
|
-
gpu: "gpu";
|
|
529
539
|
cpu: "cpu";
|
|
540
|
+
gpu: "gpu";
|
|
530
541
|
}>>;
|
|
531
542
|
}, import("zod/v4/core").$strip>;
|
|
532
543
|
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
@@ -544,8 +555,8 @@ export declare const llmPlugin: {
|
|
|
544
555
|
seq: import("zod").ZodNumber;
|
|
545
556
|
stopReason: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
546
557
|
cancelled: "cancelled";
|
|
547
|
-
eos: "eos";
|
|
548
558
|
length: "length";
|
|
559
|
+
eos: "eos";
|
|
549
560
|
stopSequence: "stopSequence";
|
|
550
561
|
}>>;
|
|
551
562
|
raw: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
@@ -71,14 +71,15 @@ import { isAddonContextOverflowError, parseContextOverflowMessage } from './ops/
|
|
|
71
71
|
import { stoppedByLength } from './ops/completion-stats.js';
|
|
72
72
|
import { isMobile } from '../../../runtime/state.js';
|
|
73
73
|
import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
|
|
74
|
+
import { resolveDeferredTools, toWireTool } from '../../../utils/tools/defer.js';
|
|
74
75
|
function createLlmModel(modelId, modelPath, llmConfig, projectionModelPath) {
|
|
75
76
|
const logger = createStreamLogger(modelId, ModelType.llamacppCompletion);
|
|
76
77
|
registerAddonLogger(modelId, ModelType.llamacppCompletion, logger);
|
|
77
78
|
const llmConfigStrings = transformLlmConfig(llmConfig);
|
|
78
79
|
if (isMobile()) {
|
|
79
|
-
const stripped = stripMultiGpuKeys(llmConfigStrings);
|
|
80
|
+
const stripped = stripMultiGpuKeys(llmConfigStrings, true);
|
|
80
81
|
if (stripped.length > 0) {
|
|
81
|
-
getEngineLogger().warn(`[${ModelType.llamacppCompletion}:${modelId}]
|
|
82
|
+
getEngineLogger().warn(`[${ModelType.llamacppCompletion}:${modelId}] Removing unsupported mobile parameters: ${stripped.join(', ')}`);
|
|
82
83
|
}
|
|
83
84
|
}
|
|
84
85
|
// Addon overflow flag: at a single slot the addon throws when driven over
|
|
@@ -116,6 +117,7 @@ export const llmPlugin = definePlugin({
|
|
|
116
117
|
displayName: 'LLM (llama.cpp)',
|
|
117
118
|
addonPackage: ADDON_LLM,
|
|
118
119
|
loadConfigSchema: llmConfigBaseSchema,
|
|
120
|
+
assessFit: LlmLlamacpp.assessFit,
|
|
119
121
|
async resolveConfig(cfg, ctx) {
|
|
120
122
|
const { projectionModelSrc, ...llmConfig } = cfg;
|
|
121
123
|
if (!projectionModelSrc) {
|
|
@@ -145,7 +147,11 @@ export const llmPlugin = definePlugin({
|
|
|
145
147
|
const dialect = request.toolDialect ?? detectToolDialect(request.modelId);
|
|
146
148
|
const modelCfg = getModelConfig(request.modelId);
|
|
147
149
|
const toolsActive = modelCfg.tools === true;
|
|
148
|
-
|
|
150
|
+
// Parse against what each prompt can call: with deferred tools that is
|
|
151
|
+
// the rendered set plus whatever an earlier `tool_search` loaded.
|
|
152
|
+
const toolsByPosition = request.prompts.map((prompt) => resolveDeferredTools(prompt.tools, prompt.history)?.callableTools ??
|
|
153
|
+
prompt.tools?.map(toWireTool) ??
|
|
154
|
+
[]);
|
|
149
155
|
const toolsById = new Map();
|
|
150
156
|
const normalizers = new Map();
|
|
151
157
|
const bufferedEvents = [];
|