@qvac/inference 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +95 -39
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/delete-cache.js +3 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.js +169 -68
- package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
- package/dist/resources/model-fit/estimators/llm.js +3 -1
- package/dist/resources/model-fit/floor.d.ts +31 -0
- package/dist/resources/model-fit/floor.js +65 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/resources/native.js +5 -4
- package/dist/resources/types.d.ts +1 -3
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +84 -0
- package/dist/schemas/assess-model-fit.js +72 -3
- package/dist/schemas/audio-gen.d.ts +482 -7
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +10 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci.d.ts +1 -1
- package/dist/schemas/common.d.ts +782 -19
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +14 -1
- package/dist/schemas/completion-event.js +4 -0
- package/dist/schemas/completion-stream.d.ts +22 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2604 -180
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/model-ops.d.ts +3 -0
- package/dist/schemas/model-ops.js +6 -0
- package/dist/schemas/plugin.d.ts +21 -11
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +30 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +29 -26
|
@@ -42,7 +42,6 @@ export declare const llmPlugin: {
|
|
|
42
42
|
'split-mode': import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
43
43
|
none: "none";
|
|
44
44
|
layer: "layer";
|
|
45
|
-
row: "row";
|
|
46
45
|
tensor: "tensor";
|
|
47
46
|
}>>;
|
|
48
47
|
'flash-attn': import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
@@ -126,7 +125,7 @@ export declare const llmPlugin: {
|
|
|
126
125
|
'cache-type-k'?: string | undefined;
|
|
127
126
|
'cache-type-v'?: string | undefined;
|
|
128
127
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
129
|
-
'split-mode'?: "none" | "layer" | "
|
|
128
|
+
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
130
129
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
131
130
|
'tensor-split'?: string | undefined;
|
|
132
131
|
openclCacheDir?: string | undefined;
|
|
@@ -159,7 +158,7 @@ export declare const llmPlugin: {
|
|
|
159
158
|
'cache-type-k'?: string | undefined;
|
|
160
159
|
'cache-type-v'?: string | undefined;
|
|
161
160
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
162
|
-
'split-mode'?: "none" | "layer" | "
|
|
161
|
+
'split-mode'?: "none" | "layer" | "tensor" | undefined;
|
|
163
162
|
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
164
163
|
'tensor-split'?: string | undefined;
|
|
165
164
|
openclCacheDir?: string | undefined;
|
|
@@ -209,6 +208,7 @@ export declare const llmPlugin: {
|
|
|
209
208
|
repeat_penalty: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
210
209
|
reasoning_budget: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
211
210
|
remove_thinking_from_context: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
211
|
+
tool_choice: import("zod").ZodOptional<import("zod").ZodString>;
|
|
212
212
|
}, import("zod/v4/core").$strict>>;
|
|
213
213
|
responseFormat: import("zod").ZodOptional<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
214
214
|
type: import("zod").ZodLiteral<"text">;
|
|
@@ -291,11 +291,13 @@ export declare const llmPlugin: {
|
|
|
291
291
|
stats: import("zod").ZodObject<{
|
|
292
292
|
timeToFirstToken: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
293
293
|
tokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
294
|
+
promptTokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
294
295
|
cacheTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
295
296
|
promptTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
296
297
|
generatedTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
297
298
|
emittedTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
298
299
|
avgConcurrentSeq: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
300
|
+
toolDefinitionsDropped: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
299
301
|
backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
300
302
|
gpu: "gpu";
|
|
301
303
|
cpu: "cpu";
|
|
@@ -328,11 +330,13 @@ export declare const llmPlugin: {
|
|
|
328
330
|
stats: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
329
331
|
timeToFirstToken: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
330
332
|
tokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
333
|
+
promptTokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
331
334
|
cacheTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
332
335
|
promptTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
333
336
|
generatedTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
334
337
|
emittedTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
335
338
|
avgConcurrentSeq: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
339
|
+
toolDefinitionsDropped: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
336
340
|
backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
337
341
|
gpu: "gpu";
|
|
338
342
|
cpu: "cpu";
|
|
@@ -382,6 +386,7 @@ export declare const llmPlugin: {
|
|
|
382
386
|
repeat_penalty: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
383
387
|
reasoning_budget: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
384
388
|
remove_thinking_from_context: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
389
|
+
tool_choice: import("zod").ZodOptional<import("zod").ZodString>;
|
|
385
390
|
}, import("zod/v4/core").$strict>>;
|
|
386
391
|
captureThinking: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
387
392
|
emitRawDeltas: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
@@ -451,11 +456,13 @@ export declare const llmPlugin: {
|
|
|
451
456
|
stats: import("zod").ZodObject<{
|
|
452
457
|
timeToFirstToken: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
453
458
|
tokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
459
|
+
promptTokensPerSecond: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
454
460
|
cacheTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
455
461
|
promptTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
456
462
|
generatedTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
457
463
|
emittedTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
458
464
|
avgConcurrentSeq: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
465
|
+
toolDefinitionsDropped: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
459
466
|
backendDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
460
467
|
gpu: "gpu";
|
|
461
468
|
cpu: "cpu";
|
|
@@ -676,7 +683,7 @@ export declare const llmPlugin: {
|
|
|
676
683
|
}, import("zod/v4/core").$strip>>;
|
|
677
684
|
};
|
|
678
685
|
logging: {
|
|
679
|
-
module: import("@qvac/llm-llamacpp/addonLogging")
|
|
686
|
+
module: () => Promise<typeof import("@qvac/llm-llamacpp/addonLogging")>;
|
|
680
687
|
namespace: "llamacpp-completion";
|
|
681
688
|
};
|
|
682
689
|
};
|
|
@@ -51,7 +51,6 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
53
|
import LlmLlamacpp from '@qvac/llm-llamacpp';
|
|
54
|
-
import llmAddonLogging from '@qvac/llm-llamacpp/addonLogging';
|
|
55
54
|
import { definePlugin, defineHandler, finetuneRequestSchema, batchCompletionStreamRequestSchema, batchCompletionStreamResponseSchema, completionStreamRequestSchema, completionStreamResponseSchema, finetuneResponseSchema, translateRequestSchema, translateResponseSchema, ModelType, llmConfigBaseSchema, ADDON_LLM } from '../../../schemas/index.js';
|
|
56
55
|
import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
|
|
57
56
|
import { getFirstShardPath } from '../../../utils/index.js';
|
|
@@ -492,7 +491,7 @@ export const llmPlugin = definePlugin({
|
|
|
492
491
|
})
|
|
493
492
|
},
|
|
494
493
|
logging: {
|
|
495
|
-
module:
|
|
494
|
+
module: () => import('@qvac/llm-llamacpp/addonLogging'),
|
|
496
495
|
namespace: ModelType.llamacppCompletion
|
|
497
496
|
}
|
|
498
497
|
});
|
|
@@ -35,7 +35,6 @@ export declare const embeddingsPlugin: {
|
|
|
35
35
|
splitMode: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
36
36
|
none: "none";
|
|
37
37
|
layer: "layer";
|
|
38
|
-
row: "row";
|
|
39
38
|
}>>;
|
|
40
39
|
tensorSplit: import("zod").ZodOptional<import("zod").ZodString>;
|
|
41
40
|
verbosity: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
@@ -74,7 +73,7 @@ export declare const embeddingsPlugin: {
|
|
|
74
73
|
turbovecIndexProvider: TurboVecIndexProvider;
|
|
75
74
|
};
|
|
76
75
|
logging: {
|
|
77
|
-
module: import("@qvac/embed-llamacpp/addonLogging")
|
|
76
|
+
module: () => Promise<typeof import("@qvac/embed-llamacpp/addonLogging")>;
|
|
78
77
|
namespace: "llamacpp-embedding";
|
|
79
78
|
};
|
|
80
79
|
};
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import EmbedLlamacpp, { IdMapIndex } from '@qvac/embed-llamacpp';
|
|
2
|
-
import embedAddonLogging from '@qvac/embed-llamacpp/addonLogging';
|
|
3
2
|
import { definePlugin, defineHandler, embedRequestSchema, embedResponseSchema, ModelType, embedConfigBaseSchema, ADDON_EMBEDDING } from '../../../schemas/index.js';
|
|
4
3
|
import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
|
|
5
4
|
import { getFirstShardPath } from '../../../utils/index.js';
|
|
@@ -7,6 +6,7 @@ import { embed } from '../../ops/embed.js';
|
|
|
7
6
|
import { forwardModelExecution } from '../../../profiling/model-execution.js';
|
|
8
7
|
import { isMobile } from '../../../runtime/state.js';
|
|
9
8
|
import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
|
|
9
|
+
import { transformEmbedConfig } from './transform.js';
|
|
10
10
|
const turbovecIndexProvider = {
|
|
11
11
|
create(options) {
|
|
12
12
|
return new IdMapIndex(options);
|
|
@@ -15,42 +15,6 @@ const turbovecIndexProvider = {
|
|
|
15
15
|
return IdMapIndex.load(snapshotPath);
|
|
16
16
|
}
|
|
17
17
|
};
|
|
18
|
-
function transformEmbedConfig(embedConfig) {
|
|
19
|
-
const config = {
|
|
20
|
-
device: embedConfig.device,
|
|
21
|
-
gpu_layers: `${embedConfig.gpuLayers}`,
|
|
22
|
-
batch_size: `${embedConfig.batchSize}`
|
|
23
|
-
};
|
|
24
|
-
if (embedConfig.flashAttention) {
|
|
25
|
-
config.flash_attn = embedConfig.flashAttention;
|
|
26
|
-
}
|
|
27
|
-
if (embedConfig.pooling) {
|
|
28
|
-
config.pooling = embedConfig.pooling;
|
|
29
|
-
}
|
|
30
|
-
if (embedConfig.attention) {
|
|
31
|
-
config.attention = embedConfig.attention;
|
|
32
|
-
}
|
|
33
|
-
if (typeof embedConfig.embdNormalize === 'number') {
|
|
34
|
-
config.embd_normalize = `${embedConfig.embdNormalize}`;
|
|
35
|
-
}
|
|
36
|
-
if (embedConfig.mainGpu !== undefined) {
|
|
37
|
-
config['main-gpu'] =
|
|
38
|
-
typeof embedConfig.mainGpu === 'number' ? `${embedConfig.mainGpu}` : embedConfig.mainGpu;
|
|
39
|
-
}
|
|
40
|
-
if (embedConfig.splitMode) {
|
|
41
|
-
config['split-mode'] = embedConfig.splitMode;
|
|
42
|
-
}
|
|
43
|
-
if (embedConfig.tensorSplit) {
|
|
44
|
-
config['tensor-split'] = embedConfig.tensorSplit;
|
|
45
|
-
}
|
|
46
|
-
if (typeof embedConfig.verbosity === 'number') {
|
|
47
|
-
config.verbosity = `${embedConfig.verbosity}`;
|
|
48
|
-
}
|
|
49
|
-
if (embedConfig.openclCacheDir) {
|
|
50
|
-
config.openclCacheDir = embedConfig.openclCacheDir;
|
|
51
|
-
}
|
|
52
|
-
return config;
|
|
53
|
-
}
|
|
54
18
|
function createEmbeddingsModel(modelId, modelPath, embedConfig) {
|
|
55
19
|
const logger = createStreamLogger(modelId, ModelType.llamacppEmbedding);
|
|
56
20
|
registerAddonLogger(modelId, ModelType.llamacppEmbedding, logger);
|
|
@@ -105,7 +69,7 @@ export const embeddingsPlugin = definePlugin({
|
|
|
105
69
|
turbovecIndexProvider
|
|
106
70
|
},
|
|
107
71
|
logging: {
|
|
108
|
-
module:
|
|
72
|
+
module: () => import('@qvac/embed-llamacpp/addonLogging'),
|
|
109
73
|
namespace: ModelType.llamacppEmbedding
|
|
110
74
|
}
|
|
111
75
|
});
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { GGMLConfig } from '@qvac/embed-llamacpp';
|
|
2
|
+
import { type EmbedConfig } from '../../../schemas/index.js';
|
|
3
|
+
/**
|
|
4
|
+
* Converts an EmbedConfig into the flat string-keyed map the C++ addon expects.
|
|
5
|
+
*
|
|
6
|
+
* Extracted from the plugin so the advisory fit check can build its request
|
|
7
|
+
* from the same transform the real embedding load uses.
|
|
8
|
+
*/
|
|
9
|
+
export declare function transformEmbedConfig(embedConfig: EmbedConfig): GGMLConfig;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Converts an EmbedConfig into the flat string-keyed map the C++ addon expects.
|
|
3
|
+
*
|
|
4
|
+
* Extracted from the plugin so the advisory fit check can build its request
|
|
5
|
+
* from the same transform the real embedding load uses.
|
|
6
|
+
*/
|
|
7
|
+
export function transformEmbedConfig(embedConfig) {
|
|
8
|
+
const config = {
|
|
9
|
+
device: embedConfig.device,
|
|
10
|
+
gpu_layers: `${embedConfig.gpuLayers}`,
|
|
11
|
+
batch_size: `${embedConfig.batchSize}`
|
|
12
|
+
};
|
|
13
|
+
if (embedConfig.flashAttention) {
|
|
14
|
+
config.flash_attn = embedConfig.flashAttention;
|
|
15
|
+
}
|
|
16
|
+
if (embedConfig.pooling) {
|
|
17
|
+
config.pooling = embedConfig.pooling;
|
|
18
|
+
}
|
|
19
|
+
if (embedConfig.attention) {
|
|
20
|
+
config.attention = embedConfig.attention;
|
|
21
|
+
}
|
|
22
|
+
if (typeof embedConfig.embdNormalize === 'number') {
|
|
23
|
+
config.embd_normalize = `${embedConfig.embdNormalize}`;
|
|
24
|
+
}
|
|
25
|
+
if (embedConfig.mainGpu !== undefined) {
|
|
26
|
+
config['main-gpu'] =
|
|
27
|
+
typeof embedConfig.mainGpu === 'number' ? `${embedConfig.mainGpu}` : embedConfig.mainGpu;
|
|
28
|
+
}
|
|
29
|
+
if (embedConfig.splitMode) {
|
|
30
|
+
config['split-mode'] = embedConfig.splitMode;
|
|
31
|
+
}
|
|
32
|
+
if (embedConfig.tensorSplit) {
|
|
33
|
+
config['tensor-split'] = embedConfig.tensorSplit;
|
|
34
|
+
}
|
|
35
|
+
if (typeof embedConfig.verbosity === 'number') {
|
|
36
|
+
config.verbosity = `${embedConfig.verbosity}`;
|
|
37
|
+
}
|
|
38
|
+
if (embedConfig.openclCacheDir) {
|
|
39
|
+
config.openclCacheDir = embedConfig.openclCacheDir;
|
|
40
|
+
}
|
|
41
|
+
return config;
|
|
42
|
+
}
|
|
@@ -451,7 +451,7 @@ export declare const nmtPlugin: {
|
|
|
451
451
|
}, import("zod/v4/core").$strip>>;
|
|
452
452
|
};
|
|
453
453
|
logging: {
|
|
454
|
-
module: import("@qvac/translation-nmtcpp/addonLogging")
|
|
454
|
+
module: () => Promise<typeof import("@qvac/translation-nmtcpp/addonLogging")>;
|
|
455
455
|
namespace: "nmtcpp-translation";
|
|
456
456
|
};
|
|
457
457
|
};
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import nmtAddonLogging from '@qvac/translation-nmtcpp/addonLogging';
|
|
2
1
|
import TranslationNmtcpp from '@qvac/translation-nmtcpp';
|
|
3
2
|
import { definePlugin, defineHandler, translateRequestSchema, translateResponseSchema, ModelType, nmtConfigBaseSchema, ADDON_NMT, BERGAMOT_CJK_LANG_PAIRS } from '../../../schemas/index.js';
|
|
4
3
|
import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
|
|
@@ -138,7 +137,7 @@ export const nmtPlugin = definePlugin({
|
|
|
138
137
|
})
|
|
139
138
|
},
|
|
140
139
|
logging: {
|
|
141
|
-
module:
|
|
140
|
+
module: () => import('@qvac/translation-nmtcpp/addonLogging'),
|
|
142
141
|
namespace: ModelType.nmtcppTranslation
|
|
143
142
|
}
|
|
144
143
|
});
|
|
@@ -127,7 +127,7 @@ export declare const parakeetPlugin: {
|
|
|
127
127
|
}, import("zod/v4/core").$strip>>;
|
|
128
128
|
};
|
|
129
129
|
logging: {
|
|
130
|
-
module: import("@qvac/asr-ggml/addonLogging")
|
|
130
|
+
module: () => Promise<typeof import("@qvac/asr-ggml/addonLogging")>;
|
|
131
131
|
namespace: "@qvac/asr-ggml";
|
|
132
132
|
};
|
|
133
133
|
};
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import asrAddonLogging from '@qvac/asr-ggml/addonLogging';
|
|
2
1
|
import ASRGgml from '@qvac/asr-ggml';
|
|
3
2
|
import { definePlugin, defineHandler, defineDuplexHandler, transcribeRequestSchema, transcribeResponseSchema, transcribeStreamRequestSchema, transcribeStreamResponseSchema, ModelType, parakeetLoadConfigSchema, LEGACY_PARAKEET_ONNX_MODEL_CONFIG_FIELDS, ADDON_ASR } from '../../../schemas/index.js';
|
|
4
3
|
import { ModelLoadFailedError, TranscriptionFailedError, LegacyParakeetModelDeprecatedError } from '../../../errors/index.js';
|
|
@@ -135,7 +134,7 @@ export const parakeetPlugin = definePlugin({
|
|
|
135
134
|
})
|
|
136
135
|
},
|
|
137
136
|
logging: {
|
|
138
|
-
module:
|
|
137
|
+
module: () => import('@qvac/asr-ggml/addonLogging'),
|
|
139
138
|
namespace: ADDON_ASR
|
|
140
139
|
}
|
|
141
140
|
});
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { VideoStableDiffusion } from '@qvac/diffusion-cpp';
|
|
2
2
|
import { type VideoRequest, type VideoStreamResponse } from '../../../../schemas/sdcpp-config.js';
|
|
3
3
|
export declare function markLtxVideoModel(model: VideoStableDiffusion): void;
|
|
4
|
+
export declare function markH3VideoModel(model: VideoStableDiffusion): void;
|
|
4
5
|
export declare function markMoeCapableVideoModel(model: VideoStableDiffusion): void;
|
|
5
6
|
export declare function video(request: VideoRequest): AsyncGenerator<VideoStreamResponse>;
|
|
@@ -60,12 +60,16 @@ import { generateRandomRequestId } from '../../../../runtime/request-id.js';
|
|
|
60
60
|
import { ModelOperationNotSupportedError, PluginRequestValidationFailedError } from '../../../../errors/index.js';
|
|
61
61
|
import { formatZodError } from '../../../../utils/zod-error.js';
|
|
62
62
|
import { ModelType } from '../../../../schemas/index.js';
|
|
63
|
-
import { ltxVideoRequestSchema, nonLtxVideoRequestSchema, sdcppConfigSchema, singleExpertVideoRequestSchema } from '../../../../schemas/sdcpp-config.js';
|
|
63
|
+
import { ltxVideoRequestSchema, h3VideoRequestSchema, wanVideoRequestSchema, nonLtxVideoRequestSchema, sdcppConfigSchema, singleExpertVideoRequestSchema } from '../../../../schemas/sdcpp-config.js';
|
|
64
64
|
const ltxVideoModels = new WeakSet();
|
|
65
|
+
const h3VideoModels = new WeakSet();
|
|
65
66
|
const moeCapableVideoModels = new WeakSet();
|
|
66
67
|
export function markLtxVideoModel(model) {
|
|
67
68
|
ltxVideoModels.add(model);
|
|
68
69
|
}
|
|
70
|
+
export function markH3VideoModel(model) {
|
|
71
|
+
h3VideoModels.add(model);
|
|
72
|
+
}
|
|
69
73
|
export function markMoeCapableVideoModel(model) {
|
|
70
74
|
moeCapableVideoModels.add(model);
|
|
71
75
|
}
|
|
@@ -117,8 +121,10 @@ export async function* video(request) {
|
|
|
117
121
|
const model = asVideoModel(getModel(request.modelId), request.modelId);
|
|
118
122
|
if (ltxVideoModels.has(model))
|
|
119
123
|
parseVideoRequest(ltxVideoRequestSchema, request);
|
|
120
|
-
else
|
|
124
|
+
else {
|
|
121
125
|
parseVideoRequest(nonLtxVideoRequestSchema, request);
|
|
126
|
+
parseVideoRequest(h3VideoModels.has(model) ? h3VideoRequestSchema : wanVideoRequestSchema, request);
|
|
127
|
+
}
|
|
122
128
|
if (!moeCapableVideoModels.has(model))
|
|
123
129
|
parseVideoRequest(singleExpertVideoRequestSchema, request);
|
|
124
130
|
validateProviderPaths(request);
|
|
@@ -38,6 +38,7 @@ export declare const diffusionPlugin: {
|
|
|
38
38
|
auto: "auto";
|
|
39
39
|
f32: "f32";
|
|
40
40
|
f16: "f16";
|
|
41
|
+
q8_0: "q8_0";
|
|
41
42
|
bf16: "bf16";
|
|
42
43
|
q2_k: "q2_k";
|
|
43
44
|
q3_k: "q3_k";
|
|
@@ -48,7 +49,6 @@ export declare const diffusionPlugin: {
|
|
|
48
49
|
q5_1: "q5_1";
|
|
49
50
|
q5_k: "q5_k";
|
|
50
51
|
q6_k: "q6_k";
|
|
51
|
-
q8_0: "q8_0";
|
|
52
52
|
}>>;
|
|
53
53
|
rng: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
54
54
|
cpu: "cpu";
|
|
@@ -60,12 +60,17 @@ export declare const diffusionPlugin: {
|
|
|
60
60
|
cuda: "cuda";
|
|
61
61
|
std_default: "std_default";
|
|
62
62
|
}>>;
|
|
63
|
-
clip_on_cpu: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
64
|
-
vae_on_cpu: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
65
63
|
vae_auto_cpu_fallback: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
66
64
|
vae_auto_cpu_fallback_memory_ratio: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
67
65
|
vae_tiling: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
68
66
|
offload_to_cpu: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
67
|
+
control_net_cpu: import("zod").ZodOptional<import("zod").ZodCustom<never, never>>;
|
|
68
|
+
clip_on_cpu: import("zod").ZodOptional<import("zod").ZodCustom<never, never>>;
|
|
69
|
+
vae_on_cpu: import("zod").ZodOptional<import("zod").ZodCustom<never, never>>;
|
|
70
|
+
backend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
71
|
+
params_backend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
72
|
+
max_vram: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodNumber, import("zod").ZodString]>>;
|
|
73
|
+
stream_layers: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
69
74
|
flash_attn: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
70
75
|
diffusion_fa: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
71
76
|
lora_apply_mode: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
@@ -899,7 +904,7 @@ export declare const diffusionPlugin: {
|
|
|
899
904
|
}, import("zod/v4/core").$strip>>;
|
|
900
905
|
};
|
|
901
906
|
logging: {
|
|
902
|
-
module: import("@qvac/diffusion-cpp/addonLogging")
|
|
907
|
+
module: () => Promise<typeof import("@qvac/diffusion-cpp/addonLogging")>;
|
|
903
908
|
namespace: "sdcpp-generation";
|
|
904
909
|
};
|
|
905
910
|
};
|
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
import ImgStableDiffusion, { EsrganUpscaler, VideoStableDiffusion } from '@qvac/diffusion-cpp';
|
|
2
|
-
import addonLogging from '@qvac/diffusion-cpp/addonLogging';
|
|
3
2
|
import { definePlugin, defineHandler, sdcppConfigSchema, diffusionRequestSchema, diffusionStreamResponseSchema, videoRequestSchema, videoStreamResponseSchema, upscaleRequestSchema, upscaleStreamResponseSchema, worldSceneRequestSchema, worldSceneStreamResponseSchema, worldStepRequestSchema, worldStepStreamResponseSchema, ModelType, ADDON_DIFFUSION } from '../../../schemas/index.js';
|
|
4
3
|
import { createStreamLogger, registerAddonLogger, getEngineLogger } from '../../../logging/index.js';
|
|
5
4
|
import { ModelLoadFailedError } from '../../../errors/index.js';
|
|
6
5
|
import { isMobile } from '../../../runtime/state.js';
|
|
7
6
|
import { stripMultiGpuKeys } from '../../../utils/multi-gpu-mobile.js';
|
|
8
7
|
import { diffusion } from './ops/diffusion.js';
|
|
9
|
-
import { markLtxVideoModel, markMoeCapableVideoModel, video } from './ops/video.js';
|
|
8
|
+
import { markLtxVideoModel, markH3VideoModel, markMoeCapableVideoModel, video } from './ops/video.js';
|
|
10
9
|
import { upscale } from './ops/upscale.js';
|
|
11
10
|
import { createWorldSession, worldCreateScene, worldScenePath, worldStep } from './ops/world.js';
|
|
12
11
|
// Single source of truth for `SdcppConfig.upscaler.*` → addon-config key
|
|
@@ -111,6 +110,7 @@ export const diffusionPlugin = definePlugin({
|
|
|
111
110
|
const worldEquivalent = {
|
|
112
111
|
device: 'world.backend',
|
|
113
112
|
'main-gpu': 'world.backend',
|
|
113
|
+
backend: 'world.backend',
|
|
114
114
|
threads: 'world.threads',
|
|
115
115
|
offload_to_cpu: 'world.offloadParamsToCpu'
|
|
116
116
|
};
|
|
@@ -145,20 +145,29 @@ export const diffusionPlugin = definePlugin({
|
|
|
145
145
|
return { config: cfg };
|
|
146
146
|
}
|
|
147
147
|
const { clipLModelSrc, clipGModelSrc, clipVisionModelSrc, t5XxlModelSrc, llmModelSrc, vaeModelSrc, highNoiseDiffusionModelSrc, uncondModelSrc, audioVaeModelSrc, embeddingsConnectorsModelSrc, taehvModelSrc, sceneSrc, upscaler, ...rest } = cfg;
|
|
148
|
-
// audioVae / embeddingsConnectors are LTX-2 video companions. Reject them
|
|
149
|
-
// up front — before any companion download — when the selected layout
|
|
150
|
-
// can't consume them, instead of silently resolving the file and then
|
|
151
|
-
// dropping it in createModel. Both fields are new, so this cannot break
|
|
152
|
-
// existing callers.
|
|
153
148
|
if (embeddingsConnectorsModelSrc && cfg.mode !== 'video') {
|
|
154
149
|
throw new ModelLoadFailedError('modelConfig.embeddingsConnectorsModelSrc selects the LTX-2 video ' +
|
|
155
150
|
"layout and is only valid with mode: 'video'.");
|
|
156
151
|
}
|
|
157
|
-
if (audioVaeModelSrc &&
|
|
158
|
-
throw new ModelLoadFailedError(
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
152
|
+
if (audioVaeModelSrc && cfg.mode !== 'video') {
|
|
153
|
+
throw new ModelLoadFailedError("modelConfig.audioVaeModelSrc requires mode: 'video' (LTX-2 or MiniMax-H3).");
|
|
154
|
+
}
|
|
155
|
+
// `t5XxlModelSrc` selects Wan and `llmModelSrc` is ignored for that layout.
|
|
156
|
+
// Without T5, `llmModelSrc` plus both VAEs selects MiniMax-H3. Reject the
|
|
157
|
+
// companions H3 cannot consume before any of them is downloaded.
|
|
158
|
+
if (cfg.mode === 'video' &&
|
|
159
|
+
!embeddingsConnectorsModelSrc &&
|
|
160
|
+
(audioVaeModelSrc || (llmModelSrc && !t5XxlModelSrc))) {
|
|
161
|
+
if (!audioVaeModelSrc || !llmModelSrc || !vaeModelSrc) {
|
|
162
|
+
throw new ModelLoadFailedError('MiniMax-H3 requires llmModelSrc, vaeModelSrc and audioVaeModelSrc.');
|
|
163
|
+
}
|
|
164
|
+
if (t5XxlModelSrc ||
|
|
165
|
+
highNoiseDiffusionModelSrc ||
|
|
166
|
+
clipVisionModelSrc ||
|
|
167
|
+
clipLModelSrc ||
|
|
168
|
+
clipGModelSrc) {
|
|
169
|
+
throw new ModelLoadFailedError('MiniMax-H3 cannot use Wan T5, high-noise, CLIP-vision or CLIP-L/G companions.');
|
|
170
|
+
}
|
|
162
171
|
}
|
|
163
172
|
// Video does not apply ESRGAN, so the whole `upscaler` object is dropped.
|
|
164
173
|
// World does not either, but it REJECTS the field above rather than dropping
|
|
@@ -287,15 +296,17 @@ export const diffusionPlugin = definePlugin({
|
|
|
287
296
|
return { model };
|
|
288
297
|
}
|
|
289
298
|
if (config.mode === 'video') {
|
|
290
|
-
//
|
|
291
|
-
//
|
|
292
|
-
//
|
|
293
|
-
//
|
|
294
|
-
//
|
|
299
|
+
// Companion layout selection, mirroring how the addon self-detects it:
|
|
300
|
+
// the LTX-2 text-embedding connectors switch to the LTX-2 layout (Gemma
|
|
301
|
+
// via llm + video VAE + connectors, optional audio VAE —
|
|
302
|
+
// `SdModel::isLtxModel_ = !embeddingsConnectorsPath.empty()`); without
|
|
303
|
+
// them, an audio VAE (or an llm with no Wan T5) means MiniMax-H3
|
|
304
|
+
// (llm + video VAE + audio VAE); everything else is Wan via t5Xxl.
|
|
305
|
+
// Native code verifies the model's tensors.
|
|
295
306
|
const embeddingsConnectorsModelPath = artifacts?.['embeddingsConnectorsModelPath'];
|
|
296
307
|
if (!artifacts?.['vaeModelPath']) {
|
|
297
308
|
throw new ModelLoadFailedError('modelConfig.vaeModelSrc is required in video mode. ' +
|
|
298
|
-
'Provide the Wan
|
|
309
|
+
'Provide the Wan, LTX-2 or MiniMax-H3 video VAE before loading the video pipeline.');
|
|
299
310
|
}
|
|
300
311
|
const vaeModelPath = artifacts['vaeModelPath'];
|
|
301
312
|
let files;
|
|
@@ -315,6 +326,25 @@ export const diffusionPlugin = definePlugin({
|
|
|
315
326
|
...(artifacts['esrganModelPath'] && { esrgan: artifacts['esrganModelPath'] })
|
|
316
327
|
};
|
|
317
328
|
}
|
|
329
|
+
else if (artifacts['audioVaeModelPath'] ||
|
|
330
|
+
(artifacts['llmModelPath'] && !artifacts['t5XxlModelPath'])) {
|
|
331
|
+
if (!artifacts['audioVaeModelPath'] || !artifacts['llmModelPath']) {
|
|
332
|
+
throw new ModelLoadFailedError('MiniMax-H3 requires llmModelSrc, vaeModelSrc and audioVaeModelSrc.');
|
|
333
|
+
}
|
|
334
|
+
if (artifacts['t5XxlModelPath'] ||
|
|
335
|
+
artifacts['highNoiseDiffusionModelPath'] ||
|
|
336
|
+
artifacts['clipVisionModelPath'] ||
|
|
337
|
+
artifacts['clipLModelPath'] ||
|
|
338
|
+
artifacts['clipGModelPath']) {
|
|
339
|
+
throw new ModelLoadFailedError('MiniMax-H3 cannot use Wan T5, high-noise, CLIP-vision or CLIP-L/G companions.');
|
|
340
|
+
}
|
|
341
|
+
files = {
|
|
342
|
+
model: modelPath,
|
|
343
|
+
vae: vaeModelPath,
|
|
344
|
+
llm: artifacts['llmModelPath'],
|
|
345
|
+
audioVae: artifacts['audioVaeModelPath']
|
|
346
|
+
};
|
|
347
|
+
}
|
|
318
348
|
else {
|
|
319
349
|
if (!artifacts['t5XxlModelPath']) {
|
|
320
350
|
throw new ModelLoadFailedError('modelConfig.t5XxlModelSrc is required in video mode. ' +
|
|
@@ -344,6 +374,8 @@ export const diffusionPlugin = definePlugin({
|
|
|
344
374
|
});
|
|
345
375
|
if (embeddingsConnectorsModelPath)
|
|
346
376
|
markLtxVideoModel(model);
|
|
377
|
+
else if (files.audioVae)
|
|
378
|
+
markH3VideoModel(model);
|
|
347
379
|
if (files.highNoiseDiffusionModel)
|
|
348
380
|
markMoeCapableVideoModel(model);
|
|
349
381
|
return { model };
|
|
@@ -425,7 +457,7 @@ export const diffusionPlugin = definePlugin({
|
|
|
425
457
|
})
|
|
426
458
|
},
|
|
427
459
|
logging: {
|
|
428
|
-
module: addonLogging,
|
|
460
|
+
module: () => import('@qvac/diffusion-cpp/addonLogging'),
|
|
429
461
|
namespace: ModelType.sdcppGeneration
|
|
430
462
|
}
|
|
431
463
|
});
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
/**
|
|
3
|
+
* The abort is persistent state, not a one-shot poke. The addon's `cancel()`
|
|
4
|
+
* only reaches a native job that is live at that instant, and every chunked
|
|
5
|
+
* path (`stream: true`, `sentenceStream`, the duplex session) runs one native
|
|
6
|
+
* job per sentence — so a cancel that lands between two jobs is a native
|
|
7
|
+
* no-op and synthesis would carry on. Ops call this on every chunk: once the
|
|
8
|
+
* signal is aborted it re-issues the cancel (the next job is live by then),
|
|
9
|
+
* and returns true so the caller drops the output instead of forwarding it.
|
|
10
|
+
*/
|
|
11
|
+
export declare function cancelIfAborted(model: unknown, signal: AbortSignal): Promise<boolean>;
|
|
12
|
+
/**
|
|
13
|
+
* Open the request-registry context that makes the model-scoped hard cancel
|
|
14
|
+
* both TTS handlers advertise actually work, and bind its abort signal to the
|
|
15
|
+
* addon's `cancel()`.
|
|
16
|
+
*
|
|
17
|
+
* `cancel: { scope: 'model', hard: true }` is a claim the runtime does not
|
|
18
|
+
* enforce on its own: `cancelByModelId` walks the request registry, so a run
|
|
19
|
+
* that never opened a context is invisible to it — a caller's `cancel()` would
|
|
20
|
+
* resolve with `cancelled: 0` while the native job ran to completion. Same
|
|
21
|
+
* shape as the transcription and audiogen ops.
|
|
22
|
+
*
|
|
23
|
+
* The returned context is an async disposable: callers must take it with
|
|
24
|
+
* `await using`, and must bail out when `ctx.signal.aborted` is already true on
|
|
25
|
+
* return — a queued request can resume from `begin()` already aborted, and it
|
|
26
|
+
* never owned the model slot, so cancelling then would interrupt the run that
|
|
27
|
+
* does.
|
|
28
|
+
*/
|
|
29
|
+
export declare function bindTtsCancel(model: unknown, modelId: string, requestId?: string): Promise<import("../../../../runtime/index.js").ManagedRequestContext>;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { getEngineLogger } from '../../../../logging/index.js';
|
|
2
|
+
import { getRequestRegistry, withRequestContext } from '../../../../runtime/index.js';
|
|
3
|
+
import { generateRequestId } from '../../../../runtime/request-id.js';
|
|
4
|
+
/**
|
|
5
|
+
* The abort is persistent state, not a one-shot poke. The addon's `cancel()`
|
|
6
|
+
* only reaches a native job that is live at that instant, and every chunked
|
|
7
|
+
* path (`stream: true`, `sentenceStream`, the duplex session) runs one native
|
|
8
|
+
* job per sentence — so a cancel that lands between two jobs is a native
|
|
9
|
+
* no-op and synthesis would carry on. Ops call this on every chunk: once the
|
|
10
|
+
* signal is aborted it re-issues the cancel (the next job is live by then),
|
|
11
|
+
* and returns true so the caller drops the output instead of forwarding it.
|
|
12
|
+
*/
|
|
13
|
+
export async function cancelIfAborted(model, signal) {
|
|
14
|
+
if (!signal.aborted)
|
|
15
|
+
return false;
|
|
16
|
+
const cancellable = model;
|
|
17
|
+
if (typeof cancellable.cancel === 'function') {
|
|
18
|
+
await cancellable.cancel().catch(() => { });
|
|
19
|
+
}
|
|
20
|
+
return true;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Open the request-registry context that makes the model-scoped hard cancel
|
|
24
|
+
* both TTS handlers advertise actually work, and bind its abort signal to the
|
|
25
|
+
* addon's `cancel()`.
|
|
26
|
+
*
|
|
27
|
+
* `cancel: { scope: 'model', hard: true }` is a claim the runtime does not
|
|
28
|
+
* enforce on its own: `cancelByModelId` walks the request registry, so a run
|
|
29
|
+
* that never opened a context is invisible to it — a caller's `cancel()` would
|
|
30
|
+
* resolve with `cancelled: 0` while the native job ran to completion. Same
|
|
31
|
+
* shape as the transcription and audiogen ops.
|
|
32
|
+
*
|
|
33
|
+
* The returned context is an async disposable: callers must take it with
|
|
34
|
+
* `await using`, and must bail out when `ctx.signal.aborted` is already true on
|
|
35
|
+
* return — a queued request can resume from `begin()` already aborted, and it
|
|
36
|
+
* never owned the model slot, so cancelling then would interrupt the run that
|
|
37
|
+
* does.
|
|
38
|
+
*/
|
|
39
|
+
export async function bindTtsCancel(model, modelId, requestId) {
|
|
40
|
+
const ctx = await getRequestRegistry().begin({
|
|
41
|
+
requestId: requestId ?? generateRequestId(),
|
|
42
|
+
kind: 'tts',
|
|
43
|
+
modelId
|
|
44
|
+
});
|
|
45
|
+
if (ctx.signal.aborted)
|
|
46
|
+
return ctx;
|
|
47
|
+
const logger = withRequestContext(getEngineLogger(), ctx);
|
|
48
|
+
const cancellable = model;
|
|
49
|
+
let cancelPromise;
|
|
50
|
+
const onAbort = () => {
|
|
51
|
+
if (typeof cancellable.cancel !== 'function')
|
|
52
|
+
return;
|
|
53
|
+
cancelPromise ??= cancellable.cancel().catch((error) => {
|
|
54
|
+
logger.warn(`[cancel] model.cancel() rejected during abort for modelId=${modelId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
55
|
+
});
|
|
56
|
+
};
|
|
57
|
+
ctx.signal.addEventListener('abort', onAbort, { once: true });
|
|
58
|
+
// Closes the window between the check above and the listener being attached.
|
|
59
|
+
if (ctx.signal.aborted)
|
|
60
|
+
onAbort();
|
|
61
|
+
ctx.scope.defer(async () => {
|
|
62
|
+
ctx.signal.removeEventListener('abort', onAbort);
|
|
63
|
+
await cancelPromise;
|
|
64
|
+
});
|
|
65
|
+
return ctx;
|
|
66
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { TTS_COSYVOICE3_EMOTIONS } from '../../../../schemas/index.js';
|
|
1
2
|
import { PluginRequestValidationFailedError } from '../../../../errors/index.js';
|
|
2
3
|
function hasEngineType(model) {
|
|
3
4
|
return (typeof model === 'object' &&
|
|
@@ -37,6 +38,15 @@ export function assertParlerJobOptionsSupported(model, options, handlerName) {
|
|
|
37
38
|
if (parlerOnlyKeys.length > 0) {
|
|
38
39
|
throw new PluginRequestValidationFailedError(handlerName, `${parlerOnlyKeys.join(', ')} ${parlerOnlyKeys.length === 1 ? 'is' : 'are'} only supported by Parler TTS models; CosyVoice3 supports emotion and pace`);
|
|
39
40
|
}
|
|
41
|
+
// The request schema validates `emotion` against Parler's 12-value
|
|
42
|
+
// vocabulary because a request carries only an opaque modelId. Now that
|
|
43
|
+
// the engine is known, narrow it to the four CosyVoice3 has a trained
|
|
44
|
+
// instruction for — the addon rejects the other eight, and rejecting them
|
|
45
|
+
// here names the supported set instead of surfacing a raw addon throw.
|
|
46
|
+
if (options.emotion !== undefined &&
|
|
47
|
+
!TTS_COSYVOICE3_EMOTIONS.includes(options.emotion)) {
|
|
48
|
+
throw new PluginRequestValidationFailedError(handlerName, `emotion "${options.emotion}" is not supported by CosyVoice3; use one of ${TTS_COSYVOICE3_EMOTIONS.join(', ')}`);
|
|
49
|
+
}
|
|
40
50
|
// CosyVoice3 is trained on one instruction per synthesis; a request that
|
|
41
51
|
// engages both channels always conflicts regardless of the loaded config.
|
|
42
52
|
if (options.emotion !== undefined &&
|