@qvac/inference 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +95 -39
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/delete-cache.js +3 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.js +169 -68
- package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
- package/dist/resources/model-fit/estimators/llm.js +3 -1
- package/dist/resources/model-fit/floor.d.ts +31 -0
- package/dist/resources/model-fit/floor.js +65 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/resources/native.js +5 -4
- package/dist/resources/types.d.ts +1 -3
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +84 -0
- package/dist/schemas/assess-model-fit.js +72 -3
- package/dist/schemas/audio-gen.d.ts +482 -7
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +10 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci.d.ts +1 -1
- package/dist/schemas/common.d.ts +782 -19
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +14 -1
- package/dist/schemas/completion-event.js +4 -0
- package/dist/schemas/completion-stream.d.ts +22 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2604 -180
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/model-ops.d.ts +3 -0
- package/dist/schemas/model-ops.js +6 -0
- package/dist/schemas/plugin.d.ts +21 -11
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +30 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +29 -26
|
@@ -1,232 +1,63 @@
|
|
|
1
|
-
var __addDisposableResource = (this && this.__addDisposableResource) || function (env, value, async) {
|
|
2
|
-
if (value !== null && value !== void 0) {
|
|
3
|
-
if (typeof value !== "object" && typeof value !== "function") throw new TypeError("Object expected.");
|
|
4
|
-
var dispose, inner;
|
|
5
|
-
if (async) {
|
|
6
|
-
if (!Symbol.asyncDispose) throw new TypeError("Symbol.asyncDispose is not defined.");
|
|
7
|
-
dispose = value[Symbol.asyncDispose];
|
|
8
|
-
}
|
|
9
|
-
if (dispose === void 0) {
|
|
10
|
-
if (!Symbol.dispose) throw new TypeError("Symbol.dispose is not defined.");
|
|
11
|
-
dispose = value[Symbol.dispose];
|
|
12
|
-
if (async) inner = dispose;
|
|
13
|
-
}
|
|
14
|
-
if (typeof dispose !== "function") throw new TypeError("Object not disposable.");
|
|
15
|
-
if (inner) dispose = function() { try { inner.call(this); } catch (e) { return Promise.reject(e); } };
|
|
16
|
-
env.stack.push({ value: value, dispose: dispose, async: async });
|
|
17
|
-
}
|
|
18
|
-
else if (async) {
|
|
19
|
-
env.stack.push({ async: true });
|
|
20
|
-
}
|
|
21
|
-
return value;
|
|
22
|
-
};
|
|
23
|
-
var __disposeResources = (this && this.__disposeResources) || (function (SuppressedError) {
|
|
24
|
-
return function (env) {
|
|
25
|
-
function fail(e) {
|
|
26
|
-
env.error = env.hasError ? new SuppressedError(e, env.error, "An error was suppressed during disposal.") : e;
|
|
27
|
-
env.hasError = true;
|
|
28
|
-
}
|
|
29
|
-
var r, s = 0;
|
|
30
|
-
function next() {
|
|
31
|
-
while (r = env.stack.pop()) {
|
|
32
|
-
try {
|
|
33
|
-
if (!r.async && s === 1) return s = 0, env.stack.push(r), Promise.resolve().then(next);
|
|
34
|
-
if (r.dispose) {
|
|
35
|
-
var result = r.dispose.call(r.value);
|
|
36
|
-
if (r.async) return s |= 2, Promise.resolve(result).then(next, function(e) { fail(e); return next(); });
|
|
37
|
-
}
|
|
38
|
-
else s |= 1;
|
|
39
|
-
}
|
|
40
|
-
catch (e) {
|
|
41
|
-
fail(e);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
if (s === 1) return env.hasError ? Promise.reject(env.error) : Promise.resolve();
|
|
45
|
-
if (env.hasError) throw env.error;
|
|
46
|
-
}
|
|
47
|
-
return next();
|
|
48
|
-
};
|
|
49
|
-
})(typeof SuppressedError === "function" ? SuppressedError : function (error, suppressed, message) {
|
|
50
|
-
var e = new Error(message);
|
|
51
|
-
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
|
-
});
|
|
53
|
-
import { AudioGen, audiogenBackendName, audiogenGpuFallbackReason } from '@qvac/audiogen-ggml';
|
|
54
|
-
import { z } from 'zod';
|
|
55
|
-
import { audioGenStatsSchema } from '../../../../schemas/audio-gen.js';
|
|
56
|
-
import { graphicsDriverSchema } from '../../../../schemas/index.js';
|
|
57
|
-
import { getEngineLogger } from '../../../../logging/index.js';
|
|
58
|
-
import { attachBackendDiagnostics } from '../../../../profiling/backend-diagnostics.js';
|
|
59
1
|
import { resolveAudioGenPcm } from './audio-gen-input.js';
|
|
60
|
-
import {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
try {
|
|
67
|
-
const ctx = __addDisposableResource(env_1, await getRequestRegistry().begin({
|
|
68
|
-
requestId: request.requestId ?? generateRequestId(),
|
|
69
|
-
kind: 'audiogen',
|
|
70
|
-
modelId: request.modelId
|
|
71
|
-
}), true);
|
|
72
|
-
const logger = withRequestContext(getEngineLogger(), ctx);
|
|
73
|
-
const candidate = getModel(request.modelId);
|
|
74
|
-
if (!(candidate instanceof AudioGen)) {
|
|
75
|
-
throw new ModelOperationNotSupportedError(request.modelId, 'audiogen-ggml', 'audioGenStream', [], []);
|
|
76
|
-
}
|
|
77
|
-
const model = candidate;
|
|
78
|
-
// A queued request can resume from begin() already aborted. It never owned
|
|
79
|
-
// the model slot, so calling the model-scoped cancel here would interrupt
|
|
80
|
-
// the earlier same-model generation that still owns it.
|
|
81
|
-
if (ctx.signal.aborted) {
|
|
82
|
-
yield {
|
|
83
|
-
type: 'audioGenStream',
|
|
84
|
-
done: true,
|
|
85
|
-
stopReason: 'cancelled'
|
|
86
|
-
};
|
|
87
|
-
return;
|
|
88
|
-
}
|
|
89
|
-
let cancelPromise;
|
|
90
|
-
const onAbort = () => {
|
|
91
|
-
cancelPromise ??= model.cancel().catch((error) => {
|
|
92
|
-
logger.warn(`[cancel] model.cancel() rejected during abort for modelId=${request.modelId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
93
|
-
});
|
|
94
|
-
};
|
|
95
|
-
ctx.signal.addEventListener('abort', onAbort, { once: true });
|
|
96
|
-
if (ctx.signal.aborted)
|
|
97
|
-
onAbort();
|
|
98
|
-
ctx.scope.defer(async () => {
|
|
99
|
-
ctx.signal.removeEventListener('abort', onAbort);
|
|
100
|
-
await cancelPromise;
|
|
101
|
-
});
|
|
102
|
-
let response;
|
|
103
|
-
try {
|
|
2
|
+
import { streamAudioGenRun } from './audio-gen-run.js';
|
|
3
|
+
export function audioGenStream(request) {
|
|
4
|
+
return streamAudioGenRun({
|
|
5
|
+
type: 'audioGenStream',
|
|
6
|
+
request,
|
|
7
|
+
async start(model, ctx, logger) {
|
|
104
8
|
// Reference/source audio is decoded before the run is admitted so the
|
|
105
9
|
// model slot is never held by a request that fails on input decoding.
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
:
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
})
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
type: 'audioGenStream',
|
|
156
|
-
data: Buffer.from(pcm).toString('base64'),
|
|
157
|
-
sampleRate: chunk.sampleRate,
|
|
158
|
-
channels: chunk.channels,
|
|
159
|
-
bitsPerSample: Int16Array.BYTES_PER_ELEMENT * 8,
|
|
160
|
-
done: false
|
|
161
|
-
};
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
}
|
|
165
|
-
catch (error) {
|
|
166
|
-
if (!ctx.signal.aborted)
|
|
167
|
-
throw error;
|
|
168
|
-
}
|
|
169
|
-
if (ctx.signal.aborted) {
|
|
170
|
-
yield {
|
|
171
|
-
type: 'audioGenStream',
|
|
172
|
-
done: true,
|
|
173
|
-
stopReason: 'cancelled'
|
|
174
|
-
};
|
|
175
|
-
return;
|
|
10
|
+
const { referenceAudio, sourceAudio } = await resolveAudioInputs(request, logger);
|
|
11
|
+
if (ctx.signal.aborted)
|
|
12
|
+
return undefined;
|
|
13
|
+
return model.run(request.caption, {
|
|
14
|
+
...(request.lyrics !== undefined && { lyrics: request.lyrics }),
|
|
15
|
+
...(request.seed !== undefined && { seed: request.seed }),
|
|
16
|
+
...(request.vocalLanguage !== undefined && { vocalLanguage: request.vocalLanguage }),
|
|
17
|
+
...(request.bpm !== undefined && { bpm: request.bpm }),
|
|
18
|
+
...(request.keyscale !== undefined && { keyscale: request.keyscale }),
|
|
19
|
+
...(request.timesignature !== undefined && { timesignature: request.timesignature }),
|
|
20
|
+
...(request.augmentCaptionWithMetadata !== undefined && {
|
|
21
|
+
augmentCaptionWithMetadata: request.augmentCaptionWithMetadata
|
|
22
|
+
}),
|
|
23
|
+
...(request.simpleMode !== undefined && { simpleMode: request.simpleMode }),
|
|
24
|
+
...(request.rewriteQuery !== undefined && { rewriteQuery: request.rewriteQuery }),
|
|
25
|
+
...(request.generateLrc !== undefined && { generateLrc: request.generateLrc }),
|
|
26
|
+
...(request.computeQualityScore !== undefined && {
|
|
27
|
+
computeQualityScore: request.computeQualityScore
|
|
28
|
+
}),
|
|
29
|
+
...(request.normalizeLoudness !== undefined && {
|
|
30
|
+
normalizeLoudness: request.normalizeLoudness
|
|
31
|
+
}),
|
|
32
|
+
...(request.guidanceScale !== undefined && { guidanceScale: request.guidanceScale }),
|
|
33
|
+
...(request.track !== undefined && { track: request.track }),
|
|
34
|
+
...(request.duration !== undefined && { duration: request.duration }),
|
|
35
|
+
...(request.maxFrames !== undefined && { maxFrames: request.maxFrames }),
|
|
36
|
+
...(request.inferenceSteps !== undefined && { inferenceSteps: request.inferenceSteps }),
|
|
37
|
+
...(request.cfgScale !== undefined && { cfgScale: request.cfgScale }),
|
|
38
|
+
...(request.lmTemperature !== undefined && { lmTemperature: request.lmTemperature }),
|
|
39
|
+
...(request.lmTopP !== undefined && { lmTopP: request.lmTopP }),
|
|
40
|
+
...(request.lmTopK !== undefined && { lmTopK: request.lmTopK }),
|
|
41
|
+
...(request.lmCfgScale !== undefined && { lmCfgScale: request.lmCfgScale }),
|
|
42
|
+
...(request.lmPhase1 !== undefined && { lmPhase1: request.lmPhase1 }),
|
|
43
|
+
...(request.dcwEnabled !== undefined && { dcwEnabled: request.dcwEnabled }),
|
|
44
|
+
...(request.dcwScaler !== undefined && { dcwScaler: request.dcwScaler }),
|
|
45
|
+
...(request.dcwHighScaler !== undefined && { dcwHighScaler: request.dcwHighScaler }),
|
|
46
|
+
...(request.audioCodes !== undefined && {
|
|
47
|
+
audioCodes: Int32Array.from(request.audioCodes)
|
|
48
|
+
}),
|
|
49
|
+
...(request.taskType !== undefined && { taskType: request.taskType }),
|
|
50
|
+
...(request.audioCoverStrength !== undefined && {
|
|
51
|
+
audioCoverStrength: request.audioCoverStrength
|
|
52
|
+
}),
|
|
53
|
+
...(request.coverNoiseStrength !== undefined && {
|
|
54
|
+
coverNoiseStrength: request.coverNoiseStrength
|
|
55
|
+
}),
|
|
56
|
+
...(referenceAudio && { referenceAudio }),
|
|
57
|
+
...(sourceAudio && { sourceAudio })
|
|
58
|
+
});
|
|
176
59
|
}
|
|
177
|
-
|
|
178
|
-
const stats = audioGenStatsSchema.parse(raw);
|
|
179
|
-
const diagnostics = buildBackendDiagnostics(audiogenStats.parse(raw));
|
|
180
|
-
const terminal = {
|
|
181
|
-
type: 'audioGenStream',
|
|
182
|
-
done: true,
|
|
183
|
-
stopReason: 'completed',
|
|
184
|
-
stats,
|
|
185
|
-
...(diagnostics && { diagnostics })
|
|
186
|
-
};
|
|
187
|
-
yield diagnostics ? attachBackendDiagnostics(terminal, diagnostics) : terminal;
|
|
188
|
-
}
|
|
189
|
-
catch (e_1) {
|
|
190
|
-
env_1.error = e_1;
|
|
191
|
-
env_1.hasError = true;
|
|
192
|
-
}
|
|
193
|
-
finally {
|
|
194
|
-
const result_1 = __disposeResources(env_1);
|
|
195
|
-
if (result_1)
|
|
196
|
-
await result_1;
|
|
197
|
-
}
|
|
198
|
-
}
|
|
199
|
-
// The addon reports gpuFallbackReason, which the wire shape deliberately does
|
|
200
|
-
// not carry: it reaches callers named, on diagnostics.fallback.reason.
|
|
201
|
-
const audiogenStats = audioGenStatsSchema.extend({
|
|
202
|
-
gpuFallbackReason: z.number().optional()
|
|
203
|
-
});
|
|
204
|
-
/** An unrecognized GPU id yields no diagnostics rather than a guessed backend name. */
|
|
205
|
-
function buildBackendDiagnostics(stats) {
|
|
206
|
-
if (stats.backendDevice === undefined)
|
|
207
|
-
return undefined;
|
|
208
|
-
if (stats.backendDevice !== 1) {
|
|
209
|
-
const fallback = gpuFallbackDetail(stats);
|
|
210
|
-
return { selectedBackend: 'cpu', selectedDevice: 'cpu', ...(fallback && { fallback }) };
|
|
211
|
-
}
|
|
212
|
-
// A 'cpu' name against backendDevice 1 is the addon contradicting itself.
|
|
213
|
-
const selectedBackend = audiogenBackendName(stats.backendId);
|
|
214
|
-
if (selectedBackend === undefined || selectedBackend === 'cpu')
|
|
215
|
-
return undefined;
|
|
216
|
-
const graphicsApi = graphicsDriverSchema.safeParse(selectedBackend);
|
|
217
|
-
return {
|
|
218
|
-
selectedBackend,
|
|
219
|
-
selectedDevice: 'gpu',
|
|
220
|
-
...(graphicsApi.success && { graphicsApi: graphicsApi.data })
|
|
221
|
-
};
|
|
222
|
-
}
|
|
223
|
-
// `none` and `not-requested` describe a run that never lost a GPU, and an
|
|
224
|
-
// unmapped code must not become a guessed reason.
|
|
225
|
-
function gpuFallbackDetail(stats) {
|
|
226
|
-
const reason = audiogenGpuFallbackReason(stats.gpuFallbackReason);
|
|
227
|
-
if (reason === undefined || reason === 'none' || reason === 'not-requested')
|
|
228
|
-
return undefined;
|
|
229
|
-
return { requestedDevice: 'gpu', reason };
|
|
60
|
+
});
|
|
230
61
|
}
|
|
231
62
|
/**
|
|
232
63
|
* Decode both optional audio inputs concurrently. When both fail, the first
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { AudioUnderstandRequest, AudioUnderstandResponse } from '../../../../schemas/audio-gen.js';
|
|
2
|
+
/**
|
|
3
|
+
* ACE-Step's reverse pipeline: encode a recording, recover its FSQ semantic
|
|
4
|
+
* codes, and let the LM describe it. Shares the generation path's admission and
|
|
5
|
+
* cancellation, and streams the description as one `understand` frame before
|
|
6
|
+
* the terminal frame repeats it on `stats`.
|
|
7
|
+
*/
|
|
8
|
+
export declare function audioUnderstand(request: AudioUnderstandRequest): AsyncGenerator<AudioUnderstandResponse>;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { resolveAudioGenPcm } from './audio-gen-input.js';
|
|
2
|
+
import { streamAudioGenRun } from './audio-gen-run.js';
|
|
3
|
+
/**
|
|
4
|
+
* ACE-Step's reverse pipeline: encode a recording, recover its FSQ semantic
|
|
5
|
+
* codes, and let the LM describe it. Shares the generation path's admission and
|
|
6
|
+
* cancellation, and streams the description as one `understand` frame before
|
|
7
|
+
* the terminal frame repeats it on `stats`.
|
|
8
|
+
*/
|
|
9
|
+
export function audioUnderstand(request) {
|
|
10
|
+
return streamAudioGenRun({
|
|
11
|
+
type: 'audioUnderstand',
|
|
12
|
+
request,
|
|
13
|
+
async start(model, ctx) {
|
|
14
|
+
// Decoded before the job is admitted, so the model slot is never held by
|
|
15
|
+
// a request that fails on input decoding.
|
|
16
|
+
const sourceAudio = await resolveAudioGenPcm(request.sourceAudio, 'sourceAudio');
|
|
17
|
+
if (ctx.signal.aborted)
|
|
18
|
+
return undefined;
|
|
19
|
+
return model.understand(sourceAudio, {
|
|
20
|
+
...(request.seed !== undefined && { seed: request.seed }),
|
|
21
|
+
...(request.vocalLanguage !== undefined && { vocalLanguage: request.vocalLanguage }),
|
|
22
|
+
...(request.lmTemperature !== undefined && { lmTemperature: request.lmTemperature }),
|
|
23
|
+
...(request.lmTopP !== undefined && { lmTopP: request.lmTopP }),
|
|
24
|
+
...(request.lmTopK !== undefined && { lmTopK: request.lmTopK })
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
}
|
|
@@ -261,6 +261,7 @@ export declare const audioGenPlugin: {
|
|
|
261
261
|
audioGenStream: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
|
|
262
262
|
type: import("zod").ZodLiteral<"audioGenStream">;
|
|
263
263
|
requestId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
264
|
+
audioCodes: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodNumber>>;
|
|
264
265
|
referenceAudio: import("zod").ZodOptional<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
265
266
|
type: import("zod").ZodLiteral<"base64">;
|
|
266
267
|
value: import("zod").ZodString;
|
|
@@ -283,6 +284,27 @@ export declare const audioGenPlugin: {
|
|
|
283
284
|
bpm: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
284
285
|
keyscale: import("zod").ZodOptional<import("zod").ZodString>;
|
|
285
286
|
timesignature: import("zod").ZodOptional<import("zod").ZodString>;
|
|
287
|
+
augmentCaptionWithMetadata: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
288
|
+
simpleMode: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
289
|
+
rewriteQuery: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
290
|
+
generateLrc: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
291
|
+
computeQualityScore: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
292
|
+
normalizeLoudness: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
293
|
+
guidanceScale: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
294
|
+
track: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
295
|
+
vocals: "vocals";
|
|
296
|
+
backing_vocals: "backing_vocals";
|
|
297
|
+
drums: "drums";
|
|
298
|
+
bass: "bass";
|
|
299
|
+
guitar: "guitar";
|
|
300
|
+
keyboard: "keyboard";
|
|
301
|
+
percussion: "percussion";
|
|
302
|
+
strings: "strings";
|
|
303
|
+
synth: "synth";
|
|
304
|
+
fx: "fx";
|
|
305
|
+
brass: "brass";
|
|
306
|
+
woodwinds: "woodwinds";
|
|
307
|
+
}>>;
|
|
286
308
|
duration: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
287
309
|
maxFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
288
310
|
inferenceSteps: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
@@ -298,11 +320,126 @@ export declare const audioGenPlugin: {
|
|
|
298
320
|
taskType: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
299
321
|
text2music: "text2music";
|
|
300
322
|
"cover-nofsq": "cover-nofsq";
|
|
323
|
+
lego: "lego";
|
|
301
324
|
}>>;
|
|
302
325
|
audioCoverStrength: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
303
326
|
coverNoiseStrength: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
304
327
|
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
328
|
+
progress: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
329
|
+
stage: import("zod").ZodString;
|
|
330
|
+
step: import("zod").ZodNumber;
|
|
331
|
+
total: import("zod").ZodNumber;
|
|
332
|
+
}, import("zod/v4/core").$strip>>;
|
|
333
|
+
data: import("zod").ZodOptional<import("zod").ZodString>;
|
|
334
|
+
sampleRate: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
335
|
+
channels: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
336
|
+
bitsPerSample: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
337
|
+
done: import("zod").ZodDefault<import("zod").ZodBoolean>;
|
|
338
|
+
stopReason: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
339
|
+
cancelled: "cancelled";
|
|
340
|
+
completed: "completed";
|
|
341
|
+
}>>;
|
|
342
|
+
stats: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
343
|
+
audioDurationMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
344
|
+
totalTimeMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
345
|
+
realTimeFactor: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
346
|
+
backendDevice: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
347
|
+
backendId: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
348
|
+
lyricsScore: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
349
|
+
lrc: import("zod").ZodOptional<import("zod").ZodString>;
|
|
350
|
+
qualityScore: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
351
|
+
understand: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
352
|
+
caption: import("zod").ZodString;
|
|
353
|
+
bpm: import("zod").ZodNumber;
|
|
354
|
+
duration: import("zod").ZodNumber;
|
|
355
|
+
keyscale: import("zod").ZodString;
|
|
356
|
+
timesignature: import("zod").ZodString;
|
|
357
|
+
vocalLanguage: import("zod").ZodString;
|
|
358
|
+
audioCodes: import("zod").ZodArray<import("zod").ZodNumber>;
|
|
359
|
+
}, import("zod/v4/core").$strip>>;
|
|
360
|
+
}, import("zod/v4/core").$strip>>;
|
|
361
|
+
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
362
|
+
selectedBackend: import("zod").ZodString;
|
|
363
|
+
selectedDevice: import("zod").ZodEnum<{
|
|
364
|
+
gpu: "gpu";
|
|
365
|
+
cpu: "cpu";
|
|
366
|
+
}>;
|
|
367
|
+
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
368
|
+
vulkan: "vulkan";
|
|
369
|
+
metal: "metal";
|
|
370
|
+
opencl: "opencl";
|
|
371
|
+
cuda: "cuda";
|
|
372
|
+
opengl: "opengl";
|
|
373
|
+
webgpu: "webgpu";
|
|
374
|
+
direct3d11: "direct3d11";
|
|
375
|
+
direct3d12: "direct3d12";
|
|
376
|
+
levelZero: "levelZero";
|
|
377
|
+
rocm: "rocm";
|
|
378
|
+
}>>;
|
|
379
|
+
driver: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
380
|
+
name: import("zod").ZodString;
|
|
381
|
+
version: import("zod").ZodOptional<import("zod").ZodString>;
|
|
382
|
+
}, import("zod/v4/core").$strip>>;
|
|
383
|
+
gpuId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
384
|
+
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
385
|
+
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
386
|
+
requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
387
|
+
gpu: "gpu";
|
|
388
|
+
cpu: "cpu";
|
|
389
|
+
}>>;
|
|
390
|
+
reason: import("zod").ZodString;
|
|
391
|
+
}, import("zod/v4/core").$strip>>;
|
|
392
|
+
probe: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
393
|
+
status: import("zod").ZodEnum<{
|
|
394
|
+
unknown: "unknown";
|
|
395
|
+
compatible: "compatible";
|
|
396
|
+
incompatible: "incompatible";
|
|
397
|
+
}>;
|
|
398
|
+
backend: import("zod").ZodString;
|
|
399
|
+
reason: import("zod").ZodOptional<import("zod").ZodString>;
|
|
400
|
+
}, import("zod/v4/core").$strip>>;
|
|
401
|
+
}, import("zod/v4/core").$strip>>;
|
|
305
402
|
type: import("zod").ZodLiteral<"audioGenStream">;
|
|
403
|
+
}, import("zod/v4/core").$strict>>;
|
|
404
|
+
audioEditStream: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
|
|
405
|
+
sourceAudio: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
406
|
+
type: import("zod").ZodLiteral<"base64">;
|
|
407
|
+
value: import("zod").ZodString;
|
|
408
|
+
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
409
|
+
type: import("zod").ZodLiteral<"filePath">;
|
|
410
|
+
value: import("zod").ZodString;
|
|
411
|
+
}, import("zod/v4/core").$strip>], "type">;
|
|
412
|
+
type: import("zod").ZodLiteral<"audioEditStream">;
|
|
413
|
+
requestId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
414
|
+
modelId: import("zod").ZodString;
|
|
415
|
+
operations: import("zod").ZodArray<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
416
|
+
type: import("zod").ZodLiteral<"flow-edit">;
|
|
417
|
+
from: import("zod").ZodObject<{
|
|
418
|
+
caption: import("zod").ZodString;
|
|
419
|
+
lyrics: import("zod").ZodOptional<import("zod").ZodString>;
|
|
420
|
+
}, import("zod/v4/core").$strict>;
|
|
421
|
+
to: import("zod").ZodObject<{
|
|
422
|
+
caption: import("zod").ZodString;
|
|
423
|
+
lyrics: import("zod").ZodOptional<import("zod").ZodString>;
|
|
424
|
+
}, import("zod/v4/core").$strict>;
|
|
425
|
+
nMin: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
426
|
+
nMax: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
427
|
+
nAvg: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
428
|
+
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
429
|
+
type: import("zod").ZodLiteral<"repaint">;
|
|
430
|
+
caption: import("zod").ZodString;
|
|
431
|
+
lyrics: import("zod").ZodOptional<import("zod").ZodString>;
|
|
432
|
+
start: import("zod").ZodNumber;
|
|
433
|
+
end: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
434
|
+
mode: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
435
|
+
conservative: "conservative";
|
|
436
|
+
balanced: "balanced";
|
|
437
|
+
aggressive: "aggressive";
|
|
438
|
+
}>>;
|
|
439
|
+
strength: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
440
|
+
}, import("zod/v4/core").$strict>], "type">>;
|
|
441
|
+
seed: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
442
|
+
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
306
443
|
progress: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
307
444
|
stage: import("zod").ZodString;
|
|
308
445
|
step: import("zod").ZodNumber;
|
|
@@ -323,6 +460,117 @@ export declare const audioGenPlugin: {
|
|
|
323
460
|
realTimeFactor: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
324
461
|
backendDevice: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
325
462
|
backendId: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
463
|
+
lyricsScore: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
464
|
+
lrc: import("zod").ZodOptional<import("zod").ZodString>;
|
|
465
|
+
qualityScore: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
466
|
+
understand: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
467
|
+
caption: import("zod").ZodString;
|
|
468
|
+
bpm: import("zod").ZodNumber;
|
|
469
|
+
duration: import("zod").ZodNumber;
|
|
470
|
+
keyscale: import("zod").ZodString;
|
|
471
|
+
timesignature: import("zod").ZodString;
|
|
472
|
+
vocalLanguage: import("zod").ZodString;
|
|
473
|
+
audioCodes: import("zod").ZodArray<import("zod").ZodNumber>;
|
|
474
|
+
}, import("zod/v4/core").$strip>>;
|
|
475
|
+
}, import("zod/v4/core").$strip>>;
|
|
476
|
+
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
477
|
+
selectedBackend: import("zod").ZodString;
|
|
478
|
+
selectedDevice: import("zod").ZodEnum<{
|
|
479
|
+
gpu: "gpu";
|
|
480
|
+
cpu: "cpu";
|
|
481
|
+
}>;
|
|
482
|
+
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
483
|
+
vulkan: "vulkan";
|
|
484
|
+
metal: "metal";
|
|
485
|
+
opencl: "opencl";
|
|
486
|
+
cuda: "cuda";
|
|
487
|
+
opengl: "opengl";
|
|
488
|
+
webgpu: "webgpu";
|
|
489
|
+
direct3d11: "direct3d11";
|
|
490
|
+
direct3d12: "direct3d12";
|
|
491
|
+
levelZero: "levelZero";
|
|
492
|
+
rocm: "rocm";
|
|
493
|
+
}>>;
|
|
494
|
+
driver: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
495
|
+
name: import("zod").ZodString;
|
|
496
|
+
version: import("zod").ZodOptional<import("zod").ZodString>;
|
|
497
|
+
}, import("zod/v4/core").$strip>>;
|
|
498
|
+
gpuId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
499
|
+
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
500
|
+
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
501
|
+
requestedDevice: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
502
|
+
gpu: "gpu";
|
|
503
|
+
cpu: "cpu";
|
|
504
|
+
}>>;
|
|
505
|
+
reason: import("zod").ZodString;
|
|
506
|
+
}, import("zod/v4/core").$strip>>;
|
|
507
|
+
probe: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
508
|
+
status: import("zod").ZodEnum<{
|
|
509
|
+
unknown: "unknown";
|
|
510
|
+
compatible: "compatible";
|
|
511
|
+
incompatible: "incompatible";
|
|
512
|
+
}>;
|
|
513
|
+
backend: import("zod").ZodString;
|
|
514
|
+
reason: import("zod").ZodOptional<import("zod").ZodString>;
|
|
515
|
+
}, import("zod/v4/core").$strip>>;
|
|
516
|
+
}, import("zod/v4/core").$strip>>;
|
|
517
|
+
type: import("zod").ZodLiteral<"audioEditStream">;
|
|
518
|
+
}, import("zod/v4/core").$strict>>;
|
|
519
|
+
audioUnderstand: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
|
|
520
|
+
sourceAudio: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
521
|
+
type: import("zod").ZodLiteral<"base64">;
|
|
522
|
+
value: import("zod").ZodString;
|
|
523
|
+
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
524
|
+
type: import("zod").ZodLiteral<"filePath">;
|
|
525
|
+
value: import("zod").ZodString;
|
|
526
|
+
}, import("zod/v4/core").$strip>], "type">;
|
|
527
|
+
type: import("zod").ZodLiteral<"audioUnderstand">;
|
|
528
|
+
requestId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
529
|
+
modelId: import("zod").ZodString;
|
|
530
|
+
seed: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
531
|
+
vocalLanguage: import("zod").ZodOptional<import("zod").ZodString>;
|
|
532
|
+
lmTemperature: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
533
|
+
lmTopP: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
534
|
+
lmTopK: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
535
|
+
}, import("zod/v4/core").$strict>, import("zod").ZodObject<{
|
|
536
|
+
type: import("zod").ZodLiteral<"audioUnderstand">;
|
|
537
|
+
progress: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
538
|
+
stage: import("zod").ZodString;
|
|
539
|
+
step: import("zod").ZodNumber;
|
|
540
|
+
total: import("zod").ZodNumber;
|
|
541
|
+
}, import("zod/v4/core").$strip>>;
|
|
542
|
+
understand: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
543
|
+
caption: import("zod").ZodString;
|
|
544
|
+
bpm: import("zod").ZodNumber;
|
|
545
|
+
duration: import("zod").ZodNumber;
|
|
546
|
+
keyscale: import("zod").ZodString;
|
|
547
|
+
timesignature: import("zod").ZodString;
|
|
548
|
+
vocalLanguage: import("zod").ZodString;
|
|
549
|
+
audioCodes: import("zod").ZodArray<import("zod").ZodNumber>;
|
|
550
|
+
}, import("zod/v4/core").$strip>>;
|
|
551
|
+
done: import("zod").ZodDefault<import("zod").ZodBoolean>;
|
|
552
|
+
stopReason: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
553
|
+
cancelled: "cancelled";
|
|
554
|
+
completed: "completed";
|
|
555
|
+
}>>;
|
|
556
|
+
stats: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
557
|
+
audioDurationMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
558
|
+
totalTimeMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
559
|
+
realTimeFactor: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
560
|
+
backendDevice: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
561
|
+
backendId: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
562
|
+
lyricsScore: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
563
|
+
lrc: import("zod").ZodOptional<import("zod").ZodString>;
|
|
564
|
+
qualityScore: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
565
|
+
understand: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
566
|
+
caption: import("zod").ZodString;
|
|
567
|
+
bpm: import("zod").ZodNumber;
|
|
568
|
+
duration: import("zod").ZodNumber;
|
|
569
|
+
keyscale: import("zod").ZodString;
|
|
570
|
+
timesignature: import("zod").ZodString;
|
|
571
|
+
vocalLanguage: import("zod").ZodString;
|
|
572
|
+
audioCodes: import("zod").ZodArray<import("zod").ZodNumber>;
|
|
573
|
+
}, import("zod/v4/core").$strip>>;
|
|
326
574
|
}, import("zod/v4/core").$strip>>;
|
|
327
575
|
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
328
576
|
selectedBackend: import("zod").ZodString;
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { AudioGen, ENGINE_ACESTEP, ENGINE_MINIMAX } from '@qvac/audiogen-ggml';
|
|
2
|
-
import { ADDON_AUDIOGEN, ModelType, audioGenConfigSchema, audioGenStreamRequestSchema, audioGenStreamResponseSchema, defineHandler, definePlugin } from '../../../schemas/index.js';
|
|
2
|
+
import { ADDON_AUDIOGEN, ModelType, audioEditStreamRequestSchema, audioEditStreamResponseSchema, audioGenConfigSchema, audioGenStreamRequestSchema, audioGenStreamResponseSchema, audioUnderstandRequestSchema, audioUnderstandResponseSchema, defineHandler, definePlugin } from '../../../schemas/index.js';
|
|
3
3
|
import { createStreamLogger, registerAddonLogger } from '../../../logging/index.js';
|
|
4
4
|
import { resolveAudioGenConfig } from './config.js';
|
|
5
|
+
import { audioEditStream } from './ops/audio-edit-stream.js';
|
|
5
6
|
import { audioGenStream } from './ops/audio-gen-stream.js';
|
|
7
|
+
import { audioUnderstand } from './ops/audio-understand.js';
|
|
6
8
|
import { ModelLoadFailedError } from '../../../errors/index.js';
|
|
7
9
|
export const audioGenPlugin = definePlugin({
|
|
8
10
|
modelType: ModelType.audiogenGgml,
|
|
@@ -35,6 +37,20 @@ export const audioGenPlugin = definePlugin({
|
|
|
35
37
|
streaming: true,
|
|
36
38
|
cancel: { scope: 'model', hard: true },
|
|
37
39
|
handler: audioGenStream
|
|
40
|
+
}),
|
|
41
|
+
audioEditStream: defineHandler({
|
|
42
|
+
requestSchema: audioEditStreamRequestSchema,
|
|
43
|
+
responseSchema: audioEditStreamResponseSchema,
|
|
44
|
+
streaming: true,
|
|
45
|
+
cancel: { scope: 'model', hard: true },
|
|
46
|
+
handler: audioEditStream
|
|
47
|
+
}),
|
|
48
|
+
audioUnderstand: defineHandler({
|
|
49
|
+
requestSchema: audioUnderstandRequestSchema,
|
|
50
|
+
responseSchema: audioUnderstandResponseSchema,
|
|
51
|
+
streaming: true,
|
|
52
|
+
cancel: { scope: 'model', hard: true },
|
|
53
|
+
handler: audioUnderstand
|
|
38
54
|
})
|
|
39
55
|
}
|
|
40
56
|
});
|