@qvac/sdk 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +111 -82
- package/dist/examples/abot-world.js +3 -3
- package/dist/examples/advisory-model-fit.d.ts +59 -0
- package/dist/examples/advisory-model-fit.js +156 -0
- package/dist/examples/asr/parakeet-nemotron-filesystem.d.ts +2 -0
- package/dist/examples/asr/parakeet-nemotron-filesystem.js +170 -0
- package/dist/examples/assess-model-fit.js +6 -1
- package/dist/examples/audiogen/edit-music.d.ts +2 -0
- package/dist/examples/audiogen/edit-music.js +121 -0
- package/dist/examples/audiogen/understand-music.d.ts +2 -0
- package/dist/examples/audiogen/understand-music.js +73 -0
- package/dist/examples/diffusion-ic-lora-ltx.js +3 -3
- package/dist/examples/diffusion-img2vid.js +1 -1
- package/dist/examples/diffusion-txt2vid-minimax-h3.d.ts +2 -0
- package/dist/examples/diffusion-txt2vid-minimax-h3.js +64 -0
- package/dist/examples/diffusion-txt2vid.js +1 -1
- package/dist/examples/kv-cache-cleanup.js +5 -2
- package/dist/examples/model-fit-ui/server.js +33 -7
- package/dist/examples/rag/rag-turbovec.d.ts +2 -0
- package/dist/examples/rag/rag-turbovec.js +58 -0
- package/dist/examples/tts/audio8.js +5 -5
- package/dist/examples/tts/chatterbox-enhanced.js +4 -5
- package/dist/examples/tts/chatterbox-multilingual.js +5 -5
- package/dist/examples/tts/chatterbox.js +5 -5
- package/dist/examples/tts/cosyvoice3.js +5 -5
- package/dist/examples/tts/llm-to-tts-streaming.js +12 -2
- package/dist/examples/tts/parler.js +5 -5
- package/dist/examples/tts/supertonic-enhanced.js +4 -5
- package/dist/examples/tts/supertonic-multilingual.js +5 -5
- package/dist/examples/tts/supertonic.js +5 -5
- package/dist/scripts/contract/build-contract.d.ts +1 -1
- package/dist/scripts/contract/constants-registry.d.ts +31 -0
- package/dist/scripts/contract/constants-registry.js +13 -2
- package/dist/scripts/contract/method-shapes.d.ts +3 -0
- package/dist/scripts/contract/method-shapes.js +3 -0
- package/dist/src/client/api/assess-model-fit.js +2 -0
- package/dist/src/client/api/audio-edit.d.ts +36 -0
- package/dist/src/client/api/audio-edit.js +38 -0
- package/dist/src/client/api/audio-gen-result.d.ts +5 -1
- package/dist/src/client/api/audio-gen-result.js +105 -52
- package/dist/src/client/api/audio-gen.d.ts +38 -1
- package/dist/src/client/api/audio-gen.js +38 -1
- package/dist/src/client/api/audio-understand.d.ts +25 -0
- package/dist/src/client/api/audio-understand.js +27 -0
- package/dist/src/client/api/delete-cache.d.ts +7 -1
- package/dist/src/client/api/delete-cache.js +11 -1
- package/dist/src/client/api/index.d.ts +3 -0
- package/dist/src/client/api/index.js +3 -0
- package/dist/src/client/api/text-to-speech.d.ts +31 -4
- package/dist/src/client/api/text-to-speech.js +118 -21
- package/dist/src/client/api/vector-index.d.ts +143 -0
- package/dist/src/client/api/vector-index.js +168 -0
- package/dist/src/client/api/world.d.ts +2 -2
- package/dist/src/commands/bundle/manifest.d.ts +2 -0
- package/dist/src/commands/verify/addon-source.d.ts +6 -0
- package/dist/src/commands/verify/bundle-source.d.ts +2 -0
- package/dist/src/commands/verify/bundle-source.js +155 -1
- package/dist/src/commands/verify/index.js +1 -0
- package/dist/src/commands/verify/prebuilds.d.ts +23 -0
- package/dist/src/commands/verify/prebuilds.js +129 -13
- package/dist/src/commands/verify/resource-collectors.js +2 -1
- package/dist/src/expo/plugins/patches/qvac-platform-addons.d.ts +23 -0
- package/dist/src/expo/plugins/patches/qvac-platform-addons.js +94 -0
- package/dist/src/expo/plugins/withMobileBundle.d.ts +12 -2
- package/dist/src/expo/plugins/withMobileBundle.js +62 -31
- package/dist/src/index.d.ts +3 -3
- package/dist/src/index.js +7 -3
- package/dist/src/schemas/sdk-errors-server.d.ts +4 -0
- package/dist/src/schemas/sdk-errors-server.js +22 -0
- package/dist/src/utils/aggregate-events.d.ts +3 -1
- package/dist/src/utils/aggregate-events.js +7 -1
- package/dist/src/utils/errors-server.d.ts +12 -0
- package/dist/src/utils/errors-server.js +21 -0
- package/docs/system-resources-support-matrix.md +19 -19
- package/package.json +22 -15
- package/src/electron-forge/index.cjs +86 -14
- package/src/expo/plugins/patches/android-link.mjs +18 -9
- package/src/expo/plugins/patches/ios-link.mjs +18 -9
- package/src/expo/plugins/patches/qvac-platform-addons.ts +124 -0
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { stream as streamRpc } from '../rpc/rpc-client.js';
|
|
2
|
+
import { createAudioEditResult } from './audio-gen-result.js';
|
|
3
|
+
/**
|
|
4
|
+
* Edits a source recording with a loaded ACE-Step AudioGen model: an ordered
|
|
5
|
+
* pipeline of `flow-edit` and `repaint` operations, executed in array order.
|
|
6
|
+
*
|
|
7
|
+
* @param params - Loaded model ID, the source audio, the ordered `operations`, and an optional `seed`.
|
|
8
|
+
* @param params.sourceAudio - The recording to edit: a file path (decoded server-side) or raw
|
|
9
|
+
* interleaved stereo 48 kHz Float32 LE PCM bytes in `[-1, 1]`.
|
|
10
|
+
* @param params.operations - `flow-edit` re-conditions the whole clip from a `from` prompt to a
|
|
11
|
+
* `to` prompt over an optional `nMin`/`nMax` diffusion window (turbo DiT variants only);
|
|
12
|
+
* `repaint` regenerates the `start`..`end` range (seconds) against a new `caption`, keeping the
|
|
13
|
+
* rest of the clip. Operations may repeat or mix.
|
|
14
|
+
* @param params.seed - Seeds the first operation; each following operation uses `seed + index`.
|
|
15
|
+
* @returns The same run shape as `audioGen()`: a synchronous request ID, generation progress stream, PCM audio promise, stats promise, and diagnostics promise.
|
|
16
|
+
*
|
|
17
|
+
* @example
|
|
18
|
+
* ```typescript
|
|
19
|
+
* const run = audioEdit({
|
|
20
|
+
* modelId,
|
|
21
|
+
* sourceAudio: "/path/to/song.wav",
|
|
22
|
+
* operations: [
|
|
23
|
+
* {
|
|
24
|
+
* type: "flow-edit",
|
|
25
|
+
* from: { caption: "original pop song", lyrics: originalLyrics },
|
|
26
|
+
* to: { caption: "guitar pop-rock", lyrics: newLyrics },
|
|
27
|
+
* },
|
|
28
|
+
* { type: "repaint", caption: "analog synth solo", start: 10, end: 20, mode: "balanced" },
|
|
29
|
+
* ],
|
|
30
|
+
* seed: 22883,
|
|
31
|
+
* });
|
|
32
|
+
* stopButton.onclick = () => cancel({ requestId: run.requestId });
|
|
33
|
+
* const { pcm, sampleRate, channels, bitsPerSample } = await run.audio;
|
|
34
|
+
* ```
|
|
35
|
+
*/
|
|
36
|
+
export function audioEdit(params) {
|
|
37
|
+
return createAudioEditResult(params, streamRpc);
|
|
38
|
+
}
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import { type AudioGenClientParams, type AudioGenResult, type AudioGenStreamRequest } from '@qvac/inference/surface';
|
|
1
|
+
import { type AudioEditClientParams, type AudioEditStreamRequest, type AudioGenClientParams, type AudioGenResult, type AudioGenStreamRequest, type AudioUnderstandClientParams, type AudioUnderstandRequest, type AudioUnderstandResult } from '@qvac/inference/surface';
|
|
2
2
|
export type AudioGenStreamFactory = (request: AudioGenStreamRequest) => AsyncGenerator<unknown>;
|
|
3
|
+
export type AudioEditStreamFactory = (request: AudioEditStreamRequest) => AsyncGenerator<unknown>;
|
|
4
|
+
export type AudioUnderstandStreamFactory = (request: AudioUnderstandRequest) => AsyncGenerator<unknown>;
|
|
3
5
|
export declare function createAudioGenResult(params: AudioGenClientParams, streamFactory: AudioGenStreamFactory): AudioGenResult;
|
|
6
|
+
export declare function createAudioEditResult(params: AudioEditClientParams, streamFactory: AudioEditStreamFactory): AudioGenResult;
|
|
7
|
+
export declare function createAudioUnderstandResult(params: AudioUnderstandClientParams, streamFactory: AudioUnderstandStreamFactory): AudioUnderstandResult;
|
|
4
8
|
//# sourceMappingURL=audio-gen-result.d.ts.map
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { audioGenClientParamsSchema, audioGenStreamResponseSchema } from '@qvac/inference/surface';
|
|
1
|
+
import { audioEditClientParamsSchema, audioEditStreamResponseSchema, audioGenClientParamsSchema, audioGenStreamResponseSchema, audioUnderstandClientParamsSchema, audioUnderstandResponseSchema } from '@qvac/inference/surface';
|
|
2
2
|
import { parseClientInput } from '../parse-input.js';
|
|
3
3
|
import { generateClientRequestId } from './client-request-id.js';
|
|
4
4
|
import { decodeBase64 } from '../../utils/encoding.js';
|
|
@@ -22,21 +22,40 @@ export function createAudioGenResult(params, streamFactory) {
|
|
|
22
22
|
type: 'audioGenStream',
|
|
23
23
|
requestId
|
|
24
24
|
};
|
|
25
|
+
return collectAudioRun(requestId, 'audioGenStream', audioGenStreamResponseSchema, () => streamFactory(request));
|
|
26
|
+
}
|
|
27
|
+
export function createAudioEditResult(params, streamFactory) {
|
|
28
|
+
const parsed = parseClientInput(audioEditClientParamsSchema, params);
|
|
29
|
+
const requestId = generateClientRequestId();
|
|
30
|
+
const request = {
|
|
31
|
+
...parsed,
|
|
32
|
+
type: 'audioEditStream',
|
|
33
|
+
requestId
|
|
34
|
+
};
|
|
35
|
+
return collectAudioRun(requestId, 'audioEditStream', audioEditStreamResponseSchema, () => streamFactory(request));
|
|
36
|
+
}
|
|
37
|
+
export function createAudioUnderstandResult(params, streamFactory) {
|
|
38
|
+
const parsed = parseClientInput(audioUnderstandClientParamsSchema, params);
|
|
39
|
+
const requestId = generateClientRequestId();
|
|
40
|
+
const request = {
|
|
41
|
+
...parsed,
|
|
42
|
+
type: 'audioUnderstand',
|
|
43
|
+
requestId
|
|
44
|
+
};
|
|
45
|
+
return collectUnderstandRun(requestId, () => streamFactory(request));
|
|
46
|
+
}
|
|
47
|
+
function collectRun(requestId, wireType, responseSchema, open, sink) {
|
|
25
48
|
const progressQueue = [];
|
|
26
|
-
const pcmChunks = [];
|
|
27
|
-
let sampleRate;
|
|
28
|
-
let channels;
|
|
29
|
-
let bitsPerSample;
|
|
30
49
|
let progressDone = false;
|
|
31
50
|
let progressError;
|
|
32
51
|
let progressResolve;
|
|
33
|
-
let
|
|
34
|
-
let
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
|
|
52
|
+
let resolvePayload = () => { };
|
|
53
|
+
let rejectPayload = () => { };
|
|
54
|
+
const payload = new Promise((resolve, reject) => {
|
|
55
|
+
resolvePayload = resolve;
|
|
56
|
+
rejectPayload = reject;
|
|
38
57
|
});
|
|
39
|
-
|
|
58
|
+
payload.catch(() => { });
|
|
40
59
|
let resolveStats = () => { };
|
|
41
60
|
let rejectStats = () => { };
|
|
42
61
|
const stats = new Promise((resolve, reject) => {
|
|
@@ -55,60 +74,47 @@ export function createAudioGenResult(params, streamFactory) {
|
|
|
55
74
|
progressResolve?.();
|
|
56
75
|
progressResolve = undefined;
|
|
57
76
|
}
|
|
77
|
+
function rejectAll(error) {
|
|
78
|
+
rejectPayload(error);
|
|
79
|
+
rejectStats(error);
|
|
80
|
+
rejectDiagnostics(error);
|
|
81
|
+
}
|
|
58
82
|
async function processResponses() {
|
|
59
83
|
let receivedDone = false;
|
|
60
84
|
try {
|
|
61
|
-
for await (const response of
|
|
85
|
+
for await (const response of open()) {
|
|
62
86
|
if (!response ||
|
|
63
87
|
typeof response !== 'object' ||
|
|
64
88
|
!('type' in response) ||
|
|
65
|
-
response.type !==
|
|
89
|
+
response.type !== wireType) {
|
|
66
90
|
continue;
|
|
67
91
|
}
|
|
68
|
-
const
|
|
69
|
-
if (
|
|
70
|
-
progressQueue.push(
|
|
92
|
+
const frame = responseSchema.parse(response);
|
|
93
|
+
if (frame.progress) {
|
|
94
|
+
progressQueue.push(frame.progress);
|
|
71
95
|
notifyProgress();
|
|
72
96
|
}
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
sampleRate = chunk.sampleRate;
|
|
76
|
-
channels = chunk.channels;
|
|
77
|
-
bitsPerSample = chunk.bitsPerSample;
|
|
78
|
-
}
|
|
79
|
-
if (chunk.done) {
|
|
97
|
+
sink.absorb(frame);
|
|
98
|
+
if (frame.done) {
|
|
80
99
|
receivedDone = true;
|
|
81
|
-
if (
|
|
82
|
-
|
|
83
|
-
rejectAudio(error);
|
|
84
|
-
rejectStats(error);
|
|
85
|
-
rejectDiagnostics(error);
|
|
100
|
+
if (frame.stopReason === 'cancelled') {
|
|
101
|
+
rejectAll(new InferenceCancelledError(requestId));
|
|
86
102
|
break;
|
|
87
103
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
pcm: concatenateChunks(pcmChunks),
|
|
93
|
-
sampleRate,
|
|
94
|
-
channels,
|
|
95
|
-
bitsPerSample
|
|
96
|
-
});
|
|
97
|
-
resolveStats(chunk.stats);
|
|
98
|
-
resolveDiagnostics(chunk.diagnostics);
|
|
104
|
+
const settled = sink.settle(frame);
|
|
105
|
+
resolvePayload(settled);
|
|
106
|
+
resolveStats(frame.stats);
|
|
107
|
+
resolveDiagnostics(frame.diagnostics);
|
|
99
108
|
break;
|
|
100
109
|
}
|
|
101
110
|
}
|
|
102
111
|
if (!receivedDone) {
|
|
103
|
-
throw new InvalidResponseError(
|
|
112
|
+
throw new InvalidResponseError(`${wireType} terminal response`);
|
|
104
113
|
}
|
|
105
114
|
}
|
|
106
115
|
catch (error) {
|
|
107
|
-
progressError =
|
|
108
|
-
|
|
109
|
-
rejectAudio(progressError);
|
|
110
|
-
rejectStats(progressError);
|
|
111
|
-
rejectDiagnostics(progressError);
|
|
116
|
+
progressError = error instanceof Error ? error : new InvalidResponseError(wireType, error);
|
|
117
|
+
rejectAll(progressError);
|
|
112
118
|
}
|
|
113
119
|
finally {
|
|
114
120
|
progressDone = true;
|
|
@@ -133,11 +139,58 @@ export function createAudioGenResult(params, streamFactory) {
|
|
|
133
139
|
}
|
|
134
140
|
}
|
|
135
141
|
void processResponses();
|
|
136
|
-
return {
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
142
|
+
return { requestId, progressStream: progressStream(), payload, stats, diagnostics };
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Consumes an `audioUnderstand()` stream: the engine streams the description
|
|
146
|
+
* as one `understand` item and repeats it on the terminal stats.
|
|
147
|
+
*/
|
|
148
|
+
function collectUnderstandRun(requestId, openStream) {
|
|
149
|
+
let seen;
|
|
150
|
+
const { progressStream, payload, stats, diagnostics } = collectRun(requestId, 'audioUnderstand', audioUnderstandResponseSchema, openStream, {
|
|
151
|
+
absorb(frame) {
|
|
152
|
+
if (frame.understand !== undefined)
|
|
153
|
+
seen = frame.understand;
|
|
154
|
+
},
|
|
155
|
+
settle(frame) {
|
|
156
|
+
const result = frame.stats?.understand ?? seen;
|
|
157
|
+
if (result === undefined) {
|
|
158
|
+
throw new InvalidResponseError('audioUnderstand description');
|
|
159
|
+
}
|
|
160
|
+
return result;
|
|
161
|
+
}
|
|
162
|
+
});
|
|
163
|
+
return { requestId, progressStream, description: payload, stats, diagnostics };
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Consumes one generation or editing stream: PCM chunks accumulate into
|
|
167
|
+
* `audio`, carrying the format reported alongside them.
|
|
168
|
+
*/
|
|
169
|
+
function collectAudioRun(requestId, wireType, responseSchema, open) {
|
|
170
|
+
const pcmChunks = [];
|
|
171
|
+
let sampleRate;
|
|
172
|
+
let channels;
|
|
173
|
+
let bitsPerSample;
|
|
174
|
+
const { progressStream, payload, stats, diagnostics } = collectRun(requestId, wireType, responseSchema, open, {
|
|
175
|
+
absorb(frame) {
|
|
176
|
+
if (frame.data === undefined)
|
|
177
|
+
return;
|
|
178
|
+
pcmChunks.push(decodeBase64(frame.data));
|
|
179
|
+
sampleRate = frame.sampleRate;
|
|
180
|
+
channels = frame.channels;
|
|
181
|
+
bitsPerSample = frame.bitsPerSample;
|
|
182
|
+
},
|
|
183
|
+
settle() {
|
|
184
|
+
if (sampleRate === undefined || channels === undefined || bitsPerSample === undefined) {
|
|
185
|
+
throw new InvalidResponseError(`${wireType} audio chunk`);
|
|
186
|
+
}
|
|
187
|
+
return {
|
|
188
|
+
pcm: concatenateChunks(pcmChunks),
|
|
189
|
+
sampleRate,
|
|
190
|
+
channels,
|
|
191
|
+
bitsPerSample
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
return { requestId, progressStream, audio: payload, stats, diagnostics };
|
|
143
196
|
}
|
|
@@ -3,6 +3,24 @@ import type { AudioGenClientParams, AudioGenResult } from '@qvac/inference/surfa
|
|
|
3
3
|
* Generates PCM audio using a loaded AudioGen model.
|
|
4
4
|
*
|
|
5
5
|
* @param params - Loaded model ID, required caption, and optional lyrics, musical controls, duration, and seed.
|
|
6
|
+
* @param params.augmentCaptionWithMetadata - ACE-Step only: append BPM/tempo, time signature, and
|
|
7
|
+
* key guidance to the internal conditioning caption (default `false`).
|
|
8
|
+
* @param params.audioCodes - ACE-Step only: frozen semantic codes (`Int32Array` or `number[]`)
|
|
9
|
+
* to synthesize instead of running the LM.
|
|
10
|
+
* @param params.simpleMode - ACE-Step only: treat `caption` as a short query and let the LM compose
|
|
11
|
+
* the full request. Requires `taskType: "text2music"`; mutually exclusive with `rewriteQuery`.
|
|
12
|
+
* @param params.rewriteQuery - ACE-Step only: the LM rewrites `caption` into a detailed description,
|
|
13
|
+
* keeping the lyrics. Needs real `lyrics`, and `taskType: "text2music"`.
|
|
14
|
+
* @param params.generateLrc - ACE-Step only: return karaoke-style LRC in `stats.lrc` with an
|
|
15
|
+
* alignment confidence in `stats.lyricsScore`. Needs lyrics, and `taskType: "text2music"`.
|
|
16
|
+
* @param params.computeQualityScore - ACE-Step only: report a weighted `[0, 1]` match of the codes
|
|
17
|
+
* against the request in `stats.qualityScore`. Requires `taskType: "text2music"`.
|
|
18
|
+
* @param params.normalizeLoudness - ACE-Step only: percentile loudness normalization on the output
|
|
19
|
+
* (default `true`). Set `false` for the raw engine output.
|
|
20
|
+
* @param params.guidanceScale - ACE-Step only: DiT classifier-free guidance scale; `0` (default)
|
|
21
|
+
* resolves per DiT variant.
|
|
22
|
+
* @param params.track - ACE-Step only: the instrument layer the `"lego"` task regenerates.
|
|
23
|
+
* Required when `taskType` is `"lego"` and rejected otherwise.
|
|
6
24
|
* @param params.maxFrames - MiniMax semantic-frame cap; mutually exclusive with `duration`.
|
|
7
25
|
* @param params.inferenceSteps - MiniMax flow steps for this run.
|
|
8
26
|
* @param params.cfgScale - MiniMax flow classifier-free guidance scale for this run.
|
|
@@ -10,7 +28,7 @@ import type { AudioGenClientParams, AudioGenResult } from '@qvac/inference/surfa
|
|
|
10
28
|
* or raw interleaved stereo 48 kHz Float32 LE PCM bytes.
|
|
11
29
|
* @param params.sourceAudio - Source audio to re-render (same forms as `referenceAudio`);
|
|
12
30
|
* required when `taskType` is `"cover-nofsq"`.
|
|
13
|
-
* @param params.taskType - `"text2music"` (default)
|
|
31
|
+
* @param params.taskType - `"text2music"` (default), `"cover-nofsq"`, or `"lego"`.
|
|
14
32
|
* @param params.audioCoverStrength - Source-context strength for cover tasks (0..1).
|
|
15
33
|
* @param params.coverNoiseStrength - Initial source/noise blend for cover tasks (0..1).
|
|
16
34
|
* @returns A run with a synchronous request ID, generation progress stream, PCM audio promise, stats promise, and diagnostics promise describing the backend the run resolved to.
|
|
@@ -60,6 +78,25 @@ import type { AudioGenClientParams, AudioGenResult } from '@qvac/inference/surfa
|
|
|
60
78
|
* referenceAudio: "/path/to/reference.mp3",
|
|
61
79
|
* coverNoiseStrength: 0.75,
|
|
62
80
|
* });
|
|
81
|
+
*
|
|
82
|
+
* // Let the LM expand a one-line query, and score and timestamp the take:
|
|
83
|
+
* const assisted = audioGen({
|
|
84
|
+
* modelId,
|
|
85
|
+
* caption: "a hopeful indie track for a road-trip montage",
|
|
86
|
+
* simpleMode: true,
|
|
87
|
+
* generateLrc: true,
|
|
88
|
+
* computeQualityScore: true,
|
|
89
|
+
* });
|
|
90
|
+
* const { lrc, lyricsScore, qualityScore } = (await assisted.stats) ?? {};
|
|
91
|
+
*
|
|
92
|
+
* // Rebuild one layer of an existing song:
|
|
93
|
+
* const newDrums = audioGen({
|
|
94
|
+
* modelId,
|
|
95
|
+
* caption: "the same song with a busier, live-sounding kit",
|
|
96
|
+
* taskType: "lego",
|
|
97
|
+
* track: "drums",
|
|
98
|
+
* sourceAudio: "/path/to/song.wav",
|
|
99
|
+
* });
|
|
63
100
|
* ```
|
|
64
101
|
*/
|
|
65
102
|
export declare function audioGen(params: AudioGenClientParams): AudioGenResult;
|
|
@@ -4,6 +4,24 @@ import { createAudioGenResult } from './audio-gen-result.js';
|
|
|
4
4
|
* Generates PCM audio using a loaded AudioGen model.
|
|
5
5
|
*
|
|
6
6
|
* @param params - Loaded model ID, required caption, and optional lyrics, musical controls, duration, and seed.
|
|
7
|
+
* @param params.augmentCaptionWithMetadata - ACE-Step only: append BPM/tempo, time signature, and
|
|
8
|
+
* key guidance to the internal conditioning caption (default `false`).
|
|
9
|
+
* @param params.audioCodes - ACE-Step only: frozen semantic codes (`Int32Array` or `number[]`)
|
|
10
|
+
* to synthesize instead of running the LM.
|
|
11
|
+
* @param params.simpleMode - ACE-Step only: treat `caption` as a short query and let the LM compose
|
|
12
|
+
* the full request. Requires `taskType: "text2music"`; mutually exclusive with `rewriteQuery`.
|
|
13
|
+
* @param params.rewriteQuery - ACE-Step only: the LM rewrites `caption` into a detailed description,
|
|
14
|
+
* keeping the lyrics. Needs real `lyrics`, and `taskType: "text2music"`.
|
|
15
|
+
* @param params.generateLrc - ACE-Step only: return karaoke-style LRC in `stats.lrc` with an
|
|
16
|
+
* alignment confidence in `stats.lyricsScore`. Needs lyrics, and `taskType: "text2music"`.
|
|
17
|
+
* @param params.computeQualityScore - ACE-Step only: report a weighted `[0, 1]` match of the codes
|
|
18
|
+
* against the request in `stats.qualityScore`. Requires `taskType: "text2music"`.
|
|
19
|
+
* @param params.normalizeLoudness - ACE-Step only: percentile loudness normalization on the output
|
|
20
|
+
* (default `true`). Set `false` for the raw engine output.
|
|
21
|
+
* @param params.guidanceScale - ACE-Step only: DiT classifier-free guidance scale; `0` (default)
|
|
22
|
+
* resolves per DiT variant.
|
|
23
|
+
* @param params.track - ACE-Step only: the instrument layer the `"lego"` task regenerates.
|
|
24
|
+
* Required when `taskType` is `"lego"` and rejected otherwise.
|
|
7
25
|
* @param params.maxFrames - MiniMax semantic-frame cap; mutually exclusive with `duration`.
|
|
8
26
|
* @param params.inferenceSteps - MiniMax flow steps for this run.
|
|
9
27
|
* @param params.cfgScale - MiniMax flow classifier-free guidance scale for this run.
|
|
@@ -11,7 +29,7 @@ import { createAudioGenResult } from './audio-gen-result.js';
|
|
|
11
29
|
* or raw interleaved stereo 48 kHz Float32 LE PCM bytes.
|
|
12
30
|
* @param params.sourceAudio - Source audio to re-render (same forms as `referenceAudio`);
|
|
13
31
|
* required when `taskType` is `"cover-nofsq"`.
|
|
14
|
-
* @param params.taskType - `"text2music"` (default)
|
|
32
|
+
* @param params.taskType - `"text2music"` (default), `"cover-nofsq"`, or `"lego"`.
|
|
15
33
|
* @param params.audioCoverStrength - Source-context strength for cover tasks (0..1).
|
|
16
34
|
* @param params.coverNoiseStrength - Initial source/noise blend for cover tasks (0..1).
|
|
17
35
|
* @returns A run with a synchronous request ID, generation progress stream, PCM audio promise, stats promise, and diagnostics promise describing the backend the run resolved to.
|
|
@@ -61,6 +79,25 @@ import { createAudioGenResult } from './audio-gen-result.js';
|
|
|
61
79
|
* referenceAudio: "/path/to/reference.mp3",
|
|
62
80
|
* coverNoiseStrength: 0.75,
|
|
63
81
|
* });
|
|
82
|
+
*
|
|
83
|
+
* // Let the LM expand a one-line query, and score and timestamp the take:
|
|
84
|
+
* const assisted = audioGen({
|
|
85
|
+
* modelId,
|
|
86
|
+
* caption: "a hopeful indie track for a road-trip montage",
|
|
87
|
+
* simpleMode: true,
|
|
88
|
+
* generateLrc: true,
|
|
89
|
+
* computeQualityScore: true,
|
|
90
|
+
* });
|
|
91
|
+
* const { lrc, lyricsScore, qualityScore } = (await assisted.stats) ?? {};
|
|
92
|
+
*
|
|
93
|
+
* // Rebuild one layer of an existing song:
|
|
94
|
+
* const newDrums = audioGen({
|
|
95
|
+
* modelId,
|
|
96
|
+
* caption: "the same song with a busier, live-sounding kit",
|
|
97
|
+
* taskType: "lego",
|
|
98
|
+
* track: "drums",
|
|
99
|
+
* sourceAudio: "/path/to/song.wav",
|
|
100
|
+
* });
|
|
64
101
|
* ```
|
|
65
102
|
*/
|
|
66
103
|
export function audioGen(params) {
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { AudioUnderstandClientParams, AudioUnderstandResult } from '@qvac/inference/surface';
|
|
2
|
+
/**
|
|
3
|
+
* Describes a recording with a loaded ACE-Step AudioGen model, running the
|
|
4
|
+
* engine's reverse pipeline: the PCM is encoded, its FSQ semantic codes are
|
|
5
|
+
* recovered, and the LM reports a caption and the clip's metadata.
|
|
6
|
+
*
|
|
7
|
+
* @param params - Loaded model ID, the source audio, and optional LM sampling controls.
|
|
8
|
+
* @param params.sourceAudio - The recording to analyse: a file path (decoded server-side) or raw
|
|
9
|
+
* interleaved stereo 48 kHz Float32 LE PCM bytes in `[-1, 1]`.
|
|
10
|
+
* @param params.vocalLanguage - Language hint forced into the result instead of the LM's guess.
|
|
11
|
+
* @param params.seed - Seeds the LM decode; omit for a random seed.
|
|
12
|
+
* @returns A synchronous request ID, a progress stream, the `description` promise, and the stats and diagnostics promises.
|
|
13
|
+
*
|
|
14
|
+
* @example
|
|
15
|
+
* ```typescript
|
|
16
|
+
* const run = audioUnderstand({ modelId, sourceAudio: "/path/to/song.wav" });
|
|
17
|
+
* stopButton.onclick = () => cancel({ requestId: run.requestId });
|
|
18
|
+
* const { caption, bpm, keyscale, timesignature, audioCodes } = await run.description;
|
|
19
|
+
*
|
|
20
|
+
* // The recovered codes re-synthesize the same piece without re-running the LM.
|
|
21
|
+
* const remake = audioGen({ modelId, caption: "the same song, brighter mix", audioCodes });
|
|
22
|
+
* ```
|
|
23
|
+
*/
|
|
24
|
+
export declare function audioUnderstand(params: AudioUnderstandClientParams): AudioUnderstandResult;
|
|
25
|
+
//# sourceMappingURL=audio-understand.d.ts.map
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { stream as streamRpc } from '../rpc/rpc-client.js';
|
|
2
|
+
import { createAudioUnderstandResult } from './audio-gen-result.js';
|
|
3
|
+
/**
|
|
4
|
+
* Describes a recording with a loaded ACE-Step AudioGen model, running the
|
|
5
|
+
* engine's reverse pipeline: the PCM is encoded, its FSQ semantic codes are
|
|
6
|
+
* recovered, and the LM reports a caption and the clip's metadata.
|
|
7
|
+
*
|
|
8
|
+
* @param params - Loaded model ID, the source audio, and optional LM sampling controls.
|
|
9
|
+
* @param params.sourceAudio - The recording to analyse: a file path (decoded server-side) or raw
|
|
10
|
+
* interleaved stereo 48 kHz Float32 LE PCM bytes in `[-1, 1]`.
|
|
11
|
+
* @param params.vocalLanguage - Language hint forced into the result instead of the LM's guess.
|
|
12
|
+
* @param params.seed - Seeds the LM decode; omit for a random seed.
|
|
13
|
+
* @returns A synchronous request ID, a progress stream, the `description` promise, and the stats and diagnostics promises.
|
|
14
|
+
*
|
|
15
|
+
* @example
|
|
16
|
+
* ```typescript
|
|
17
|
+
* const run = audioUnderstand({ modelId, sourceAudio: "/path/to/song.wav" });
|
|
18
|
+
* stopButton.onclick = () => cancel({ requestId: run.requestId });
|
|
19
|
+
* const { caption, bpm, keyscale, timesignature, audioCodes } = await run.description;
|
|
20
|
+
*
|
|
21
|
+
* // The recovered codes re-synthesize the same piece without re-running the LM.
|
|
22
|
+
* const remake = audioGen({ modelId, caption: "the same song, brighter mix", audioCodes });
|
|
23
|
+
* ```
|
|
24
|
+
*/
|
|
25
|
+
export function audioUnderstand(params) {
|
|
26
|
+
return createAudioUnderstandResult(params, streamRpc);
|
|
27
|
+
}
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
* Deletes KV cache files.
|
|
3
3
|
*
|
|
4
4
|
* @param params - The delete cache parameters
|
|
5
|
-
* @param params.all - If true, deletes all cache files
|
|
5
|
+
* @param params.all - If true, deletes all cache files, including caller-owned named caches
|
|
6
|
+
* @param params.auto - If true, reclaims only automatic caches (`completion({ kvCache: true })`) that no request is using. Named caches are untouched. Covers the whole cache directory, which is shared by every QVAC process using the same home directory
|
|
6
7
|
* @param params.kvCacheKey - The cache key to delete
|
|
7
8
|
* @param params.modelId - Optional: specific model ID to delete within the cache key. If not provided, deletes entire cache key.
|
|
8
9
|
* @returns Promise resolving to success status
|
|
@@ -12,6 +13,9 @@
|
|
|
12
13
|
* // Delete all caches
|
|
13
14
|
* await deleteCache({ all: true });
|
|
14
15
|
*
|
|
16
|
+
* // Reclaim disk from automatic caches, leaving named caches alone
|
|
17
|
+
* await deleteCache({ auto: true });
|
|
18
|
+
*
|
|
15
19
|
* // Delete entire cache key (all models)
|
|
16
20
|
* await deleteCache({ kvCacheKey: "my-session" });
|
|
17
21
|
*
|
|
@@ -21,6 +25,8 @@
|
|
|
21
25
|
*/
|
|
22
26
|
export declare function deleteCache(params: {
|
|
23
27
|
all: true;
|
|
28
|
+
} | {
|
|
29
|
+
auto: true;
|
|
24
30
|
} | {
|
|
25
31
|
kvCacheKey: string;
|
|
26
32
|
modelId?: string;
|
|
@@ -4,7 +4,8 @@ import { InvalidDeleteCacheParamsError, DeleteCacheFailedError } from '../../uti
|
|
|
4
4
|
* Deletes KV cache files.
|
|
5
5
|
*
|
|
6
6
|
* @param params - The delete cache parameters
|
|
7
|
-
* @param params.all - If true, deletes all cache files
|
|
7
|
+
* @param params.all - If true, deletes all cache files, including caller-owned named caches
|
|
8
|
+
* @param params.auto - If true, reclaims only automatic caches (`completion({ kvCache: true })`) that no request is using. Named caches are untouched. Covers the whole cache directory, which is shared by every QVAC process using the same home directory
|
|
8
9
|
* @param params.kvCacheKey - The cache key to delete
|
|
9
10
|
* @param params.modelId - Optional: specific model ID to delete within the cache key. If not provided, deletes entire cache key.
|
|
10
11
|
* @returns Promise resolving to success status
|
|
@@ -14,6 +15,9 @@ import { InvalidDeleteCacheParamsError, DeleteCacheFailedError } from '../../uti
|
|
|
14
15
|
* // Delete all caches
|
|
15
16
|
* await deleteCache({ all: true });
|
|
16
17
|
*
|
|
18
|
+
* // Reclaim disk from automatic caches, leaving named caches alone
|
|
19
|
+
* await deleteCache({ auto: true });
|
|
20
|
+
*
|
|
17
21
|
* // Delete entire cache key (all models)
|
|
18
22
|
* await deleteCache({ kvCacheKey: "my-session" });
|
|
19
23
|
*
|
|
@@ -29,6 +33,12 @@ export async function deleteCache(params) {
|
|
|
29
33
|
all: true
|
|
30
34
|
};
|
|
31
35
|
}
|
|
36
|
+
else if ('auto' in params && params.auto) {
|
|
37
|
+
req = {
|
|
38
|
+
type: 'deleteCache',
|
|
39
|
+
auto: true
|
|
40
|
+
};
|
|
41
|
+
}
|
|
32
42
|
else if ('kvCacheKey' in params) {
|
|
33
43
|
req = {
|
|
34
44
|
type: 'deleteCache',
|
|
@@ -14,6 +14,7 @@ export { finetune, type FinetuneHandle } from './finetune.js';
|
|
|
14
14
|
export { translate } from './translate.js';
|
|
15
15
|
export { cancel } from './cancel.js';
|
|
16
16
|
export { ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace } from './rag.js';
|
|
17
|
+
export { createVectorIndex, loadVectorIndex, type VectorIndex } from './vector-index.js';
|
|
17
18
|
export { textToSpeech, textToSpeechStream } from './text-to-speech.js';
|
|
18
19
|
export { getModelInfo } from './get-model-info.js';
|
|
19
20
|
export { getLoadedModelInfo } from './get-loaded-model-info.js';
|
|
@@ -34,4 +35,6 @@ export { state } from './state.js';
|
|
|
34
35
|
export { vla, vlaHparams, vlaSetEmbodiment } from './vla.js';
|
|
35
36
|
export { vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE } from './vla-helpers.js';
|
|
36
37
|
export { audioGen } from './audio-gen.js';
|
|
38
|
+
export { audioEdit } from './audio-edit.js';
|
|
39
|
+
export { audioUnderstand } from './audio-understand.js';
|
|
37
40
|
//# sourceMappingURL=index.d.ts.map
|
|
@@ -14,6 +14,7 @@ export { finetune } from './finetune.js';
|
|
|
14
14
|
export { translate } from './translate.js';
|
|
15
15
|
export { cancel } from './cancel.js';
|
|
16
16
|
export { ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace } from './rag.js';
|
|
17
|
+
export { createVectorIndex, loadVectorIndex } from './vector-index.js';
|
|
17
18
|
export { textToSpeech, textToSpeechStream } from './text-to-speech.js';
|
|
18
19
|
export { getModelInfo } from './get-model-info.js';
|
|
19
20
|
export { getLoadedModelInfo } from './get-loaded-model-info.js';
|
|
@@ -34,3 +35,5 @@ export { state } from './state.js';
|
|
|
34
35
|
export { vla, vlaHparams, vlaSetEmbodiment } from './vla.js';
|
|
35
36
|
export { vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE } from './vla-helpers.js';
|
|
36
37
|
export { audioGen } from './audio-gen.js';
|
|
38
|
+
export { audioEdit } from './audio-edit.js';
|
|
39
|
+
export { audioUnderstand } from './audio-understand.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type TtsClientParamsInput, type RPCOptions, type TtsResponse, type TextToSpeechStreamClientParams, type TextToSpeechStreamSession, type TextToSpeechStreamResult } from '@qvac/inference/surface';
|
|
1
|
+
import { type TtsClientParamsInput, type RPCOptions, type TtsResponse, type TextToSpeechStreamClientParams, type TextToSpeechStreamSession, type TextToSpeechStreamResult, type TtsStats } from '@qvac/inference/surface';
|
|
2
2
|
/**
|
|
3
3
|
* Fan-out queue that lets multiple consumers iterate the same TTS response
|
|
4
4
|
* stream independently. Items are retained only until every active subscriber
|
|
@@ -33,9 +33,9 @@ export declare class TtsMulticast {
|
|
|
33
33
|
*
|
|
34
34
|
* Three modes selected by `params.stream` and `params.sentenceStream`:
|
|
35
35
|
*
|
|
36
|
-
* - `stream: false`
|
|
36
|
+
* - `stream: false` — collect all PCM samples and resolve once via
|
|
37
37
|
* `result.buffer` (`Promise<number[]>`). `bufferStream` is empty.
|
|
38
|
-
* - `stream: true` — yield PCM samples through `result.bufferStream`
|
|
38
|
+
* - `stream: true` (default) — yield PCM samples through `result.bufferStream`
|
|
39
39
|
* (`AsyncGenerator<number>`) as they arrive. `buffer` resolves to an empty
|
|
40
40
|
* array.
|
|
41
41
|
* - `stream: true, sentenceStream: true` — also exposes `result.chunkUpdates`
|
|
@@ -47,14 +47,41 @@ export declare class TtsMulticast {
|
|
|
47
47
|
* if the consumer breaks out before the terminal frame, or rejects on a
|
|
48
48
|
* pipeline error. Awaiting `done` is safe even when no stream is iterated.
|
|
49
49
|
*
|
|
50
|
+
* `result.sampleRate` resolves from the first audio frame (so it can be awaited
|
|
51
|
+
* before draining the audio); `result.stats` and `result.stopReason` resolve
|
|
52
|
+
* from the terminal frame. All three resolve — with `undefined` when the run
|
|
53
|
+
* produced none — rather than reject. In `stream: true` mode (the default) the
|
|
54
|
+
* run starts lazily: nothing is sent until `bufferStream` is first iterated, so
|
|
55
|
+
* `done`, `sampleRate`, `stats` and `stopReason` stay pending until then. In
|
|
56
|
+
* `stream: false` and `sentenceStream: true` modes the run starts immediately.
|
|
57
|
+
* Read `sampleRate` rather than assuming the engine default: `outputSampleRate`
|
|
58
|
+
* and the LavaSR enhancer both change it.
|
|
59
|
+
*
|
|
50
60
|
* @param params - TTS request parameters (see `TtsClientParamsInput`).
|
|
51
61
|
* @param options - Optional RPC options (timeout, profiling, force new connection).
|
|
52
62
|
* @returns A `TextToSpeechStreamResult` with `bufferStream`, `buffer`, `done`,
|
|
53
|
-
* and (when `sentenceStream: true`)
|
|
63
|
+
* `sampleRate`, `stats`, and (when `sentenceStream: true`)
|
|
64
|
+
* `chunkUpdates`.
|
|
54
65
|
* @throws {TextToSpeechStreamFailedError} When `sentenceStream: true` is paired
|
|
55
66
|
* with `stream: false`, or when the underlying RPC stream errors.
|
|
56
67
|
*/
|
|
57
68
|
export declare function textToSpeech(params: TtsClientParamsInput, options?: RPCOptions): TextToSpeechStreamResult;
|
|
69
|
+
/**
|
|
70
|
+
* The output rate and the runtime stats ride the response frames, but the
|
|
71
|
+
* three result shapes below expose PCM through generators a caller may abandon
|
|
72
|
+
* early. This collects both off the frames as they pass and hands them back as
|
|
73
|
+
* promises that always RESOLVE — with `undefined` when the run produced none —
|
|
74
|
+
* so `await result.sampleRate` can never hang or reject on its own.
|
|
75
|
+
*/
|
|
76
|
+
export type TtsSideChannel = {
|
|
77
|
+
sampleRate: Promise<number | undefined>;
|
|
78
|
+
stats: Promise<TtsStats | undefined>;
|
|
79
|
+
stopReason: Promise<'completed' | 'cancelled' | undefined>;
|
|
80
|
+
observe(response: TtsResponse): void;
|
|
81
|
+
settle(): void;
|
|
82
|
+
};
|
|
83
|
+
/** Exported for test use; not part of the public SDK surface. */
|
|
84
|
+
export declare function createTtsSideChannel(): TtsSideChannel;
|
|
58
85
|
export declare function encodeTextToSpeechFragment(textFragment: string | Uint8Array): Uint8Array<ArrayBufferLike>;
|
|
59
86
|
/**
|
|
60
87
|
* Duplex session: write UTF-8 text fragments (e.g. LLM token deltas) via `write`. Each string or
|