@qvac/sdk 0.19.1 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +78 -55
- package/README.md +3 -1
- package/dist/examples/abot-world.js +3 -3
- package/dist/examples/advisory-model-fit.d.ts +59 -0
- package/dist/examples/advisory-model-fit.js +156 -0
- package/dist/examples/asr/parakeet-nemotron-filesystem.d.ts +2 -0
- package/dist/examples/asr/parakeet-nemotron-filesystem.js +170 -0
- package/dist/examples/audiogen/edit-music.d.ts +2 -0
- package/dist/examples/audiogen/edit-music.js +121 -0
- package/dist/examples/audiogen/understand-music.d.ts +2 -0
- package/dist/examples/audiogen/understand-music.js +73 -0
- package/dist/examples/diffusion-ic-lora-ltx.js +3 -3
- package/dist/examples/diffusion-img2vid.js +1 -1
- package/dist/examples/diffusion-txt2vid-minimax-h3.d.ts +2 -0
- package/dist/examples/diffusion-txt2vid-minimax-h3.js +64 -0
- package/dist/examples/diffusion-txt2vid.js +1 -1
- package/dist/examples/rag/rag-turbovec.d.ts +2 -0
- package/dist/examples/rag/rag-turbovec.js +58 -0
- package/dist/examples/tts/audio8.js +5 -5
- package/dist/examples/tts/chatterbox-enhanced.js +4 -5
- package/dist/examples/tts/chatterbox-multilingual.js +5 -5
- package/dist/examples/tts/chatterbox.js +5 -5
- package/dist/examples/tts/cosyvoice3.js +5 -5
- package/dist/examples/tts/llm-to-tts-streaming.js +12 -2
- package/dist/examples/tts/parler.js +5 -5
- package/dist/examples/tts/supertonic-enhanced.js +4 -5
- package/dist/examples/tts/supertonic-multilingual.js +5 -5
- package/dist/examples/tts/supertonic.js +5 -5
- package/dist/scripts/contract/build-contract.d.ts +1 -1
- package/dist/scripts/contract/constants-registry.d.ts +31 -0
- package/dist/scripts/contract/constants-registry.js +13 -2
- package/dist/scripts/contract/method-shapes.d.ts +3 -0
- package/dist/scripts/contract/method-shapes.js +3 -0
- package/dist/src/client/api/assess-model-fit.d.ts +9 -4
- package/dist/src/client/api/assess-model-fit.js +9 -4
- package/dist/src/client/api/audio-edit.d.ts +36 -0
- package/dist/src/client/api/audio-edit.js +38 -0
- package/dist/src/client/api/audio-gen-result.d.ts +5 -1
- package/dist/src/client/api/audio-gen-result.js +105 -52
- package/dist/src/client/api/audio-gen.d.ts +38 -1
- package/dist/src/client/api/audio-gen.js +38 -1
- package/dist/src/client/api/audio-understand.d.ts +25 -0
- package/dist/src/client/api/audio-understand.js +27 -0
- package/dist/src/client/api/index.d.ts +3 -0
- package/dist/src/client/api/index.js +3 -0
- package/dist/src/client/api/text-to-speech.d.ts +31 -4
- package/dist/src/client/api/text-to-speech.js +118 -21
- package/dist/src/client/api/vector-index.d.ts +143 -0
- package/dist/src/client/api/vector-index.js +168 -0
- package/dist/src/client/api/world.d.ts +2 -2
- package/dist/src/commands/bundle/manifest.d.ts +2 -0
- package/dist/src/commands/verify/addon-source.d.ts +6 -0
- package/dist/src/commands/verify/bundle-source.d.ts +2 -0
- package/dist/src/commands/verify/bundle-source.js +155 -1
- package/dist/src/commands/verify/index.js +1 -0
- package/dist/src/commands/verify/prebuilds.d.ts +23 -0
- package/dist/src/commands/verify/prebuilds.js +129 -13
- package/dist/src/commands/verify/resource-collectors.js +2 -1
- package/dist/src/expo/plugins/patches/qvac-platform-addons.d.ts +23 -0
- package/dist/src/expo/plugins/patches/qvac-platform-addons.js +94 -0
- package/dist/src/expo/plugins/withMobileBundle.d.ts +9 -6
- package/dist/src/expo/plugins/withMobileBundle.js +54 -34
- package/dist/src/index.d.ts +3 -3
- package/dist/src/index.js +7 -3
- package/dist/src/schemas/sdk-errors-server.d.ts +4 -0
- package/dist/src/schemas/sdk-errors-server.js +22 -0
- package/dist/src/utils/aggregate-events.d.ts +3 -1
- package/dist/src/utils/aggregate-events.js +7 -1
- package/dist/src/utils/errors-server.d.ts +12 -0
- package/dist/src/utils/errors-server.js +21 -0
- package/package.json +23 -17
- package/src/electron-forge/index.cjs +86 -14
- package/src/expo/plugins/patches/android-link.mjs +18 -9
- package/src/expo/plugins/patches/ios-link.mjs +18 -9
- package/src/expo/plugins/patches/qvac-platform-addons.ts +124 -0
|
@@ -4,6 +4,8 @@ import { createWav, playAudio, int16ArrayToBuffer, createWavHeader } from './uti
|
|
|
4
4
|
// synthesis (emotion, pace, or instruct). The LLM GGUF's registry companion
|
|
5
5
|
// set downloads the flow/HiFT models, baked voice and tokenizer alongside it.
|
|
6
6
|
// Native output is 24 kHz.
|
|
7
|
+
// Only a fallback: the engine reports the rate it actually produced, and
|
|
8
|
+
// `outputSampleRate` (plus the LavaSR enhancer) can move it off this default.
|
|
7
9
|
const COSYVOICE3_SAMPLE_RATE = 24000;
|
|
8
10
|
try {
|
|
9
11
|
const modelId = await loadModel({
|
|
@@ -30,16 +32,14 @@ try {
|
|
|
30
32
|
emotion: 'happy'
|
|
31
33
|
});
|
|
32
34
|
const audioBuffer = await result.buffer;
|
|
35
|
+
const sampleRate = (await result.sampleRate) ?? COSYVOICE3_SAMPLE_RATE;
|
|
33
36
|
console.log(`▸ TTS complete. Total samples: ${audioBuffer.length}`);
|
|
34
37
|
console.log('▸ Saving audio to file...');
|
|
35
|
-
createWav(audioBuffer,
|
|
38
|
+
createWav(audioBuffer, sampleRate, 'cosyvoice3-output.wav');
|
|
36
39
|
console.log('▸ Audio saved to cosyvoice3-output.wav');
|
|
37
40
|
console.log('▸ Playing audio...');
|
|
38
41
|
const audioData = int16ArrayToBuffer(audioBuffer);
|
|
39
|
-
const wavBuffer = Buffer.concat([
|
|
40
|
-
createWavHeader(audioData.length, COSYVOICE3_SAMPLE_RATE),
|
|
41
|
-
audioData
|
|
42
|
-
]);
|
|
42
|
+
const wavBuffer = Buffer.concat([createWavHeader(audioData.length, sampleRate), audioData]);
|
|
43
43
|
playAudio(wavBuffer);
|
|
44
44
|
console.log('▸ Audio playback complete');
|
|
45
45
|
await unloadModel({ modelId });
|
|
@@ -14,6 +14,9 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import { completion, loadModel, textToSpeechStream, unloadModel, LLAMA_3_2_1B_INST_Q4_0, TTS_MULTILINGUAL_SUPERTONIC3_Q8_0 } from '@qvac/sdk';
|
|
16
16
|
import { createWav, playPcmInt16Chunk } from './utils.js';
|
|
17
|
+
// Only a fallback, for the case where no frame ever reports a rate: each
|
|
18
|
+
// audio frame carries the rate the engine actually produced, and
|
|
19
|
+
// `outputSampleRate` can move it off this default.
|
|
17
20
|
const SUPERTONIC_SAMPLE_RATE = 44100;
|
|
18
21
|
/**
|
|
19
22
|
* Drop angle-bracket spans (e.g. `<think>…</think>`, `<|foo|>`) even when split across tokens,
|
|
@@ -107,16 +110,23 @@ try {
|
|
|
107
110
|
});
|
|
108
111
|
let phraseIndex = 0;
|
|
109
112
|
const filterChunk = createAngleBracketAndStarFilter();
|
|
113
|
+
// Playback and the WAV header both follow what the engine reports rather
|
|
114
|
+
// than the engine's documented default, so a configured `outputSampleRate`
|
|
115
|
+
// cannot leave the audio playing back at the wrong speed.
|
|
116
|
+
let sampleRate = SUPERTONIC_SAMPLE_RATE;
|
|
110
117
|
const drainPcm = (async () => {
|
|
111
118
|
for await (const m of ttsSession) {
|
|
112
119
|
if (m.buffer.length > 0) {
|
|
120
|
+
if (m.sampleRate !== undefined) {
|
|
121
|
+
sampleRate = m.sampleRate;
|
|
122
|
+
}
|
|
113
123
|
appendPcmSamples(combinedPcm, m.buffer);
|
|
114
124
|
phraseIndex += 1;
|
|
115
125
|
const preview = typeof m.sentenceChunk === 'string'
|
|
116
126
|
? m.sentenceChunk.replace(/\s+/g, ' ').trim().slice(0, 72)
|
|
117
127
|
: '';
|
|
118
128
|
console.log(`\n▸ [TTS phrase ${phraseIndex}] ${m.buffer.length} samples${preview ? ` — "${preview}${preview.length >= 72 ? '...' : ''}"` : ''}`);
|
|
119
|
-
await playPcmInt16Chunk(m.buffer,
|
|
129
|
+
await playPcmInt16Chunk(m.buffer, sampleRate);
|
|
120
130
|
}
|
|
121
131
|
}
|
|
122
132
|
})();
|
|
@@ -135,7 +145,7 @@ try {
|
|
|
135
145
|
}
|
|
136
146
|
const outWav = 'llm-to-tts-streaming-output.wav';
|
|
137
147
|
console.log(`\n▸ Writing ${combinedPcm.length} samples to ${outWav} (full utterance; phrases were already played above).`);
|
|
138
|
-
createWav(combinedPcm,
|
|
148
|
+
createWav(combinedPcm, sampleRate, outWav);
|
|
139
149
|
await unloadModel({ modelId: llmModelId, clearStorage: false });
|
|
140
150
|
await unloadModel({ modelId: ttsModelId, clearStorage: false });
|
|
141
151
|
console.log('▸ Models unloaded.');
|
|
@@ -2,6 +2,8 @@ import { loadModel, textToSpeech, unloadModel, TTS_MINI_V1_EN_PARLER_TTS_Q8_0 }
|
|
|
2
2
|
import { createWav, playAudio, int16ArrayToBuffer, createWavHeader } from './utils.js';
|
|
3
3
|
// Parler-TTS (GGML): description-conditioned speech with per-call voice controls.
|
|
4
4
|
// Uses the registry-hosted Mini v1 Q8_0 model and its native 44.1 kHz output.
|
|
5
|
+
// Only a fallback: the engine reports the rate it actually produced, and
|
|
6
|
+
// `outputSampleRate` (plus the LavaSR enhancer) can move it off this default.
|
|
5
7
|
const PARLER_SAMPLE_RATE = 44100;
|
|
6
8
|
try {
|
|
7
9
|
const modelId = await loadModel({
|
|
@@ -29,16 +31,14 @@ try {
|
|
|
29
31
|
emotion: 'happy'
|
|
30
32
|
});
|
|
31
33
|
const audioBuffer = await result.buffer;
|
|
34
|
+
const sampleRate = (await result.sampleRate) ?? PARLER_SAMPLE_RATE;
|
|
32
35
|
console.log(`▸ TTS complete. Total samples: ${audioBuffer.length}`);
|
|
33
36
|
console.log('▸ Saving audio to file...');
|
|
34
|
-
createWav(audioBuffer,
|
|
37
|
+
createWav(audioBuffer, sampleRate, 'parler-output.wav');
|
|
35
38
|
console.log('▸ Audio saved to parler-output.wav');
|
|
36
39
|
console.log('▸ Playing audio...');
|
|
37
40
|
const audioData = int16ArrayToBuffer(audioBuffer);
|
|
38
|
-
const wavBuffer = Buffer.concat([
|
|
39
|
-
createWavHeader(audioData.length, PARLER_SAMPLE_RATE),
|
|
40
|
-
audioData
|
|
41
|
-
]);
|
|
41
|
+
const wavBuffer = Buffer.concat([createWavHeader(audioData.length, sampleRate), audioData]);
|
|
42
42
|
playAudio(wavBuffer);
|
|
43
43
|
console.log('▸ Audio playback complete');
|
|
44
44
|
await unloadModel({ modelId });
|
|
@@ -5,6 +5,7 @@ import { createWav, playAudio, int16ArrayToBuffer, createWavHeader } from './uti
|
|
|
5
5
|
// Supplying the enhancer GGUF is what enables enhancement — there is no on/off
|
|
6
6
|
// flag — and it forces the output to 48 kHz regardless of the engine's native
|
|
7
7
|
// rate.
|
|
8
|
+
// Only a fallback; the engine reports the rate it actually produced.
|
|
8
9
|
const ENHANCED_SAMPLE_RATE = 48000;
|
|
9
10
|
try {
|
|
10
11
|
const modelId = await loadModel({
|
|
@@ -35,16 +36,14 @@ try {
|
|
|
35
36
|
stream: false
|
|
36
37
|
});
|
|
37
38
|
const audioBuffer = await result.buffer;
|
|
39
|
+
const sampleRate = (await result.sampleRate) ?? ENHANCED_SAMPLE_RATE;
|
|
38
40
|
console.log(`▸ TTS complete. Total samples: ${audioBuffer.length}`);
|
|
39
41
|
console.log('▸ Saving audio to file...');
|
|
40
|
-
createWav(audioBuffer,
|
|
42
|
+
createWav(audioBuffer, sampleRate, 'supertonic-enhanced-output.wav');
|
|
41
43
|
console.log('▸ Audio saved to supertonic-enhanced-output.wav');
|
|
42
44
|
console.log('▸ Playing audio...');
|
|
43
45
|
const audioData = int16ArrayToBuffer(audioBuffer);
|
|
44
|
-
const wavBuffer = Buffer.concat([
|
|
45
|
-
createWavHeader(audioData.length, ENHANCED_SAMPLE_RATE),
|
|
46
|
-
audioData
|
|
47
|
-
]);
|
|
46
|
+
const wavBuffer = Buffer.concat([createWavHeader(audioData.length, sampleRate), audioData]);
|
|
48
47
|
playAudio(wavBuffer);
|
|
49
48
|
console.log('▸ Audio playback complete');
|
|
50
49
|
await unloadModel({ modelId });
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { loadModel, textToSpeech, unloadModel, TTS_MULTILINGUAL_SUPERTONIC3_Q8_0 } from '@qvac/sdk';
|
|
2
2
|
import { createWav, playAudio, int16ArrayToBuffer, createWavHeader } from './utils.js';
|
|
3
3
|
// Supertonic 3 TTS (GGML): multilingual synthesis across 31 languages.
|
|
4
|
+
// Only a fallback: the engine reports the rate it actually produced, and
|
|
5
|
+
// `outputSampleRate` (plus the LavaSR enhancer) can move it off this default.
|
|
4
6
|
const SUPERTONIC_SAMPLE_RATE = 44100;
|
|
5
7
|
try {
|
|
6
8
|
const modelId = await loadModel({
|
|
@@ -29,16 +31,14 @@ try {
|
|
|
29
31
|
stream: false
|
|
30
32
|
});
|
|
31
33
|
const audioBuffer = await result.buffer;
|
|
34
|
+
const sampleRate = (await result.sampleRate) ?? SUPERTONIC_SAMPLE_RATE;
|
|
32
35
|
console.log(`▸ TTS complete. Total samples: ${audioBuffer.length}`);
|
|
33
36
|
console.log('▸ Saving audio to file...');
|
|
34
|
-
createWav(audioBuffer,
|
|
37
|
+
createWav(audioBuffer, sampleRate, 'supertonic-multilingual-output.wav');
|
|
35
38
|
console.log('▸ Audio saved to supertonic-multilingual-output.wav');
|
|
36
39
|
console.log('▸ Playing audio...');
|
|
37
40
|
const audioData = int16ArrayToBuffer(audioBuffer);
|
|
38
|
-
const wavBuffer = Buffer.concat([
|
|
39
|
-
createWavHeader(audioData.length, SUPERTONIC_SAMPLE_RATE),
|
|
40
|
-
audioData
|
|
41
|
-
]);
|
|
41
|
+
const wavBuffer = Buffer.concat([createWavHeader(audioData.length, sampleRate), audioData]);
|
|
42
42
|
playAudio(wavBuffer);
|
|
43
43
|
console.log('▸ Audio playback complete');
|
|
44
44
|
await unloadModel({ modelId });
|
|
@@ -2,6 +2,8 @@ import { loadModel, textToSpeech, unloadModel, TTS_MULTILINGUAL_SUPERTONIC3_Q8_0
|
|
|
2
2
|
import { createWav, playAudio, int16ArrayToBuffer, createWavHeader } from './utils.js';
|
|
3
3
|
// Supertonic 3 TTS (GGML): fast multilingual synthesis with baked-in voices.
|
|
4
4
|
// Uses registry model constants — downloads automatically from QVAC Registry.
|
|
5
|
+
// Only a fallback: the engine reports the rate it actually produced, and
|
|
6
|
+
// `outputSampleRate` (plus the LavaSR enhancer) can move it off this default.
|
|
5
7
|
const SUPERTONIC_SAMPLE_RATE = 44100;
|
|
6
8
|
try {
|
|
7
9
|
const modelId = await loadModel({
|
|
@@ -30,16 +32,14 @@ try {
|
|
|
30
32
|
stream: false
|
|
31
33
|
});
|
|
32
34
|
const audioBuffer = await result.buffer;
|
|
35
|
+
const sampleRate = (await result.sampleRate) ?? SUPERTONIC_SAMPLE_RATE;
|
|
33
36
|
console.log(`▸ TTS complete. Total samples: ${audioBuffer.length}`);
|
|
34
37
|
console.log('▸ Saving audio to file...');
|
|
35
|
-
createWav(audioBuffer,
|
|
38
|
+
createWav(audioBuffer, sampleRate, 'supertonic-output.wav');
|
|
36
39
|
console.log('▸ Audio saved to supertonic-output.wav');
|
|
37
40
|
console.log('▸ Playing audio...');
|
|
38
41
|
const audioData = int16ArrayToBuffer(audioBuffer);
|
|
39
|
-
const wavBuffer = Buffer.concat([
|
|
40
|
-
createWavHeader(audioData.length, SUPERTONIC_SAMPLE_RATE),
|
|
41
|
-
audioData
|
|
42
|
-
]);
|
|
42
|
+
const wavBuffer = Buffer.concat([createWavHeader(audioData.length, sampleRate), audioData]);
|
|
43
43
|
playAudio(wavBuffer);
|
|
44
44
|
console.log('▸ Audio playback complete');
|
|
45
45
|
await unloadModel({ modelId });
|
|
@@ -20,7 +20,7 @@ export declare function buildContract(): {
|
|
|
20
20
|
condition: string;
|
|
21
21
|
responseSchema: string;
|
|
22
22
|
};
|
|
23
|
-
name: "audioGenStream" | "heartbeat" | "loadModel" | "downloadAsset" | "completionStream" | "completionOrchestrate" | "batchCompletionStream" | "unloadModel" | "transcribe" | "transcribeStream" | "bciTranscribe" | "bciTranscribeStream" | "loggingStream" | "embed" | "translate" | "textToSpeech" | "textToSpeechStream" | "cancel" | "finetune" | "rag" | "deleteCache" | "getModelInfo" | "getLoadedModelInfo" | "getSystemResources" | "assessModelFit" | "ocrStream" | "diffusionStream" | "videoStream" | "upscaleStream" | "worldSceneStream" | "worldStepStream" | "resume" | "pluginInvoke" | "pluginInvokeStream" | "modelRegistryList" | "modelRegistrySearch" | "modelRegistryGetModel" | "suspend" | "state" | "classify";
|
|
23
|
+
name: "audioGenStream" | "audioEditStream" | "audioUnderstand" | "heartbeat" | "loadModel" | "downloadAsset" | "completionStream" | "completionOrchestrate" | "batchCompletionStream" | "unloadModel" | "transcribe" | "transcribeStream" | "bciTranscribe" | "bciTranscribeStream" | "loggingStream" | "embed" | "translate" | "textToSpeech" | "textToSpeechStream" | "cancel" | "finetune" | "rag" | "vectorIndex" | "deleteCache" | "getModelInfo" | "getLoadedModelInfo" | "getSystemResources" | "assessModelFit" | "ocrStream" | "diffusionStream" | "videoStream" | "upscaleStream" | "worldSceneStream" | "worldStepStream" | "resume" | "pluginInvoke" | "pluginInvokeStream" | "modelRegistryList" | "modelRegistrySearch" | "modelRegistryGetModel" | "suspend" | "state" | "classify";
|
|
24
24
|
callShape: "duplex" | "request-reply" | "server-stream";
|
|
25
25
|
requestSchema: string;
|
|
26
26
|
responseSchema: string;
|
|
@@ -61,9 +61,24 @@ export declare const constantsRegistry: {
|
|
|
61
61
|
readonly SupportedAudioFormat: z.ZodEnum<{
|
|
62
62
|
[k: string]: string;
|
|
63
63
|
}>;
|
|
64
|
+
readonly TtsEngine: z.ZodEnum<{
|
|
65
|
+
[k: string]: string;
|
|
66
|
+
}>;
|
|
64
67
|
readonly TtsPace: z.ZodEnum<{
|
|
65
68
|
[k: string]: string;
|
|
66
69
|
}>;
|
|
70
|
+
readonly TtsParlerEmotion: z.ZodEnum<{
|
|
71
|
+
[k: string]: "command" | "anger" | "narration" | "conversation" | "disgust" | "fear" | "happy" | "neutral" | "proper noun" | "news" | "sad" | "surprise";
|
|
72
|
+
}>;
|
|
73
|
+
readonly TtsSentenceDelimiterPreset: z.ZodEnum<{
|
|
74
|
+
[k: string]: string;
|
|
75
|
+
}>;
|
|
76
|
+
readonly TtsChatterboxLanguage: z.ZodEnum<{
|
|
77
|
+
[k: string]: string;
|
|
78
|
+
}>;
|
|
79
|
+
readonly TtsSupertonicLanguage: z.ZodEnum<{
|
|
80
|
+
[k: string]: string;
|
|
81
|
+
}>;
|
|
67
82
|
readonly TtsCosyvoice3Emotion: z.ZodEnum<{
|
|
68
83
|
[k: string]: string;
|
|
69
84
|
}>;
|
|
@@ -82,5 +97,21 @@ export declare const constantsRegistry: {
|
|
|
82
97
|
readonly AudioGenTaskType: z.ZodEnum<{
|
|
83
98
|
[k: string]: string;
|
|
84
99
|
}>;
|
|
100
|
+
readonly AudioGenEditOperation: z.ZodEnum<{
|
|
101
|
+
[k: string]: string;
|
|
102
|
+
}>;
|
|
103
|
+
readonly AudioGenRepaintMode: z.ZodEnum<{
|
|
104
|
+
[k: string]: string;
|
|
105
|
+
}>;
|
|
106
|
+
readonly AudioGenTrack: z.ZodEnum<{
|
|
107
|
+
[k: string]: string;
|
|
108
|
+
}>;
|
|
109
|
+
readonly VectorIndexStorage: z.ZodEnum<{
|
|
110
|
+
readonly F32: "f32";
|
|
111
|
+
readonly Q8: "q8";
|
|
112
|
+
readonly Q4: "q4";
|
|
113
|
+
readonly TURBOVEC_Q4: "turbovec-q4";
|
|
114
|
+
readonly TURBOVEC_Q2: "turbovec-q2";
|
|
115
|
+
}>;
|
|
85
116
|
};
|
|
86
117
|
//# sourceMappingURL=constants-registry.d.ts.map
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import { ModelType, VERBOSITY, SUPPORTED_AUDIO_FORMATS, AUDIOGEN_ENGINES, AUDIOGEN_TASK_TYPES, TTS_PACES, TTS_COSYVOICE3_EMOTIONS, TTS_COSYVOICE3_INSTRUCT_DIALECTS, TTS_COSYVOICE3_INSTRUCT_VOLUMES, TTS_COSYVOICE3_INSTRUCT_STYLES } from '@qvac/inference/surface';
|
|
2
|
+
import { ModelType, VERBOSITY, SUPPORTED_AUDIO_FORMATS, AUDIOGEN_ENGINES, AUDIOGEN_TASK_TYPES, AUDIOGEN_EDIT_OPERATIONS, AUDIOGEN_REPAINT_MODES, AUDIOGEN_TRACKS, TTS_ENGINES, TTS_PACES, TTS_PARLER_EMOTIONS, TTS_SENTENCE_DELIMITER_PRESETS, TTS_CHATTERBOX_LANGUAGES, TTS_SUPERTONIC_LANGUAGES, TTS_COSYVOICE3_EMOTIONS, TTS_COSYVOICE3_INSTRUCT_DIALECTS, TTS_COSYVOICE3_INSTRUCT_VOLUMES, TTS_COSYVOICE3_INSTRUCT_STYLES, VectorIndexStorage } from '@qvac/inference/surface';
|
|
3
3
|
import { PLUGIN_LLM, PLUGIN_EMBEDDING, PLUGIN_WHISPER, PLUGIN_BCI, PLUGIN_NMT, PLUGIN_TTS, PLUGIN_OCR, PLUGIN_DIFFUSION, PLUGIN_AUDIOGEN, PLUGIN_VLA, PLUGIN_CLASSIFICATION } from '../../src/plugin-ids.js';
|
|
4
4
|
// Uppercases each vocabulary value into an identifier-safe varname (e.g.
|
|
5
5
|
// 'slow' -> 'SLOW', 'cover-nofsq' -> 'COVER_NOFSQ'), mirroring the
|
|
@@ -52,11 +52,22 @@ export const constantsRegistry = {
|
|
|
52
52
|
// than a hand-written record, so the varnames can't drift from the one
|
|
53
53
|
// source of truth every other consumer of SUPPORTED_AUDIO_FORMATS shares.
|
|
54
54
|
SupportedAudioFormat: z.enum(Object.fromEntries(SUPPORTED_AUDIO_FORMATS.map((ext) => [ext.slice(1).toUpperCase(), ext]))),
|
|
55
|
+
TtsEngine: enumFromVocabulary(TTS_ENGINES),
|
|
55
56
|
TtsPace: enumFromVocabulary(TTS_PACES),
|
|
57
|
+
// 'proper noun' carries a space; enumFromVocabulary only rewrites '-', so
|
|
58
|
+
// map it explicitly to keep the varname identifier-safe.
|
|
59
|
+
TtsParlerEmotion: z.enum(Object.fromEntries(TTS_PARLER_EMOTIONS.map((value) => [value.toUpperCase().replace(/[\s-]/g, '_'), value]))),
|
|
60
|
+
TtsSentenceDelimiterPreset: enumFromVocabulary(TTS_SENTENCE_DELIMITER_PRESETS),
|
|
61
|
+
TtsChatterboxLanguage: enumFromVocabulary(TTS_CHATTERBOX_LANGUAGES),
|
|
62
|
+
TtsSupertonicLanguage: enumFromVocabulary(TTS_SUPERTONIC_LANGUAGES),
|
|
56
63
|
TtsCosyvoice3Emotion: enumFromVocabulary(TTS_COSYVOICE3_EMOTIONS),
|
|
57
64
|
TtsCosyvoice3InstructDialect: enumFromVocabulary(TTS_COSYVOICE3_INSTRUCT_DIALECTS),
|
|
58
65
|
TtsCosyvoice3InstructVolume: enumFromVocabulary(TTS_COSYVOICE3_INSTRUCT_VOLUMES),
|
|
59
66
|
TtsCosyvoice3InstructStyle: enumFromVocabulary(TTS_COSYVOICE3_INSTRUCT_STYLES),
|
|
60
67
|
AudioGenEngine: enumFromVocabulary(AUDIOGEN_ENGINES),
|
|
61
|
-
AudioGenTaskType: enumFromVocabulary(AUDIOGEN_TASK_TYPES)
|
|
68
|
+
AudioGenTaskType: enumFromVocabulary(AUDIOGEN_TASK_TYPES),
|
|
69
|
+
AudioGenEditOperation: enumFromVocabulary(AUDIOGEN_EDIT_OPERATIONS),
|
|
70
|
+
AudioGenRepaintMode: enumFromVocabulary(AUDIOGEN_REPAINT_MODES),
|
|
71
|
+
AudioGenTrack: enumFromVocabulary(AUDIOGEN_TRACKS),
|
|
72
|
+
VectorIndexStorage: z.enum(VectorIndexStorage)
|
|
62
73
|
};
|
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
*/
|
|
9
9
|
export declare const methodShapes: {
|
|
10
10
|
readonly assessModelFit: "reply";
|
|
11
|
+
readonly audioEditStream: "stream";
|
|
11
12
|
readonly audioGenStream: "stream";
|
|
13
|
+
readonly audioUnderstand: "stream";
|
|
12
14
|
readonly batchCompletionStream: "stream";
|
|
13
15
|
readonly bciTranscribe: "stream";
|
|
14
16
|
readonly bciTranscribeStream: "duplex";
|
|
@@ -44,6 +46,7 @@ export declare const methodShapes: {
|
|
|
44
46
|
readonly translate: "stream";
|
|
45
47
|
readonly unloadModel: "reply";
|
|
46
48
|
readonly upscaleStream: "stream";
|
|
49
|
+
readonly vectorIndex: "reply";
|
|
47
50
|
readonly videoStream: "stream";
|
|
48
51
|
readonly worldSceneStream: "stream";
|
|
49
52
|
readonly worldStepStream: "stream";
|
|
@@ -8,7 +8,9 @@
|
|
|
8
8
|
*/
|
|
9
9
|
export const methodShapes = {
|
|
10
10
|
assessModelFit: 'reply',
|
|
11
|
+
audioEditStream: 'stream',
|
|
11
12
|
audioGenStream: 'stream',
|
|
13
|
+
audioUnderstand: 'stream',
|
|
12
14
|
batchCompletionStream: 'stream',
|
|
13
15
|
bciTranscribe: 'stream',
|
|
14
16
|
bciTranscribeStream: 'duplex',
|
|
@@ -44,6 +46,7 @@ export const methodShapes = {
|
|
|
44
46
|
translate: 'stream',
|
|
45
47
|
unloadModel: 'reply',
|
|
46
48
|
upscaleStream: 'stream',
|
|
49
|
+
vectorIndex: 'reply',
|
|
47
50
|
videoStream: 'stream',
|
|
48
51
|
worldSceneStream: 'stream',
|
|
49
52
|
worldStepStream: 'stream'
|
|
@@ -3,10 +3,15 @@ import { type AssessModelFitInput, type AssessModelFitResult } from '@qvac/infer
|
|
|
3
3
|
* Assesses, before anything is downloaded, whether the given models are likely
|
|
4
4
|
* to fit in this device's memory.
|
|
5
5
|
*
|
|
6
|
-
* Advisory only: it does not download weights,
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
6
|
+
* Advisory only: it does not download weights, block `loadModel`, reserve
|
|
7
|
+
* memory, or make a performance claim. `unknown` is a real answer — it means
|
|
8
|
+
* the available evidence does not support a verdict either way, and callers
|
|
9
|
+
* should treat it as "cannot say", not "no".
|
|
10
|
+
*
|
|
11
|
+
* For a single candidate it fetches the registry's weightless description of
|
|
12
|
+
* the artifact — tens of KB, never the weights — and runs the engine's own
|
|
13
|
+
* fitter against it, reported as `native-fit` evidence. Where that is
|
|
14
|
+
* unavailable, and for a set of candidates, the calibrated estimate stands.
|
|
10
15
|
*
|
|
11
16
|
* @param input - Candidates with their intended workloads, the declared
|
|
12
17
|
* execution mode, and the headroom policy.
|
|
@@ -5,10 +5,15 @@ import { InvalidResponseError } from '../../utils/errors-client.js';
|
|
|
5
5
|
* Assesses, before anything is downloaded, whether the given models are likely
|
|
6
6
|
* to fit in this device's memory.
|
|
7
7
|
*
|
|
8
|
-
* Advisory only: it does not download weights,
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* Advisory only: it does not download weights, block `loadModel`, reserve
|
|
9
|
+
* memory, or make a performance claim. `unknown` is a real answer — it means
|
|
10
|
+
* the available evidence does not support a verdict either way, and callers
|
|
11
|
+
* should treat it as "cannot say", not "no".
|
|
12
|
+
*
|
|
13
|
+
* For a single candidate it fetches the registry's weightless description of
|
|
14
|
+
* the artifact — tens of KB, never the weights — and runs the engine's own
|
|
15
|
+
* fitter against it, reported as `native-fit` evidence. Where that is
|
|
16
|
+
* unavailable, and for a set of candidates, the calibrated estimate stands.
|
|
12
17
|
*
|
|
13
18
|
* @param input - Candidates with their intended workloads, the declared
|
|
14
19
|
* execution mode, and the headroom policy.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { AudioEditClientParams, AudioGenResult } from '@qvac/inference/surface';
|
|
2
|
+
/**
|
|
3
|
+
* Edits a source recording with a loaded ACE-Step AudioGen model: an ordered
|
|
4
|
+
* pipeline of `flow-edit` and `repaint` operations, executed in array order.
|
|
5
|
+
*
|
|
6
|
+
* @param params - Loaded model ID, the source audio, the ordered `operations`, and an optional `seed`.
|
|
7
|
+
* @param params.sourceAudio - The recording to edit: a file path (decoded server-side) or raw
|
|
8
|
+
* interleaved stereo 48 kHz Float32 LE PCM bytes in `[-1, 1]`.
|
|
9
|
+
* @param params.operations - `flow-edit` re-conditions the whole clip from a `from` prompt to a
|
|
10
|
+
* `to` prompt over an optional `nMin`/`nMax` diffusion window (turbo DiT variants only);
|
|
11
|
+
* `repaint` regenerates the `start`..`end` range (seconds) against a new `caption`, keeping the
|
|
12
|
+
* rest of the clip. Operations may repeat or mix.
|
|
13
|
+
* @param params.seed - Seeds the first operation; each following operation uses `seed + index`.
|
|
14
|
+
* @returns The same run shape as `audioGen()`: a synchronous request ID, generation progress stream, PCM audio promise, stats promise, and diagnostics promise.
|
|
15
|
+
*
|
|
16
|
+
* @example
|
|
17
|
+
* ```typescript
|
|
18
|
+
* const run = audioEdit({
|
|
19
|
+
* modelId,
|
|
20
|
+
* sourceAudio: "/path/to/song.wav",
|
|
21
|
+
* operations: [
|
|
22
|
+
* {
|
|
23
|
+
* type: "flow-edit",
|
|
24
|
+
* from: { caption: "original pop song", lyrics: originalLyrics },
|
|
25
|
+
* to: { caption: "guitar pop-rock", lyrics: newLyrics },
|
|
26
|
+
* },
|
|
27
|
+
* { type: "repaint", caption: "analog synth solo", start: 10, end: 20, mode: "balanced" },
|
|
28
|
+
* ],
|
|
29
|
+
* seed: 22883,
|
|
30
|
+
* });
|
|
31
|
+
* stopButton.onclick = () => cancel({ requestId: run.requestId });
|
|
32
|
+
* const { pcm, sampleRate, channels, bitsPerSample } = await run.audio;
|
|
33
|
+
* ```
|
|
34
|
+
*/
|
|
35
|
+
export declare function audioEdit(params: AudioEditClientParams): AudioGenResult;
|
|
36
|
+
//# sourceMappingURL=audio-edit.d.ts.map
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import { stream as streamRpc } from '../rpc/rpc-client.js';
|
|
2
|
+
import { createAudioEditResult } from './audio-gen-result.js';
|
|
3
|
+
/**
|
|
4
|
+
* Edits a source recording with a loaded ACE-Step AudioGen model: an ordered
|
|
5
|
+
* pipeline of `flow-edit` and `repaint` operations, executed in array order.
|
|
6
|
+
*
|
|
7
|
+
* @param params - Loaded model ID, the source audio, the ordered `operations`, and an optional `seed`.
|
|
8
|
+
* @param params.sourceAudio - The recording to edit: a file path (decoded server-side) or raw
|
|
9
|
+
* interleaved stereo 48 kHz Float32 LE PCM bytes in `[-1, 1]`.
|
|
10
|
+
* @param params.operations - `flow-edit` re-conditions the whole clip from a `from` prompt to a
|
|
11
|
+
* `to` prompt over an optional `nMin`/`nMax` diffusion window (turbo DiT variants only);
|
|
12
|
+
* `repaint` regenerates the `start`..`end` range (seconds) against a new `caption`, keeping the
|
|
13
|
+
* rest of the clip. Operations may repeat or mix.
|
|
14
|
+
* @param params.seed - Seeds the first operation; each following operation uses `seed + index`.
|
|
15
|
+
* @returns The same run shape as `audioGen()`: a synchronous request ID, generation progress stream, PCM audio promise, stats promise, and diagnostics promise.
|
|
16
|
+
*
|
|
17
|
+
* @example
|
|
18
|
+
* ```typescript
|
|
19
|
+
* const run = audioEdit({
|
|
20
|
+
* modelId,
|
|
21
|
+
* sourceAudio: "/path/to/song.wav",
|
|
22
|
+
* operations: [
|
|
23
|
+
* {
|
|
24
|
+
* type: "flow-edit",
|
|
25
|
+
* from: { caption: "original pop song", lyrics: originalLyrics },
|
|
26
|
+
* to: { caption: "guitar pop-rock", lyrics: newLyrics },
|
|
27
|
+
* },
|
|
28
|
+
* { type: "repaint", caption: "analog synth solo", start: 10, end: 20, mode: "balanced" },
|
|
29
|
+
* ],
|
|
30
|
+
* seed: 22883,
|
|
31
|
+
* });
|
|
32
|
+
* stopButton.onclick = () => cancel({ requestId: run.requestId });
|
|
33
|
+
* const { pcm, sampleRate, channels, bitsPerSample } = await run.audio;
|
|
34
|
+
* ```
|
|
35
|
+
*/
|
|
36
|
+
export function audioEdit(params) {
|
|
37
|
+
return createAudioEditResult(params, streamRpc);
|
|
38
|
+
}
|
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import { type AudioGenClientParams, type AudioGenResult, type AudioGenStreamRequest } from '@qvac/inference/surface';
|
|
1
|
+
import { type AudioEditClientParams, type AudioEditStreamRequest, type AudioGenClientParams, type AudioGenResult, type AudioGenStreamRequest, type AudioUnderstandClientParams, type AudioUnderstandRequest, type AudioUnderstandResult } from '@qvac/inference/surface';
|
|
2
2
|
export type AudioGenStreamFactory = (request: AudioGenStreamRequest) => AsyncGenerator<unknown>;
|
|
3
|
+
export type AudioEditStreamFactory = (request: AudioEditStreamRequest) => AsyncGenerator<unknown>;
|
|
4
|
+
export type AudioUnderstandStreamFactory = (request: AudioUnderstandRequest) => AsyncGenerator<unknown>;
|
|
3
5
|
export declare function createAudioGenResult(params: AudioGenClientParams, streamFactory: AudioGenStreamFactory): AudioGenResult;
|
|
6
|
+
export declare function createAudioEditResult(params: AudioEditClientParams, streamFactory: AudioEditStreamFactory): AudioGenResult;
|
|
7
|
+
export declare function createAudioUnderstandResult(params: AudioUnderstandClientParams, streamFactory: AudioUnderstandStreamFactory): AudioUnderstandResult;
|
|
4
8
|
//# sourceMappingURL=audio-gen-result.d.ts.map
|