@qvac/tts-ggml 0.9.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +317 -33
- package/docs/pocket-tts.md +201 -0
- package/examples/moss-dialogue-tts.js +113 -0
- package/examples/moss-tts.js +154 -0
- package/examples/pocket-tts.js +51 -0
- package/examples/supertonic-chunk-stream-tts.js +156 -0
- package/examples/supertonic-sentence-stream-tts.js +4 -4
- package/index.d.ts +352 -29
- package/index.js +698 -52
- package/lib/fit.d.ts +127 -0
- package/lib/fit.js +29 -0
- package/lib/pocketConfig.d.ts +10 -0
- package/lib/pocketConfig.js +145 -0
- package/package.json +14 -9
- package/test/integration/addon.test.js +5 -0
- package/test/integration/audio8.test.js +7 -0
- package/test/integration/chatterbox-mtl.test.js +6 -0
- package/test/integration/chatterbox-speed.test.js +13 -2
- package/test/integration/cosyvoice3-clone.test.js +16 -1
- package/test/integration/cosyvoice3-lavasr.test.js +6 -0
- package/test/integration/cosyvoice3.test.js +19 -0
- package/test/integration/fit.test.js +174 -0
- package/test/integration/gpu-smoke.test.js +13 -0
- package/test/integration/lavasr-enhancer.test.js +104 -3
- package/test/integration/moss.test.js +282 -0
- package/test/integration/multiple-runs.test.js +18 -2
- package/test/integration/output-sample-rate.test.js +6 -0
- package/test/integration/parler-wer.test.js +7 -1
- package/test/integration/parler.test.js +27 -3
- package/test/integration/pocket.integration.test.js +156 -0
- package/test/integration/supertonic-mtl.test.js +2 -0
- package/test/integration/supertonic.test.js +70 -0
- package/test/integration/supertonic3-quant.test.js +2 -0
- package/test/integration/voice-controls.test.js +56 -0
- package/test/mobile/integration.auto.cjs +16 -1
- package/test/mobile/pocket-worklet-host.cpp +92 -0
- package/test/mobile/pocket-worklet.cjs +47 -0
- package/test/mobile/test-groups.json +8 -4
- package/test/utils/runCosyvoiceTTS.js +1 -0
- package/test/utils/runTTS.js +6 -11
- package/tts.d.ts +13 -2
- package/tts.js +3 -2
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const fs = require('bare-fs')
|
|
4
|
+
const os = require('bare-os')
|
|
5
|
+
const path = require('bare-path')
|
|
6
|
+
const proc = require('bare-process')
|
|
7
|
+
const test = require('brittle')
|
|
8
|
+
|
|
9
|
+
const TTSGgml = require('@qvac/tts-ggml')
|
|
10
|
+
const { recordTtsStats } = require('../utils/perf-helper')
|
|
11
|
+
const { resolveRefWavPath } = require('../utils/runChatterboxTTS')
|
|
12
|
+
const { readWavAsFloat32, createWav, resampleLinear } = require('../utils/wav-helper')
|
|
13
|
+
const { TTS_TEST_THREADS } = require('../utils/testThreads')
|
|
14
|
+
|
|
15
|
+
const MOSS_SAMPLE_RATE = 24000
|
|
16
|
+
const MOSS_STREAM_FRAMES = 10
|
|
17
|
+
const MOSS_SAMPLES_PER_FRAME = 1920
|
|
18
|
+
const TEST_TIMEOUT_MS = 1800000
|
|
19
|
+
const SYNTHESIS_TEXT = 'The MOSS integration test checks this generated voice.'
|
|
20
|
+
const MODEL_DIR_ENV = 'QVAC_TEST_MOSS_MODEL_DIR'
|
|
21
|
+
const ENCODER_RE = /^moss-codec-encoder(-[a-z0-9_]+)?\.gguf$/i
|
|
22
|
+
const DIALOGUE_BACKBONE_RE = /^moss-ttsd(-[a-z0-9_]+)?\.gguf$/i
|
|
23
|
+
const DIALOGUE_TEXT =
|
|
24
|
+
'[S1] And so, my fellow Americans, [S2] ask what you can do for your country. ' +
|
|
25
|
+
'[S1] Did the build finish this morning? [S2] Yes, every single test passed.'
|
|
26
|
+
const SPEAKER_ONE_SECONDS = [0, 3.29]
|
|
27
|
+
const SPEAKER_TWO_SECONDS = [7.6, 10.6]
|
|
28
|
+
const SHORT_DURATION_TOKENS = 25
|
|
29
|
+
const LONG_DURATION_TOKENS = 100
|
|
30
|
+
const INT16_FULL_SCALE = 32767
|
|
31
|
+
const CPU_DEVICE = 0
|
|
32
|
+
const CPU_BACKEND = 0
|
|
33
|
+
const GPU_DEVICE = 1
|
|
34
|
+
|
|
35
|
+
const modelDir = (proc.env && proc.env[MODEL_DIR_ENV]) || ''
|
|
36
|
+
const skipWithoutModels = modelDir === ''
|
|
37
|
+
const noGpu = proc.env && proc.env.NO_GPU === 'true'
|
|
38
|
+
|
|
39
|
+
function createMossModel(extra = {}, useGPU = false) {
|
|
40
|
+
return new TTSGgml({
|
|
41
|
+
threads: TTS_TEST_THREADS,
|
|
42
|
+
engine: TTSGgml.ENGINE_MOSS,
|
|
43
|
+
files: { modelDir: path.resolve(modelDir) },
|
|
44
|
+
config: { language: 'en', useGPU },
|
|
45
|
+
opts: { stats: true },
|
|
46
|
+
...extra
|
|
47
|
+
})
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function toInt16(samples) {
|
|
51
|
+
return Array.from(samples, (sample) => Math.round(sample * INT16_FULL_SCALE))
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function resampleToMossRate(sourcePath) {
|
|
55
|
+
const source = readWavAsFloat32(sourcePath)
|
|
56
|
+
return resampleLinear(source.samples, source.sampleRate, MOSS_SAMPLE_RATE)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function writeMossWav(samples, tag) {
|
|
60
|
+
const target = path.join(os.tmpdir(), `moss-${tag}-${Date.now()}.wav`)
|
|
61
|
+
createWav(toInt16(samples), MOSS_SAMPLE_RATE, target)
|
|
62
|
+
return target
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function writeReferenceAtMossRate(sourcePath) {
|
|
66
|
+
return writeMossWav(resampleToMossRate(sourcePath), 'reference')
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function sliceSeconds(samples, [from, to]) {
|
|
70
|
+
return samples.slice(Math.round(from * MOSS_SAMPLE_RATE), Math.round(to * MOSS_SAMPLE_RATE))
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function writeSpeakerReferences(sourcePath) {
|
|
74
|
+
const samples = resampleToMossRate(sourcePath)
|
|
75
|
+
return [
|
|
76
|
+
writeMossWav(sliceSeconds(samples, SPEAKER_ONE_SECONDS), 'speaker-1'),
|
|
77
|
+
writeMossWav(sliceSeconds(samples, SPEAKER_TWO_SECONDS), 'speaker-2')
|
|
78
|
+
]
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function removeFiles(paths) {
|
|
82
|
+
for (const file of paths) fs.unlinkSync(file)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function modelDirHas(pattern) {
|
|
86
|
+
return !skipWithoutModels && fs.readdirSync(modelDir).some((entry) => pattern.test(entry))
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function hasEncoder() {
|
|
90
|
+
return modelDirHas(ENCODER_RE)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function hasDialogueBackbone() {
|
|
94
|
+
return modelDirHas(DIALOGUE_BACKBONE_RE)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function collectChunk(result, data) {
|
|
98
|
+
if (!data || !data.outputArray) return
|
|
99
|
+
result.chunks.push(data.outputArray.length)
|
|
100
|
+
result.samples = result.samples.concat(Array.from(data.outputArray))
|
|
101
|
+
if (data.sampleRate) result.sampleRate = data.sampleRate
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async function synthesize(model, text) {
|
|
105
|
+
const response = await model.run({ input: text, type: 'text' })
|
|
106
|
+
const result = { samples: [], chunks: [], sampleRate: null }
|
|
107
|
+
await response.onUpdate((data) => collectChunk(result, data)).await()
|
|
108
|
+
result.stats = response.stats
|
|
109
|
+
return result
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function assertAudio(t, label, result) {
|
|
113
|
+
t.ok(result.samples.length > 0, `${label} produced audio`)
|
|
114
|
+
t.is(result.sampleRate, MOSS_SAMPLE_RATE, `${label} reports 24 kHz`)
|
|
115
|
+
t.ok(result.stats, `${label} returns runtime stats`)
|
|
116
|
+
t.is(result.stats.totalSamples, result.samples.length, `${label} reports the emitted samples`)
|
|
117
|
+
t.ok(result.stats.generatedFrames > 0, `${label} reports generated codec frames`)
|
|
118
|
+
t.ok(result.stats.tokensPerSecond > 0, `${label} reports codec frames per second`)
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function nonEmptyChunks(result) {
|
|
122
|
+
return result.chunks.filter((length) => length > 0)
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function recordMoss(t, label, result, wallMs) {
|
|
126
|
+
t.comment(
|
|
127
|
+
recordTtsStats(label, result.stats, {
|
|
128
|
+
wallMs,
|
|
129
|
+
sampleCount: result.samples.length,
|
|
130
|
+
model: 'moss',
|
|
131
|
+
output: label
|
|
132
|
+
})
|
|
133
|
+
)
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function assertCpuBackend(t, label, stats) {
|
|
137
|
+
t.is(stats.backendDevice, CPU_DEVICE, `${label} reports a CPU device`)
|
|
138
|
+
t.is(stats.backendId, CPU_BACKEND, `${label} reports the CPU backend`)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
async function withLoadedModel(extra, body, useGPU = false) {
|
|
142
|
+
const model = createMossModel(extra, useGPU)
|
|
143
|
+
await model.load()
|
|
144
|
+
try {
|
|
145
|
+
await body(model)
|
|
146
|
+
} finally {
|
|
147
|
+
await model.unload()
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
test(
|
|
152
|
+
'MOSS TTS: batch synthesis through the public JS API',
|
|
153
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels },
|
|
154
|
+
async (t) => {
|
|
155
|
+
await withLoadedModel({}, async (model) => {
|
|
156
|
+
const started = Date.now()
|
|
157
|
+
const result = await synthesize(model, SYNTHESIS_TEXT)
|
|
158
|
+
assertAudio(t, 'MOSS batch', result)
|
|
159
|
+
assertCpuBackend(t, 'MOSS batch', result.stats)
|
|
160
|
+
t.is(nonEmptyChunks(result).length, 1, 'batch synthesis emits one buffer')
|
|
161
|
+
recordMoss(t, 'moss batch', result, Date.now() - started)
|
|
162
|
+
})
|
|
163
|
+
}
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
test(
|
|
167
|
+
'MOSS TTS: native chunk streaming emits frame-sized chunks',
|
|
168
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels },
|
|
169
|
+
async (t) => {
|
|
170
|
+
await withLoadedModel({ streamChunkTokens: MOSS_STREAM_FRAMES }, async (model) => {
|
|
171
|
+
const started = Date.now()
|
|
172
|
+
const result = await synthesize(model, SYNTHESIS_TEXT)
|
|
173
|
+
assertAudio(t, 'MOSS stream', result)
|
|
174
|
+
const chunks = nonEmptyChunks(result)
|
|
175
|
+
t.ok(chunks.length > 1, `streaming emits several chunks (got ${chunks.length})`)
|
|
176
|
+
t.ok(
|
|
177
|
+
chunks.every((length) => length <= MOSS_STREAM_FRAMES * MOSS_SAMPLES_PER_FRAME),
|
|
178
|
+
'no chunk exceeds the configured frame count'
|
|
179
|
+
)
|
|
180
|
+
recordMoss(t, 'moss stream', result, Date.now() - started)
|
|
181
|
+
})
|
|
182
|
+
}
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
test(
|
|
186
|
+
'MOSS TTS: useGPU=true synthesizes on a GPU backend',
|
|
187
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels || noGpu },
|
|
188
|
+
async (t) => {
|
|
189
|
+
await withLoadedModel(
|
|
190
|
+
{},
|
|
191
|
+
async (model) => {
|
|
192
|
+
const started = Date.now()
|
|
193
|
+
const result = await synthesize(model, SYNTHESIS_TEXT)
|
|
194
|
+
assertAudio(t, 'MOSS GPU', result)
|
|
195
|
+
t.is(result.stats.backendDevice, GPU_DEVICE, 'MOSS GPU reports a GPU device')
|
|
196
|
+
t.not(result.stats.backendId, CPU_BACKEND, 'MOSS GPU does not report the CPU backend')
|
|
197
|
+
t.is(result.stats.gpuUnsupported, 0, 'MOSS GPU does not report a GPU fallback')
|
|
198
|
+
recordMoss(t, 'moss gpu', result, Date.now() - started)
|
|
199
|
+
},
|
|
200
|
+
true
|
|
201
|
+
)
|
|
202
|
+
}
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
test(
|
|
206
|
+
'MOSS TTS: voice cloning synthesizes from a reference recording',
|
|
207
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels },
|
|
208
|
+
async (t) => {
|
|
209
|
+
const sourceAudio = resolveRefWavPath({})
|
|
210
|
+
if (!hasEncoder()) {
|
|
211
|
+
t.comment(`skipping: no moss-codec-encoder GGUF in ${modelDir}`)
|
|
212
|
+
return
|
|
213
|
+
}
|
|
214
|
+
if (!fs.existsSync(sourceAudio)) {
|
|
215
|
+
t.fail(`MOSS reference audio is missing: ${sourceAudio}`)
|
|
216
|
+
return
|
|
217
|
+
}
|
|
218
|
+
const referenceAudio = writeReferenceAtMossRate(sourceAudio)
|
|
219
|
+
try {
|
|
220
|
+
await withLoadedModel({ referenceAudio, seed: 7 }, async (model) => {
|
|
221
|
+
const started = Date.now()
|
|
222
|
+
const result = await synthesize(model, SYNTHESIS_TEXT)
|
|
223
|
+
assertAudio(t, 'MOSS clone', result)
|
|
224
|
+
recordMoss(t, 'moss clone', result, Date.now() - started)
|
|
225
|
+
})
|
|
226
|
+
} finally {
|
|
227
|
+
fs.unlinkSync(referenceAudio)
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
test(
|
|
233
|
+
'MOSS TTS: durationTokens steers the length of the synthesized speech',
|
|
234
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels },
|
|
235
|
+
async (t) => {
|
|
236
|
+
await withLoadedModel({ durationTokens: SHORT_DURATION_TOKENS, seed: 7 }, async (model) => {
|
|
237
|
+
const short = await synthesize(model, SYNTHESIS_TEXT)
|
|
238
|
+
assertAudio(t, 'MOSS short target', short)
|
|
239
|
+
await model.reload({ durationTokens: LONG_DURATION_TOKENS })
|
|
240
|
+
const long = await synthesize(model, SYNTHESIS_TEXT)
|
|
241
|
+
assertAudio(t, 'MOSS long target', long)
|
|
242
|
+
t.ok(
|
|
243
|
+
long.samples.length > short.samples.length,
|
|
244
|
+
`a longer target yields longer speech (${short.samples.length} -> ${long.samples.length} samples)`
|
|
245
|
+
)
|
|
246
|
+
})
|
|
247
|
+
}
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
test(
|
|
251
|
+
'MOSS TTS: dialogue synthesis clones one reference per speaker',
|
|
252
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels },
|
|
253
|
+
async (t) => {
|
|
254
|
+
if (!hasDialogueBackbone() || !hasEncoder()) {
|
|
255
|
+
t.comment(`skipping: the TTSD backbone or the codec encoder is not in ${modelDir}`)
|
|
256
|
+
return
|
|
257
|
+
}
|
|
258
|
+
const sourceAudio = resolveRefWavPath({})
|
|
259
|
+
if (!fs.existsSync(sourceAudio)) {
|
|
260
|
+
t.fail(`MOSS reference audio is missing: ${sourceAudio}`)
|
|
261
|
+
return
|
|
262
|
+
}
|
|
263
|
+
const references = writeSpeakerReferences(sourceAudio)
|
|
264
|
+
try {
|
|
265
|
+
await withLoadedModel(
|
|
266
|
+
{ dialogueReferences: references, seed: 7, files: { modelDir: path.resolve(modelDir) } },
|
|
267
|
+
async (model) => {
|
|
268
|
+
t.ok(
|
|
269
|
+
DIALOGUE_BACKBONE_RE.test(path.basename(model._mossBackbonePath)),
|
|
270
|
+
'dialogue picks the TTSD backbone'
|
|
271
|
+
)
|
|
272
|
+
const started = Date.now()
|
|
273
|
+
const result = await synthesize(model, DIALOGUE_TEXT)
|
|
274
|
+
assertAudio(t, 'MOSS dialogue', result)
|
|
275
|
+
recordMoss(t, 'moss dialogue', result, Date.now() - started)
|
|
276
|
+
}
|
|
277
|
+
)
|
|
278
|
+
} finally {
|
|
279
|
+
removeFiles(references)
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
)
|
|
@@ -29,6 +29,7 @@ const {
|
|
|
29
29
|
} = require('../utils/runChatterboxTTS')
|
|
30
30
|
const { loadSupertonicTTS, runSupertonicTTS } = require('../utils/runSupertonicTTS')
|
|
31
31
|
const { ensureChatterboxModels, ensureSupertonicModel } = require('../utils/downloadModel')
|
|
32
|
+
const { TTS_TEST_THREADS } = require('../utils/testThreads')
|
|
32
33
|
|
|
33
34
|
const platform = os.platform()
|
|
34
35
|
const isMobile = platform === 'ios' || platform === 'android'
|
|
@@ -73,6 +74,7 @@ test(
|
|
|
73
74
|
}
|
|
74
75
|
|
|
75
76
|
const model = await loadChatterboxTTS({
|
|
77
|
+
threads: TTS_TEST_THREADS,
|
|
76
78
|
modelDir: download.targetDir,
|
|
77
79
|
refWavPath,
|
|
78
80
|
language: 'en'
|
|
@@ -137,6 +139,7 @@ test(
|
|
|
137
139
|
for (let i = 0; i < N; i++) {
|
|
138
140
|
const t0 = Date.now()
|
|
139
141
|
const model = await loadChatterboxTTS({
|
|
142
|
+
threads: TTS_TEST_THREADS,
|
|
140
143
|
modelDir: download.targetDir,
|
|
141
144
|
refWavPath,
|
|
142
145
|
language: 'en'
|
|
@@ -195,6 +198,7 @@ test('Chatterbox: reload() between runs preserves stability', { timeout: 1800000
|
|
|
195
198
|
}
|
|
196
199
|
|
|
197
200
|
const model = await loadChatterboxTTS({
|
|
201
|
+
threads: TTS_TEST_THREADS,
|
|
198
202
|
modelDir: download.targetDir,
|
|
199
203
|
refWavPath,
|
|
200
204
|
language: 'en'
|
|
@@ -235,6 +239,7 @@ test('Supertonic: reload() between runs preserves stability', { timeout: 1800000
|
|
|
235
239
|
}
|
|
236
240
|
|
|
237
241
|
const model = await loadSupertonicTTS({
|
|
242
|
+
threads: TTS_TEST_THREADS,
|
|
238
243
|
supertonicModelPath: download.path,
|
|
239
244
|
voice: 'F1',
|
|
240
245
|
language: 'en',
|
|
@@ -291,7 +296,12 @@ test(
|
|
|
291
296
|
return
|
|
292
297
|
}
|
|
293
298
|
|
|
294
|
-
const c1 = await loadChatterboxTTS({
|
|
299
|
+
const c1 = await loadChatterboxTTS({
|
|
300
|
+
threads: TTS_TEST_THREADS,
|
|
301
|
+
modelDir: cb.targetDir,
|
|
302
|
+
refWavPath,
|
|
303
|
+
language: 'en'
|
|
304
|
+
})
|
|
295
305
|
try {
|
|
296
306
|
const r = await runChatterboxTTS(c1, { text: 'Hello from chatterbox.' }, { minSamples: 5000 })
|
|
297
307
|
t.ok(r.passed, 'first chatterbox instance OK')
|
|
@@ -302,6 +312,7 @@ test(
|
|
|
302
312
|
}
|
|
303
313
|
|
|
304
314
|
const s1 = await loadSupertonicTTS({
|
|
315
|
+
threads: TTS_TEST_THREADS,
|
|
305
316
|
supertonicModelPath: st.path,
|
|
306
317
|
voice: 'F1',
|
|
307
318
|
language: 'en',
|
|
@@ -316,7 +327,12 @@ test(
|
|
|
316
327
|
} catch (_e) {}
|
|
317
328
|
}
|
|
318
329
|
|
|
319
|
-
const c2 = await loadChatterboxTTS({
|
|
330
|
+
const c2 = await loadChatterboxTTS({
|
|
331
|
+
threads: TTS_TEST_THREADS,
|
|
332
|
+
modelDir: cb.targetDir,
|
|
333
|
+
refWavPath,
|
|
334
|
+
language: 'en'
|
|
335
|
+
})
|
|
320
336
|
try {
|
|
321
337
|
const r = await runChatterboxTTS(
|
|
322
338
|
c2,
|
|
@@ -12,6 +12,7 @@ const test = require('brittle')
|
|
|
12
12
|
const TTSGgml = require('@qvac/tts-ggml')
|
|
13
13
|
|
|
14
14
|
const { ensureSupertonicModel, ensureParlerModel } = require('../utils/downloadModel')
|
|
15
|
+
const { TTS_TEST_THREADS } = require('../utils/testThreads')
|
|
15
16
|
|
|
16
17
|
const platform = os.platform()
|
|
17
18
|
const isMobile = platform === 'ios' || platform === 'android'
|
|
@@ -40,6 +41,7 @@ async function collect(model, text) {
|
|
|
40
41
|
// JS half, which must forward the combination rather than reject it up front.
|
|
41
42
|
test('CosyVoice3: enhancer + streaming forwards a non-native outputSampleRate', (t) => {
|
|
42
43
|
const model = new TTSGgml({
|
|
44
|
+
threads: TTS_TEST_THREADS,
|
|
43
45
|
engine: TTSGgml.ENGINE_COSYVOICE3,
|
|
44
46
|
files: {
|
|
45
47
|
cosyvoiceModelDir: './models/cosyvoice3',
|
|
@@ -72,6 +74,7 @@ test(
|
|
|
72
74
|
const text = 'Output rate selection resamples the synthesized audio.'
|
|
73
75
|
|
|
74
76
|
const native = new TTSGgml({
|
|
77
|
+
threads: TTS_TEST_THREADS,
|
|
75
78
|
engine: TTSGgml.ENGINE_SUPERTONIC,
|
|
76
79
|
files: { supertonicModel: dl.path },
|
|
77
80
|
voice: 'F1',
|
|
@@ -91,6 +94,7 @@ test(
|
|
|
91
94
|
}
|
|
92
95
|
|
|
93
96
|
const resampled = new TTSGgml({
|
|
97
|
+
threads: TTS_TEST_THREADS,
|
|
94
98
|
engine: TTSGgml.ENGINE_SUPERTONIC,
|
|
95
99
|
files: { supertonicModel: dl.path },
|
|
96
100
|
voice: 'F1',
|
|
@@ -131,6 +135,7 @@ test(
|
|
|
131
135
|
const text = 'Output rate selection resamples the synthesized audio.'
|
|
132
136
|
|
|
133
137
|
const native = new TTSGgml({
|
|
138
|
+
threads: TTS_TEST_THREADS,
|
|
134
139
|
engine: TTSGgml.ENGINE_PARLER,
|
|
135
140
|
files: { parlerModel: dl.path },
|
|
136
141
|
voice: 'Laura',
|
|
@@ -150,6 +155,7 @@ test(
|
|
|
150
155
|
}
|
|
151
156
|
|
|
152
157
|
const resampled = new TTSGgml({
|
|
158
|
+
threads: TTS_TEST_THREADS,
|
|
153
159
|
engine: TTSGgml.ENGINE_PARLER,
|
|
154
160
|
files: { parlerModel: dl.path },
|
|
155
161
|
voice: 'Laura',
|
|
@@ -19,6 +19,7 @@ const test = require('brittle')
|
|
|
19
19
|
const { loadParlerTTS, runParlerTTS } = require('../utils/runParlerTTS')
|
|
20
20
|
const { ensureParlerModel, ensureWhisperModel } = require('../utils/downloadModel')
|
|
21
21
|
const { loadWhisper, runWhisper } = require('../utils/runWhisper')
|
|
22
|
+
const { TTS_TEST_THREADS } = require('../utils/testThreads')
|
|
22
23
|
|
|
23
24
|
const platform = os.platform()
|
|
24
25
|
const isDarwin = platform === 'darwin'
|
|
@@ -77,7 +78,12 @@ for (const { variant } of PARLER_WER_TIERS) {
|
|
|
77
78
|
}
|
|
78
79
|
|
|
79
80
|
const useGPU = isApple && !NO_GPU
|
|
80
|
-
const model = await loadParlerTTS({
|
|
81
|
+
const model = await loadParlerTTS({
|
|
82
|
+
threads: TTS_TEST_THREADS,
|
|
83
|
+
parlerModelPath: download.path,
|
|
84
|
+
seed: 42,
|
|
85
|
+
useGPU
|
|
86
|
+
})
|
|
81
87
|
const entries = []
|
|
82
88
|
try {
|
|
83
89
|
for (const text of WER_SENTENCES) {
|
|
@@ -13,6 +13,7 @@ const test = require('brittle')
|
|
|
13
13
|
const { loadParlerTTS, runParlerTTS } = require('../utils/runParlerTTS')
|
|
14
14
|
const { ensureParlerModel } = require('../utils/downloadModel')
|
|
15
15
|
const { recordTtsStats } = require('../utils/perf-helper')
|
|
16
|
+
const { TTS_TEST_THREADS } = require('../utils/testThreads')
|
|
16
17
|
|
|
17
18
|
const platform = os.platform()
|
|
18
19
|
const isMobile = platform === 'ios' || platform === 'android'
|
|
@@ -39,7 +40,11 @@ test(
|
|
|
39
40
|
|
|
40
41
|
// Deliberately NO description/voice/emotion: the engine renders the
|
|
41
42
|
// models' recommended fallback caption — everything works on defaults.
|
|
42
|
-
const model = await loadParlerTTS({
|
|
43
|
+
const model = await loadParlerTTS({
|
|
44
|
+
threads: TTS_TEST_THREADS,
|
|
45
|
+
parlerModelPath: download.path,
|
|
46
|
+
seed: 42
|
|
47
|
+
})
|
|
43
48
|
try {
|
|
44
49
|
const wavPath = isMobile ? undefined : path.join(baseDir, 'test', 'output', 'parler-en.wav')
|
|
45
50
|
const text = 'The parler engine speaks with a voice controlled by a text description.'
|
|
@@ -100,7 +105,11 @@ test(
|
|
|
100
105
|
return
|
|
101
106
|
}
|
|
102
107
|
|
|
103
|
-
const model = await loadParlerTTS({
|
|
108
|
+
const model = await loadParlerTTS({
|
|
109
|
+
threads: TTS_TEST_THREADS,
|
|
110
|
+
parlerModelPath: download.path,
|
|
111
|
+
voice: 'Laura'
|
|
112
|
+
})
|
|
104
113
|
try {
|
|
105
114
|
const response = await model.run({
|
|
106
115
|
type: 'text',
|
|
@@ -143,6 +152,7 @@ test(
|
|
|
143
152
|
t.exception(
|
|
144
153
|
() =>
|
|
145
154
|
new TTSGgml({
|
|
155
|
+
threads: TTS_TEST_THREADS,
|
|
146
156
|
engine: TTSGgml.ENGINE_PARLER,
|
|
147
157
|
files: { parlerModel: download.path },
|
|
148
158
|
description: 'A calm female voice.',
|
|
@@ -157,6 +167,7 @@ test(
|
|
|
157
167
|
t.exception(
|
|
158
168
|
() =>
|
|
159
169
|
new TTSGgml({
|
|
170
|
+
threads: TTS_TEST_THREADS,
|
|
160
171
|
engine: TTSGgml.ENGINE_SUPERTONIC,
|
|
161
172
|
files: { supertonicModel: download.path },
|
|
162
173
|
emotion: 'happy'
|
|
@@ -167,6 +178,7 @@ test(
|
|
|
167
178
|
t.exception(
|
|
168
179
|
() =>
|
|
169
180
|
new TTSGgml({
|
|
181
|
+
threads: TTS_TEST_THREADS,
|
|
170
182
|
engine: TTSGgml.ENGINE_SUPERTONIC,
|
|
171
183
|
files: { supertonicModel: download.path },
|
|
172
184
|
pitch: 'high'
|
|
@@ -176,6 +188,7 @@ test(
|
|
|
176
188
|
)
|
|
177
189
|
|
|
178
190
|
const model = await loadParlerTTS({
|
|
191
|
+
threads: TTS_TEST_THREADS,
|
|
179
192
|
parlerModelPath: download.path,
|
|
180
193
|
voice: 'Laura',
|
|
181
194
|
seed: 42
|
|
@@ -218,6 +231,7 @@ test(
|
|
|
218
231
|
|
|
219
232
|
// Constructor-level free-text description + per-call template rejects.
|
|
220
233
|
const descModel = await loadParlerTTS({
|
|
234
|
+
threads: TTS_TEST_THREADS,
|
|
221
235
|
parlerModelPath: download.path,
|
|
222
236
|
description: 'A calm female voice, very clear audio.',
|
|
223
237
|
seed: 42
|
|
@@ -248,6 +262,7 @@ test(
|
|
|
248
262
|
}
|
|
249
263
|
|
|
250
264
|
const model = await loadParlerTTS({
|
|
265
|
+
threads: TTS_TEST_THREADS,
|
|
251
266
|
parlerModelPath: download.path,
|
|
252
267
|
voice: 'Laura',
|
|
253
268
|
seed: 42
|
|
@@ -295,6 +310,7 @@ test(
|
|
|
295
310
|
}
|
|
296
311
|
|
|
297
312
|
const model = await loadParlerTTS({
|
|
313
|
+
threads: TTS_TEST_THREADS,
|
|
298
314
|
parlerModelPath: download.path,
|
|
299
315
|
voice: 'Laura',
|
|
300
316
|
emotion: 'neutral',
|
|
@@ -340,6 +356,7 @@ test(
|
|
|
340
356
|
|
|
341
357
|
const TTSGgml = require('@qvac/tts-ggml')
|
|
342
358
|
const model = new TTSGgml({
|
|
359
|
+
threads: TTS_TEST_THREADS,
|
|
343
360
|
engine: TTSGgml.ENGINE_PARLER,
|
|
344
361
|
files: { parlerModel: download.path },
|
|
345
362
|
seed: 42,
|
|
@@ -392,6 +409,7 @@ test(
|
|
|
392
409
|
|
|
393
410
|
async function synth(extra) {
|
|
394
411
|
const model = new TTSGgml({
|
|
412
|
+
threads: TTS_TEST_THREADS,
|
|
395
413
|
engine: TTSGgml.ENGINE_PARLER,
|
|
396
414
|
files: { parlerModel: download.path },
|
|
397
415
|
seed: 42,
|
|
@@ -481,6 +499,7 @@ test(
|
|
|
481
499
|
}
|
|
482
500
|
|
|
483
501
|
const model = await loadParlerTTS({
|
|
502
|
+
threads: TTS_TEST_THREADS,
|
|
484
503
|
parlerModelPath: download.path,
|
|
485
504
|
voice: 'Rohit',
|
|
486
505
|
emotion: 'happy',
|
|
@@ -542,7 +561,12 @@ for (const { variant, quant } of PARLER_QUANT_MATRIX) {
|
|
|
542
561
|
}
|
|
543
562
|
|
|
544
563
|
const useGPU = isApple && !NO_GPU
|
|
545
|
-
const model = await loadParlerTTS({
|
|
564
|
+
const model = await loadParlerTTS({
|
|
565
|
+
threads: TTS_TEST_THREADS,
|
|
566
|
+
parlerModelPath: download.path,
|
|
567
|
+
seed: 42,
|
|
568
|
+
useGPU
|
|
569
|
+
})
|
|
546
570
|
try {
|
|
547
571
|
const text =
|
|
548
572
|
variant === 'indic'
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
const test = require('brittle')
|
|
3
|
+
const TTSGgml = require('../../index')
|
|
4
|
+
const process = require('bare-process')
|
|
5
|
+
global.process = process
|
|
6
|
+
const bundle = process.env.QVAC_POCKET_MODEL_DIR
|
|
7
|
+
const text = 'Hello! We can generate speech with Fabric.'
|
|
8
|
+
|
|
9
|
+
// Opt in with QVAC_POCKET_MODEL_DIR pointing at the converted four-file bundle.
|
|
10
|
+
module.exports = test(
|
|
11
|
+
'Pocket native addon streams, cancels, recovers and resamples on reload',
|
|
12
|
+
{ skip: !bundle },
|
|
13
|
+
async (t) => {
|
|
14
|
+
const model = new TTSGgml({
|
|
15
|
+
engine: 'pocket',
|
|
16
|
+
files: { modelDir: bundle },
|
|
17
|
+
opts: { stats: true }
|
|
18
|
+
})
|
|
19
|
+
try {
|
|
20
|
+
await model.load()
|
|
21
|
+
const run = async (rate) => {
|
|
22
|
+
const response = await model.run({ input: text })
|
|
23
|
+
const pcm = []
|
|
24
|
+
const indices = []
|
|
25
|
+
let last = 0
|
|
26
|
+
for await (const chunk of response.iterate()) {
|
|
27
|
+
t.is(chunk.sampleRate, rate)
|
|
28
|
+
indices.push(chunk.chunkIndex)
|
|
29
|
+
pcm.push(...chunk.outputArray)
|
|
30
|
+
if (chunk.isLast) {
|
|
31
|
+
last++
|
|
32
|
+
t.is(chunk.outputArray.length, 0)
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
t.is(last, 1)
|
|
36
|
+
t.alike(
|
|
37
|
+
indices,
|
|
38
|
+
indices.map((_, i) => i)
|
|
39
|
+
)
|
|
40
|
+
t.is(pcm.length, response.stats.totalSamples)
|
|
41
|
+
t.ok(pcm.some((x) => x !== 0))
|
|
42
|
+
t.ok(response.stats.firstAudioMs > 0)
|
|
43
|
+
return pcm
|
|
44
|
+
}
|
|
45
|
+
const original = await run(24000)
|
|
46
|
+
const sentenceStream = await model.runStream('Hello. Speech is ready.', {
|
|
47
|
+
maxChunkScalars: 16
|
|
48
|
+
})
|
|
49
|
+
await sentenceStream.onUpdate(() => {}).await()
|
|
50
|
+
t.ok(sentenceStream.stats.firstAudioMs > 0, 'runStream preserves native first-audio latency')
|
|
51
|
+
const textStream = await model.runStreaming(
|
|
52
|
+
(async function* () {
|
|
53
|
+
yield 'Hello.'
|
|
54
|
+
yield 'Speech is ready.'
|
|
55
|
+
})(),
|
|
56
|
+
{ accumulateSentences: false }
|
|
57
|
+
)
|
|
58
|
+
await textStream.onUpdate(() => {}).await()
|
|
59
|
+
t.ok(textStream.stats.firstAudioMs > 0, 'runStreaming preserves native first-audio latency')
|
|
60
|
+
|
|
61
|
+
// Check that omitted steps match the explicit upstream/native default
|
|
62
|
+
// through the complete addon path. PCM parity validates configuration,
|
|
63
|
+
// not subjective quality or the absence of model-generated artifacts.
|
|
64
|
+
const explicit = new TTSGgml({
|
|
65
|
+
engine: 'pocket',
|
|
66
|
+
files: { modelDir: bundle },
|
|
67
|
+
steps: 1
|
|
68
|
+
})
|
|
69
|
+
try {
|
|
70
|
+
await explicit.load()
|
|
71
|
+
const reference = await explicit.run({ input: text })
|
|
72
|
+
const pcm = []
|
|
73
|
+
for await (const chunk of reference.iterate()) pcm.push(...chunk.outputArray)
|
|
74
|
+
t.is(pcm.length, original.length, 'default and explicit one-step lengths match')
|
|
75
|
+
t.ok(
|
|
76
|
+
pcm.every((v, i) => Math.abs(v - original[i]) <= 4),
|
|
77
|
+
'default native PCM matches explicit one-step synthesis'
|
|
78
|
+
)
|
|
79
|
+
} finally {
|
|
80
|
+
await explicit.destroy()
|
|
81
|
+
}
|
|
82
|
+
if (process.env.QVAC_POCKET_AUDIO_OUTPUT) {
|
|
83
|
+
require('../utils/wav-helper').createWav(
|
|
84
|
+
new Int16Array(original),
|
|
85
|
+
24000,
|
|
86
|
+
process.env.QVAC_POCKET_AUDIO_OUTPUT
|
|
87
|
+
)
|
|
88
|
+
}
|
|
89
|
+
const response = await model.run({ input: text.repeat(5) })
|
|
90
|
+
let cancelPromise
|
|
91
|
+
const outcomePromise = response.await().then(
|
|
92
|
+
() => ({ completed: true }),
|
|
93
|
+
(error) => ({ error })
|
|
94
|
+
)
|
|
95
|
+
response.onUpdate((chunk) => {
|
|
96
|
+
if (!cancelPromise && chunk.outputArray.length) {
|
|
97
|
+
cancelPromise = response.cancel()
|
|
98
|
+
// Attach a rejection handler immediately; await the original promise
|
|
99
|
+
// below so cancellation failures still fail the integration test.
|
|
100
|
+
cancelPromise.catch(() => {})
|
|
101
|
+
}
|
|
102
|
+
})
|
|
103
|
+
const outcome = await outcomePromise
|
|
104
|
+
t.ok(cancelPromise, 'requested cancellation after first PCM')
|
|
105
|
+
if (cancelPromise) await cancelPromise
|
|
106
|
+
t.ok(
|
|
107
|
+
outcome.error && /cancel|abort/i.test(String(outcome.error)),
|
|
108
|
+
'cancelled synthesis rejects instead of completing'
|
|
109
|
+
)
|
|
110
|
+
const recovered = await run(24000)
|
|
111
|
+
t.is(recovered.length, original.length)
|
|
112
|
+
t.ok(
|
|
113
|
+
recovered.every((v, i) => Math.abs(v - original[i]) <= 4),
|
|
114
|
+
'deterministic PCM after cancellation'
|
|
115
|
+
)
|
|
116
|
+
await model.reload({ outputSampleRate: 44100 })
|
|
117
|
+
const resampled = await run(44100)
|
|
118
|
+
t.ok(Math.abs(resampled.length / 44100 - original.length / 24000) < 1 / 44100)
|
|
119
|
+
await t.exception(model.reload({ referenceAudio: `${bundle}/missing-reload-test.wav` }))
|
|
120
|
+
t.is(
|
|
121
|
+
model.getState().weightsLoaded,
|
|
122
|
+
false,
|
|
123
|
+
'failed native activation leaves the model unloaded'
|
|
124
|
+
)
|
|
125
|
+
await t.exception(model.run({ input: text }), /not loaded/)
|
|
126
|
+
await model.load()
|
|
127
|
+
const restored = await run(44100)
|
|
128
|
+
t.is(
|
|
129
|
+
restored.length,
|
|
130
|
+
resampled.length,
|
|
131
|
+
'load restores the last successful sample rate and voice'
|
|
132
|
+
)
|
|
133
|
+
t.ok(
|
|
134
|
+
restored.every((v, i) => Math.abs(v - resampled[i]) <= 4),
|
|
135
|
+
'PCM recovers after failed reload'
|
|
136
|
+
)
|
|
137
|
+
const empty = await model.run({ input: ' ' })
|
|
138
|
+
await t.exception(empty.await())
|
|
139
|
+
await run(44100)
|
|
140
|
+
// Verify the packaged native dependency includes the merged EOS-tail
|
|
141
|
+
// fix: the old token-only budget threw after emitting partial audio.
|
|
142
|
+
await model.reload({ outputSampleRate: 24000, framesAfterEos: 100, eosThreshold: -1e30 })
|
|
143
|
+
const tail = await model.run({ input: 'Hi.' })
|
|
144
|
+
let tailSamples = 0
|
|
145
|
+
let tailCompleted = 0
|
|
146
|
+
for await (const chunk of tail.iterate()) {
|
|
147
|
+
tailSamples += chunk.outputArray.length
|
|
148
|
+
if (chunk.isLast) tailCompleted++
|
|
149
|
+
}
|
|
150
|
+
t.is(tailCompleted, 1, 'explicit EOS tail completes normally')
|
|
151
|
+
t.is(tailSamples, 100 * 1920, 'all 100 EOS-tail frames are delivered')
|
|
152
|
+
} finally {
|
|
153
|
+
await model.destroy()
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
)
|