@qvac/tts-ggml 0.9.2 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +330 -36
- package/docs/pocket-tts.md +201 -0
- package/examples/moss-dialogue-tts.js +113 -0
- package/examples/moss-tts.js +154 -0
- package/examples/pocket-tts.js +51 -0
- package/examples/supertonic-chunk-stream-tts.js +156 -0
- package/examples/supertonic-sentence-stream-tts.js +4 -4
- package/index.d.ts +352 -29
- package/index.js +698 -52
- package/lib/fit.d.ts +127 -0
- package/lib/fit.js +29 -0
- package/lib/pocketConfig.d.ts +10 -0
- package/lib/pocketConfig.js +145 -0
- package/package.json +16 -9
- package/test/integration/addon.test.js +13 -7
- package/test/integration/audio8.test.js +7 -0
- package/test/integration/chatterbox-mtl.test.js +20 -9
- package/test/integration/chatterbox-speed.test.js +16 -3
- package/test/integration/cosyvoice3-clone.test.js +17 -2
- package/test/integration/cosyvoice3-lavasr.test.js +21 -11
- package/test/integration/cosyvoice3.test.js +33 -8
- package/test/integration/fit.test.js +174 -0
- package/test/integration/gpu-smoke.test.js +13 -0
- package/test/integration/lavasr-enhancer.test.js +107 -5
- package/test/integration/moss.test.js +287 -0
- package/test/integration/multiple-runs.test.js +84 -57
- package/test/integration/output-sample-rate.test.js +10 -2
- package/test/integration/parler-wer.test.js +14 -9
- package/test/integration/parler.test.js +54 -24
- package/test/integration/pocket.integration.test.js +156 -0
- package/test/integration/supertonic-mtl.test.js +2 -0
- package/test/integration/supertonic.test.js +70 -0
- package/test/integration/supertonic3-quant.test.js +2 -0
- package/test/integration/voice-controls.test.js +56 -0
- package/test/mobile/integration.auto.cjs +16 -1
- package/test/mobile/pocket-worklet-host.cpp +92 -0
- package/test/mobile/pocket-worklet.cjs +47 -0
- package/test/mobile/test-groups.json +8 -4
- package/test/utils/runCosyvoiceTTS.js +1 -0
- package/test/utils/runTTS.js +6 -11
- package/tts.d.ts +13 -2
- package/tts.js +3 -2
|
@@ -32,6 +32,8 @@ const {
|
|
|
32
32
|
ensureParlerModel
|
|
33
33
|
} = require('../utils/downloadModel')
|
|
34
34
|
const { resolveRefWavPath } = require('../utils/runChatterboxTTS')
|
|
35
|
+
const { TTS_TEST_THREADS } = require('../utils/testThreads')
|
|
36
|
+
const { GPU_ONLY_USE_GPU } = require('../utils/gpuOnly')
|
|
35
37
|
|
|
36
38
|
const platform = os.platform()
|
|
37
39
|
const isMobile = platform === 'ios' || platform === 'android'
|
|
@@ -159,6 +161,13 @@ async function runAndCollect(model, text) {
|
|
|
159
161
|
return { samples, sampleRate, stats: response.stats || null }
|
|
160
162
|
}
|
|
161
163
|
|
|
164
|
+
// A CPU engine keeps the denoiser on its scalar core, which runtimeStats
|
|
165
|
+
// reports as CPU (denoiserBackendDevice 0, denoiserBackendId 0).
|
|
166
|
+
function assertDenoiserOnCpu(t, stats, label) {
|
|
167
|
+
t.is(stats.denoiserBackendDevice, 0, `${label}: useGPU:false -> denoiser on CPU`)
|
|
168
|
+
t.is(stats.denoiserBackendId, 0, `${label}: denoiser backend reported as CPU`)
|
|
169
|
+
}
|
|
170
|
+
|
|
162
171
|
// Every chunk that carries audio must be tagged at the enhanced 48 kHz rate
|
|
163
172
|
// rather than the engine's native rate — the mislabel this feature prevents.
|
|
164
173
|
function assertStreamedChunksReportEnhancedRate(t, updates) {
|
|
@@ -202,6 +211,7 @@ function assertStreamTerminatesOnce(t, updates, terminal) {
|
|
|
202
211
|
|
|
203
212
|
async function runParlerBatch(files) {
|
|
204
213
|
const model = new TTSGgml({
|
|
214
|
+
threads: TTS_TEST_THREADS,
|
|
205
215
|
engine: TTSGgml.ENGINE_PARLER,
|
|
206
216
|
files,
|
|
207
217
|
voice: 'Laura',
|
|
@@ -225,6 +235,7 @@ test('Chatterbox: enhancer + streamChunkTokens constructs and forwards both', (t
|
|
|
225
235
|
// construct and forward both knobs to the addon (which runs the streaming
|
|
226
236
|
// enhancer per chunk).
|
|
227
237
|
const model = new TTSGgml({
|
|
238
|
+
threads: TTS_TEST_THREADS,
|
|
228
239
|
engine: TTSGgml.ENGINE_CHATTERBOX,
|
|
229
240
|
files: {
|
|
230
241
|
t3Model: './models/chatterbox-t3-turbo.gguf',
|
|
@@ -247,6 +258,7 @@ test('Parler: enhancer + streamChunkTokens constructs and forwards both', (t) =>
|
|
|
247
258
|
// Parler used to reject the enhancer outright; it now enhances both the batch
|
|
248
259
|
// path and native chunk streaming (seam-free via the streaming enhancer).
|
|
249
260
|
const model = new TTSGgml({
|
|
261
|
+
threads: TTS_TEST_THREADS,
|
|
250
262
|
engine: TTSGgml.ENGINE_PARLER,
|
|
251
263
|
files: {
|
|
252
264
|
parlerModel: './models/parler-mini-v1-q8_0.gguf',
|
|
@@ -266,6 +278,7 @@ test('Parler: enhancer + streamChunkTokens constructs and forwards both', (t) =>
|
|
|
266
278
|
|
|
267
279
|
test('Parler: denoiser forwards on the batch path and is rejected while streaming', (t) => {
|
|
268
280
|
const model = new TTSGgml({
|
|
281
|
+
threads: TTS_TEST_THREADS,
|
|
269
282
|
engine: TTSGgml.ENGINE_PARLER,
|
|
270
283
|
files: {
|
|
271
284
|
parlerModel: './models/parler-mini-v1-q8_0.gguf',
|
|
@@ -281,6 +294,7 @@ test('Parler: denoiser forwards on the batch path and is rejected while streamin
|
|
|
281
294
|
t.exception(
|
|
282
295
|
() =>
|
|
283
296
|
new TTSGgml({
|
|
297
|
+
threads: TTS_TEST_THREADS,
|
|
284
298
|
engine: TTSGgml.ENGINE_PARLER,
|
|
285
299
|
files: {
|
|
286
300
|
parlerModel: './models/parler-mini-v1-q8_0.gguf',
|
|
@@ -299,6 +313,7 @@ test('CosyVoice3: enhancer + streamChunkTokens constructs and forwards both', (t
|
|
|
299
313
|
// batch path and on native chunk streaming, so both knobs must reach the
|
|
300
314
|
// addon together.
|
|
301
315
|
const model = new TTSGgml({
|
|
316
|
+
threads: TTS_TEST_THREADS,
|
|
302
317
|
engine: TTSGgml.ENGINE_COSYVOICE3,
|
|
303
318
|
files: {
|
|
304
319
|
cosyvoiceModelDir: './models/cosyvoice3',
|
|
@@ -318,6 +333,7 @@ test('CosyVoice3: enhancer + streamChunkTokens constructs and forwards both', (t
|
|
|
318
333
|
|
|
319
334
|
test('CosyVoice3: denoiser is forwarded for batch but rejected with streaming', (t) => {
|
|
320
335
|
const batch = new TTSGgml({
|
|
336
|
+
threads: TTS_TEST_THREADS,
|
|
321
337
|
engine: TTSGgml.ENGINE_COSYVOICE3,
|
|
322
338
|
files: {
|
|
323
339
|
cosyvoiceModelDir: './models/cosyvoice3',
|
|
@@ -334,6 +350,7 @@ test('CosyVoice3: denoiser is forwarded for batch but rejected with streaming',
|
|
|
334
350
|
t.exception(
|
|
335
351
|
() =>
|
|
336
352
|
new TTSGgml({
|
|
353
|
+
threads: TTS_TEST_THREADS,
|
|
337
354
|
engine: TTSGgml.ENGINE_COSYVOICE3,
|
|
338
355
|
files: {
|
|
339
356
|
cosyvoiceModelDir: './models/cosyvoice3',
|
|
@@ -351,6 +368,7 @@ test('enhancer with an unknown type is rejected at construction', (t) => {
|
|
|
351
368
|
t.exception(
|
|
352
369
|
() =>
|
|
353
370
|
new TTSGgml({
|
|
371
|
+
threads: TTS_TEST_THREADS,
|
|
354
372
|
engine: TTSGgml.ENGINE_SUPERTONIC,
|
|
355
373
|
files: {
|
|
356
374
|
supertonicModel: './models/supertonic.gguf',
|
|
@@ -366,6 +384,7 @@ test('enhancer with an unknown type is rejected at construction', (t) => {
|
|
|
366
384
|
|
|
367
385
|
test('enhancer block with no GGUF path leaves enhancement off (no throw)', (t) => {
|
|
368
386
|
const model = new TTSGgml({
|
|
387
|
+
threads: TTS_TEST_THREADS,
|
|
369
388
|
engine: TTSGgml.ENGINE_SUPERTONIC,
|
|
370
389
|
files: { supertonicModel: './models/supertonic.gguf' },
|
|
371
390
|
enhancer: { type: 'lavasr' },
|
|
@@ -420,7 +439,12 @@ for (const [engineName, engine, files] of [
|
|
|
420
439
|
]
|
|
421
440
|
]) {
|
|
422
441
|
test(`${engineName}: enhancer + useGPU:true forwards useGPU alongside the enhancer path`, (t) => {
|
|
423
|
-
const model = new TTSGgml({
|
|
442
|
+
const model = new TTSGgml({
|
|
443
|
+
threads: TTS_TEST_THREADS,
|
|
444
|
+
engine,
|
|
445
|
+
files,
|
|
446
|
+
config: { language: 'en', useGPU: true }
|
|
447
|
+
})
|
|
424
448
|
const params = model._buildTtsParams()
|
|
425
449
|
t.is(params.useGPU, true, 'useGPU:true forwarded to the addon (drives EnhancerOptions.use_gpu)')
|
|
426
450
|
t.is(
|
|
@@ -431,7 +455,12 @@ for (const [engineName, engine, files] of [
|
|
|
431
455
|
})
|
|
432
456
|
|
|
433
457
|
test(`${engineName}: enhancer + useGPU:false keeps the enhancer on CPU`, (t) => {
|
|
434
|
-
const model = new TTSGgml({
|
|
458
|
+
const model = new TTSGgml({
|
|
459
|
+
threads: TTS_TEST_THREADS,
|
|
460
|
+
engine,
|
|
461
|
+
files,
|
|
462
|
+
config: { language: 'en', useGPU: false }
|
|
463
|
+
})
|
|
435
464
|
const params = model._buildTtsParams()
|
|
436
465
|
t.is(
|
|
437
466
|
params.useGPU,
|
|
@@ -446,7 +475,13 @@ for (const [engineName, engine, files] of [
|
|
|
446
475
|
})
|
|
447
476
|
|
|
448
477
|
test(`${engineName}: enhancer + nGpuLayers!=0 forwards the GPU layer count`, (t) => {
|
|
449
|
-
const model = new TTSGgml({
|
|
478
|
+
const model = new TTSGgml({
|
|
479
|
+
threads: TTS_TEST_THREADS,
|
|
480
|
+
engine,
|
|
481
|
+
files,
|
|
482
|
+
nGpuLayers: 99,
|
|
483
|
+
config: { language: 'en' }
|
|
484
|
+
})
|
|
450
485
|
const params = model._buildTtsParams()
|
|
451
486
|
t.is(params.nGpuLayers, 99, 'nGpuLayers forwarded (non-zero => enhancer requests the GPU)')
|
|
452
487
|
t.is(
|
|
@@ -479,6 +514,7 @@ test(
|
|
|
479
514
|
}
|
|
480
515
|
|
|
481
516
|
const model = new TTSGgml({
|
|
517
|
+
threads: TTS_TEST_THREADS,
|
|
482
518
|
engine: TTSGgml.ENGINE_SUPERTONIC,
|
|
483
519
|
files: { supertonicModel: dl.path, lavasrEnhancer: enh.path },
|
|
484
520
|
voice: 'F1',
|
|
@@ -525,6 +561,7 @@ test(
|
|
|
525
561
|
}
|
|
526
562
|
|
|
527
563
|
const model = new TTSGgml({
|
|
564
|
+
threads: TTS_TEST_THREADS,
|
|
528
565
|
engine: TTSGgml.ENGINE_SUPERTONIC,
|
|
529
566
|
files: { supertonicModel: dl.path },
|
|
530
567
|
voice: 'F1',
|
|
@@ -566,6 +603,7 @@ test(
|
|
|
566
603
|
const dir = dl.targetDir || modelsDir
|
|
567
604
|
|
|
568
605
|
const model = new TTSGgml({
|
|
606
|
+
threads: TTS_TEST_THREADS,
|
|
569
607
|
engine: TTSGgml.ENGINE_CHATTERBOX,
|
|
570
608
|
files: {
|
|
571
609
|
modelDir: dir,
|
|
@@ -595,7 +633,7 @@ test(
|
|
|
595
633
|
|
|
596
634
|
test(
|
|
597
635
|
'Chatterbox + LavaSR enhancer + native chunk streaming emits 48 kHz chunks',
|
|
598
|
-
{ timeout: 900000 },
|
|
636
|
+
{ timeout: 900000, skip: NO_GPU },
|
|
599
637
|
async (t) => {
|
|
600
638
|
const baseDir = getBaseDir()
|
|
601
639
|
const enh = await ensureLavaSREnhancerGguf({
|
|
@@ -615,6 +653,7 @@ test(
|
|
|
615
653
|
const dir = dl.targetDir || modelsDir
|
|
616
654
|
|
|
617
655
|
const model = new TTSGgml({
|
|
656
|
+
threads: TTS_TEST_THREADS,
|
|
618
657
|
engine: TTSGgml.ENGINE_CHATTERBOX,
|
|
619
658
|
files: {
|
|
620
659
|
modelDir: dir,
|
|
@@ -624,7 +663,7 @@ test(
|
|
|
624
663
|
},
|
|
625
664
|
referenceAudio: resolveRefWavPath({}),
|
|
626
665
|
streamChunkTokens: 25, // native chunk streaming + enhancer (the path)
|
|
627
|
-
config: { language: 'en', useGPU:
|
|
666
|
+
config: { language: 'en', useGPU: GPU_ONLY_USE_GPU },
|
|
628
667
|
opts: { stats: true }
|
|
629
668
|
})
|
|
630
669
|
await model.load()
|
|
@@ -675,6 +714,7 @@ test(
|
|
|
675
714
|
}
|
|
676
715
|
|
|
677
716
|
const model = new TTSGgml({
|
|
717
|
+
threads: TTS_TEST_THREADS,
|
|
678
718
|
engine: TTSGgml.ENGINE_PARLER,
|
|
679
719
|
files: { parlerModel: dl.path, lavasrEnhancer: enh.path },
|
|
680
720
|
voice: 'Laura',
|
|
@@ -718,6 +758,7 @@ test(
|
|
|
718
758
|
}
|
|
719
759
|
|
|
720
760
|
const model = new TTSGgml({
|
|
761
|
+
threads: TTS_TEST_THREADS,
|
|
721
762
|
engine: TTSGgml.ENGINE_PARLER,
|
|
722
763
|
files: { parlerModel: dl.path },
|
|
723
764
|
voice: 'Laura',
|
|
@@ -731,6 +772,8 @@ test(
|
|
|
731
772
|
t.ok(r.samples > 0, 'synthesis produced audio')
|
|
732
773
|
t.ok(r.stats, 'runtimeStats returned (constructed with stats:true)')
|
|
733
774
|
t.is(r.stats.enhancerBackendDevice, -1, 'no enhancer loaded -> enhancerBackendDevice=-1')
|
|
775
|
+
t.is(r.stats.denoiserBackendDevice, -1, 'no denoiser loaded -> denoiserBackendDevice=-1')
|
|
776
|
+
t.is(r.stats.denoiserBackendId, -1, 'no denoiser loaded -> denoiserBackendId=-1')
|
|
734
777
|
} finally {
|
|
735
778
|
try {
|
|
736
779
|
await model.unload()
|
|
@@ -759,6 +802,7 @@ test(
|
|
|
759
802
|
}
|
|
760
803
|
|
|
761
804
|
const model = new TTSGgml({
|
|
805
|
+
threads: TTS_TEST_THREADS,
|
|
762
806
|
engine: TTSGgml.ENGINE_PARLER,
|
|
763
807
|
files: { parlerModel: dl.path, lavasrEnhancer: enh.path },
|
|
764
808
|
voice: 'Laura',
|
|
@@ -815,6 +859,7 @@ test(
|
|
|
815
859
|
}
|
|
816
860
|
|
|
817
861
|
const model = new TTSGgml({
|
|
862
|
+
threads: TTS_TEST_THREADS,
|
|
818
863
|
engine: TTSGgml.ENGINE_PARLER,
|
|
819
864
|
files: { parlerModel: dl.path, lavasrEnhancer: enh.path },
|
|
820
865
|
voice: 'Laura',
|
|
@@ -882,6 +927,7 @@ test(
|
|
|
882
927
|
-1,
|
|
883
928
|
'denoiser alone loads no enhancer (enhancerBackendDevice=-1)'
|
|
884
929
|
)
|
|
930
|
+
assertDenoiserOnCpu(t, denoisedOnly.stats, 'denoiser alone')
|
|
885
931
|
|
|
886
932
|
const enh = await ensureLavaSREnhancerGguf({
|
|
887
933
|
targetDir: path.join(baseDir, 'models', 'lavasr')
|
|
@@ -909,6 +955,7 @@ test(
|
|
|
909
955
|
0,
|
|
910
956
|
'useGPU:false -> enhancer on CPU (enhancerBackendDevice=0)'
|
|
911
957
|
)
|
|
958
|
+
assertDenoiserOnCpu(t, denoisedAndEnhanced.stats, 'denoiser + enhancer')
|
|
912
959
|
}
|
|
913
960
|
)
|
|
914
961
|
|
|
@@ -941,6 +988,7 @@ test(
|
|
|
941
988
|
}
|
|
942
989
|
|
|
943
990
|
const model = new TTSGgml({
|
|
991
|
+
threads: TTS_TEST_THREADS,
|
|
944
992
|
engine: TTSGgml.ENGINE_SUPERTONIC,
|
|
945
993
|
files: { supertonicModel: dl.path, lavasrEnhancer: enh.path },
|
|
946
994
|
voice: 'F1',
|
|
@@ -964,6 +1012,59 @@ test(
|
|
|
964
1012
|
}
|
|
965
1013
|
)
|
|
966
1014
|
|
|
1015
|
+
test(
|
|
1016
|
+
'Supertonic + LavaSR denoiser on GPU (useGPU:true) runs the denoiser on the engine device',
|
|
1017
|
+
{ timeout: 600000, skip: NO_GPU },
|
|
1018
|
+
async (t) => {
|
|
1019
|
+
const baseDir = getBaseDir()
|
|
1020
|
+
const den = await ensureLavaSRDenoiserGguf({
|
|
1021
|
+
targetDir: path.join(baseDir, 'models', 'lavasr')
|
|
1022
|
+
})
|
|
1023
|
+
if (!den.success) {
|
|
1024
|
+
t.comment('LavaSR denoiser GGUF not staged; skipping.')
|
|
1025
|
+
t.pass('skipped — no denoiser GGUF')
|
|
1026
|
+
return
|
|
1027
|
+
}
|
|
1028
|
+
const dl = await ensureSupertonicModel({ targetDir: path.join(baseDir, 'models') })
|
|
1029
|
+
if (!dl.success) {
|
|
1030
|
+
t.fail('Supertonic GGUF not available — registry fetch failed.')
|
|
1031
|
+
return
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
const model = new TTSGgml({
|
|
1035
|
+
threads: TTS_TEST_THREADS,
|
|
1036
|
+
engine: TTSGgml.ENGINE_SUPERTONIC,
|
|
1037
|
+
files: { supertonicModel: dl.path, lavasrDenoiser: den.path },
|
|
1038
|
+
voice: 'F1',
|
|
1039
|
+
config: { language: 'en', useGPU: true },
|
|
1040
|
+
opts: { stats: true }
|
|
1041
|
+
})
|
|
1042
|
+
await model.load()
|
|
1043
|
+
try {
|
|
1044
|
+
const r = await runAndCollect(model, 'The denoiser follows the engine onto the GPU.')
|
|
1045
|
+
t.is(r.sampleRate, 44100, 'denoising preserves the native 44.1 kHz')
|
|
1046
|
+
t.ok(r.samples > 0, 'denoised synthesis produced audio')
|
|
1047
|
+
const { backendDevice, backendId, denoiserBackendDevice, denoiserBackendId } = r.stats
|
|
1048
|
+
console.log(
|
|
1049
|
+
`[denoiser/GPU] backendDevice=${backendDevice} denoiserBackendDevice=` +
|
|
1050
|
+
`${denoiserBackendDevice} denoiserBackendId=${denoiserBackendId} ` +
|
|
1051
|
+
`(${backendIdToName(denoiserBackendId)})`
|
|
1052
|
+
)
|
|
1053
|
+
t.not(denoiserBackendDevice, -1, 'denoiser was loaded (denoiserBackendDevice != -1)')
|
|
1054
|
+
t.is(denoiserBackendDevice, backendDevice, 'denoiser runs on the engine device')
|
|
1055
|
+
if (backendDevice === 1) {
|
|
1056
|
+
t.is(denoiserBackendId, backendId, 'GPU denoiser uses the engine backend')
|
|
1057
|
+
} else {
|
|
1058
|
+
t.is(denoiserBackendId, 0, 'a CPU-fallback engine keeps the denoiser on CPU')
|
|
1059
|
+
}
|
|
1060
|
+
} finally {
|
|
1061
|
+
try {
|
|
1062
|
+
await model.unload()
|
|
1063
|
+
} catch (_e) {}
|
|
1064
|
+
}
|
|
1065
|
+
}
|
|
1066
|
+
)
|
|
1067
|
+
|
|
967
1068
|
test(
|
|
968
1069
|
'Chatterbox + LavaSR enhancer on GPU (useGPU:true, batch) runs the enhancer on the GPU and reports 48 kHz',
|
|
969
1070
|
{ timeout: 900000, skip: NO_GPU },
|
|
@@ -986,6 +1087,7 @@ test(
|
|
|
986
1087
|
const dir = dl.targetDir || modelsDir
|
|
987
1088
|
|
|
988
1089
|
const model = new TTSGgml({
|
|
1090
|
+
threads: TTS_TEST_THREADS,
|
|
989
1091
|
engine: TTSGgml.ENGINE_CHATTERBOX,
|
|
990
1092
|
files: {
|
|
991
1093
|
modelDir: dir,
|
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const fs = require('bare-fs')
|
|
4
|
+
const os = require('bare-os')
|
|
5
|
+
const path = require('bare-path')
|
|
6
|
+
const proc = require('bare-process')
|
|
7
|
+
const test = require('brittle')
|
|
8
|
+
|
|
9
|
+
const TTSGgml = require('@qvac/tts-ggml')
|
|
10
|
+
const { recordTtsStats } = require('../utils/perf-helper')
|
|
11
|
+
const { resolveRefWavPath } = require('../utils/runChatterboxTTS')
|
|
12
|
+
const { readWavAsFloat32, createWav, resampleLinear } = require('../utils/wav-helper')
|
|
13
|
+
const { TTS_TEST_THREADS } = require('../utils/testThreads')
|
|
14
|
+
const { NO_GPU, GPU_ONLY_USE_GPU } = require('../utils/gpuOnly')
|
|
15
|
+
|
|
16
|
+
const MOSS_SAMPLE_RATE = 24000
|
|
17
|
+
const MOSS_STREAM_FRAMES = 10
|
|
18
|
+
const MOSS_SAMPLES_PER_FRAME = 1920
|
|
19
|
+
const TEST_TIMEOUT_MS = 1800000
|
|
20
|
+
const SYNTHESIS_TEXT = 'The MOSS integration test checks this generated voice.'
|
|
21
|
+
const MODEL_DIR_ENV = 'QVAC_TEST_MOSS_MODEL_DIR'
|
|
22
|
+
const ENCODER_RE = /^moss-codec-encoder(-[a-z0-9_]+)?\.gguf$/i
|
|
23
|
+
const DIALOGUE_BACKBONE_RE = /^moss-ttsd(-[a-z0-9_]+)?\.gguf$/i
|
|
24
|
+
const DIALOGUE_TEXT =
|
|
25
|
+
'[S1] And so, my fellow Americans, [S2] ask what you can do for your country. ' +
|
|
26
|
+
'[S1] Did the build finish this morning? [S2] Yes, every single test passed.'
|
|
27
|
+
const SPEAKER_ONE_SECONDS = [0, 3.29]
|
|
28
|
+
const SPEAKER_TWO_SECONDS = [7.6, 10.6]
|
|
29
|
+
const SHORT_DURATION_TOKENS = 25
|
|
30
|
+
const LONG_DURATION_TOKENS = 100
|
|
31
|
+
const INT16_FULL_SCALE = 32767
|
|
32
|
+
const CPU_DEVICE = 0
|
|
33
|
+
const CPU_BACKEND = 0
|
|
34
|
+
const GPU_DEVICE = 1
|
|
35
|
+
|
|
36
|
+
const modelDir = (proc.env && proc.env[MODEL_DIR_ENV]) || ''
|
|
37
|
+
const skipWithoutModels = modelDir === ''
|
|
38
|
+
|
|
39
|
+
function createMossModel(extra = {}, useGPU = false) {
|
|
40
|
+
return new TTSGgml({
|
|
41
|
+
threads: TTS_TEST_THREADS,
|
|
42
|
+
engine: TTSGgml.ENGINE_MOSS,
|
|
43
|
+
files: { modelDir: path.resolve(modelDir) },
|
|
44
|
+
config: { language: 'en', useGPU },
|
|
45
|
+
opts: { stats: true },
|
|
46
|
+
...extra
|
|
47
|
+
})
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function toInt16(samples) {
|
|
51
|
+
return Array.from(samples, (sample) => Math.round(sample * INT16_FULL_SCALE))
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function resampleToMossRate(sourcePath) {
|
|
55
|
+
const source = readWavAsFloat32(sourcePath)
|
|
56
|
+
return resampleLinear(source.samples, source.sampleRate, MOSS_SAMPLE_RATE)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function writeMossWav(samples, tag) {
|
|
60
|
+
const target = path.join(os.tmpdir(), `moss-${tag}-${Date.now()}.wav`)
|
|
61
|
+
createWav(toInt16(samples), MOSS_SAMPLE_RATE, target)
|
|
62
|
+
return target
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function writeReferenceAtMossRate(sourcePath) {
|
|
66
|
+
return writeMossWav(resampleToMossRate(sourcePath), 'reference')
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function sliceSeconds(samples, [from, to]) {
|
|
70
|
+
return samples.slice(Math.round(from * MOSS_SAMPLE_RATE), Math.round(to * MOSS_SAMPLE_RATE))
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function writeSpeakerReferences(sourcePath) {
|
|
74
|
+
const samples = resampleToMossRate(sourcePath)
|
|
75
|
+
return [
|
|
76
|
+
writeMossWav(sliceSeconds(samples, SPEAKER_ONE_SECONDS), 'speaker-1'),
|
|
77
|
+
writeMossWav(sliceSeconds(samples, SPEAKER_TWO_SECONDS), 'speaker-2')
|
|
78
|
+
]
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function removeFiles(paths) {
|
|
82
|
+
for (const file of paths) fs.unlinkSync(file)
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function modelDirHas(pattern) {
|
|
86
|
+
return !skipWithoutModels && fs.readdirSync(modelDir).some((entry) => pattern.test(entry))
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function hasEncoder() {
|
|
90
|
+
return modelDirHas(ENCODER_RE)
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function hasDialogueBackbone() {
|
|
94
|
+
return modelDirHas(DIALOGUE_BACKBONE_RE)
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function collectChunk(result, data) {
|
|
98
|
+
if (!data || !data.outputArray) return
|
|
99
|
+
result.chunks.push(data.outputArray.length)
|
|
100
|
+
result.samples = result.samples.concat(Array.from(data.outputArray))
|
|
101
|
+
if (data.sampleRate) result.sampleRate = data.sampleRate
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async function synthesize(model, text) {
|
|
105
|
+
const response = await model.run({ input: text, type: 'text' })
|
|
106
|
+
const result = { samples: [], chunks: [], sampleRate: null }
|
|
107
|
+
await response.onUpdate((data) => collectChunk(result, data)).await()
|
|
108
|
+
result.stats = response.stats
|
|
109
|
+
return result
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
function assertAudio(t, label, result) {
|
|
113
|
+
t.ok(result.samples.length > 0, `${label} produced audio`)
|
|
114
|
+
t.is(result.sampleRate, MOSS_SAMPLE_RATE, `${label} reports 24 kHz`)
|
|
115
|
+
t.ok(result.stats, `${label} returns runtime stats`)
|
|
116
|
+
t.is(result.stats.totalSamples, result.samples.length, `${label} reports the emitted samples`)
|
|
117
|
+
t.ok(result.stats.generatedFrames > 0, `${label} reports generated codec frames`)
|
|
118
|
+
t.ok(result.stats.tokensPerSecond > 0, `${label} reports codec frames per second`)
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function nonEmptyChunks(result) {
|
|
122
|
+
return result.chunks.filter((length) => length > 0)
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function recordMoss(t, label, result, wallMs) {
|
|
126
|
+
t.comment(
|
|
127
|
+
recordTtsStats(label, result.stats, {
|
|
128
|
+
wallMs,
|
|
129
|
+
sampleCount: result.samples.length,
|
|
130
|
+
model: 'moss',
|
|
131
|
+
output: label
|
|
132
|
+
})
|
|
133
|
+
)
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function assertCpuBackend(t, label, stats) {
|
|
137
|
+
t.is(stats.backendDevice, CPU_DEVICE, `${label} reports a CPU device`)
|
|
138
|
+
t.is(stats.backendId, CPU_BACKEND, `${label} reports the CPU backend`)
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
async function withLoadedModel(extra, body, useGPU = false) {
|
|
142
|
+
const model = createMossModel(extra, useGPU)
|
|
143
|
+
await model.load()
|
|
144
|
+
try {
|
|
145
|
+
await body(model)
|
|
146
|
+
} finally {
|
|
147
|
+
await model.unload()
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
test(
|
|
152
|
+
'MOSS TTS: batch synthesis through the public JS API',
|
|
153
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels },
|
|
154
|
+
async (t) => {
|
|
155
|
+
await withLoadedModel({}, async (model) => {
|
|
156
|
+
const started = Date.now()
|
|
157
|
+
const result = await synthesize(model, SYNTHESIS_TEXT)
|
|
158
|
+
assertAudio(t, 'MOSS batch', result)
|
|
159
|
+
assertCpuBackend(t, 'MOSS batch', result.stats)
|
|
160
|
+
t.is(nonEmptyChunks(result).length, 1, 'batch synthesis emits one buffer')
|
|
161
|
+
recordMoss(t, 'moss batch', result, Date.now() - started)
|
|
162
|
+
})
|
|
163
|
+
}
|
|
164
|
+
)
|
|
165
|
+
|
|
166
|
+
test(
|
|
167
|
+
'MOSS TTS: native chunk streaming emits frame-sized chunks',
|
|
168
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels },
|
|
169
|
+
async (t) => {
|
|
170
|
+
await withLoadedModel({ streamChunkTokens: MOSS_STREAM_FRAMES }, async (model) => {
|
|
171
|
+
const started = Date.now()
|
|
172
|
+
const result = await synthesize(model, SYNTHESIS_TEXT)
|
|
173
|
+
assertAudio(t, 'MOSS stream', result)
|
|
174
|
+
const chunks = nonEmptyChunks(result)
|
|
175
|
+
t.ok(chunks.length > 1, `streaming emits several chunks (got ${chunks.length})`)
|
|
176
|
+
t.ok(
|
|
177
|
+
chunks.every((length) => length <= MOSS_STREAM_FRAMES * MOSS_SAMPLES_PER_FRAME),
|
|
178
|
+
'no chunk exceeds the configured frame count'
|
|
179
|
+
)
|
|
180
|
+
recordMoss(t, 'moss stream', result, Date.now() - started)
|
|
181
|
+
})
|
|
182
|
+
}
|
|
183
|
+
)
|
|
184
|
+
|
|
185
|
+
test(
|
|
186
|
+
'MOSS TTS: useGPU=true synthesizes on a GPU backend',
|
|
187
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels || NO_GPU },
|
|
188
|
+
async (t) => {
|
|
189
|
+
await withLoadedModel(
|
|
190
|
+
{},
|
|
191
|
+
async (model) => {
|
|
192
|
+
const started = Date.now()
|
|
193
|
+
const result = await synthesize(model, SYNTHESIS_TEXT)
|
|
194
|
+
assertAudio(t, 'MOSS GPU', result)
|
|
195
|
+
t.is(result.stats.backendDevice, GPU_DEVICE, 'MOSS GPU reports a GPU device')
|
|
196
|
+
t.not(result.stats.backendId, CPU_BACKEND, 'MOSS GPU does not report the CPU backend')
|
|
197
|
+
t.is(result.stats.gpuUnsupported, 0, 'MOSS GPU does not report a GPU fallback')
|
|
198
|
+
recordMoss(t, 'moss gpu', result, Date.now() - started)
|
|
199
|
+
},
|
|
200
|
+
true
|
|
201
|
+
)
|
|
202
|
+
}
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
test(
|
|
206
|
+
'MOSS TTS: voice cloning synthesizes from a reference recording',
|
|
207
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels },
|
|
208
|
+
async (t) => {
|
|
209
|
+
const sourceAudio = resolveRefWavPath({})
|
|
210
|
+
if (!hasEncoder()) {
|
|
211
|
+
t.comment(`skipping: no moss-codec-encoder GGUF in ${modelDir}`)
|
|
212
|
+
return
|
|
213
|
+
}
|
|
214
|
+
if (!fs.existsSync(sourceAudio)) {
|
|
215
|
+
t.fail(`MOSS reference audio is missing: ${sourceAudio}`)
|
|
216
|
+
return
|
|
217
|
+
}
|
|
218
|
+
const referenceAudio = writeReferenceAtMossRate(sourceAudio)
|
|
219
|
+
try {
|
|
220
|
+
await withLoadedModel({ referenceAudio, seed: 7 }, async (model) => {
|
|
221
|
+
const started = Date.now()
|
|
222
|
+
const result = await synthesize(model, SYNTHESIS_TEXT)
|
|
223
|
+
assertAudio(t, 'MOSS clone', result)
|
|
224
|
+
recordMoss(t, 'moss clone', result, Date.now() - started)
|
|
225
|
+
})
|
|
226
|
+
} finally {
|
|
227
|
+
fs.unlinkSync(referenceAudio)
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
)
|
|
231
|
+
|
|
232
|
+
test(
|
|
233
|
+
'MOSS TTS: durationTokens steers the length of the synthesized speech',
|
|
234
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels || NO_GPU },
|
|
235
|
+
async (t) => {
|
|
236
|
+
await withLoadedModel(
|
|
237
|
+
{ durationTokens: SHORT_DURATION_TOKENS, seed: 7 },
|
|
238
|
+
async (model) => {
|
|
239
|
+
const short = await synthesize(model, SYNTHESIS_TEXT)
|
|
240
|
+
assertAudio(t, 'MOSS short target', short)
|
|
241
|
+
await model.reload({ durationTokens: LONG_DURATION_TOKENS })
|
|
242
|
+
const long = await synthesize(model, SYNTHESIS_TEXT)
|
|
243
|
+
assertAudio(t, 'MOSS long target', long)
|
|
244
|
+
t.ok(
|
|
245
|
+
long.samples.length > short.samples.length,
|
|
246
|
+
`a longer target yields longer speech (${short.samples.length} -> ${long.samples.length} samples)`
|
|
247
|
+
)
|
|
248
|
+
},
|
|
249
|
+
GPU_ONLY_USE_GPU
|
|
250
|
+
)
|
|
251
|
+
}
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
test(
|
|
255
|
+
'MOSS TTS: dialogue synthesis clones one reference per speaker',
|
|
256
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels || NO_GPU },
|
|
257
|
+
async (t) => {
|
|
258
|
+
if (!hasDialogueBackbone() || !hasEncoder()) {
|
|
259
|
+
t.comment(`skipping: the TTSD backbone or the codec encoder is not in ${modelDir}`)
|
|
260
|
+
return
|
|
261
|
+
}
|
|
262
|
+
const sourceAudio = resolveRefWavPath({})
|
|
263
|
+
if (!fs.existsSync(sourceAudio)) {
|
|
264
|
+
t.fail(`MOSS reference audio is missing: ${sourceAudio}`)
|
|
265
|
+
return
|
|
266
|
+
}
|
|
267
|
+
const references = writeSpeakerReferences(sourceAudio)
|
|
268
|
+
try {
|
|
269
|
+
await withLoadedModel(
|
|
270
|
+
{ dialogueReferences: references, seed: 7, files: { modelDir: path.resolve(modelDir) } },
|
|
271
|
+
async (model) => {
|
|
272
|
+
t.ok(
|
|
273
|
+
DIALOGUE_BACKBONE_RE.test(path.basename(model._mossBackbonePath)),
|
|
274
|
+
'dialogue picks the TTSD backbone'
|
|
275
|
+
)
|
|
276
|
+
const started = Date.now()
|
|
277
|
+
const result = await synthesize(model, DIALOGUE_TEXT)
|
|
278
|
+
assertAudio(t, 'MOSS dialogue', result)
|
|
279
|
+
recordMoss(t, 'moss dialogue', result, Date.now() - started)
|
|
280
|
+
},
|
|
281
|
+
GPU_ONLY_USE_GPU
|
|
282
|
+
)
|
|
283
|
+
} finally {
|
|
284
|
+
removeFiles(references)
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
)
|