@qvac/tts-ggml 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +86 -6
- package/examples/moss-sfx-fit.js +14 -7
- package/lib/fit.d.ts +3 -3
- package/lib/fit.js +4 -4
- package/package.json +14 -9
- package/test/integration/fit.test.js +115 -18
- package/test/integration/moss-fit.test.js +13 -13
- package/test/integration/moss-sfx-fit.test.js +14 -14
- package/test/integration/moss-speech.test.js +19 -5
- package/test/integration/moss.test.js +92 -55
- package/test/mobile/integration.auto.cjs +10 -0
- package/test/mobile/test-groups.json +6 -2
- package/test/utils/gpuOnly.js +15 -0
- package/test/utils/mossBackend.js +34 -0
- package/test/utils/mossGpuValidation.js +70 -0
- package/test/utils/perf-helper.js +275 -0
- package/test/utils/testThreads.js +5 -0
|
@@ -12,6 +12,12 @@ const { resolveRefWavPath } = require('../utils/runChatterboxTTS')
|
|
|
12
12
|
const { readWavAsFloat32, createWav, resampleLinear } = require('../utils/wav-helper')
|
|
13
13
|
const { TTS_TEST_THREADS } = require('../utils/testThreads')
|
|
14
14
|
const { NO_GPU, GPU_ONLY_USE_GPU } = require('../utils/gpuOnly')
|
|
15
|
+
const { assertMossBackend } = require('../utils/mossBackend')
|
|
16
|
+
const {
|
|
17
|
+
createMossGpuValidation,
|
|
18
|
+
ENCODER_RE,
|
|
19
|
+
DIALOGUE_BACKBONE_RE
|
|
20
|
+
} = require('../utils/mossGpuValidation')
|
|
15
21
|
|
|
16
22
|
const MOSS_SAMPLE_RATE = 24000
|
|
17
23
|
const MOSS_STREAM_FRAMES = 10
|
|
@@ -19,8 +25,6 @@ const MOSS_SAMPLES_PER_FRAME = 1920
|
|
|
19
25
|
const TEST_TIMEOUT_MS = 1800000
|
|
20
26
|
const SYNTHESIS_TEXT = 'The MOSS integration test checks this generated voice.'
|
|
21
27
|
const MODEL_DIR_ENV = 'QVAC_TEST_MOSS_MODEL_DIR'
|
|
22
|
-
const ENCODER_RE = /^moss-codec-encoder(-[a-z0-9_]+)?\.gguf$/i
|
|
23
|
-
const DIALOGUE_BACKBONE_RE = /^moss-ttsd(-[a-z0-9_]+)?\.gguf$/i
|
|
24
28
|
const DIALOGUE_TEXT =
|
|
25
29
|
'[S1] And so, my fellow Americans, [S2] ask what you can do for your country. ' +
|
|
26
30
|
'[S1] Did the build finish this morning? [S2] Yes, every single test passed.'
|
|
@@ -29,12 +33,12 @@ const SPEAKER_TWO_SECONDS = [7.6, 10.6]
|
|
|
29
33
|
const SHORT_DURATION_TOKENS = 25
|
|
30
34
|
const LONG_DURATION_TOKENS = 100
|
|
31
35
|
const INT16_FULL_SCALE = 32767
|
|
32
|
-
const
|
|
33
|
-
const CPU_BACKEND = 0
|
|
34
|
-
const GPU_DEVICE = 1
|
|
36
|
+
const PINNED_GPU_BACKEND = (proc.env && proc.env.TTS_CPP_GPU_BACKEND) || ''
|
|
35
37
|
|
|
36
38
|
const modelDir = (proc.env && proc.env[MODEL_DIR_ENV]) || ''
|
|
37
39
|
const skipWithoutModels = modelDir === ''
|
|
40
|
+
const sourceAudio = resolveRefWavPath({})
|
|
41
|
+
const validation = createMossGpuValidation(proc.env || {}, sourceAudio, GPU_ONLY_USE_GPU)
|
|
38
42
|
|
|
39
43
|
function createMossModel(extra = {}, useGPU = false) {
|
|
40
44
|
return new TTSGgml({
|
|
@@ -133,9 +137,8 @@ function recordMoss(t, label, result, wallMs) {
|
|
|
133
137
|
)
|
|
134
138
|
}
|
|
135
139
|
|
|
136
|
-
function
|
|
137
|
-
t
|
|
138
|
-
t.is(stats.backendId, CPU_BACKEND, `${label} reports the CPU backend`)
|
|
140
|
+
function assertBackend(t, label, stats, useGPU = false) {
|
|
141
|
+
assertMossBackend(t, label, stats, useGPU, PINNED_GPU_BACKEND)
|
|
139
142
|
}
|
|
140
143
|
|
|
141
144
|
async function withLoadedModel(extra, body, useGPU = false) {
|
|
@@ -156,31 +159,40 @@ test(
|
|
|
156
159
|
const started = Date.now()
|
|
157
160
|
const result = await synthesize(model, SYNTHESIS_TEXT)
|
|
158
161
|
assertAudio(t, 'MOSS batch', result)
|
|
159
|
-
|
|
162
|
+
assertBackend(t, 'MOSS batch', result.stats)
|
|
160
163
|
t.is(nonEmptyChunks(result).length, 1, 'batch synthesis emits one buffer')
|
|
161
164
|
recordMoss(t, 'moss batch', result, Date.now() - started)
|
|
162
165
|
})
|
|
166
|
+
validation.completed('CPU batch')
|
|
163
167
|
}
|
|
164
168
|
)
|
|
165
169
|
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
170
|
+
for (const useGPU of [false, true]) {
|
|
171
|
+
test(
|
|
172
|
+
`MOSS TTS: native chunk streaming emits frame-sized chunks (${useGPU ? 'GPU' : 'CPU'})`,
|
|
173
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels || (useGPU && NO_GPU) },
|
|
174
|
+
async (t) => {
|
|
175
|
+
await withLoadedModel(
|
|
176
|
+
{ streamChunkTokens: MOSS_STREAM_FRAMES },
|
|
177
|
+
async (model) => {
|
|
178
|
+
const started = Date.now()
|
|
179
|
+
const result = await synthesize(model, SYNTHESIS_TEXT)
|
|
180
|
+
assertAudio(t, 'MOSS stream', result)
|
|
181
|
+
assertBackend(t, 'MOSS stream', result.stats, useGPU)
|
|
182
|
+
const chunks = nonEmptyChunks(result)
|
|
183
|
+
t.ok(chunks.length > 1, `streaming emits several chunks (got ${chunks.length})`)
|
|
184
|
+
t.ok(
|
|
185
|
+
chunks.every((length) => length <= MOSS_STREAM_FRAMES * MOSS_SAMPLES_PER_FRAME),
|
|
186
|
+
'no chunk exceeds the configured frame count'
|
|
187
|
+
)
|
|
188
|
+
recordMoss(t, 'moss stream', result, Date.now() - started)
|
|
189
|
+
},
|
|
190
|
+
useGPU
|
|
179
191
|
)
|
|
180
|
-
|
|
181
|
-
}
|
|
182
|
-
|
|
183
|
-
|
|
192
|
+
validation.completed(`${useGPU ? 'GPU' : 'CPU'} streaming`)
|
|
193
|
+
}
|
|
194
|
+
)
|
|
195
|
+
}
|
|
184
196
|
|
|
185
197
|
test(
|
|
186
198
|
'MOSS TTS: useGPU=true synthesizes on a GPU backend',
|
|
@@ -192,42 +204,48 @@ test(
|
|
|
192
204
|
const started = Date.now()
|
|
193
205
|
const result = await synthesize(model, SYNTHESIS_TEXT)
|
|
194
206
|
assertAudio(t, 'MOSS GPU', result)
|
|
195
|
-
t
|
|
196
|
-
t.not(result.stats.backendId, CPU_BACKEND, 'MOSS GPU does not report the CPU backend')
|
|
197
|
-
t.is(result.stats.gpuUnsupported, 0, 'MOSS GPU does not report a GPU fallback')
|
|
207
|
+
assertBackend(t, 'MOSS GPU', result.stats, true)
|
|
198
208
|
recordMoss(t, 'moss gpu', result, Date.now() - started)
|
|
199
209
|
},
|
|
200
210
|
true
|
|
201
211
|
)
|
|
212
|
+
validation.completed('GPU batch')
|
|
202
213
|
}
|
|
203
214
|
)
|
|
204
215
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
216
|
+
for (const useGPU of [false, true]) {
|
|
217
|
+
test(
|
|
218
|
+
`MOSS TTS: voice cloning synthesizes from a reference recording (${useGPU ? 'GPU' : 'CPU'})`,
|
|
219
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels || (useGPU && NO_GPU) },
|
|
220
|
+
async (t) => {
|
|
221
|
+
if (!hasEncoder()) {
|
|
222
|
+
t.comment(`skipping: no moss-codec-encoder GGUF in ${modelDir}`)
|
|
223
|
+
return
|
|
224
|
+
}
|
|
225
|
+
if (!fs.existsSync(sourceAudio)) {
|
|
226
|
+
t.fail(`MOSS reference audio is missing: ${sourceAudio}`)
|
|
227
|
+
return
|
|
228
|
+
}
|
|
229
|
+
const referenceAudio = writeReferenceAtMossRate(sourceAudio)
|
|
230
|
+
try {
|
|
231
|
+
await withLoadedModel(
|
|
232
|
+
{ referenceAudio, seed: 7 },
|
|
233
|
+
async (model) => {
|
|
234
|
+
const started = Date.now()
|
|
235
|
+
const result = await synthesize(model, SYNTHESIS_TEXT)
|
|
236
|
+
assertAudio(t, 'MOSS clone', result)
|
|
237
|
+
assertBackend(t, 'MOSS clone', result.stats, useGPU)
|
|
238
|
+
recordMoss(t, 'moss clone', result, Date.now() - started)
|
|
239
|
+
},
|
|
240
|
+
useGPU
|
|
241
|
+
)
|
|
242
|
+
} finally {
|
|
243
|
+
fs.unlinkSync(referenceAudio)
|
|
244
|
+
}
|
|
245
|
+
validation.completed(`${useGPU ? 'GPU' : 'CPU'} cloning`)
|
|
228
246
|
}
|
|
229
|
-
|
|
230
|
-
|
|
247
|
+
)
|
|
248
|
+
}
|
|
231
249
|
|
|
232
250
|
test(
|
|
233
251
|
'MOSS TTS: durationTokens steers the length of the synthesized speech',
|
|
@@ -238,9 +256,11 @@ test(
|
|
|
238
256
|
async (model) => {
|
|
239
257
|
const short = await synthesize(model, SYNTHESIS_TEXT)
|
|
240
258
|
assertAudio(t, 'MOSS short target', short)
|
|
259
|
+
assertBackend(t, 'MOSS short target', short.stats, GPU_ONLY_USE_GPU)
|
|
241
260
|
await model.reload({ durationTokens: LONG_DURATION_TOKENS })
|
|
242
261
|
const long = await synthesize(model, SYNTHESIS_TEXT)
|
|
243
262
|
assertAudio(t, 'MOSS long target', long)
|
|
263
|
+
assertBackend(t, 'MOSS long target', long.stats, GPU_ONLY_USE_GPU)
|
|
244
264
|
t.ok(
|
|
245
265
|
long.samples.length > short.samples.length,
|
|
246
266
|
`a longer target yields longer speech (${short.samples.length} -> ${long.samples.length} samples)`
|
|
@@ -248,6 +268,7 @@ test(
|
|
|
248
268
|
},
|
|
249
269
|
GPU_ONLY_USE_GPU
|
|
250
270
|
)
|
|
271
|
+
validation.completed('GPU duration/reload')
|
|
251
272
|
}
|
|
252
273
|
)
|
|
253
274
|
|
|
@@ -259,7 +280,6 @@ test(
|
|
|
259
280
|
t.comment(`skipping: the TTSD backbone or the codec encoder is not in ${modelDir}`)
|
|
260
281
|
return
|
|
261
282
|
}
|
|
262
|
-
const sourceAudio = resolveRefWavPath({})
|
|
263
283
|
if (!fs.existsSync(sourceAudio)) {
|
|
264
284
|
t.fail(`MOSS reference audio is missing: ${sourceAudio}`)
|
|
265
285
|
return
|
|
@@ -276,6 +296,7 @@ test(
|
|
|
276
296
|
const started = Date.now()
|
|
277
297
|
const result = await synthesize(model, DIALOGUE_TEXT)
|
|
278
298
|
assertAudio(t, 'MOSS dialogue', result)
|
|
299
|
+
assertBackend(t, 'MOSS dialogue', result.stats, GPU_ONLY_USE_GPU)
|
|
279
300
|
recordMoss(t, 'moss dialogue', result, Date.now() - started)
|
|
280
301
|
},
|
|
281
302
|
GPU_ONLY_USE_GPU
|
|
@@ -283,5 +304,21 @@ test(
|
|
|
283
304
|
} finally {
|
|
284
305
|
removeFiles(references)
|
|
285
306
|
}
|
|
307
|
+
validation.completed('GPU TTSD dialogue')
|
|
286
308
|
}
|
|
287
309
|
)
|
|
310
|
+
|
|
311
|
+
if (validation.enabled) {
|
|
312
|
+
test('MOSS TTS: full GPU validation completed every required case', (t) => {
|
|
313
|
+
validation.assertComplete(t, [
|
|
314
|
+
'CPU batch',
|
|
315
|
+
'CPU streaming',
|
|
316
|
+
'GPU streaming',
|
|
317
|
+
'GPU batch',
|
|
318
|
+
'CPU cloning',
|
|
319
|
+
'GPU cloning',
|
|
320
|
+
'GPU duration/reload',
|
|
321
|
+
'GPU TTSD dialogue'
|
|
322
|
+
])
|
|
323
|
+
})
|
|
324
|
+
}
|
|
@@ -50,6 +50,14 @@ async function runLavasrEnhancerTest (options = {}) { // eslint-disable-line no-
|
|
|
50
50
|
return runIntegrationModule('../integration/lavasr-enhancer.test.js', options)
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
+
async function runMossFitTest (options = {}) { // eslint-disable-line no-unused-vars
|
|
54
|
+
return runIntegrationModule('../integration/moss-fit.test.js', options)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function runMossSfxFitTest (options = {}) { // eslint-disable-line no-unused-vars
|
|
58
|
+
return runIntegrationModule('../integration/moss-sfx-fit.test.js', options)
|
|
59
|
+
}
|
|
60
|
+
|
|
53
61
|
async function runMossSfxTest (options = {}) { // eslint-disable-line no-unused-vars
|
|
54
62
|
return runIntegrationModule('../integration/moss-sfx.test.js', options)
|
|
55
63
|
}
|
|
@@ -118,6 +126,8 @@ module.exports = {
|
|
|
118
126
|
runFitTest,
|
|
119
127
|
runGpuSmokeTest,
|
|
120
128
|
runLavasrEnhancerTest,
|
|
129
|
+
runMossFitTest,
|
|
130
|
+
runMossSfxFitTest,
|
|
121
131
|
runMossSfxTest,
|
|
122
132
|
runMossSpeechTest,
|
|
123
133
|
runMossTest,
|
|
@@ -33,7 +33,9 @@
|
|
|
33
33
|
"runMultipleRunsTest",
|
|
34
34
|
"runGpuSmokeTest",
|
|
35
35
|
"runOutputSampleRateTest",
|
|
36
|
-
"runVoiceControlsTest"
|
|
36
|
+
"runVoiceControlsTest",
|
|
37
|
+
"runMossFitTest",
|
|
38
|
+
"runMossSfxFitTest"
|
|
37
39
|
]
|
|
38
40
|
},
|
|
39
41
|
"android": {
|
|
@@ -69,7 +71,9 @@
|
|
|
69
71
|
"runMultipleRunsTest",
|
|
70
72
|
"runGpuSmokeTest",
|
|
71
73
|
"runOutputSampleRateTest",
|
|
72
|
-
"runVoiceControlsTest"
|
|
74
|
+
"runVoiceControlsTest",
|
|
75
|
+
"runMossFitTest",
|
|
76
|
+
"runMossSfxFitTest"
|
|
73
77
|
]
|
|
74
78
|
}
|
|
75
79
|
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const os = require('bare-os')
|
|
4
|
+
const proc = require('bare-process')
|
|
5
|
+
|
|
6
|
+
const platform = os.platform()
|
|
7
|
+
const isMobile = platform === 'ios' || platform === 'android'
|
|
8
|
+
const isApple = platform === 'darwin' || platform === 'ios'
|
|
9
|
+
|
|
10
|
+
const NO_GPU = !!proc.env && proc.env.NO_GPU === 'true'
|
|
11
|
+
const GPU_ONLY_USE_GPU = !isMobile
|
|
12
|
+
const SKIP_PARLER_GPU_ONLY = NO_GPU || (!isMobile && !isApple)
|
|
13
|
+
const PARLER_GPU_ONLY_USE_GPU = isApple && !isMobile
|
|
14
|
+
|
|
15
|
+
module.exports = { NO_GPU, GPU_ONLY_USE_GPU, SKIP_PARLER_GPU_ONLY, PARLER_GPU_ONLY_USE_GPU }
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const GPU_BACKEND_IDS = { metal: 1, cuda: 2, vulkan: 3, opencl: 4 }
|
|
4
|
+
|
|
5
|
+
// The native engine reads TTS_CPP_GPU_BACKEND. Check its reported backend as
|
|
6
|
+
// well: a different GPU must not make a CUDA/Vulkan validation lane pass.
|
|
7
|
+
function assertMossBackend(t, label, stats, useGPU, pinnedBackend = '') {
|
|
8
|
+
if (!stats) {
|
|
9
|
+
t.fail(`${label}: missing runtime stats; cannot verify the backend`)
|
|
10
|
+
return
|
|
11
|
+
}
|
|
12
|
+
t.is(stats.backendDevice, useGPU ? 1 : 0, `${label}: requested compute device`)
|
|
13
|
+
if (!useGPU) {
|
|
14
|
+
t.is(stats.backendId, 0, `${label}: CPU requested explicitly`)
|
|
15
|
+
return
|
|
16
|
+
}
|
|
17
|
+
t.is(stats.gpuUnsupported, 0, `${label}: no GPU fallback`)
|
|
18
|
+
pinnedBackend = pinnedBackend.toLowerCase()
|
|
19
|
+
if (pinnedBackend) {
|
|
20
|
+
if (!Object.prototype.hasOwnProperty.call(GPU_BACKEND_IDS, pinnedBackend)) {
|
|
21
|
+
t.fail(`${label}: unknown TTS_CPP_GPU_BACKEND=${pinnedBackend}`)
|
|
22
|
+
return
|
|
23
|
+
}
|
|
24
|
+
t.is(
|
|
25
|
+
stats.backendId,
|
|
26
|
+
GPU_BACKEND_IDS[pinnedBackend],
|
|
27
|
+
`${label}: TTS_CPP_GPU_BACKEND=${pinnedBackend}`
|
|
28
|
+
)
|
|
29
|
+
} else {
|
|
30
|
+
t.ok(Object.values(GPU_BACKEND_IDS).includes(stats.backendId), `${label}: known GPU backend`)
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
module.exports = { assertMossBackend }
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const fs = require('bare-fs')
|
|
4
|
+
const path = require('bare-path')
|
|
5
|
+
|
|
6
|
+
const ENCODER_RE = /^moss-codec-encoder(-[a-z0-9_]+)?\.gguf$/i
|
|
7
|
+
const DIALOGUE_BACKBONE_RE = /^moss-ttsd(-[a-z0-9_]+)?\.gguf$/i
|
|
8
|
+
const REQUIRED_MODELS = {
|
|
9
|
+
QVAC_TEST_MOSS_MODEL_DIR: [
|
|
10
|
+
['TTS backbone', /^moss-tts-delay(-[a-z0-9_]+)?\.gguf$/i],
|
|
11
|
+
['TTSD backbone', DIALOGUE_BACKBONE_RE],
|
|
12
|
+
['codec decoder', /^moss-codec-decoder(-[a-z0-9_]+)?\.gguf$/i],
|
|
13
|
+
['codec encoder', ENCODER_RE]
|
|
14
|
+
],
|
|
15
|
+
QVAC_TEST_MOSS_SPEECH_MODEL_DIR: [
|
|
16
|
+
['Speech backbone', /^moss-speech(-(?!codec)[a-z0-9_]+)?\.gguf$/i],
|
|
17
|
+
['Speech codec', /^moss-speech-codec(-[a-z0-9_]+)?\.gguf$/i]
|
|
18
|
+
]
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function requireValidation(condition, message) {
|
|
22
|
+
if (!condition) throw new Error(`MOSS full GPU validation: ${message}`)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function isFile(file) {
|
|
26
|
+
return fs.existsSync(file) && fs.statSync(file).isFile()
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
// Called before either suite registers tests. Reuse each suite's reference WAV
|
|
30
|
+
// resolver so preflight and inference cannot silently select different audio.
|
|
31
|
+
function createMossGpuValidation(env, referenceAudio, gpuOnlyUseGPU = true) {
|
|
32
|
+
const enabled = env.QVAC_TEST_MOSS_FULL_GPU === 'true'
|
|
33
|
+
const completed = new Set()
|
|
34
|
+
if (enabled) {
|
|
35
|
+
requireValidation(env.NO_GPU !== 'true', 'NO_GPU=true conflicts with full GPU validation')
|
|
36
|
+
requireValidation(gpuOnlyUseGPU, 'full GPU validation requires the desktop GPU cases')
|
|
37
|
+
requireValidation(
|
|
38
|
+
['cuda', 'vulkan', 'metal', 'opencl'].includes((env.TTS_CPP_GPU_BACKEND || '').toLowerCase()),
|
|
39
|
+
'set TTS_CPP_GPU_BACKEND to cuda, vulkan, metal or opencl'
|
|
40
|
+
)
|
|
41
|
+
for (const [variable, requirements] of Object.entries(REQUIRED_MODELS)) {
|
|
42
|
+
const directory = env[variable]
|
|
43
|
+
requireValidation(
|
|
44
|
+
directory && fs.existsSync(directory) && fs.statSync(directory).isDirectory(),
|
|
45
|
+
`${variable} must name an existing model directory`
|
|
46
|
+
)
|
|
47
|
+
const files = fs.readdirSync(directory).filter((entry) => isFile(path.join(directory, entry)))
|
|
48
|
+
for (const [label, pattern] of requirements) {
|
|
49
|
+
requireValidation(
|
|
50
|
+
files.some((file) => pattern.test(file)),
|
|
51
|
+
`${variable}: missing ${label}`
|
|
52
|
+
)
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
requireValidation(isFile(referenceAudio), `reference WAV is missing: ${referenceAudio}`)
|
|
56
|
+
}
|
|
57
|
+
return {
|
|
58
|
+
enabled,
|
|
59
|
+
completed(caseName) {
|
|
60
|
+
completed.add(caseName)
|
|
61
|
+
},
|
|
62
|
+
assertComplete(t, expected) {
|
|
63
|
+
for (const caseName of expected) {
|
|
64
|
+
t.ok(completed.has(caseName), `full GPU validation executed ${caseName}`)
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
module.exports = { createMossGpuValidation, ENCODER_RE, DIALOGUE_BACKBONE_RE }
|
|
@@ -0,0 +1,275 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
// Performance-recording helper shared by the tts-ggml integration tests
|
|
3
|
+
// (Chatterbox English/MTL, Supertonic, Supertonic MTL, GPU smoke). Holds the
|
|
4
|
+
// singleton perf-reporter, the mobile-safe inline fallback, and
|
|
5
|
+
// `recordTtsStats()` which turns an addon `response.stats` payload into a perf
|
|
6
|
+
// row.
|
|
7
|
+
//
|
|
8
|
+
// Lives in test/utils/ (not test/integration/) so it sits next to the other
|
|
9
|
+
// shared runners (runChatterboxTTS.js, runSupertonicTTS.js, ...) and is never
|
|
10
|
+
// picked up by the mobile test generator or the brittle test runner, which
|
|
11
|
+
// only discover test/integration/*.test.js.
|
|
12
|
+
//
|
|
13
|
+
// Mirrors the structure of the Parakeet/LLM helpers: on desktop we require the
|
|
14
|
+
// shared scripts/test-utils/performance-reporter directly; on mobile that path
|
|
15
|
+
// lives outside the addon package and bare-pack can't bundle it, so we fall
|
|
16
|
+
// back to an inline lightweight reporter that emits [PERF_REPORT_START]/
|
|
17
|
+
// [PERF_CHUNK] markers extract-from-log.js already understands.
|
|
18
|
+
|
|
19
|
+
const fs = require('bare-fs')
|
|
20
|
+
const path = require('bare-path')
|
|
21
|
+
const os = require('bare-os')
|
|
22
|
+
const process = require('bare-process')
|
|
23
|
+
|
|
24
|
+
const platform = os.platform()
|
|
25
|
+
const arch = os.arch()
|
|
26
|
+
const platformLabel = `${platform}-${arch}`
|
|
27
|
+
const isMobile = platform === 'ios' || platform === 'android'
|
|
28
|
+
|
|
29
|
+
// Inject bare-subprocess so performance-reporter.js's _detectGpu() can shell
|
|
30
|
+
// out to nvidia-smi / vulkaninfo / system_profiler on desktop runners.
|
|
31
|
+
// Resolving from this caller file works (it lives next to the addon's
|
|
32
|
+
// node_modules); resolving from inside scripts/test-utils/ does not because
|
|
33
|
+
// require('child_process') throws under Bare. Without this, device.gpu stays
|
|
34
|
+
// null and the perf report shows "N/A" for the GPU. Mirrors the LLM addon's
|
|
35
|
+
// _perf-helper.js. Mobile doesn't need it — the inline fallback below leaves
|
|
36
|
+
// gpu null on Device Farm where the probes wouldn't work anyway.
|
|
37
|
+
let _subprocess = null
|
|
38
|
+
try {
|
|
39
|
+
_subprocess = require('bare-subprocess')
|
|
40
|
+
} catch (_) {}
|
|
41
|
+
|
|
42
|
+
let createPerformanceReporter
|
|
43
|
+
const _scriptBase = path.join('..', '..', '..', '..', 'scripts', 'test-utils')
|
|
44
|
+
try {
|
|
45
|
+
const perfReporterMod = require(path.join(_scriptBase, 'performance-reporter'))
|
|
46
|
+
perfReporterMod.configure({ fs, path, process, os, subprocess: _subprocess })
|
|
47
|
+
createPerformanceReporter = perfReporterMod.createPerformanceReporter
|
|
48
|
+
} catch (_) {
|
|
49
|
+
createPerformanceReporter = function (opts) {
|
|
50
|
+
const _results = []
|
|
51
|
+
const _startedAt = new Date().toISOString()
|
|
52
|
+
const _addon = (opts && opts.addon) || 'tts-ggml'
|
|
53
|
+
const _addonType = (opts && opts.addonType) || 'tts'
|
|
54
|
+
const _device = {
|
|
55
|
+
name: platform,
|
|
56
|
+
platform,
|
|
57
|
+
os_version: '',
|
|
58
|
+
arch: os.arch ? os.arch() : '',
|
|
59
|
+
// GPU label only filled in for desktop runners by
|
|
60
|
+
// performance-reporter.js's _detectGpu(). On Device Farm we leave it
|
|
61
|
+
// null and let the aggregator fall back to the device name.
|
|
62
|
+
gpu: null,
|
|
63
|
+
runner: 'device-farm'
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return {
|
|
67
|
+
record(testName, metrics, extra) {
|
|
68
|
+
const entry = {
|
|
69
|
+
test: testName,
|
|
70
|
+
execution_provider: (extra && extra.execution_provider) || null,
|
|
71
|
+
model: (extra && extra.model) || null,
|
|
72
|
+
metrics: Object.assign(
|
|
73
|
+
{
|
|
74
|
+
total_time_ms: null,
|
|
75
|
+
tps: null,
|
|
76
|
+
real_time_factor: null,
|
|
77
|
+
sample_count: null,
|
|
78
|
+
audio_duration_ms: null
|
|
79
|
+
},
|
|
80
|
+
metrics
|
|
81
|
+
),
|
|
82
|
+
input: (extra && extra.input) || null,
|
|
83
|
+
output: (extra && extra.output) || null
|
|
84
|
+
}
|
|
85
|
+
_results.push(entry)
|
|
86
|
+
},
|
|
87
|
+
toJSON() {
|
|
88
|
+
return {
|
|
89
|
+
schema_version: '1.0',
|
|
90
|
+
addon: _addon,
|
|
91
|
+
addon_type: _addonType,
|
|
92
|
+
timestamp: _startedAt,
|
|
93
|
+
device: _device,
|
|
94
|
+
results: _results
|
|
95
|
+
}
|
|
96
|
+
},
|
|
97
|
+
writeReport() {
|
|
98
|
+
const json = JSON.stringify(this.toJSON())
|
|
99
|
+
const dirs = []
|
|
100
|
+
if (global.testDir) dirs.push(global.testDir)
|
|
101
|
+
if (platform === 'android') {
|
|
102
|
+
dirs.push('/sdcard/Android/data/io.tether.test.qvac/files')
|
|
103
|
+
dirs.push('/storage/emulated/0/Android/data/io.tether.test.qvac/files')
|
|
104
|
+
dirs.push('/data/local/tmp')
|
|
105
|
+
}
|
|
106
|
+
dirs.push('/tmp')
|
|
107
|
+
for (let di = 0; di < dirs.length; di++) {
|
|
108
|
+
try {
|
|
109
|
+
try {
|
|
110
|
+
fs.mkdirSync(dirs[di], { recursive: true })
|
|
111
|
+
} catch (_) {}
|
|
112
|
+
const p = path.join(dirs[di], 'perf-report.json')
|
|
113
|
+
fs.writeFileSync(p, json)
|
|
114
|
+
console.log('[PERF_REPORT_PATH]' + p)
|
|
115
|
+
} catch (e) {
|
|
116
|
+
console.log('[perf-reporter] write to ' + dirs[di] + ' failed: ' + e.message)
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
},
|
|
120
|
+
writeStepSummary() {},
|
|
121
|
+
writeToConsole() {
|
|
122
|
+
try {
|
|
123
|
+
const json = JSON.stringify(this.toJSON())
|
|
124
|
+
const CHUNK = 800
|
|
125
|
+
if (json.length <= CHUNK) {
|
|
126
|
+
console.log('[PERF_REPORT_START]' + json + '[PERF_REPORT_END]')
|
|
127
|
+
} else {
|
|
128
|
+
const id = Date.now().toString(36)
|
|
129
|
+
const n = Math.ceil(json.length / CHUNK)
|
|
130
|
+
for (let i = 0; i < n; i++) {
|
|
131
|
+
console.log(
|
|
132
|
+
'[PERF_CHUNK:' +
|
|
133
|
+
id +
|
|
134
|
+
':' +
|
|
135
|
+
i +
|
|
136
|
+
':' +
|
|
137
|
+
n +
|
|
138
|
+
']' +
|
|
139
|
+
json.substring(i * CHUNK, (i + 1) * CHUNK)
|
|
140
|
+
)
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
} catch (err) {
|
|
144
|
+
console.log('[perf-reporter] mobile console write failed: ' + err.message)
|
|
145
|
+
}
|
|
146
|
+
},
|
|
147
|
+
get length() {
|
|
148
|
+
return _results.length
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
const _perfReporter = createPerformanceReporter({
|
|
155
|
+
addon: 'tts-ggml',
|
|
156
|
+
addonType: 'tts'
|
|
157
|
+
})
|
|
158
|
+
|
|
159
|
+
const _reportPath = path.resolve('.', 'test/results/performance-report.json')
|
|
160
|
+
let _reportScheduled = false
|
|
161
|
+
|
|
162
|
+
function _flushPerfReport() {
|
|
163
|
+
if (_perfReporter.length === 0) return
|
|
164
|
+
try {
|
|
165
|
+
_perfReporter.writeReport(_reportPath)
|
|
166
|
+
} catch (_) {}
|
|
167
|
+
try {
|
|
168
|
+
_perfReporter.writeStepSummary()
|
|
169
|
+
} catch (_) {}
|
|
170
|
+
try {
|
|
171
|
+
_perfReporter.writeToConsole()
|
|
172
|
+
} catch (_) {}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function _scheduleReportWrite() {
|
|
176
|
+
if (_reportScheduled) return
|
|
177
|
+
_reportScheduled = true
|
|
178
|
+
process.on('exit', _flushPerfReport)
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function _num(v) {
|
|
182
|
+
return typeof v === 'number' && Number.isFinite(v) ? v : null
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Record a Chatterbox (ggml) synthesis stats row through the shared perf
|
|
187
|
+
* reporter.
|
|
188
|
+
*
|
|
189
|
+
* @param {string} label - Test label, e.g. 'chatterbox english 1'. A leading
|
|
190
|
+
* [CPU]/[GPU] is optional and only used as a fallback
|
|
191
|
+
* hint — the row is tagged from the resolved backend
|
|
192
|
+
* (stats.backendDevice) when available, and the prefix
|
|
193
|
+
* is normalized to match.
|
|
194
|
+
* @param {Object} stats - response.stats from the addon:
|
|
195
|
+
* { realTimeFactor, audioDurationMs, totalSamples,
|
|
196
|
+
* backendDevice, ... }
|
|
197
|
+
* @param {Object} [extra] - Optional overrides:
|
|
198
|
+
* { wallMs, sampleCount, model, output,
|
|
199
|
+
* executionProvider }.
|
|
200
|
+
*/
|
|
201
|
+
function recordTtsStats(label, stats, extra) {
|
|
202
|
+
const s = stats || {}
|
|
203
|
+
const resolvedEp = s.backendDevice === 1 ? 'gpu' : s.backendDevice === 0 ? 'cpu' : null
|
|
204
|
+
const epOverride = extra && extra.executionProvider
|
|
205
|
+
const labelEp = /\[gpu\]/i.test(label) ? 'gpu' : /\[cpu\]/i.test(label) ? 'cpu' : null
|
|
206
|
+
const ep = resolvedEp || epOverride || labelEp
|
|
207
|
+
|
|
208
|
+
const baseLabel = label.replace(/^\s*\[(?:cpu|gpu)\]\s*/i, '')
|
|
209
|
+
const finalLabel = ep ? `[${ep.toUpperCase()}] ${baseLabel}` : baseLabel
|
|
210
|
+
|
|
211
|
+
const rtf = _num(s.realTimeFactor)
|
|
212
|
+
const sampleCount =
|
|
213
|
+
(extra && _num(extra.sampleCount)) != null ? _num(extra.sampleCount) : _num(s.totalSamples)
|
|
214
|
+
const audioMs = _num(s.audioDurationMs) != null ? Math.round(_num(s.audioDurationMs)) : null
|
|
215
|
+
const wallMs = extra && _num(extra.wallMs) != null ? Math.round(_num(extra.wallMs)) : null
|
|
216
|
+
const tps = _num(s.tokensPerSecond)
|
|
217
|
+
|
|
218
|
+
// When the addon doesn't report a positive realTimeFactor (e.g. Supertonic /
|
|
219
|
+
// some GPU paths), derive it from the measured wall time and audio duration
|
|
220
|
+
// — the same fallback the RTF benchmark uses — so the row still carries an
|
|
221
|
+
// RTF instead of n/a. Callers that already pass a derived realTimeFactor
|
|
222
|
+
// (addon.test.js) keep it untouched.
|
|
223
|
+
//
|
|
224
|
+
// The two bases aren't comparable (compute-RTF excludes JS/marshalling
|
|
225
|
+
// overhead that wall-RTF includes), so record rtf_source per row: anyone
|
|
226
|
+
// comparing numbers across rows in a single report can see which is which.
|
|
227
|
+
const usingComputeRtf = rtf != null && rtf > 0
|
|
228
|
+
const effRtf = usingComputeRtf ? rtf : wallMs != null && audioMs ? wallMs / audioMs : null
|
|
229
|
+
const rtfSource = usingComputeRtf ? 'compute' : effRtf != null ? 'wall' : null
|
|
230
|
+
|
|
231
|
+
_perfReporter.record(
|
|
232
|
+
finalLabel,
|
|
233
|
+
{
|
|
234
|
+
total_time_ms: wallMs,
|
|
235
|
+
tps,
|
|
236
|
+
real_time_factor: effRtf,
|
|
237
|
+
rtf_source: rtfSource,
|
|
238
|
+
sample_count: sampleCount,
|
|
239
|
+
audio_duration_ms: audioMs
|
|
240
|
+
},
|
|
241
|
+
{
|
|
242
|
+
execution_provider: ep,
|
|
243
|
+
model: (extra && extra.model) || null,
|
|
244
|
+
output: extra && extra.output ? String(extra.output) : null
|
|
245
|
+
}
|
|
246
|
+
)
|
|
247
|
+
_scheduleReportWrite()
|
|
248
|
+
|
|
249
|
+
if (isMobile) {
|
|
250
|
+
try {
|
|
251
|
+
_perfReporter.writeReport()
|
|
252
|
+
} catch (_) {}
|
|
253
|
+
try {
|
|
254
|
+
_perfReporter.writeToConsole()
|
|
255
|
+
} catch (_) {}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
const lines = [
|
|
259
|
+
`${finalLabel} Performance Metrics (platform=${platformLabel}):`,
|
|
260
|
+
` - Wall time: ${wallMs !== null ? wallMs + 'ms' : 'n/a'}`,
|
|
261
|
+
` - RTF: ${effRtf !== null ? effRtf.toFixed(4) + ' (' + rtfSource + ')' : 'n/a'}`,
|
|
262
|
+
` - Samples: ${sampleCount !== null ? sampleCount : 'n/a'}`,
|
|
263
|
+
` - Audio: ${audioMs !== null ? audioMs + 'ms' : 'n/a'}`
|
|
264
|
+
]
|
|
265
|
+
return lines.join('\n')
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
module.exports = {
|
|
269
|
+
platform,
|
|
270
|
+
arch,
|
|
271
|
+
platformLabel,
|
|
272
|
+
isMobile,
|
|
273
|
+
recordTtsStats,
|
|
274
|
+
flushTtsPerfReport: _flushPerfReport
|
|
275
|
+
}
|