@qvac/tts-ggml 0.10.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +238 -20
- package/examples/moss-sfx-fit.js +18 -0
- package/examples/moss-sound-effect.js +125 -0
- package/examples/moss-speech.js +136 -0
- package/examples/pcm-chunk-player.js +35 -10
- package/index.d.ts +116 -6
- package/index.js +391 -2
- package/lib/fit.d.ts +22 -2
- package/package.json +31 -16
- package/test/integration/fit.test.js +7 -3
- package/test/integration/moss-fit.test.js +57 -0
- package/test/integration/moss-sfx-fit.test.js +63 -0
- package/test/integration/moss-sfx.test.js +96 -0
- package/test/integration/moss-speech.test.js +110 -0
- package/test/mobile/integration.auto.cjs +15 -0
- package/tts.d.ts +20 -0
- /package/test/{mobile → pocket-worklet}/pocket-worklet-host.cpp +0 -0
- /package/test/{mobile → pocket-worklet}/pocket-worklet.cjs +0 -0
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qvac/tts-ggml",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "Text to Speech (TTS) addon for qvac (ggml backend, wrapping the chatterbox + supertonic + parler + cosyvoice3 + audio8 + moss + Pocket engines from tts-cpp)",
|
|
3
|
+
"version": "0.11.0",
|
|
4
|
+
"description": "Text to Speech (TTS) addon for qvac (ggml backend, wrapping the chatterbox + supertonic + parler + cosyvoice3 + audio8 + moss + moss-sfx + moss-speech + Pocket engines from tts-cpp)",
|
|
5
5
|
"addon": true,
|
|
6
6
|
"engines": {
|
|
7
7
|
"bare": ">=1.19.0"
|
|
@@ -53,8 +53,8 @@
|
|
|
53
53
|
},
|
|
54
54
|
"scripts": {
|
|
55
55
|
"build": "npm run build:ts && npm run build:native",
|
|
56
|
-
"build:native": "bare-make generate && bare-make build && bare-make install",
|
|
57
|
-
"build:native:cuda": "bare-make generate -D ENABLE_CUDA=ON && bare-make build && bare-make install",
|
|
56
|
+
"build:native": "bare-make generate -D BUILD_FUZZING=OFF && bare-make build && bare-make install",
|
|
57
|
+
"build:native:cuda": "bare-make generate -D ENABLE_CUDA=ON -D BUILD_FUZZING=OFF && bare-make build && bare-make install",
|
|
58
58
|
"build:cuda": "npm run build:ts && npm run build:native:cuda",
|
|
59
59
|
"build:ts": "tsc -p tsconfig.build.json",
|
|
60
60
|
"build:pack": "npm run build:ts && mkdir -p dist && npm pack --pack-destination dist",
|
|
@@ -80,8 +80,10 @@
|
|
|
80
80
|
"example:parler-enhanced": "bare examples/parler-enhanced.js \"Hello from enhanced Parler.\"",
|
|
81
81
|
"example:audio8": "bare examples/audio8-tts.js \"Audio eight speaks on-device.\"",
|
|
82
82
|
"example:moss": "bare examples/moss-tts.js \"MOSS speaks on-device.\"",
|
|
83
|
+
"example:moss-sfx": "bare examples/moss-sound-effect.js \"Heavy rain falling on a tin roof with distant thunder.\" 5",
|
|
84
|
+
"example:moss-speech": "bare examples/moss-speech.js test/reference-audio/jfk.wav",
|
|
83
85
|
"lint": "npm run lint:js && npm run lint:ts && npm run test:types",
|
|
84
|
-
"lint:js": "prettier --check \"test/**/*.js\" \"scripts/__tests__/package-contract.test.js\" \"scripts/__tests__/binding-loader.test.js\" \"binding.js\" \"addon-unavailable.js\" && lunte \"test/**/*.js\" \"scripts/__tests__/package-contract.test.js\" \"scripts/__tests__/binding-loader.test.js\" \"binding.js\" \"addon-unavailable.js\"",
|
|
86
|
+
"lint:js": "prettier --check \"test/**/*.js\" \"scripts/run-cpp-fuzz.js\" \"scripts/__tests__/package-contract.test.js\" \"scripts/__tests__/binding-loader.test.js\" \"scripts/__tests__/run-cpp-fuzz.test.js\" \"binding.js\" \"addon-unavailable.js\" && lunte \"test/**/*.js\" \"scripts/run-cpp-fuzz.js\" \"scripts/__tests__/package-contract.test.js\" \"scripts/__tests__/binding-loader.test.js\" \"scripts/__tests__/run-cpp-fuzz.test.js\" \"binding.js\" \"addon-unavailable.js\"",
|
|
85
87
|
"lint:ts": "eslint \"src/**/*.ts\" --max-warnings=0",
|
|
86
88
|
"lint:ts:fix": "eslint \"src/**/*.ts\" --fix",
|
|
87
89
|
"format": "prettier --write \"test/**/*.js\" \"scripts/__tests__/package-contract.test.js\" \"scripts/__tests__/binding-loader.test.js\" \"binding.js\" \"addon-unavailable.js\"",
|
|
@@ -95,10 +97,22 @@
|
|
|
95
97
|
"test:benchmark:streaming": "npm run build:ts && brittle-bare test/benchmark/streaming-benchmark.test.js",
|
|
96
98
|
"test:mobile:generate": "node scripts/generate-mobile-integration-tests.js",
|
|
97
99
|
"test:mobile:validate": "node scripts/validate-mobile-tests.js",
|
|
98
|
-
"test:cpp:build": "bare-make generate -D BUILD_TESTING=ON && bare-make build --target tts_ggml_tests && bare-make install",
|
|
100
|
+
"test:cpp:build": "bare-make generate -D BUILD_TESTING=ON -D BUILD_FUZZING=OFF && bare-make build --target tts_ggml_tests && bare-make install",
|
|
99
101
|
"test:cpp:run": "cd build && LLVM_PROFILE_FILE=default.profraw ./tts_ggml_tests --gtest_output=xml:cpp-test-results.xml",
|
|
100
102
|
"test:cpp": "npm run test:cpp:build && npm run test:cpp:run",
|
|
101
|
-
"
|
|
103
|
+
"fuzz:build": "bare-make generate -D BUILD_FUZZING=ON -D BUILD_TESTING=OFF -D FUZZTEST_FUZZING_MODE=OFF && bare-make build --target tts-config-parse-fuzz",
|
|
104
|
+
"fuzz:build:continuous": "bare-make generate -D BUILD_FUZZING=ON -D BUILD_TESTING=OFF -D FUZZTEST_FUZZING_MODE=ON && bare-make build --target tts-config-parse-fuzz",
|
|
105
|
+
"fuzz:run": "node scripts/run-cpp-fuzz.js",
|
|
106
|
+
"fuzz": "npm run fuzz:build && npm run fuzz:run",
|
|
107
|
+
"fuzz:continuous": "npm run fuzz:build:continuous && node scripts/run-cpp-fuzz.js --continuous",
|
|
108
|
+
"test:cpp:fuzz:configure": "bare-make generate -D BUILD_TESTING=ON -D BUILD_FUZZING=ON -D FUZZTEST_FUZZING_MODE=OFF",
|
|
109
|
+
"test:cpp:fuzz:build:tests": "bare-make build --target tts_ggml_tests",
|
|
110
|
+
"test:cpp:fuzz:build:fuzz": "bare-make build --target tts-config-parse-fuzz",
|
|
111
|
+
"test:cpp:fuzz:build": "npm run test:cpp:fuzz:configure && npm run test:cpp:fuzz:build:tests && npm run test:cpp:fuzz:build:fuzz",
|
|
112
|
+
"test:cpp:fuzz:run": "npm run test:scripts && npm run test:cpp:run && node scripts/run-cpp-fuzz.js",
|
|
113
|
+
"test:cpp:fuzz": "npm run test:cpp:fuzz:build && npm run test:cpp:fuzz:run",
|
|
114
|
+
"test:scripts": "node --test scripts/__tests__/run-cpp-fuzz.test.js",
|
|
115
|
+
"coverage:cpp:build": "bare-make generate -D BUILD_TESTING=ON -D ENABLE_COVERAGE=ON -D BUILD_FUZZING=OFF && bare-make build --target tts_ggml_tests",
|
|
102
116
|
"coverage:cpp:summary": "cd build && llvm-cov report ./tts_ggml_tests --instr-profile=coverage.profdata -ignore-filename-regex='(tests|build|node_modules|gtest|gmock|\\.vcpkg|/usr)/' > coverage-summary.txt",
|
|
103
117
|
"coverage:cpp:report": "cd build && ls -lha && llvm-profdata merge -sparse default.profraw -o coverage.profdata && llvm-cov show ./tts_ggml_tests -instr-profile=coverage.profdata -format=html -output-dir=coverage-html -ignore-filename-regex='(tests|build|node_modules|gtest|gmock|\\.vcpkg|/usr)/' && llvm-cov export ./tts_ggml_tests -instr-profile=coverage.profdata -format=lcov -ignore-filename-regex='(tests|build|node_modules|gtest|gmock|\\.vcpkg|/usr)/' > lcov.info && npm run coverage:cpp:summary",
|
|
104
118
|
"coverage:cpp": "npm run coverage:cpp:build && npm run test:cpp:run && npm run coverage:cpp:report",
|
|
@@ -127,6 +141,7 @@
|
|
|
127
141
|
"scripts/download-tts-ggml-models.js",
|
|
128
142
|
"test/integration",
|
|
129
143
|
"test/mobile",
|
|
144
|
+
"test/pocket-worklet",
|
|
130
145
|
"test/reference-audio",
|
|
131
146
|
"test/utils/runTTS.js",
|
|
132
147
|
"test/utils/runChatterboxTTS.js",
|
|
@@ -197,7 +212,9 @@
|
|
|
197
212
|
"cmake-npm": "^1.1.0",
|
|
198
213
|
"bare-url": "^2.1.6",
|
|
199
214
|
"bare-https": "^3.0.0",
|
|
200
|
-
"bare-stream": "^2.7.0"
|
|
215
|
+
"bare-stream": "^2.7.0",
|
|
216
|
+
"bare-subprocess": "^6.2.1",
|
|
217
|
+
"brittle": "^3.17.0"
|
|
201
218
|
},
|
|
202
219
|
"dependencies": {
|
|
203
220
|
"@qvac/error": "^0.1.0",
|
|
@@ -210,16 +227,14 @@
|
|
|
210
227
|
"bare-path": "^3.0.0",
|
|
211
228
|
"bare-process": "^4.2.2",
|
|
212
229
|
"bare-stream": "^2.7.0",
|
|
213
|
-
"bare-
|
|
214
|
-
"bare-url": "^2.4.3",
|
|
215
|
-
"brittle": "^3.17.0"
|
|
230
|
+
"bare-url": "^2.4.3"
|
|
216
231
|
},
|
|
217
232
|
"types": "index.d.ts",
|
|
218
233
|
"optionalDependencies": {
|
|
219
|
-
"@qvac/tts-ggml-linux-x64": "0.
|
|
220
|
-
"@qvac/tts-ggml-linux-arm64": "0.
|
|
221
|
-
"@qvac/tts-ggml-darwin-arm64": "0.
|
|
222
|
-
"@qvac/tts-ggml-darwin-x64": "0.
|
|
223
|
-
"@qvac/tts-ggml-win32-x64": "0.
|
|
234
|
+
"@qvac/tts-ggml-linux-x64": "0.11.0",
|
|
235
|
+
"@qvac/tts-ggml-linux-arm64": "0.11.0",
|
|
236
|
+
"@qvac/tts-ggml-darwin-arm64": "0.11.0",
|
|
237
|
+
"@qvac/tts-ggml-darwin-x64": "0.11.0",
|
|
238
|
+
"@qvac/tts-ggml-win32-x64": "0.11.0"
|
|
224
239
|
}
|
|
225
240
|
}
|
|
@@ -141,14 +141,18 @@ test('a LavaSR stage is sized beside the projection', { timeout: TEST_TIMEOUT_MS
|
|
|
141
141
|
t.ok(fit.lavasrFileBytes < fit.deviceBytes, 'reported beside the projection')
|
|
142
142
|
})
|
|
143
143
|
|
|
144
|
-
test('
|
|
144
|
+
test('MOSS routes to its fitter when a model cannot be read', (t) => {
|
|
145
145
|
const fit = TTSGgml.assessFit({
|
|
146
146
|
engineType: 'moss',
|
|
147
|
-
mossBackbonePath: '/models/moss-tts-delay.gguf'
|
|
147
|
+
mossBackbonePath: '/models/moss-tts-delay.gguf',
|
|
148
|
+
mossCodecDecoderPath: '/models/moss-codec-decoder.gguf',
|
|
149
|
+
promptRows: 128,
|
|
150
|
+
referenceSamples: 0,
|
|
151
|
+
streaming: false
|
|
148
152
|
})
|
|
149
153
|
|
|
150
154
|
t.is(fit.status, 'error')
|
|
151
|
-
t.is(fit.reason, '
|
|
155
|
+
t.is(fit.reason, 'model-unreadable')
|
|
152
156
|
t.is(fit.modelVariant, 'moss')
|
|
153
157
|
})
|
|
154
158
|
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const path = require('bare-path')
|
|
4
|
+
const process = require('bare-process')
|
|
5
|
+
const test = require('brittle')
|
|
6
|
+
const TTSGgml = require('@qvac/tts-ggml')
|
|
7
|
+
|
|
8
|
+
test('MOSS fit requires explicit, representable workload counts', (t) => {
|
|
9
|
+
const request = {
|
|
10
|
+
engineType: 'moss',
|
|
11
|
+
mossBackbonePath: '/models/moss-tts-delay.gguf',
|
|
12
|
+
mossCodecDecoderPath: '/models/moss-codec-decoder.gguf',
|
|
13
|
+
promptRows: 128,
|
|
14
|
+
referenceSamples: 0,
|
|
15
|
+
streaming: false
|
|
16
|
+
}
|
|
17
|
+
for (const invalid of [NaN, Infinity, -1, 0.5, 2147483648]) {
|
|
18
|
+
t.exception(() => TTSGgml.assessFit({ ...request, promptRows: invalid }))
|
|
19
|
+
t.exception(() => TTSGgml.assessFit({ ...request, referenceSamples: invalid }))
|
|
20
|
+
}
|
|
21
|
+
t.exception(() => TTSGgml.assessFit({ ...request, promptRows: undefined }))
|
|
22
|
+
t.exception(() => TTSGgml.assessFit({ ...request, streaming: undefined }))
|
|
23
|
+
t.exception(() => TTSGgml.assessFit({ ...request, threads: -1 }))
|
|
24
|
+
t.exception(() => TTSGgml.assessFit({ ...request, durationTokens: -1 }))
|
|
25
|
+
t.exception(() => TTSGgml.assessFit({ ...request, streamChunkTokens: -1 }))
|
|
26
|
+
t.exception(() => TTSGgml.assessFit({ ...request, useGPU: true, nGpuLayers: 0 }))
|
|
27
|
+
})
|
|
28
|
+
|
|
29
|
+
test('MOSS metadata-only fit covers batch, streaming and cloning', (t) => {
|
|
30
|
+
const modelDir = process.env.QVAC_TEST_MOSS_MODEL_DIR
|
|
31
|
+
if (!modelDir) {
|
|
32
|
+
t.comment('Set QVAC_TEST_MOSS_MODEL_DIR to exercise real MOSS GGUFs')
|
|
33
|
+
return
|
|
34
|
+
}
|
|
35
|
+
const request = {
|
|
36
|
+
engineType: 'moss',
|
|
37
|
+
mossBackbonePath: path.join(modelDir, 'moss-tts-delay-f16.gguf'),
|
|
38
|
+
mossCodecDecoderPath: path.join(modelDir, 'moss-codec-decoder-f16.gguf'),
|
|
39
|
+
promptRows: 128,
|
|
40
|
+
referenceSamples: 0,
|
|
41
|
+
streaming: false
|
|
42
|
+
}
|
|
43
|
+
const batch = TTSGgml.assessFit(request)
|
|
44
|
+
t.not(batch.status, 'error', batch.report)
|
|
45
|
+
t.ok(batch.weightsBytes > 0)
|
|
46
|
+
t.ok(batch.lmComputeBytes > 0)
|
|
47
|
+
t.ok(batch.codecComputeBytes > 0)
|
|
48
|
+
const stream = TTSGgml.assessFit({ ...request, streaming: true })
|
|
49
|
+
t.not(stream.status, 'error', stream.report)
|
|
50
|
+
const cloning = TTSGgml.assessFit({
|
|
51
|
+
...request,
|
|
52
|
+
mossCodecEncoderPath: path.join(modelDir, 'moss-codec-encoder-f16.gguf'),
|
|
53
|
+
referenceSamples: 24000
|
|
54
|
+
})
|
|
55
|
+
t.not(cloning.status, 'error', cloning.report)
|
|
56
|
+
t.ok(cloning.weightsBytes > batch.weightsBytes)
|
|
57
|
+
})
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const test = require('brittle')
|
|
4
|
+
const process = require('bare-process')
|
|
5
|
+
const TTSGgml = require('@qvac/tts-ggml')
|
|
6
|
+
|
|
7
|
+
function request(overrides = {}) {
|
|
8
|
+
return {
|
|
9
|
+
engineType: 'moss-sfx',
|
|
10
|
+
mossSoundEffectPath: '/nonexistent/moss-sfx.gguf',
|
|
11
|
+
prompt: 'Rain on a tin roof.',
|
|
12
|
+
seconds: 8,
|
|
13
|
+
useGPU: false,
|
|
14
|
+
...overrides
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
test('MOSS sound-effect fit rejects missing workload and invalid load controls', (t) => {
|
|
19
|
+
t.exception(() => TTSGgml.assessFit(request({ prompt: undefined })))
|
|
20
|
+
t.exception(() => TTSGgml.assessFit(request({ seconds: undefined })))
|
|
21
|
+
t.exception(() => TTSGgml.assessFit(request({ seconds: NaN })))
|
|
22
|
+
t.exception(() => TTSGgml.assessFit(request({ threads: -1 })))
|
|
23
|
+
t.exception(() => TTSGgml.assessFit(request({ useGPU: true, nGpuLayers: 0 })))
|
|
24
|
+
t.exception(() => TTSGgml.assessFit(request({ marginBytes: -1 })))
|
|
25
|
+
})
|
|
26
|
+
|
|
27
|
+
test('MOSS sound-effect fit reports unreadable models', (t) => {
|
|
28
|
+
const fit = TTSGgml.assessFit(request())
|
|
29
|
+
t.is(fit.status, 'error')
|
|
30
|
+
t.is(fit.reason, 'model-unreadable')
|
|
31
|
+
t.is(fit.modelVariant, 'moss-sfx')
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
test(
|
|
35
|
+
'MOSS sound-effect fit projects real-model CPU workloads',
|
|
36
|
+
{
|
|
37
|
+
skip: !process.env.QVAC_TEST_MOSS_SFX_GGUF
|
|
38
|
+
},
|
|
39
|
+
(t) => {
|
|
40
|
+
const model = process.env.QVAC_TEST_MOSS_SFX_GGUF
|
|
41
|
+
const cfg = { mossSoundEffectPath: model }
|
|
42
|
+
const fit = TTSGgml.assessFit(request(cfg))
|
|
43
|
+
t.ok(fit.status === 'fits' || fit.status === 'does-not-fit')
|
|
44
|
+
t.is(fit.modelVariant, 'moss-sfx')
|
|
45
|
+
t.ok(fit.deviceIsCpu)
|
|
46
|
+
t.ok(fit.deviceSharesHostMemory)
|
|
47
|
+
t.ok(fit.weightsBytes > 0)
|
|
48
|
+
t.ok(fit.lmComputeBytes > 0)
|
|
49
|
+
t.ok(fit.hostBytes > 0)
|
|
50
|
+
t.is(
|
|
51
|
+
fit.deviceBytes,
|
|
52
|
+
fit.weightsBytes + fit.stateBytes + fit.lmComputeBytes + fit.codecComputeBytes
|
|
53
|
+
)
|
|
54
|
+
const shorter = TTSGgml.assessFit(request({ ...cfg, seconds: 1 }))
|
|
55
|
+
t.is(shorter.weightsBytes, fit.weightsBytes)
|
|
56
|
+
t.is(shorter.lmComputeBytes, fit.lmComputeBytes)
|
|
57
|
+
const noHeadroom = TTSGgml.assessFit(request({ ...cfg, marginBytes: Number.MAX_VALUE }))
|
|
58
|
+
t.is(noHeadroom.status, 'does-not-fit')
|
|
59
|
+
const invalid = TTSGgml.assessFit(request({ ...cfg, seconds: 0 }))
|
|
60
|
+
t.is(invalid.status, 'error')
|
|
61
|
+
t.is(invalid.reason, 'invalid-arguments')
|
|
62
|
+
}
|
|
63
|
+
)
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const path = require('bare-path')
|
|
4
|
+
const proc = require('bare-process')
|
|
5
|
+
const test = require('brittle')
|
|
6
|
+
|
|
7
|
+
const TTSGgml = require('@qvac/tts-ggml')
|
|
8
|
+
|
|
9
|
+
const SFX_SAMPLE_RATE = 48000
|
|
10
|
+
const TENTHS_PER_SECOND = 10
|
|
11
|
+
const TEST_TIMEOUT_MS = 1800000
|
|
12
|
+
const PROMPT = 'A glass falls and shatters on a tile floor.'
|
|
13
|
+
const MODEL_DIR_ENV = 'QVAC_TEST_MOSS_SFX_MODEL_DIR'
|
|
14
|
+
const QUICK_STEPS = 2
|
|
15
|
+
const SHORT_SECONDS = 1.5
|
|
16
|
+
const LONGER_SECONDS = 2.5
|
|
17
|
+
const CPU_DEVICE = 0
|
|
18
|
+
const GPU_DEVICE = 1
|
|
19
|
+
|
|
20
|
+
const modelDir = (proc.env && proc.env[MODEL_DIR_ENV]) || ''
|
|
21
|
+
const skipWithoutModels = modelDir === ''
|
|
22
|
+
const noGpu = proc.env && proc.env.NO_GPU === 'true'
|
|
23
|
+
|
|
24
|
+
function createSfxModel(useGPU = false) {
|
|
25
|
+
return new TTSGgml({
|
|
26
|
+
engine: TTSGgml.ENGINE_MOSS_SFX,
|
|
27
|
+
files: { modelDir: path.resolve(modelDir) },
|
|
28
|
+
config: { useGPU },
|
|
29
|
+
seed: 0,
|
|
30
|
+
opts: { stats: true }
|
|
31
|
+
})
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function collectChunk(result, data) {
|
|
35
|
+
if (!data || !data.outputArray) return
|
|
36
|
+
result.samples += data.outputArray.length
|
|
37
|
+
if (data.sampleRate) result.sampleRate = data.sampleRate
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
async function generate(model, fields) {
|
|
41
|
+
const response = await model.run({ input: PROMPT, type: 'text', steps: QUICK_STEPS, ...fields })
|
|
42
|
+
const result = { samples: 0, sampleRate: null }
|
|
43
|
+
await response.onUpdate((data) => collectChunk(result, data)).await()
|
|
44
|
+
result.stats = response.stats
|
|
45
|
+
return result
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function samplesAtNativeDuration(seconds) {
|
|
49
|
+
return Math.round(seconds * TENTHS_PER_SECOND) * (SFX_SAMPLE_RATE / TENTHS_PER_SECOND)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function assertClip(t, label, result, seconds) {
|
|
53
|
+
t.is(result.sampleRate, SFX_SAMPLE_RATE, `${label} reports 48 kHz`)
|
|
54
|
+
t.is(
|
|
55
|
+
result.samples,
|
|
56
|
+
samplesAtNativeDuration(seconds),
|
|
57
|
+
`${label} lasts exactly ${seconds} s, rounded to 0.1 s as the engine documents`
|
|
58
|
+
)
|
|
59
|
+
t.ok(result.stats, `${label} returns runtime stats`)
|
|
60
|
+
t.is(result.stats.totalSamples, result.samples, `${label} reports the emitted samples`)
|
|
61
|
+
t.ok(result.stats.totalTime > 0, `${label} reports its generation time`)
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
test(
|
|
65
|
+
'MOSS-SFX integration: a prompt generates the requested duration on CPU',
|
|
66
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels },
|
|
67
|
+
async (t) => {
|
|
68
|
+
const model = createSfxModel()
|
|
69
|
+
await model.load()
|
|
70
|
+
try {
|
|
71
|
+
const result = await generate(model, { seconds: SHORT_SECONDS })
|
|
72
|
+
assertClip(t, 'cpu clip', result, SHORT_SECONDS)
|
|
73
|
+
t.is(result.stats.backendDevice, CPU_DEVICE, 'the clip ran on the CPU')
|
|
74
|
+
} finally {
|
|
75
|
+
await model.unload()
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
test(
|
|
81
|
+
'MOSS-SFX integration: seconds and the negative prompt change per call',
|
|
82
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels || noGpu },
|
|
83
|
+
async (t) => {
|
|
84
|
+
const model = createSfxModel(true)
|
|
85
|
+
await model.load()
|
|
86
|
+
try {
|
|
87
|
+
const short = await generate(model, { seconds: SHORT_SECONDS })
|
|
88
|
+
const longer = await generate(model, { seconds: LONGER_SECONDS, negativePrompt: 'music' })
|
|
89
|
+
assertClip(t, 'short clip', short, SHORT_SECONDS)
|
|
90
|
+
assertClip(t, 'longer clip', longer, LONGER_SECONDS)
|
|
91
|
+
t.is(short.stats.backendDevice, GPU_DEVICE, 'the clips ran on the GPU')
|
|
92
|
+
} finally {
|
|
93
|
+
await model.unload()
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
)
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
const path = require('bare-path')
|
|
4
|
+
const proc = require('bare-process')
|
|
5
|
+
const test = require('brittle')
|
|
6
|
+
|
|
7
|
+
const TTSGgml = require('@qvac/tts-ggml')
|
|
8
|
+
const { resolveRefWavPath } = require('../utils/runChatterboxTTS')
|
|
9
|
+
const { readWavAsFloat32 } = require('../utils/wav-helper')
|
|
10
|
+
const { TTS_TEST_THREADS } = require('../utils/testThreads')
|
|
11
|
+
|
|
12
|
+
const SPEECH_SAMPLE_RATE = 24000
|
|
13
|
+
const TEST_TIMEOUT_MS = 1800000
|
|
14
|
+
const MODEL_DIR_ENV = 'QVAC_TEST_MOSS_SPEECH_MODEL_DIR'
|
|
15
|
+
const MAX_NEW_TOKENS = 120
|
|
16
|
+
const CUT_SECONDS = 2
|
|
17
|
+
const CUT_MARGIN_SECONDS = 0.5
|
|
18
|
+
const CPU_DEVICE = 0
|
|
19
|
+
const GPU_DEVICE = 1
|
|
20
|
+
|
|
21
|
+
const modelDir = (proc.env && proc.env[MODEL_DIR_ENV]) || ''
|
|
22
|
+
const skipWithoutModels = modelDir === ''
|
|
23
|
+
const noGpu = proc.env && proc.env.NO_GPU === 'true'
|
|
24
|
+
|
|
25
|
+
function createSpeechModel(useGPU = false) {
|
|
26
|
+
return new TTSGgml({
|
|
27
|
+
threads: TTS_TEST_THREADS,
|
|
28
|
+
engine: TTSGgml.ENGINE_MOSS_SPEECH,
|
|
29
|
+
files: { modelDir: path.resolve(modelDir) },
|
|
30
|
+
config: { useGPU },
|
|
31
|
+
seed: 0,
|
|
32
|
+
opts: { stats: true }
|
|
33
|
+
})
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function spokenQuestion() {
|
|
37
|
+
const wav = readWavAsFloat32(resolveRefWavPath({}))
|
|
38
|
+
return { audio: wav.samples, sampleRate: wav.sampleRate }
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function collectReply(result, data) {
|
|
42
|
+
if (!data || !data.outputArray) return
|
|
43
|
+
result.samples += data.outputArray.length
|
|
44
|
+
if (data.sampleRate) result.sampleRate = data.sampleRate
|
|
45
|
+
if (typeof data.text === 'string') result.text += data.text
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async function respond(model, fields) {
|
|
49
|
+
const response = await model.run({
|
|
50
|
+
type: 'text',
|
|
51
|
+
input: '',
|
|
52
|
+
maxNewTokens: MAX_NEW_TOKENS,
|
|
53
|
+
...fields
|
|
54
|
+
})
|
|
55
|
+
const result = { samples: 0, sampleRate: null, text: '' }
|
|
56
|
+
await response.onUpdate((data) => collectReply(result, data)).await()
|
|
57
|
+
result.stats = response.stats
|
|
58
|
+
return result
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function assertSpokenReply(t, label, result) {
|
|
62
|
+
t.is(result.sampleRate, SPEECH_SAMPLE_RATE, `${label} reports 24 kHz`)
|
|
63
|
+
t.ok(result.samples > 0, `${label} carries speech`)
|
|
64
|
+
t.ok(result.stats, `${label} returns runtime stats`)
|
|
65
|
+
t.is(result.stats.totalSamples, result.samples, `${label} reports the emitted samples`)
|
|
66
|
+
t.ok(result.stats.promptTokens > 0, `${label} reports its prompt length`)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
test(
|
|
70
|
+
'MOSS-Speech integration: a spoken question gets a spoken answer on CPU',
|
|
71
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels },
|
|
72
|
+
async (t) => {
|
|
73
|
+
const model = createSpeechModel()
|
|
74
|
+
await model.load()
|
|
75
|
+
try {
|
|
76
|
+
const result = await respond(model, spokenQuestion())
|
|
77
|
+
assertSpokenReply(t, 'cpu reply', result)
|
|
78
|
+
t.is(result.stats.backendDevice, CPU_DEVICE, 'the reply ran on the CPU')
|
|
79
|
+
} finally {
|
|
80
|
+
await model.unload()
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
test(
|
|
86
|
+
'MOSS-Speech integration: text replies, reply cuts and a text turn on the GPU',
|
|
87
|
+
{ timeout: TEST_TIMEOUT_MS, skip: skipWithoutModels || noGpu },
|
|
88
|
+
async (t) => {
|
|
89
|
+
const model = createSpeechModel(true)
|
|
90
|
+
await model.load()
|
|
91
|
+
try {
|
|
92
|
+
const textReply = await respond(model, { ...spokenQuestion(), textReply: true })
|
|
93
|
+
t.is(textReply.samples, 0, 'a text reply carries no audio')
|
|
94
|
+
t.ok(textReply.text.trim().length > 0, 'a text reply carries the answer')
|
|
95
|
+
|
|
96
|
+
const cut = await respond(model, { ...spokenQuestion(), maxReplySeconds: CUT_SECONDS })
|
|
97
|
+
assertSpokenReply(t, 'cut reply', cut)
|
|
98
|
+
t.ok(
|
|
99
|
+
cut.samples <= (CUT_SECONDS + CUT_MARGIN_SECONDS) * SPEECH_SAMPLE_RATE,
|
|
100
|
+
'maxReplySeconds bounds the spoken reply'
|
|
101
|
+
)
|
|
102
|
+
t.is(cut.stats.backendDevice, GPU_DEVICE, 'the replies ran on the GPU')
|
|
103
|
+
|
|
104
|
+
const typed = await respond(model, { input: 'What is the capital of France?' })
|
|
105
|
+
assertSpokenReply(t, 'reply to a typed question', typed)
|
|
106
|
+
} finally {
|
|
107
|
+
await model.unload()
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
)
|
|
@@ -50,6 +50,14 @@ async function runLavasrEnhancerTest (options = {}) { // eslint-disable-line no-
|
|
|
50
50
|
return runIntegrationModule('../integration/lavasr-enhancer.test.js', options)
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
+
async function runMossSfxTest (options = {}) { // eslint-disable-line no-unused-vars
|
|
54
|
+
return runIntegrationModule('../integration/moss-sfx.test.js', options)
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function runMossSpeechTest (options = {}) { // eslint-disable-line no-unused-vars
|
|
58
|
+
return runIntegrationModule('../integration/moss-speech.test.js', options)
|
|
59
|
+
}
|
|
60
|
+
|
|
53
61
|
async function runMossTest (options = {}) { // eslint-disable-line no-unused-vars
|
|
54
62
|
return runIntegrationModule('../integration/moss.test.js', options)
|
|
55
63
|
}
|
|
@@ -70,6 +78,10 @@ async function runParlerTest (options = {}) { // eslint-disable-line no-unused-v
|
|
|
70
78
|
return runIntegrationModule('../integration/parler.test.js', options)
|
|
71
79
|
}
|
|
72
80
|
|
|
81
|
+
async function runPocketIntegrationTest (options = {}) { // eslint-disable-line no-unused-vars
|
|
82
|
+
return runIntegrationModule('../integration/pocket.integration.test.js', options)
|
|
83
|
+
}
|
|
84
|
+
|
|
73
85
|
async function runRtfBenchmarkTest (options = {}) { // eslint-disable-line no-unused-vars
|
|
74
86
|
return runIntegrationModule('../integration/rtf-benchmark.test.js', options)
|
|
75
87
|
}
|
|
@@ -106,11 +118,14 @@ module.exports = {
|
|
|
106
118
|
runFitTest,
|
|
107
119
|
runGpuSmokeTest,
|
|
108
120
|
runLavasrEnhancerTest,
|
|
121
|
+
runMossSfxTest,
|
|
122
|
+
runMossSpeechTest,
|
|
109
123
|
runMossTest,
|
|
110
124
|
runMultipleRunsTest,
|
|
111
125
|
runOutputSampleRateTest,
|
|
112
126
|
runParlerWerTest,
|
|
113
127
|
runParlerTest,
|
|
128
|
+
runPocketIntegrationTest,
|
|
114
129
|
runRtfBenchmarkTest,
|
|
115
130
|
runStreamingBenchmarkTest,
|
|
116
131
|
runSupertonicMtlTest,
|
package/tts.d.ts
CHANGED
|
@@ -17,6 +17,26 @@ export interface TTSJobData {
|
|
|
17
17
|
instruct?: string;
|
|
18
18
|
referenceAudio?: string;
|
|
19
19
|
referenceText?: string;
|
|
20
|
+
seconds?: number;
|
|
21
|
+
negativePrompt?: string;
|
|
22
|
+
steps?: number;
|
|
23
|
+
guidance?: number;
|
|
24
|
+
shift?: number;
|
|
25
|
+
messages?: Array<{
|
|
26
|
+
role: string;
|
|
27
|
+
text?: string;
|
|
28
|
+
audio?: Int16Array | Float32Array;
|
|
29
|
+
sampleRate?: number;
|
|
30
|
+
}>;
|
|
31
|
+
replyVoice?: Int16Array | Float32Array;
|
|
32
|
+
replyVoiceSampleRate?: number;
|
|
33
|
+
textReply?: boolean;
|
|
34
|
+
maxReplySeconds?: number;
|
|
35
|
+
maxNewTokens?: number;
|
|
36
|
+
greedy?: boolean;
|
|
37
|
+
temperature?: number;
|
|
38
|
+
topP?: number;
|
|
39
|
+
topK?: number;
|
|
20
40
|
}
|
|
21
41
|
export interface TTSWeightData {
|
|
22
42
|
filename: string;
|
|
File without changes
|
|
File without changes
|