@qvac/tts-ggml 0.9.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +317 -33
- package/docs/pocket-tts.md +201 -0
- package/examples/moss-dialogue-tts.js +113 -0
- package/examples/moss-tts.js +154 -0
- package/examples/pocket-tts.js +51 -0
- package/examples/supertonic-chunk-stream-tts.js +156 -0
- package/examples/supertonic-sentence-stream-tts.js +4 -4
- package/index.d.ts +352 -29
- package/index.js +698 -52
- package/lib/fit.d.ts +127 -0
- package/lib/fit.js +29 -0
- package/lib/pocketConfig.d.ts +10 -0
- package/lib/pocketConfig.js +145 -0
- package/package.json +14 -9
- package/test/integration/addon.test.js +5 -0
- package/test/integration/audio8.test.js +7 -0
- package/test/integration/chatterbox-mtl.test.js +6 -0
- package/test/integration/chatterbox-speed.test.js +13 -2
- package/test/integration/cosyvoice3-clone.test.js +16 -1
- package/test/integration/cosyvoice3-lavasr.test.js +6 -0
- package/test/integration/cosyvoice3.test.js +19 -0
- package/test/integration/fit.test.js +174 -0
- package/test/integration/gpu-smoke.test.js +13 -0
- package/test/integration/lavasr-enhancer.test.js +104 -3
- package/test/integration/moss.test.js +282 -0
- package/test/integration/multiple-runs.test.js +18 -2
- package/test/integration/output-sample-rate.test.js +6 -0
- package/test/integration/parler-wer.test.js +7 -1
- package/test/integration/parler.test.js +27 -3
- package/test/integration/pocket.integration.test.js +156 -0
- package/test/integration/supertonic-mtl.test.js +2 -0
- package/test/integration/supertonic.test.js +70 -0
- package/test/integration/supertonic3-quant.test.js +2 -0
- package/test/integration/voice-controls.test.js +56 -0
- package/test/mobile/integration.auto.cjs +16 -1
- package/test/mobile/pocket-worklet-host.cpp +92 -0
- package/test/mobile/pocket-worklet.cjs +47 -0
- package/test/mobile/test-groups.json +8 -4
- package/test/utils/runCosyvoiceTTS.js +1 -0
- package/test/utils/runTTS.js +6 -11
- package/tts.d.ts +13 -2
- package/tts.js +3 -2
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
# Pocket TTS
|
|
2
|
+
|
|
3
|
+
Pocket is an English, CPU-only TTS engine implemented in the Fabric speech
|
|
4
|
+
library. It streams PCM16 audio through the `@qvac/tts-ggml` addon. Python
|
|
5
|
+
is used only to convert weights and benchmark the upstream reference; it is
|
|
6
|
+
not needed for native synthesis. Inference plugin and public SDK integration
|
|
7
|
+
will follow in a separate change after the addon release.
|
|
8
|
+
|
|
9
|
+
## Model bundle
|
|
10
|
+
|
|
11
|
+
Convert the supported Kyutai English 2026-04 checkpoint using
|
|
12
|
+
`engines/tts/scripts/convert-pocket-to-gguf.py` in qvac-fabric-speech.cpp.
|
|
13
|
+
The bundle contains `flow-lm.gguf`, `mimi.gguf`, `frontend.json` and `voice.gguf`.
|
|
14
|
+
The native repository's `engines/tts/docs/pocket-tts.md` documents conversion,
|
|
15
|
+
asset revisions, validation, CLI usage and the benchmark scripts.
|
|
16
|
+
|
|
17
|
+
The public checkpoint without voice cloning works with a prepared voice
|
|
18
|
+
embedding. Reference-WAV conditioning requires weights with a functioning
|
|
19
|
+
Mimi encoder; the public checkpoint with zero encoder weights is rejected
|
|
20
|
+
for that operation. Do not advertise voice cloning from those weights.
|
|
21
|
+
|
|
22
|
+
## Addon usage
|
|
23
|
+
|
|
24
|
+
```js
|
|
25
|
+
const TTSGgml = require('@qvac/tts-ggml')
|
|
26
|
+
const model = new TTSGgml({
|
|
27
|
+
engine: 'pocket',
|
|
28
|
+
files: { modelDir: '/absolute/path/to/pocket-bundle' },
|
|
29
|
+
config: { language: 'en', useGPU: false },
|
|
30
|
+
threads: 1,
|
|
31
|
+
seed: 1234,
|
|
32
|
+
temperature: 0.3,
|
|
33
|
+
steps: 1
|
|
34
|
+
})
|
|
35
|
+
try {
|
|
36
|
+
await model.load()
|
|
37
|
+
const response = await model.run({ input: 'Hello from Pocket TTS in Fabric.' })
|
|
38
|
+
for await (const chunk of response.iterate()) {
|
|
39
|
+
// chunk.outputArray: Int16Array; chunk.sampleRate defaults to 24000.
|
|
40
|
+
// Feed samples to an audio device or collect them in a WAV file.
|
|
41
|
+
}
|
|
42
|
+
} finally {
|
|
43
|
+
await model.destroy()
|
|
44
|
+
}
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
`npm run example:pocket -- /absolute/bundle "Hello from Fabric." /tmp/pocket.wav`
|
|
48
|
+
saves a playable WAV. The addon also accepts explicit `files.pocketFlowModel`,
|
|
49
|
+
`files.pocketMimiModel`, `files.pocketFrontend` and `files.pocketVoice` paths.
|
|
50
|
+
|
|
51
|
+
`run()` streams native chunks without sentence splitting, including when
|
|
52
|
+
`streamOutput: true` is requested. `runStream()` explicitly splits sentences;
|
|
53
|
+
`runStreaming()` accepts incremental text. `cancel()` invalidates pending text
|
|
54
|
+
and waits for native completion before another request is admitted.
|
|
55
|
+
A `run()` AbortSignal also stops native work. Invalid reload options preserve
|
|
56
|
+
the loaded model. A valid `reload()` drains native work and releases the old
|
|
57
|
+
model before loading its replacement, so two model instances never overlap.
|
|
58
|
+
If replacement loading fails, the instance is left unloaded and retains its
|
|
59
|
+
last successful configuration; call `load()` to restore it or `reload()` to
|
|
60
|
+
retry with new options. Overlapping lifecycle operations are rejected;
|
|
61
|
+
`unload()` permits a later load/reload, while `destroy()` is terminal.
|
|
62
|
+
|
|
63
|
+
When stats are enabled with `opts: { stats: true }`, `firstAudioMs` reports
|
|
64
|
+
the native latency to first audio. For `runStream()` and `runStreaming()`,
|
|
65
|
+
it preserves the first reported native-job value rather than summing sentence
|
|
66
|
+
latencies; it excludes time waiting for incoming text. `chunkIndex` and `isLast`
|
|
67
|
+
are optional public output fields. Pocket's `run()` emits a final empty PCM
|
|
68
|
+
chunk with `isLast: true`; other engines may omit this metadata.
|
|
69
|
+
|
|
70
|
+
## Supported controls
|
|
71
|
+
|
|
72
|
+
English only; CPU only; `threads` defaults to one per worker (two workers).
|
|
73
|
+
`seed` accepts unsigned 32-bit integers. `temperature`, `steps`, `nCtx`,
|
|
74
|
+
`maxTokens`, `noiseClamp`, `eosThreshold`, `framesAfterEos` and
|
|
75
|
+
`outputSampleRate` are validated before native loading. Output resampling
|
|
76
|
+
supports 8–192 kHz. Other engines' sampling, emotional, speed, GPU and
|
|
77
|
+
LavaSR settings are rejected instead of silently ignored.
|
|
78
|
+
|
|
79
|
+
Fabric defaults to one sampling step, matching the native CLI and upstream
|
|
80
|
+
reference. An explicit `steps: 4` (or
|
|
81
|
+
addon `numInferenceSteps: 4`) offers a quality option at additional compute
|
|
82
|
+
cost; use `--steps 4` for an equivalent native CLI comparison.
|
|
83
|
+
|
|
84
|
+
In a listening comparison of six prompts from 0.88 to 70 seconds, the listener
|
|
85
|
+
reported an artifact on “speech” only in the one-step render of “Hello! We can
|
|
86
|
+
generate speech with Fabric.” The listener also heard the artifact in the
|
|
87
|
+
upstream PyTorch one-step reproduction with matching random inputs, while the
|
|
88
|
+
four-step render sounded clean. The faster native build and earlier build
|
|
89
|
+
produced nearly identical PCM, and matched-noise PyTorch synthesis closely
|
|
90
|
+
matched Fabric. These findings support an upstream sampling artifact for this
|
|
91
|
+
take, not a Fabric-specific one-step implementation bug. Four steps mitigated
|
|
92
|
+
this example; they are not a guarantee against all artifacts or a port fix.
|
|
93
|
+
|
|
94
|
+
On an Apple M2, the paired renders took about 20–25% longer at four steps,
|
|
95
|
+
while remaining faster than playback (the 70-second passage took 16.0 seconds
|
|
96
|
+
versus 13.0 seconds with one step, excluding model loading). These were single
|
|
97
|
+
renders after warmup, rather than repeated benchmark measurements.
|
|
98
|
+
|
|
99
|
+
### Why choose four steps for audio quality?
|
|
100
|
+
|
|
101
|
+
Pocket's flow sampler transforms noise into the next audio latent. One step
|
|
102
|
+
predicts a single update over the full interval from 0 to 1. Four steps use
|
|
103
|
+
four quarter-interval updates, evaluating the flow network again on the
|
|
104
|
+
updated latent each time. This gives the model intermediate refinement
|
|
105
|
+
opportunities instead of relying on one full-interval prediction. It is a
|
|
106
|
+
mechanistic reason to try additional steps when a take has an artifact, not
|
|
107
|
+
proof that the reported sound was caused by a particular numerical error.
|
|
108
|
+
|
|
109
|
+
The recommendation here is supported by the controlled listening result:
|
|
110
|
+
the same text, voice, seed and temperature produced an audible artifact with
|
|
111
|
+
one step in both Fabric and upstream, while the four-step take sounded clean
|
|
112
|
+
to the listener. Use `steps: 4` when avoiding this artifact matters more than
|
|
113
|
+
minimum generation latency. Keep `steps: 1` for the upstream performance
|
|
114
|
+
default. The other eleven files in the listening set had no reported artifact;
|
|
115
|
+
this is evidence of an improvement for one take, not a general quality score.
|
|
116
|
+
|
|
117
|
+
Only the flow sampling stage repeats; text/voice conditioning and Mimi audio
|
|
118
|
+
decoding are not each run four times. That is why the observed total latency
|
|
119
|
+
increase was about 20–25%, rather than fourfold:
|
|
120
|
+
|
|
121
|
+
| Passage | One-step generation | Four-step generation |
|
|
122
|
+
| --- | ---: | ---: |
|
|
123
|
+
| Original sentence (2.64 s audio) | 0.46 s | 0.56 s |
|
|
124
|
+
| Extended story (69.92 s audio) | 13.03 s | 15.96 s |
|
|
125
|
+
|
|
126
|
+
## Validation
|
|
127
|
+
|
|
128
|
+
Build the native addon with `npm run build:native`, then run `npm run lint`
|
|
129
|
+
and `npm run test:unit` in `packages/tts-ggml`. Set `QVAC_POCKET_MODEL_DIR`
|
|
130
|
+
to the converted bundle and run `npm run test:pocket` for real audio coverage.
|
|
131
|
+
Set `QVAC_POCKET_AUDIO_OUTPUT` to save a WAV. The real-model test skips if
|
|
132
|
+
its model variable is absent; pass it explicitly when validating synthesis.
|
|
133
|
+
|
|
134
|
+
The tests cover configuration, native streaming, first-audio statistics for
|
|
135
|
+
both streaming APIs, cancellation/recovery, reload, resampling and the
|
|
136
|
+
100-frame EOS-tail regression. Declaration tests cover the exported stats
|
|
137
|
+
and completion metadata on ordinary `run()` output. Physical iOS/Android
|
|
138
|
+
audio and broad subjective quality evaluations remain outside this coverage.
|
|
139
|
+
|
|
140
|
+
## Native dependencies and measured performance
|
|
141
|
+
|
|
142
|
+
The manifest requires speech-cpp and ggml-speech `2026-09-15` without changing
|
|
143
|
+
`vcpkg-configuration.json`. These registered versions select the merged source
|
|
144
|
+
commits `a976c4d63195601fcff8bfc0c2cdf3ef36fba539` (speech-cpp) and
|
|
145
|
+
`59a0ca2c2bcd7de36c27d930f79e0c94cc0c2ff3` (ggml-speech).
|
|
146
|
+
[Registry #367](https://github.com/tetherto/qvac-registry-vcpkg/pull/367)
|
|
147
|
+
contains these packages. Pocket resolves CPU memory planning through the
|
|
148
|
+
backend registry, fixing the Linux/Android unresolved `ggml_graph_plan` import.
|
|
149
|
+
The native fixes are merged in [ggml #92](https://github.com/tetherto/qvac-ext-ggml/pull/92)
|
|
150
|
+
and [speech #251](https://github.com/tetherto/qvac-fabric-speech.cpp/pull/251).
|
|
151
|
+
|
|
152
|
+
Before the addon-only review revision, these exact native port trees passed
|
|
153
|
+
[the full prebuild and integration run](https://github.com/tetherto/qvac/actions/runs/34975293366)
|
|
154
|
+
at Fabric `49d864d2e1a2070a4cb9872cd0f62319fdb9e839`: nine platform prebuilds
|
|
155
|
+
and seven desktop integration lanes covering CPU, Metal, Vulkan and CUDA.
|
|
156
|
+
That run validates the native dependencies; it does not validate subsequent
|
|
157
|
+
JavaScript review changes. The same native pins also passed all 10 Pocket
|
|
158
|
+
CTests with upstream numerical fixtures, static/dynamic CPU planner tests,
|
|
159
|
+
and 116 selected Supertonic Metal operations against CPU. Six paired
|
|
160
|
+
old-versus-merged native audio cases produced bit-identical PCM; this is
|
|
161
|
+
regression coverage, not proof of artifact-free output.
|
|
162
|
+
|
|
163
|
+
The recorded September 10 native benchmark linked the exact libraries installed
|
|
164
|
+
by the previous speech-cpp 2026-09-10#1 registry pin (`470e678f` source) and
|
|
165
|
+
normal addon build. It has not been rerun against the September 15 package;
|
|
166
|
+
current validation above covers builds and functional audio checks. It uses Apple M2 CPU, one thread per worker,
|
|
167
|
+
two workers, **one sampling step**, one warmup and three measured runs per
|
|
168
|
+
prompt, matching Fabric's default. Medians:
|
|
169
|
+
|
|
170
|
+
| Prompt | Upstream generation | Fabric generation | Upstream / Fabric audio length |
|
|
171
|
+
| --- | ---: | ---: | ---: |
|
|
172
|
+
| Short | 0.94 s | 0.98 s | 5.52 / 5.52 s |
|
|
173
|
+
| Numbers | 1.23 s | 1.42 s | 7.36 / 7.44 s |
|
|
174
|
+
| Punctuation | 1.18 s | 1.24 s | 6.88 / 6.72 s |
|
|
175
|
+
| Accented names | 1.23 s | 1.38 s | 7.20 / 7.36 s |
|
|
176
|
+
| Long | 6.16 s | 6.55 s | 36.32 / 36.48 s |
|
|
177
|
+
|
|
178
|
+
Fabric RTF is 0.178–0.191 (about 5.2–5.6× real time), versus upstream
|
|
179
|
+
0.167–0.172. Generation time is 5–16% above upstream across these prompts.
|
|
180
|
+
First audio arrives in 74–116 ms in Fabric versus 53–69 ms upstream.
|
|
181
|
+
Loading takes 0.23–0.31 seconds during the matrix; an earlier first launch
|
|
182
|
+
measured 9.61 seconds, so warmed launch timings do not establish cold-cache
|
|
183
|
+
startup performance. The previous manual development build was slower; these
|
|
184
|
+
final numbers supersede its approximately 2× slowdown.
|
|
185
|
+
These are native steady-state measurements; public SDK startup/IPC latency
|
|
186
|
+
is additional. Seeds do not produce equivalent randomness across the two
|
|
187
|
+
implementations at temperature 0.3. All ten WAVs pass signal checks. ASR
|
|
188
|
+
recovers short and punctuation prompts exactly, shares the long prompt's
|
|
189
|
+
single assistance/assistants mismatch, and finds a contraction difference in
|
|
190
|
+
the accented-name prompt. The earlier zero-temperature follow-up resolves
|
|
191
|
+
that contraction and gives matching transcriptions. No subjective listening
|
|
192
|
+
score or equal-naturalness claim is made.
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
To reproduce the packaged iOS worklet after building the simulator prebuild,
|
|
196
|
+
run `python3 scripts/run-pocket-ios-worklet.py --help`. The runner needs a
|
|
197
|
+
booted arm64 simulator, the bundle, Bare Kit, bare-pack >= 2 and bare-link >= 3.
|
|
198
|
+
It resolves pnpm package paths canonically, uses the actual TTS framework,
|
|
199
|
+
and leaves a fresh output directory containing logs, result JSON and WAV.
|
|
200
|
+
It neither boots/shuts down a simulator nor edits an app checkout. Functional
|
|
201
|
+
coverage runs with brittle's optional Node-only coverage reporter deferred.
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* MOSS-TTSD multi-speaker dialogue for @qvac/tts-ggml.
|
|
5
|
+
*
|
|
6
|
+
* Pass one reference recording per speaker, sampled at 24 kHz, in the order
|
|
7
|
+
* the text tags them with [S1], [S2], and so on. The model continues the
|
|
8
|
+
* references, so the text must open with what each recording says, under its
|
|
9
|
+
* tag, followed by the lines to generate; only the new lines come out as audio.
|
|
10
|
+
*
|
|
11
|
+
* Usage:
|
|
12
|
+
* bare examples/moss-dialogue-tts.js "<[S1] ... [S2] ...>" speaker1.wav [speaker2.wav ...]
|
|
13
|
+
*
|
|
14
|
+
* Examples:
|
|
15
|
+
* bare examples/moss-dialogue-tts.js "[S1] What alice.wav says. [S2] What bob.wav says. [S1] Did the build finish? [S2] Yes, every test passed." alice.wav bob.wav
|
|
16
|
+
* QVAC_TTS_MOSS_GPU=1 bare examples/moss-dialogue-tts.js "..." alice.wav bob.wav
|
|
17
|
+
*
|
|
18
|
+
* Expects the MOSS-TTSD backbone (moss-ttsd-f16.gguf) and both codec halves
|
|
19
|
+
* (moss-codec-decoder-f16.gguf, moss-codec-encoder-f16.gguf) under:
|
|
20
|
+
* models/
|
|
21
|
+
* Produce them with the converters in qvac-fabric-speech.cpp
|
|
22
|
+
* (engines/tts/scripts/convert-moss-delay-to-gguf.py and
|
|
23
|
+
* convert-moss-codec-to-gguf.py) until they are published to the model
|
|
24
|
+
* registry.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
const path = require('bare-path')
|
|
28
|
+
const proc = require('bare-process')
|
|
29
|
+
const TTSGgml = require('../')
|
|
30
|
+
const { createWav } = require('./wav-helper')
|
|
31
|
+
const { setLogger, releaseLogger } = require('../addonLogging')
|
|
32
|
+
|
|
33
|
+
const MOSS_SAMPLE_RATE = 24000
|
|
34
|
+
const DEFAULT_LANGUAGE = 'en'
|
|
35
|
+
|
|
36
|
+
const argv = global.Bare ? global.Bare.argv : process.argv
|
|
37
|
+
const textArg = argv[2]
|
|
38
|
+
const referenceArgs = argv.slice(3)
|
|
39
|
+
|
|
40
|
+
function fail(message) {
|
|
41
|
+
console.error(message)
|
|
42
|
+
if (global.Bare) global.Bare.exit(1)
|
|
43
|
+
else process.exit(1)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
if (!textArg || typeof textArg !== 'string' || referenceArgs.length === 0) {
|
|
47
|
+
fail('Usage: moss-dialogue-tts.js "<[S1] ... [S2] ...>" speaker1.wav [speaker2.wav ...]')
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const modelDir = path.join(__dirname, '..', 'models')
|
|
51
|
+
|
|
52
|
+
function buildModel() {
|
|
53
|
+
return new TTSGgml({
|
|
54
|
+
engine: TTSGgml.ENGINE_MOSS,
|
|
55
|
+
files: { modelDir },
|
|
56
|
+
dialogueReferences: referenceArgs.map((file) => path.resolve(file)),
|
|
57
|
+
config: {
|
|
58
|
+
language: DEFAULT_LANGUAGE,
|
|
59
|
+
useGPU: proc.env.QVAC_TTS_MOSS_GPU === '1'
|
|
60
|
+
},
|
|
61
|
+
logger: console,
|
|
62
|
+
opts: { stats: true }
|
|
63
|
+
})
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function reportStats(stats) {
|
|
67
|
+
if (!stats) return
|
|
68
|
+
console.log(
|
|
69
|
+
`Inference stats: totalTime=${stats.totalTime.toFixed(2)}s, ` +
|
|
70
|
+
`framesPerSecond=${stats.tokensPerSecond.toFixed(2)}, ` +
|
|
71
|
+
`realTimeFactor=${stats.realTimeFactor.toFixed(3)}, ` +
|
|
72
|
+
`audioDuration=${stats.audioDurationMs}ms, ` +
|
|
73
|
+
`backendDevice=${stats.backendDevice}, backendId=${stats.backendId}`
|
|
74
|
+
)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function collectPcm(response) {
|
|
78
|
+
let buffer = []
|
|
79
|
+
await response
|
|
80
|
+
.onUpdate((data) => {
|
|
81
|
+
if (data && data.outputArray) buffer = buffer.concat(Array.from(data.outputArray))
|
|
82
|
+
})
|
|
83
|
+
.await()
|
|
84
|
+
return buffer
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
async function main() {
|
|
88
|
+
setLogger((priority, message) => {
|
|
89
|
+
if (priority > 1) return
|
|
90
|
+
const names = { 0: 'ERROR', 1: 'WARNING', 2: 'INFO', 3: 'DEBUG', 4: 'OFF' }
|
|
91
|
+
const name = names[priority] || 'UNKNOWN'
|
|
92
|
+
console.log(`[${new Date().toISOString()}] [C++ log] [${name}]: ${message}`)
|
|
93
|
+
})
|
|
94
|
+
|
|
95
|
+
const outputFile = path.join(__dirname, 'moss-dialogue-output.wav')
|
|
96
|
+
const model = buildModel()
|
|
97
|
+
|
|
98
|
+
try {
|
|
99
|
+
console.log('Loading MOSS-TTSD model...')
|
|
100
|
+
await model.load()
|
|
101
|
+
console.log(`Synthesizing a ${referenceArgs.length}-speaker dialogue...`)
|
|
102
|
+
const response = await model.run({ input: textArg, type: 'text' })
|
|
103
|
+
const buffer = await collectPcm(response)
|
|
104
|
+
reportStats(response.stats)
|
|
105
|
+
createWav(buffer, MOSS_SAMPLE_RATE, outputFile)
|
|
106
|
+
console.log(`Finished writing to ${outputFile}`)
|
|
107
|
+
} finally {
|
|
108
|
+
await model.unload()
|
|
109
|
+
releaseLogger()
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
main().catch((err) => fail(err))
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* MOSS batch and streaming synthesis for @qvac/tts-ggml.
|
|
5
|
+
*
|
|
6
|
+
* MOSS Delay is an autoregressive model that predicts 32 RVQ codebooks per
|
|
7
|
+
* 80 ms frame on a delay pattern, and a transformer codec turns those codes
|
|
8
|
+
* back into 24 kHz audio. It ships as three GGUFs -- the backbone, the
|
|
9
|
+
* codec's synthesis half, and the codec's analysis half -- and text-only
|
|
10
|
+
* synthesis never touches the analysis half.
|
|
11
|
+
*
|
|
12
|
+
* Voice cloning is fully in-process: pass a reference recording and the
|
|
13
|
+
* analysis half encodes it into the prompt. Set QVAC_TTS_MOSS_STREAM_FRAMES
|
|
14
|
+
* to stream fixed-size chunks (codec frames, 12.5 per second) instead of
|
|
15
|
+
* waiting for the whole utterance.
|
|
16
|
+
*
|
|
17
|
+
* Usage:
|
|
18
|
+
* bare examples/moss-tts.js "text to synthesize" [reference.wav]
|
|
19
|
+
*
|
|
20
|
+
* Examples:
|
|
21
|
+
* bare examples/moss-tts.js "Hello from a fully on-device pipeline."
|
|
22
|
+
* bare examples/moss-tts.js "Cloned speech." voice.wav
|
|
23
|
+
* QVAC_TTS_MOSS_STREAM_FRAMES=25 bare examples/moss-tts.js "Streamed speech."
|
|
24
|
+
* QVAC_TTS_MOSS_GPU=1 bare examples/moss-tts.js "GPU synthesis."
|
|
25
|
+
* QVAC_TTS_MOSS_DURATION=38 bare examples/moss-tts.js "About three seconds [pause 0.5s] long."
|
|
26
|
+
*
|
|
27
|
+
* Expects the MOSS GGUFs (moss-tts-delay-f16.gguf,
|
|
28
|
+
* moss-codec-decoder-f16.gguf, and, to clone, moss-codec-encoder-f16.gguf)
|
|
29
|
+
* under:
|
|
30
|
+
* models/
|
|
31
|
+
* Produce them with the converters in qvac-fabric-speech.cpp
|
|
32
|
+
* (engines/tts/scripts/convert-moss-delay-to-gguf.py and
|
|
33
|
+
* convert-moss-codec-to-gguf.py) until they are published to the model
|
|
34
|
+
* registry. A reference recording must be sampled at 24 kHz.
|
|
35
|
+
*/
|
|
36
|
+
|
|
37
|
+
const path = require('bare-path')
|
|
38
|
+
const proc = require('bare-process')
|
|
39
|
+
const TTSGgml = require('../')
|
|
40
|
+
const { createWav } = require('./wav-helper')
|
|
41
|
+
const { setLogger, releaseLogger } = require('../addonLogging')
|
|
42
|
+
|
|
43
|
+
const MOSS_SAMPLE_RATE = 24000
|
|
44
|
+
const DEFAULT_LANGUAGE = 'en'
|
|
45
|
+
|
|
46
|
+
const argv = global.Bare ? global.Bare.argv : process.argv
|
|
47
|
+
const textArg = argv[2]
|
|
48
|
+
const referenceAudioArg = argv[3]
|
|
49
|
+
const streamFrames = Number(proc.env.QVAC_TTS_MOSS_STREAM_FRAMES || 0)
|
|
50
|
+
const durationTokens = Number(proc.env.QVAC_TTS_MOSS_DURATION || 0)
|
|
51
|
+
|
|
52
|
+
function fail(message) {
|
|
53
|
+
console.error(message)
|
|
54
|
+
if (global.Bare) global.Bare.exit(1)
|
|
55
|
+
else process.exit(1)
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (!textArg || typeof textArg !== 'string' || textArg.trim().length === 0) {
|
|
59
|
+
fail('Usage: moss-tts.js "<text to synthesize>" [reference.wav]')
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const pkgRoot = path.join(__dirname, '..')
|
|
63
|
+
const modelDir = path.join(pkgRoot, 'models')
|
|
64
|
+
|
|
65
|
+
function buildModel() {
|
|
66
|
+
const voice = referenceAudioArg ? { referenceAudio: path.resolve(referenceAudioArg) } : {}
|
|
67
|
+
const streaming = streamFrames > 0 ? { streamChunkTokens: streamFrames } : {}
|
|
68
|
+
const duration = durationTokens > 0 ? { durationTokens } : {}
|
|
69
|
+
return new TTSGgml({
|
|
70
|
+
engine: TTSGgml.ENGINE_MOSS,
|
|
71
|
+
files: { modelDir },
|
|
72
|
+
...voice,
|
|
73
|
+
...streaming,
|
|
74
|
+
...duration,
|
|
75
|
+
config: {
|
|
76
|
+
language: DEFAULT_LANGUAGE,
|
|
77
|
+
useGPU: proc.env.QVAC_TTS_MOSS_GPU === '1'
|
|
78
|
+
},
|
|
79
|
+
logger: console,
|
|
80
|
+
opts: { stats: true }
|
|
81
|
+
})
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function reportStats(stats) {
|
|
85
|
+
if (!stats) return
|
|
86
|
+
console.log(
|
|
87
|
+
`Inference stats: totalTime=${stats.totalTime.toFixed(2)}s, ` +
|
|
88
|
+
`framesPerSecond=${stats.tokensPerSecond.toFixed(2)}, ` +
|
|
89
|
+
`realTimeFactor=${stats.realTimeFactor.toFixed(3)}, ` +
|
|
90
|
+
`audioDuration=${stats.audioDurationMs}ms, totalSamples=${stats.totalSamples}, ` +
|
|
91
|
+
`backendDevice=${stats.backendDevice}, backendId=${stats.backendId}`
|
|
92
|
+
)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function appendChunk(buffer, data) {
|
|
96
|
+
if (!data || !data.outputArray) return buffer
|
|
97
|
+
if (streamFrames > 0) console.log(`chunk: ${data.outputArray.length} samples`)
|
|
98
|
+
return buffer.concat(Array.from(data.outputArray))
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function collectPcm(response) {
|
|
102
|
+
let buffer = []
|
|
103
|
+
await response
|
|
104
|
+
.onUpdate((data) => {
|
|
105
|
+
buffer = appendChunk(buffer, data)
|
|
106
|
+
})
|
|
107
|
+
.await()
|
|
108
|
+
return buffer
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async function main() {
|
|
112
|
+
setLogger((priority, message) => {
|
|
113
|
+
if (priority > 1) return
|
|
114
|
+
const names = { 0: 'ERROR', 1: 'WARNING', 2: 'INFO', 3: 'DEBUG', 4: 'OFF' }
|
|
115
|
+
const name = names[priority] || 'UNKNOWN'
|
|
116
|
+
console.log(`[${new Date().toISOString()}] [C++ log] [${name}]: ${message}`)
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
const outputFile = path.join(__dirname, 'moss-output.wav')
|
|
120
|
+
const model = buildModel()
|
|
121
|
+
|
|
122
|
+
try {
|
|
123
|
+
console.log('Loading MOSS TTS model...')
|
|
124
|
+
await model.load()
|
|
125
|
+
console.log('Model loaded.')
|
|
126
|
+
|
|
127
|
+
const how = referenceAudioArg ? `cloning ${referenceAudioArg}` : 'default speaker'
|
|
128
|
+
console.log(`Running TTS on: "${textArg}" (${how})`)
|
|
129
|
+
|
|
130
|
+
const response = await model.run({ input: textArg, type: 'text' })
|
|
131
|
+
const buffer = await collectPcm(response)
|
|
132
|
+
|
|
133
|
+
console.log('TTS finished!')
|
|
134
|
+
reportStats(response.stats)
|
|
135
|
+
|
|
136
|
+
console.log('\nWriting to .wav file...')
|
|
137
|
+
createWav(buffer, MOSS_SAMPLE_RATE, outputFile)
|
|
138
|
+
console.log(`Finished writing to ${outputFile}`)
|
|
139
|
+
} catch (err) {
|
|
140
|
+
console.error('Error during TTS processing:', err)
|
|
141
|
+
throw err
|
|
142
|
+
} finally {
|
|
143
|
+
console.log('Unloading model...')
|
|
144
|
+
await model.unload()
|
|
145
|
+
console.log('Model unloaded.')
|
|
146
|
+
releaseLogger()
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
main().catch((err) => {
|
|
151
|
+
console.error(err)
|
|
152
|
+
if (global.Bare) global.Bare.exit(1)
|
|
153
|
+
else process.exit(1)
|
|
154
|
+
})
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
// bare examples/pocket-tts.js /absolute/bundle "Text to speak." /absolute/output.wav
|
|
4
|
+
// Convert the bundle with the speech repository's convert-pocket-to-gguf.py.
|
|
5
|
+
const TTSGgml = require('../')
|
|
6
|
+
const { createWav } = require('./wav-helper')
|
|
7
|
+
const path = require('bare-path')
|
|
8
|
+
const process = require('bare-process')
|
|
9
|
+
global.process = process
|
|
10
|
+
|
|
11
|
+
async function main() {
|
|
12
|
+
const [, , bundle, text = 'Hello from Pocket TTS in Fabric.', output = 'pocket-output.wav'] =
|
|
13
|
+
global.Bare.argv
|
|
14
|
+
if (!bundle) throw new Error('Usage: pocket-tts.js <bundle-directory> [text] [output.wav]')
|
|
15
|
+
const model = new TTSGgml({
|
|
16
|
+
engine: TTSGgml.ENGINE_POCKET,
|
|
17
|
+
files: { modelDir: path.resolve(bundle) },
|
|
18
|
+
config: { language: 'en', useGPU: false },
|
|
19
|
+
threads: 1,
|
|
20
|
+
seed: 1234,
|
|
21
|
+
steps: 1, // Set to 4 for the slower quality option.
|
|
22
|
+
opts: { stats: true }
|
|
23
|
+
})
|
|
24
|
+
try {
|
|
25
|
+
const start = Date.now()
|
|
26
|
+
await model.load()
|
|
27
|
+
console.log('Load milliseconds:', Date.now() - start)
|
|
28
|
+
const response = await model.run({ input: text })
|
|
29
|
+
const chunks = []
|
|
30
|
+
let rate = 24000
|
|
31
|
+
for await (const chunk of response.iterate()) {
|
|
32
|
+
rate = chunk.sampleRate
|
|
33
|
+
if (chunk.outputArray.length) chunks.push(chunk.outputArray)
|
|
34
|
+
}
|
|
35
|
+
const pcm = new Int16Array(chunks.reduce((n, c) => n + c.length, 0))
|
|
36
|
+
let offset = 0
|
|
37
|
+
for (const chunk of chunks) {
|
|
38
|
+
pcm.set(chunk, offset)
|
|
39
|
+
offset += chunk.length
|
|
40
|
+
}
|
|
41
|
+
createWav(pcm, rate, path.resolve(output))
|
|
42
|
+
console.log(JSON.stringify(response.stats, null, 2))
|
|
43
|
+
console.log('Saved:', path.resolve(output))
|
|
44
|
+
} finally {
|
|
45
|
+
await model.destroy()
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
main().catch((err) => {
|
|
49
|
+
console.error(err)
|
|
50
|
+
global.Bare.exit(1)
|
|
51
|
+
})
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
'use strict'
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Supertonic (ggml) — native chunk streaming.
|
|
5
|
+
*
|
|
6
|
+
* Input is a single string; the *C++ Engine* splits it into chunks of about
|
|
7
|
+
* `streamChunkTokens` text tokens (Unicode code points), snapping each boundary
|
|
8
|
+
* to a sentence end, clause or space, synthesizes them one after another and
|
|
9
|
+
* emits each chunk's PCM to JS via `onUpdate` as soon as it is ready. Seams get
|
|
10
|
+
* a short raised-cosine fade, so audio can start playing before the whole
|
|
11
|
+
* paragraph has been rendered.
|
|
12
|
+
*
|
|
13
|
+
* Contrast with `supertonic-sentence-stream-tts.js`, which streams *sentences*
|
|
14
|
+
* in and runs one engine job per sentence; that path also works with the LavaSR
|
|
15
|
+
* enhancer / denoiser, which native streaming rejects.
|
|
16
|
+
*
|
|
17
|
+
* Usage:
|
|
18
|
+
* bare examples/supertonic-chunk-stream-tts.js [voice]
|
|
19
|
+
*
|
|
20
|
+
* Expects the Supertonic GGUF at:
|
|
21
|
+
* models/supertonic.gguf
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
const fs = require('bare-fs')
|
|
25
|
+
const path = require('bare-path')
|
|
26
|
+
const TTSGgml = require('../')
|
|
27
|
+
const { createWav } = require('./wav-helper')
|
|
28
|
+
const { setLogger, releaseLogger } = require('../addonLogging')
|
|
29
|
+
const { canPlayPcmChunks, createStreamingPlayer } = require('./pcm-chunk-player')
|
|
30
|
+
|
|
31
|
+
const SUPERTONIC_SAMPLE_RATE = 44100
|
|
32
|
+
|
|
33
|
+
const argv = global.Bare ? global.Bare.argv : process.argv
|
|
34
|
+
const voiceArg = argv[2]
|
|
35
|
+
|
|
36
|
+
const pkgRoot = path.join(__dirname, '..')
|
|
37
|
+
const modelDir = path.join(pkgRoot, 'models')
|
|
38
|
+
const supertonicModel = path.join(modelDir, 'supertonic.gguf')
|
|
39
|
+
|
|
40
|
+
if (!fs.existsSync(supertonicModel)) {
|
|
41
|
+
console.error(`Missing model file: ${supertonicModel}`)
|
|
42
|
+
console.error('Run "npm run download-models:registry" to fetch the Supertonic GGUF.')
|
|
43
|
+
if (global.Bare) global.Bare.exit(1)
|
|
44
|
+
else process.exit(1)
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async function main () {
|
|
48
|
+
setLogger((priority, message) => {
|
|
49
|
+
if (priority > 1) return
|
|
50
|
+
const names = { 0: 'ERROR', 1: 'WARNING', 2: 'INFO', 3: 'DEBUG', 4: 'OFF' }
|
|
51
|
+
const name = names[priority] || 'UNKNOWN'
|
|
52
|
+
console.log(`[${new Date().toISOString()}] [C++ log] [${name}]: ${message}`)
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
const text =
|
|
56
|
+
'Native streaming splits one request inside the engine. Each chunk is ' +
|
|
57
|
+
'synthesized and emitted as soon as it is ready, so playback can start ' +
|
|
58
|
+
'well before the whole paragraph has been rendered.'
|
|
59
|
+
|
|
60
|
+
// streamChunkTokens turns on native streaming (text tokens per chunk; about
|
|
61
|
+
// 50 suits English, 25-30 CJK). streamFirstChunkTokens keeps the first chunk
|
|
62
|
+
// small for a quick first audio; streamMinChunkTokens (engine default 30) is
|
|
63
|
+
// the floor below which the model starts dropping phonemes.
|
|
64
|
+
const model = new TTSGgml({
|
|
65
|
+
files: { supertonicModel },
|
|
66
|
+
voice: voiceArg || 'F1',
|
|
67
|
+
streamChunkTokens: 50,
|
|
68
|
+
streamFirstChunkTokens: 30,
|
|
69
|
+
config: { language: 'en' },
|
|
70
|
+
logger: console,
|
|
71
|
+
opts: { stats: true }
|
|
72
|
+
})
|
|
73
|
+
|
|
74
|
+
const outputFile = path.join(__dirname, 'supertonic-chunk-stream-output.wav')
|
|
75
|
+
|
|
76
|
+
try {
|
|
77
|
+
console.log('Loading Supertonic TTS model (native streaming)...')
|
|
78
|
+
await model.load()
|
|
79
|
+
console.log('Model loaded.')
|
|
80
|
+
|
|
81
|
+
const player = canPlayPcmChunks()
|
|
82
|
+
? createStreamingPlayer({ sampleRate: SUPERTONIC_SAMPLE_RATE })
|
|
83
|
+
: null
|
|
84
|
+
if (player) {
|
|
85
|
+
console.log(`Streaming playback via ${player.backend}: chunks flow to stdin as they arrive.`)
|
|
86
|
+
} else {
|
|
87
|
+
console.warn(
|
|
88
|
+
'No supported player found (install ffmpeg / sox / alsa-utils). Chunks will be logged only.'
|
|
89
|
+
)
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
console.log(`\nSynthesizing: "${text.slice(0, 80)}${text.length > 80 ? '…' : ''}"\n`)
|
|
93
|
+
|
|
94
|
+
const t0 = Date.now()
|
|
95
|
+
let firstChunkMs = -1
|
|
96
|
+
let pcmConcat = []
|
|
97
|
+
|
|
98
|
+
const response = await model.run({ input: text, type: 'text' })
|
|
99
|
+
|
|
100
|
+
await response
|
|
101
|
+
.onUpdate(data => {
|
|
102
|
+
if (data && data.outputArray) {
|
|
103
|
+
if (firstChunkMs < 0) firstChunkMs = Date.now() - t0
|
|
104
|
+
const samples = Array.from(data.outputArray)
|
|
105
|
+
pcmConcat = pcmConcat.concat(samples)
|
|
106
|
+
const chunkMs = (samples.length / SUPERTONIC_SAMPLE_RATE) * 1000
|
|
107
|
+
console.log(
|
|
108
|
+
`[native chunk ${data.chunkIndex}${data.isLast ? ', last' : ''}] ${samples.length} samples ` +
|
|
109
|
+
`(${chunkMs.toFixed(0)} ms of audio) at t+${Date.now() - t0} ms`
|
|
110
|
+
)
|
|
111
|
+
if (player) player.write(samples)
|
|
112
|
+
}
|
|
113
|
+
})
|
|
114
|
+
.await()
|
|
115
|
+
|
|
116
|
+
const totalMs = Date.now() - t0
|
|
117
|
+
const audioMs = (pcmConcat.length / SUPERTONIC_SAMPLE_RATE) * 1000
|
|
118
|
+
console.log(
|
|
119
|
+
`\nSynthesis done: ${pcmConcat.length} samples (${audioMs.toFixed(0)} ms of audio), ` +
|
|
120
|
+
`first-audio-out ${firstChunkMs} ms, total ${totalMs} ms, RTF ${(totalMs / audioMs).toFixed(3)}`
|
|
121
|
+
)
|
|
122
|
+
|
|
123
|
+
if (player) {
|
|
124
|
+
console.log('Waiting for playback to finish...')
|
|
125
|
+
await player.end()
|
|
126
|
+
console.log('Playback finished!')
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
if (response.stats) {
|
|
130
|
+
const s = response.stats
|
|
131
|
+
console.log(
|
|
132
|
+
`Stats: totalTime=${s.totalTime?.toFixed(2)}s rtf=${s.realTimeFactor?.toFixed(2)} ` +
|
|
133
|
+
`audio=${s.audioDurationMs?.toFixed(0)}ms samples=${s.totalSamples}`
|
|
134
|
+
)
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
if (pcmConcat.length > 0) {
|
|
138
|
+
console.log(`\nWriting concatenated PCM to ${outputFile}`)
|
|
139
|
+
createWav(pcmConcat, SUPERTONIC_SAMPLE_RATE, outputFile)
|
|
140
|
+
}
|
|
141
|
+
} catch (err) {
|
|
142
|
+
console.error('Error during TTS processing:', err)
|
|
143
|
+
throw err
|
|
144
|
+
} finally {
|
|
145
|
+
console.log('Unloading model...')
|
|
146
|
+
await model.unload()
|
|
147
|
+
console.log('Model unloaded.')
|
|
148
|
+
releaseLogger()
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
main().catch(err => {
|
|
153
|
+
console.error(err)
|
|
154
|
+
if (global.Bare) global.Bare.exit(1)
|
|
155
|
+
else process.exit(1)
|
|
156
|
+
})
|