@qvac/tts-ggml 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -10
- package/docs/pocket-tts.md +12 -10
- package/index.d.ts +43 -1130
- package/index.js +127 -3042
- package/lib/addonEvents.d.ts +26 -0
- package/lib/addonEvents.js +51 -0
- package/lib/conditioning.d.ts +50 -0
- package/lib/conditioning.js +184 -0
- package/lib/engineDetection.d.ts +14 -0
- package/lib/engineDetection.js +110 -0
- package/lib/engineIds.d.ts +15 -0
- package/lib/engineIds.js +39 -0
- package/lib/engineParams.d.ts +3 -0
- package/lib/engineParams.js +34 -0
- package/lib/engineSetup.d.ts +153 -0
- package/lib/engineSetup.js +2 -0
- package/lib/engines/audio8.d.ts +83 -0
- package/lib/engines/audio8.js +262 -0
- package/lib/engines/chatterbox.d.ts +10 -0
- package/lib/engines/chatterbox.js +112 -0
- package/lib/engines/cosyvoice.d.ts +87 -0
- package/lib/engines/cosyvoice.js +267 -0
- package/lib/engines/moss.d.ts +26 -0
- package/lib/engines/moss.js +177 -0
- package/lib/engines/mossSoundEffect.d.ts +41 -0
- package/lib/engines/mossSoundEffect.js +117 -0
- package/lib/engines/mossSpeech.d.ts +86 -0
- package/lib/engines/mossSpeech.js +228 -0
- package/lib/engines/parler.d.ts +71 -0
- package/lib/engines/parler.js +287 -0
- package/lib/engines/supertonic.d.ts +11 -0
- package/lib/engines/supertonic.js +115 -0
- package/lib/exclusiveQueue.d.ts +16 -0
- package/lib/exclusiveQueue.js +45 -0
- package/lib/jobFields.d.ts +20 -0
- package/lib/jobFields.js +73 -0
- package/lib/modelFiles.d.ts +207 -0
- package/lib/modelFiles.js +117 -0
- package/lib/nativeParams.d.ts +12 -0
- package/lib/nativeParams.js +73 -0
- package/lib/optionValidation.d.ts +2 -0
- package/lib/optionValidation.js +103 -0
- package/lib/options.d.ts +4 -0
- package/lib/options.js +17 -0
- package/lib/pocketLifecycle.d.ts +67 -0
- package/lib/pocketLifecycle.js +170 -0
- package/lib/reload.d.ts +12 -0
- package/lib/reload.js +76 -0
- package/lib/sentenceStream.d.ts +85 -0
- package/lib/sentenceStream.js +367 -0
- package/lib/setup.d.ts +7 -0
- package/lib/setup.js +169 -0
- package/lib/streamingSupport.d.ts +8 -0
- package/lib/streamingSupport.js +77 -0
- package/lib/types.d.ts +465 -0
- package/lib/types.js +2 -0
- package/lib/voiceControls.d.ts +12 -0
- package/lib/voiceControls.js +23 -0
- package/package.json +11 -10
- package/test/integration/audio8.test.js +6 -12
- package/test/integration/gpu-smoke.test.js +2 -11
- package/test/integration/lavasr-enhancer.test.js +2 -15
- package/test/integration/moss-sfx.test.js +29 -4
- package/test/integration/moss.test.js +1 -1
- package/test/utils/desktopGpuBackend.js +21 -0
- package/test/utils/runTTS.js +1 -1
package/lib/types.d.ts
ADDED
|
@@ -0,0 +1,465 @@
|
|
|
1
|
+
import type { TTSConditioningFields } from "./conditioning";
|
|
2
|
+
import type { EngineType } from "./engineIds";
|
|
3
|
+
import type { TTSGgmlRuntimeConfig } from "./engineSetup";
|
|
4
|
+
import type { Audio8VoiceFields } from "./engines/audio8";
|
|
5
|
+
import type { CosyvoiceInstruct, CosyvoiceJobFields } from "./engines/cosyvoice";
|
|
6
|
+
import type { MossSoundEffectFields } from "./engines/mossSoundEffect";
|
|
7
|
+
import type { MossSpeechFields } from "./engines/mossSpeech";
|
|
8
|
+
import type { ParlerDescriptionFields } from "./engines/parler";
|
|
9
|
+
import type { TTSGgmlFiles } from "./modelFiles";
|
|
10
|
+
import type { SentenceDelimiterPreset } from "./textStreamAccumulator";
|
|
11
|
+
export type { Audio8VoiceFields, CosyvoiceInstruct, CosyvoiceJobFields, MossSoundEffectFields, MossSpeechFields, ParlerDescriptionFields, TTSConditioningFields, TTSGgmlFiles, TTSGgmlRuntimeConfig, };
|
|
12
|
+
export type { MossSpeechJobFields, MossSpeechMessage, MossSpeechPcm, } from "./engines/mossSpeech";
|
|
13
|
+
export type { Audio8SamplingFields } from "./engines/audio8";
|
|
14
|
+
/**
|
|
15
|
+
* LavaSR enhancer config. The discriminated `type` leaves room for future
|
|
16
|
+
* enhancer kinds; v1 ships `lavasr`. Enhancement is enabled by providing a
|
|
17
|
+
* GGUF path here or via `files.lavasrEnhancer`.
|
|
18
|
+
*/
|
|
19
|
+
export interface LavaSREnhancerOptions {
|
|
20
|
+
type: "lavasr";
|
|
21
|
+
/** Enhancer GGUF path (alternative to `files.lavasrEnhancer`). */
|
|
22
|
+
enhancerPath?: string;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* LavaSR denoiser config. V1 ships the `lavasr` UL-UNAS denoiser. Denoising
|
|
26
|
+
* is enabled by providing a GGUF path here or via `files.lavasrDenoiser`.
|
|
27
|
+
* It runs before the enhancer and preserves the sample rate.
|
|
28
|
+
*/
|
|
29
|
+
export interface LavaSRDenoiserOptions {
|
|
30
|
+
type: "lavasr";
|
|
31
|
+
/** Denoiser GGUF path (alternative to `files.lavasrDenoiser`). */
|
|
32
|
+
denoiserPath?: string;
|
|
33
|
+
}
|
|
34
|
+
export interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields {
|
|
35
|
+
/** Pocket: generation/context controls; native sampling uses a portable RNG. */
|
|
36
|
+
maxTokens?: number;
|
|
37
|
+
noiseClamp?: number;
|
|
38
|
+
eosThreshold?: number;
|
|
39
|
+
framesAfterEos?: number;
|
|
40
|
+
files?: TTSGgmlFiles;
|
|
41
|
+
config?: TTSGgmlRuntimeConfig;
|
|
42
|
+
logger?: object;
|
|
43
|
+
lazySessionLoading?: boolean;
|
|
44
|
+
/** Explicit engine selection. Auto-detected from `files` when omitted. */
|
|
45
|
+
engine?: EngineType;
|
|
46
|
+
/** Chatterbox: directory of baked voice-conditioning tensors. */
|
|
47
|
+
voiceDir?: string;
|
|
48
|
+
/**
|
|
49
|
+
* RNG seed for Chatterbox CFM/SineGen, Supertonic latent generation,
|
|
50
|
+
* the MOSS-SoundEffect diffusion noise (0 when unset, so an unseeded prompt
|
|
51
|
+
* repeats its clip), or Pocket's portable sampling RNG. Pocket accepts
|
|
52
|
+
* integers from 0 to 4294967295 (inclusive).
|
|
53
|
+
*/
|
|
54
|
+
seed?: number;
|
|
55
|
+
/**
|
|
56
|
+
* Move N layers to the GPU backend. Chatterbox: pass 99 to move everything.
|
|
57
|
+
* Supertonic: pass 99 to offload on GPU-capable hosts, including Android.
|
|
58
|
+
* Audio8: pass 99 to use Vulkan on Linux and Windows.
|
|
59
|
+
* MOSS: any non-zero value selects the GPU backend.
|
|
60
|
+
* CosyVoice3: pass 99 to offload on Metal (macOS / iOS), Vulkan (desktop
|
|
61
|
+
* Linux / Windows), or OpenCL/Adreno (Android); other hosts fall back to
|
|
62
|
+
* CPU by policy.
|
|
63
|
+
*/
|
|
64
|
+
nGpuLayers?: number;
|
|
65
|
+
/**
|
|
66
|
+
* Chatterbox: cap on the T3 context length (prompt + generated speech
|
|
67
|
+
* tokens, 25 tokens ~= 1 second of audio). The KV cache is allocated up
|
|
68
|
+
* front at this length, so the cap directly bounds memory. Pass 0 to use
|
|
69
|
+
* the GGUF's full context; negative values are rejected.
|
|
70
|
+
* Pocket: FlowLM context capacity; accepts integers from 1 to 8192.
|
|
71
|
+
*/
|
|
72
|
+
nCtx?: number;
|
|
73
|
+
/**
|
|
74
|
+
* Chatterbox-only T3 KV-cache storage dtype: `f32` | `f16` | `q8_0`.
|
|
75
|
+
* `f16` is the safe cross-backend default. `q8_0` is smaller and faster on
|
|
76
|
+
* supported backends, but is opt-in because not every backend implements
|
|
77
|
+
* its required operations.
|
|
78
|
+
*/
|
|
79
|
+
kvCacheType?: "f32" | "f16" | "q8_0";
|
|
80
|
+
/** Override `std::thread::hardware_concurrency()`. */
|
|
81
|
+
threads?: number;
|
|
82
|
+
/**
|
|
83
|
+
* Native streaming chunk size; 0 disables. Chatterbox / CosyVoice3: speech
|
|
84
|
+
* tokens per chunk. MOSS: codec frames per chunk (12.5 per second).
|
|
85
|
+
* Supertonic: text tokens (Unicode code points) per chunk, about 50 for
|
|
86
|
+
* English (CJK is denser, 25-30). Supertonic native streaming cannot be
|
|
87
|
+
* combined with the LavaSR enhancer or denoiser.
|
|
88
|
+
*/
|
|
89
|
+
streamChunkTokens?: number;
|
|
90
|
+
/**
|
|
91
|
+
* Chatterbox / CosyVoice3 / Supertonic smaller first chunk for low
|
|
92
|
+
* first-audio-out latency.
|
|
93
|
+
*/
|
|
94
|
+
streamFirstChunkTokens?: number;
|
|
95
|
+
/**
|
|
96
|
+
* Supertonic-only: boundary-snap window for clause and whitespace chunk
|
|
97
|
+
* boundaries, as a percentage of the chunk target (engine default 20).
|
|
98
|
+
*/
|
|
99
|
+
streamChunkTolerancePct?: number;
|
|
100
|
+
/**
|
|
101
|
+
* Supertonic-only: floor on every streamed chunk's size in text tokens
|
|
102
|
+
* (engine default 30); shorter trailing chunks merge into the previous one.
|
|
103
|
+
*/
|
|
104
|
+
streamMinChunkTokens?: number;
|
|
105
|
+
/**
|
|
106
|
+
* Chatterbox: sliding-window S3Gen for native streaming — keep only this many
|
|
107
|
+
* already-emitted speech tokens of left context per chunk, which bounds the
|
|
108
|
+
* per-chunk cost on long utterances (typical 25-50; 0 = cumulative prefix).
|
|
109
|
+
* CosyVoice3: reserved / not yet effective — the pinned tts-cpp engine
|
|
110
|
+
* accepts the value but does not read it.
|
|
111
|
+
*/
|
|
112
|
+
streamLeftContextTokens?: number;
|
|
113
|
+
/**
|
|
114
|
+
* Chatterbox CFM Euler step count for native streaming chunks. CosyVoice3:
|
|
115
|
+
* reserved / not yet effective — the engine runs a fixed 10-step schedule
|
|
116
|
+
* and ignores this.
|
|
117
|
+
*/
|
|
118
|
+
cfmSteps?: number;
|
|
119
|
+
/**
|
|
120
|
+
* Chatterbox-only CFM Euler step count for batch synthesis; 0 = library
|
|
121
|
+
* default (2-step meanflow on Turbo, the model's full schedule on
|
|
122
|
+
* multilingual). The multilingual engine does not floor this, so values well
|
|
123
|
+
* below its schedule under-integrate.
|
|
124
|
+
*/
|
|
125
|
+
batchCfmSteps?: number;
|
|
126
|
+
/**
|
|
127
|
+
* Chatterbox-only cap on speech tokens T3 decodes per synthesis (25 tokens ~=
|
|
128
|
+
* 1 second); engine default 1000 (~40 s). Longer single-call audio also needs
|
|
129
|
+
* `nCtx` to cover prompt plus generated tokens, or use `maxSentenceChars`.
|
|
130
|
+
*/
|
|
131
|
+
nPredict?: number;
|
|
132
|
+
/**
|
|
133
|
+
* Chatterbox-only sentence auto-split: run T3 + S3Gen per segment of at most
|
|
134
|
+
* this many bytes (the reference CLI uses 180; 0 = off). Bounds each T3
|
|
135
|
+
* sequence, so `nPredict` applies per segment.
|
|
136
|
+
*/
|
|
137
|
+
maxSentenceChars?: number;
|
|
138
|
+
/**
|
|
139
|
+
* Chatterbox-only raised-cosine crossfade between auto-split batch segments,
|
|
140
|
+
* in milliseconds (engine default 30). Native streaming stays gapless.
|
|
141
|
+
*/
|
|
142
|
+
crossfadeMs?: number;
|
|
143
|
+
/** Chatterbox-only T3 repetition penalty (engine default 1.2; 1 = off). */
|
|
144
|
+
repeatPenalty?: number;
|
|
145
|
+
/**
|
|
146
|
+
* Chatterbox multilingual only: prosody intensity, above 0 more expressive,
|
|
147
|
+
* below 0 flatter (engine default 0.5). The Turbo variant ignores it.
|
|
148
|
+
*/
|
|
149
|
+
exaggeration?: number;
|
|
150
|
+
/**
|
|
151
|
+
* Chatterbox multilingual only: T3 classifier-free guidance scale (engine
|
|
152
|
+
* default 0.5; 0 disables). Separate from `cfgRate`, which steers S3Gen.
|
|
153
|
+
*/
|
|
154
|
+
cfgWeight?: number;
|
|
155
|
+
/** Chatterbox multilingual only: min-p sampling threshold (0 = off). */
|
|
156
|
+
minP?: number;
|
|
157
|
+
/**
|
|
158
|
+
* Chatterbox-only S3Gen classifier-free-guidance rate. The diffusion loop
|
|
159
|
+
* normally runs a batched conditioned + unconditioned pass combined by this
|
|
160
|
+
* rate. `0` skips the unconditioned pass; a positive value overrides the
|
|
161
|
+
* model's baked rate. Omit it to retain the baked rate.
|
|
162
|
+
*/
|
|
163
|
+
cfgRate?: number;
|
|
164
|
+
/**
|
|
165
|
+
* CosyVoice3: verbatim transcript of `referenceAudio`, selecting the
|
|
166
|
+
* cloning mode per the upstream frontends — set it for zero-shot (the LM
|
|
167
|
+
* is prompted with transcript + reference speech tokens; best fidelity in
|
|
168
|
+
* the reference's own language), omit it for cross-lingual (timbre-only
|
|
169
|
+
* conditioning; best when synthesizing a different language than the
|
|
170
|
+
* reference). Without `referenceAudio` it still overrides the baked
|
|
171
|
+
* voice's transcript metadata for the LM prompt.
|
|
172
|
+
*/
|
|
173
|
+
promptText?: string;
|
|
174
|
+
/**
|
|
175
|
+
* CosyVoice3: natural-language control (instruct2) — Chinese dialect, emotion,
|
|
176
|
+
* speed, volume, or style. Pass a structured object (e.g. `{ dialect:
|
|
177
|
+
* 'cantonese' }`, `{ emotion: 'happy' }`) which renders to the trained
|
|
178
|
+
* instruction, or a raw string for an arbitrary instruction. Applied on top of
|
|
179
|
+
* the selected voice's timbre; one control takes effect per synthesis.
|
|
180
|
+
*/
|
|
181
|
+
instruct?: string | CosyvoiceInstruct;
|
|
182
|
+
/**
|
|
183
|
+
* Supertonic voice id baked into the GGUF, such as `F1` or `M1`. CosyVoice3:
|
|
184
|
+
* reserved / not yet effective — named-voice selection is not yet wired.
|
|
185
|
+
*/
|
|
186
|
+
voice?: string;
|
|
187
|
+
/** Alias for `voice` for compatibility with `@qvac/tts-onnx`. */
|
|
188
|
+
voiceName?: string;
|
|
189
|
+
/**
|
|
190
|
+
* Supertonic-only: external voice JSON (`{ style_ttl, style_dp }`, e.g. a
|
|
191
|
+
* cloned voice) that overrides the baked `voice` preset. The engine checks
|
|
192
|
+
* the tensor sizes against the loaded model.
|
|
193
|
+
*/
|
|
194
|
+
voiceJsonPath?: string;
|
|
195
|
+
/**
|
|
196
|
+
* Supertonic-only: text synthesized once at load so GPU pipelines (and a
|
|
197
|
+
* Core ML vocoder sidecar) are ready before the first `run()`; skipped on a
|
|
198
|
+
* plain CPU run. Use a sentence of representative length. Wins over the
|
|
199
|
+
* `vulkanCacheDir` default pre-warm.
|
|
200
|
+
*/
|
|
201
|
+
prewarmText?: string;
|
|
202
|
+
/**
|
|
203
|
+
* Supertonic / CosyVoice3 Vulkan adapter index, also used by their LavaSR
|
|
204
|
+
* enhancer: 0 (default) = first adapter, N = the Nth, -1 = auto-pick by free
|
|
205
|
+
* VRAM preferring a discrete adapter. Only consulted on Vulkan.
|
|
206
|
+
*/
|
|
207
|
+
vulkanDevice?: number;
|
|
208
|
+
/**
|
|
209
|
+
* CosyVoice3-only: treat the baked-voice prompt frames as attention-only
|
|
210
|
+
* conditioning in the flow, for a faster flow whose output deviates slightly
|
|
211
|
+
* from the reference (engine default off).
|
|
212
|
+
*/
|
|
213
|
+
flowCutPrompt?: boolean;
|
|
214
|
+
/** Supertonic CFM steps (0 uses GGUF default); Pocket sampling steps (1–64, default 1). */
|
|
215
|
+
steps?: number;
|
|
216
|
+
/** Alias for `steps` for compatibility with `@qvac/tts-onnx`. */
|
|
217
|
+
numInferenceSteps?: number;
|
|
218
|
+
/**
|
|
219
|
+
* Speech-rate / duration multiplier (1.0 = unchanged, less than 1 slower,
|
|
220
|
+
* greater than 1 faster). Supertonic scales its native duration predictor.
|
|
221
|
+
* Chatterbox applies pitch-preserving WSOLA time-stretch, bounded to
|
|
222
|
+
* [0.25, 4.0].
|
|
223
|
+
*/
|
|
224
|
+
speed?: number;
|
|
225
|
+
/** Supertonic optional `.npy` initial-noise tensor path. */
|
|
226
|
+
noiseNpyPath?: string;
|
|
227
|
+
/**
|
|
228
|
+
* LavaSR neural speech enhancement. Opt-in CPU/GGML bandwidth extension to
|
|
229
|
+
* 48 kHz, enabled by a GGUF path here or through `files.lavasrEnhancer`.
|
|
230
|
+
* Works for every engine, including the native chunk streaming of
|
|
231
|
+
* Chatterbox, Parler and CosyVoice3.
|
|
232
|
+
*/
|
|
233
|
+
enhancer?: LavaSREnhancerOptions;
|
|
234
|
+
/**
|
|
235
|
+
* LavaSR neural speech denoiser (UL-UNAS). Opt-in preprocessing that runs
|
|
236
|
+
* before the enhancer and preserves the sample rate. Enabled by a GGUF path
|
|
237
|
+
* here or through `files.lavasrDenoiser`; rejected with native chunk
|
|
238
|
+
* streaming (batch synthesis only).
|
|
239
|
+
*/
|
|
240
|
+
denoiser?: LavaSRDenoiserOptions;
|
|
241
|
+
/**
|
|
242
|
+
* Directory the addon scans for dynamically loaded ggml backends, as given.
|
|
243
|
+
* Unset scans the addon's own: the one its platform package or the app
|
|
244
|
+
* installed next to the native module.
|
|
245
|
+
*/
|
|
246
|
+
backendsDir?: string;
|
|
247
|
+
/** Directory where ggml-opencl persists its compiled program binary. */
|
|
248
|
+
openclCacheDir?: string;
|
|
249
|
+
/**
|
|
250
|
+
* Supertonic + `useGPU: true` only: directory where the Vulkan backend
|
|
251
|
+
* persists its compiled pipeline cache (`GGML_VK_PIPELINE_CACHE_DIR`).
|
|
252
|
+
* Unset means no cross-process cache or load-time pre-warm.
|
|
253
|
+
*/
|
|
254
|
+
vulkanCacheDir?: string;
|
|
255
|
+
/** Chatterbox MTL MeCab/IPAdic dictionary directory for Japanese. */
|
|
256
|
+
mecabDictPath?: string;
|
|
257
|
+
mecabDictDir?: string;
|
|
258
|
+
/** Chatterbox MTL Cangjie TSV path for Chinese. */
|
|
259
|
+
cangjieTsvPath?: string;
|
|
260
|
+
/**
|
|
261
|
+
* Parler, Audio8 and Chatterbox sampling knobs; each unset defers to the
|
|
262
|
+
* engine's own defaults (Parler: temperature 1.0, top-k 50; Audio8:
|
|
263
|
+
* temperature 0.7, top-k 50, top-p 0.9; Chatterbox: temperature 0.8,
|
|
264
|
+
* top-k 1000, top-p 0.95, where temperature 0 decodes greedily and top-k 0
|
|
265
|
+
* disables the cutoff). Audio8 filters by top-k/top-p on the raw logits and
|
|
266
|
+
* only then applies the temperature, following its reference.
|
|
267
|
+
* Pocket: sampling temperature; accepts finite values from 0 to 10.
|
|
268
|
+
*/
|
|
269
|
+
temperature?: number;
|
|
270
|
+
topK?: number;
|
|
271
|
+
topP?: number;
|
|
272
|
+
/**
|
|
273
|
+
* Generation-length cap in decoder frames; 0 = engine default. Parler runs
|
|
274
|
+
* ~86 frames/s, Audio8 ~21.5.
|
|
275
|
+
*/
|
|
276
|
+
maxFrames?: number;
|
|
277
|
+
/** Audio8: take the argmax instead of sampling. */
|
|
278
|
+
greedy?: boolean;
|
|
279
|
+
/**
|
|
280
|
+
* MOSS: target length in codec frames (12.5 per second), from 0 to 2015
|
|
281
|
+
* (about 161 s); 0 or unset keeps the length free. Set at construction or
|
|
282
|
+
* with `reload()`, not per call.
|
|
283
|
+
*/
|
|
284
|
+
durationTokens?: number;
|
|
285
|
+
/**
|
|
286
|
+
* MOSS-TTSD dialogue: one 24 kHz reference recording per speaker, in the
|
|
287
|
+
* order the text tags them (`[S1]`, `[S2]`, ...). The model continues the
|
|
288
|
+
* references, so the input text must open with each reference's transcript
|
|
289
|
+
* under its tag, followed by the lines to generate; sentence streaming is
|
|
290
|
+
* therefore rejected (use `run()` or `streamChunkTokens`). Needs
|
|
291
|
+
* `files.mossCodecEncoder`, excludes `referenceAudio`, and is fixed for the
|
|
292
|
+
* instance. With a `modelDir`, requires a `moss-ttsd-*.gguf` backbone.
|
|
293
|
+
*/
|
|
294
|
+
dialogueReferences?: string[];
|
|
295
|
+
minNewTokens?: number;
|
|
296
|
+
/** Parler prompt digit expansion (engine default: enabled). */
|
|
297
|
+
normalizeNumbers?: boolean;
|
|
298
|
+
opts?: object;
|
|
299
|
+
exclusiveRun?: boolean;
|
|
300
|
+
}
|
|
301
|
+
export interface InferenceState {
|
|
302
|
+
configLoaded: boolean;
|
|
303
|
+
weightsLoaded: boolean;
|
|
304
|
+
destroyed: boolean;
|
|
305
|
+
}
|
|
306
|
+
export interface TTSOutputChunk {
|
|
307
|
+
chunkIndex?: number;
|
|
308
|
+
isLast?: boolean;
|
|
309
|
+
/** Signed 16-bit mono PCM audio payload. */
|
|
310
|
+
outputArray: Int16Array;
|
|
311
|
+
/** MOSS-Speech: the reply's text channel (the whole answer with `textReply`). */
|
|
312
|
+
text?: string;
|
|
313
|
+
/**
|
|
314
|
+
* Output sample rate. The native engine rate (24000 for Chatterbox,
|
|
315
|
+
* CosyVoice3 and MOSS; 44100 for Supertonic, Parler, and Audio8), or 48000
|
|
316
|
+
* when the LavaSR enhancer is active.
|
|
317
|
+
*/
|
|
318
|
+
sampleRate?: number;
|
|
319
|
+
}
|
|
320
|
+
export interface NativeOutputChunk {
|
|
321
|
+
outputArray: Int16Array;
|
|
322
|
+
text?: string;
|
|
323
|
+
isLast?: boolean;
|
|
324
|
+
sampleRate?: number;
|
|
325
|
+
}
|
|
326
|
+
export interface RuntimeStats {
|
|
327
|
+
firstAudioMs?: number;
|
|
328
|
+
totalTime: number;
|
|
329
|
+
tokensPerSecond: number;
|
|
330
|
+
realTimeFactor: number;
|
|
331
|
+
audioDurationMs: number;
|
|
332
|
+
totalSamples: number;
|
|
333
|
+
/** Active compute device after load-time backend selection. 0 = CPU, 1 = GPU. */
|
|
334
|
+
backendDevice?: number;
|
|
335
|
+
/** Stable backend code: 0=CPU, 1=Metal, 2=CUDA, 3=Vulkan, 4=OpenCL, 99=other GPU. */
|
|
336
|
+
backendId?: number;
|
|
337
|
+
/** LavaSR enhancer compute device. -1 = not loaded, 0 = CPU, 1 = GPU. */
|
|
338
|
+
enhancerBackendDevice?: number;
|
|
339
|
+
/** LavaSR enhancer backend code, using the same values as `backendId`. */
|
|
340
|
+
enhancerBackendId?: number;
|
|
341
|
+
/** LavaSR denoiser compute device. -1 = not loaded, 0 = CPU, 1 = GPU. */
|
|
342
|
+
denoiserBackendDevice?: number;
|
|
343
|
+
/** LavaSR denoiser backend code, using the same values as `backendId`. */
|
|
344
|
+
denoiserBackendId?: number;
|
|
345
|
+
/** 1 when a present GPU is unsupported by engine policy; 0 otherwise. */
|
|
346
|
+
gpuUnsupported?: number;
|
|
347
|
+
/**
|
|
348
|
+
* Audio8 and MOSS only: codec frames generated (Audio8 on a fixed 46 ms
|
|
349
|
+
* grid, MOSS at 12.5 per second). This is the unit their `tokensPerSecond`
|
|
350
|
+
* counts, in batch and in streaming alike.
|
|
351
|
+
*/
|
|
352
|
+
generatedFrames?: number;
|
|
353
|
+
/**
|
|
354
|
+
* Audio8 only, macOS / iOS: 1 while an Apple Core ML sidecar for the codec's
|
|
355
|
+
* synthesis stack is attached (a compiled `audio8-codec-decoder.mlmodelc`
|
|
356
|
+
* next to the decoder GGUF); 0 without one, or once a failing sidecar has
|
|
357
|
+
* been retired. Streams report the last chunk that supplied this field.
|
|
358
|
+
*/
|
|
359
|
+
codecSidecarLoaded?: number;
|
|
360
|
+
/**
|
|
361
|
+
* Audio8 only: 1 when this synthesis ran the codec's synthesis stack on the
|
|
362
|
+
* Apple Core ML sidecar -- a compiled `audio8-codec-decoder.mlmodelc` next
|
|
363
|
+
* to the decoder GGUF on macOS / iOS -- 0 when it ran on the ggml backend
|
|
364
|
+
* `backendId` reports (which the language model always uses). A loaded
|
|
365
|
+
* sidecar that cannot serve a call falls back to ggml and reports 0 for it.
|
|
366
|
+
* Streams report the last chunk that supplied this field, not a sum.
|
|
367
|
+
*/
|
|
368
|
+
codecOnCoreml?: number;
|
|
369
|
+
/** Chatterbox only: T3 decode wall time of the last synthesis, in ms. */
|
|
370
|
+
t3Ms?: number;
|
|
371
|
+
/** Chatterbox only: S3Gen + HiFT wall time of the last synthesis, in ms. */
|
|
372
|
+
s3genMs?: number;
|
|
373
|
+
/** Chatterbox only: speech tokens T3 emitted in the last synthesis. */
|
|
374
|
+
t3Tokens?: number;
|
|
375
|
+
/**
|
|
376
|
+
* CosyVoice3 / Audio8: the engine's own wall time for the last synthesis, in
|
|
377
|
+
* ms (the sum of its stages), excluding the addon's post-processing.
|
|
378
|
+
*/
|
|
379
|
+
stageTotalMs?: number;
|
|
380
|
+
/** CosyVoice3 stage wall times of the last synthesis, in ms. */
|
|
381
|
+
lmPrefillMs?: number;
|
|
382
|
+
lmDecodeMs?: number;
|
|
383
|
+
flowFrontendMs?: number;
|
|
384
|
+
ditEulerMs?: number;
|
|
385
|
+
hiftF0Ms?: number;
|
|
386
|
+
hiftSourceMs?: number;
|
|
387
|
+
hiftStftMs?: number;
|
|
388
|
+
hiftDecodeMs?: number;
|
|
389
|
+
/**
|
|
390
|
+
* CosyVoice3 work counters of the last synthesis: LM decode steps, speech
|
|
391
|
+
* tokens generated, flow frames in (prompt + generated), mel frames out,
|
|
392
|
+
* text ids in the LM prefill, and prompt speech tokens the LM continued from.
|
|
393
|
+
*/
|
|
394
|
+
decodeSteps?: number;
|
|
395
|
+
speechTokens?: number;
|
|
396
|
+
flowFrames?: number;
|
|
397
|
+
melFrames?: number;
|
|
398
|
+
textIds?: number;
|
|
399
|
+
promptSpeechTokens?: number;
|
|
400
|
+
/** Audio8 stage wall times of the last synthesis, in ms. */
|
|
401
|
+
voiceEncodeMs?: number;
|
|
402
|
+
promptMs?: number;
|
|
403
|
+
prefillMs?: number;
|
|
404
|
+
sampleMs?: number;
|
|
405
|
+
fastDecodeMs?: number;
|
|
406
|
+
slowDecodeMs?: number;
|
|
407
|
+
codecLatentMs?: number;
|
|
408
|
+
codecSynthMs?: number;
|
|
409
|
+
resampleMs?: number;
|
|
410
|
+
}
|
|
411
|
+
/**
|
|
412
|
+
* tts-cpp's canonical conditioning vocabulary and each engine's supported
|
|
413
|
+
* subset, keyed by engine type (`TTSGgml.getVoiceControls()`).
|
|
414
|
+
*/
|
|
415
|
+
export interface VoiceControlsCatalog {
|
|
416
|
+
emotions: string[];
|
|
417
|
+
paces: string[];
|
|
418
|
+
engines: Record<string, {
|
|
419
|
+
emotions: string[];
|
|
420
|
+
paces: string[];
|
|
421
|
+
}>;
|
|
422
|
+
}
|
|
423
|
+
export interface NativeStats {
|
|
424
|
+
firstAudioMs?: number;
|
|
425
|
+
totalTime?: number;
|
|
426
|
+
audioDurationMs?: number;
|
|
427
|
+
totalSamples?: number;
|
|
428
|
+
[key: string]: unknown;
|
|
429
|
+
}
|
|
430
|
+
export interface SentenceStreamChunkMeta {
|
|
431
|
+
chunkIndex?: number;
|
|
432
|
+
sentenceChunk?: string;
|
|
433
|
+
/**
|
|
434
|
+
* True on the final pre-chunked synthesis output. Undefined for async
|
|
435
|
+
* iterator streaming where the final chunk is not known up front.
|
|
436
|
+
*/
|
|
437
|
+
isLast?: boolean;
|
|
438
|
+
}
|
|
439
|
+
export interface SentenceStreamOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields, CosyvoiceJobFields {
|
|
440
|
+
/** BCP-47 locale for `Intl.Segmenter` when available. */
|
|
441
|
+
locale?: string;
|
|
442
|
+
/** Maximum graphemes per chunk; defaults to 300, or 120 for Korean. */
|
|
443
|
+
maxChunkScalars?: number;
|
|
444
|
+
}
|
|
445
|
+
export interface RunStreamingOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields, CosyvoiceJobFields {
|
|
446
|
+
accumulateSentences?: boolean;
|
|
447
|
+
sentenceDelimiter?: RegExp;
|
|
448
|
+
sentenceDelimiterPreset?: SentenceDelimiterPreset;
|
|
449
|
+
maxBufferScalars?: number;
|
|
450
|
+
flushAfterMs?: number;
|
|
451
|
+
}
|
|
452
|
+
/** Input accepted by `runStreaming`. */
|
|
453
|
+
export type TextStreamInput = string | string[] | Iterable<string> | AsyncIterable<string>;
|
|
454
|
+
export interface TTSRunInput extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields, CosyvoiceJobFields, MossSoundEffectFields, MossSpeechFields {
|
|
455
|
+
type?: string;
|
|
456
|
+
input: string;
|
|
457
|
+
streamOutput?: boolean;
|
|
458
|
+
locale?: string;
|
|
459
|
+
maxChunkScalars?: number;
|
|
460
|
+
/**
|
|
461
|
+
* Cancels non-streaming `run()`. An already-aborted signal rejects without
|
|
462
|
+
* native dispatch. Ignored by all streaming paths.
|
|
463
|
+
*/
|
|
464
|
+
signal?: AbortSignal;
|
|
465
|
+
}
|
package/lib/types.js
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `TTSGgml.getVoiceControls()`: tts-cpp's conditioning vocabulary, re-keyed
|
|
3
|
+
* from tts-cpp's engine names to this package's engine types.
|
|
4
|
+
*/
|
|
5
|
+
import type { NativeVoiceControls } from "../tts";
|
|
6
|
+
import type { VoiceControlsCatalog } from "./types";
|
|
7
|
+
/**
|
|
8
|
+
* Copies the native catalog so callers cannot change the binding's tables.
|
|
9
|
+
* tts-cpp's `cosyvoice` appears as `cosyvoice3`; an engine this package does
|
|
10
|
+
* not wrap keeps its tts-cpp name.
|
|
11
|
+
*/
|
|
12
|
+
export declare function toVoiceControlsCatalog(native: NativeVoiceControls): VoiceControlsCatalog;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.toVoiceControlsCatalog = toVoiceControlsCatalog;
|
|
4
|
+
const engineIds_1 = require("./engineIds");
|
|
5
|
+
/**
|
|
6
|
+
* Copies the native catalog so callers cannot change the binding's tables.
|
|
7
|
+
* tts-cpp's `cosyvoice` appears as `cosyvoice3`; an engine this package does
|
|
8
|
+
* not wrap keeps its tts-cpp name.
|
|
9
|
+
*/
|
|
10
|
+
function toVoiceControlsCatalog(native) {
|
|
11
|
+
const engines = {};
|
|
12
|
+
for (const [name, controls] of Object.entries(native.engines)) {
|
|
13
|
+
engines[engineIds_1.FABRIC_ENGINE_TYPES[name] ?? name] = {
|
|
14
|
+
emotions: [...controls.emotions],
|
|
15
|
+
paces: [...controls.paces],
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
return {
|
|
19
|
+
emotions: [...native.emotions],
|
|
20
|
+
paces: [...native.paces],
|
|
21
|
+
engines,
|
|
22
|
+
};
|
|
23
|
+
}
|
package/package.json
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qvac/tts-ggml",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "Text to Speech (TTS) addon for qvac (ggml backend, wrapping the chatterbox + supertonic + parler + cosyvoice3 + audio8 + moss + moss-sfx + moss-speech + Pocket engines from tts-cpp)",
|
|
5
5
|
"qvac": {
|
|
6
|
-
"ggml-speech": "2026-10-
|
|
6
|
+
"ggml-speech": "2026-10-23"
|
|
7
7
|
},
|
|
8
8
|
"engines": {
|
|
9
9
|
"bare": ">=1.19.0"
|
|
@@ -79,7 +79,7 @@
|
|
|
79
79
|
"test:benchmark:streaming": "npm run build:ts && brittle-bare test/benchmark/streaming-benchmark.test.js",
|
|
80
80
|
"test:mobile:generate": "node scripts/generate-mobile-integration-tests.js",
|
|
81
81
|
"test:mobile:validate": "node scripts/validate-mobile-tests.js",
|
|
82
|
-
"test:cpp:build": "bare-make generate -D BUILD_TESTING=ON -D BUILD_FUZZING=OFF && bare-make build --target tts_ggml_tests
|
|
82
|
+
"test:cpp:build": "bare-make generate -D BUILD_TESTING=ON -D BUILD_FUZZING=OFF && bare-make build --target tts_ggml_tests",
|
|
83
83
|
"test:cpp:run": "cd build && LLVM_PROFILE_FILE=default.profraw ./tts_ggml_tests --gtest_output=xml:cpp-test-results.xml",
|
|
84
84
|
"test:cpp": "npm run test:cpp:build && npm run test:cpp:run",
|
|
85
85
|
"fuzz:build": "bare-make generate -D BUILD_FUZZING=ON -D BUILD_TESTING=OFF -D FUZZTEST_FUZZING_MODE=OFF && bare-make build --target tts-config-parse-fuzz",
|
|
@@ -129,6 +129,7 @@
|
|
|
129
129
|
"test/utils/mossBackend.js",
|
|
130
130
|
"test/utils/mossGpuValidation.js",
|
|
131
131
|
"test/utils/gpuOnly.js",
|
|
132
|
+
"test/utils/desktopGpuBackend.js",
|
|
132
133
|
"test/utils/testThreads.js",
|
|
133
134
|
"test/utils/perf-helper.js",
|
|
134
135
|
"test/utils/runChatterboxTTS.js",
|
|
@@ -217,16 +218,16 @@
|
|
|
217
218
|
"bare-url": "^2.4.3"
|
|
218
219
|
},
|
|
219
220
|
"optionalDependencies": {
|
|
220
|
-
"@qvac/tts-ggml-linux-x64": "0.
|
|
221
|
-
"@qvac/tts-ggml-linux-arm64": "0.
|
|
222
|
-
"@qvac/tts-ggml-darwin-arm64": "0.
|
|
223
|
-
"@qvac/tts-ggml-darwin-x64": "0.
|
|
224
|
-
"@qvac/tts-ggml-win32-x64": "0.
|
|
221
|
+
"@qvac/tts-ggml-linux-x64": "0.14.0",
|
|
222
|
+
"@qvac/tts-ggml-linux-arm64": "0.14.0",
|
|
223
|
+
"@qvac/tts-ggml-darwin-arm64": "0.14.0",
|
|
224
|
+
"@qvac/tts-ggml-darwin-x64": "0.14.0",
|
|
225
|
+
"@qvac/tts-ggml-win32-x64": "0.14.0"
|
|
225
226
|
},
|
|
226
227
|
"types": "index.d.ts",
|
|
227
228
|
"peerDependencies": {
|
|
228
|
-
"@qvac/tts-ggml-android-arm64": "0.
|
|
229
|
-
"@qvac/tts-ggml-ios": "0.
|
|
229
|
+
"@qvac/tts-ggml-android-arm64": "0.14.0",
|
|
230
|
+
"@qvac/tts-ggml-ios": "0.14.0"
|
|
230
231
|
},
|
|
231
232
|
"peerDependenciesMeta": {
|
|
232
233
|
"@qvac/tts-ggml-android-arm64": {
|
|
@@ -11,16 +11,14 @@ const { ensureAudio8Models } = require('../utils/downloadModel')
|
|
|
11
11
|
const { recordTtsStats } = require('../utils/perf-helper')
|
|
12
12
|
const { resolveRefWavPath } = require('../utils/runChatterboxTTS')
|
|
13
13
|
const { TTS_TEST_THREADS } = require('../utils/testThreads')
|
|
14
|
+
const { assertDesktopGpuBackend } = require('../utils/desktopGpuBackend')
|
|
14
15
|
|
|
15
16
|
const AUDIO8_SAMPLE_RATE = 44100
|
|
16
17
|
const AUDIO8_MAX_FRAMES = 16
|
|
17
18
|
const AUDIO8_QUANT = 'q8_0'
|
|
18
19
|
const CPU_TEST_TIMEOUT_MS = 900000
|
|
19
20
|
const GPU_TEST_TIMEOUT_MS = 600000
|
|
20
|
-
const GPU_DEVICE = 1
|
|
21
21
|
const CPU_DEVICE = 0
|
|
22
|
-
const CUDA_BACKEND = 2
|
|
23
|
-
const VULKAN_BACKEND = 3
|
|
24
22
|
const CPU_BACKEND = 0
|
|
25
23
|
const REFERENCE_TEXT =
|
|
26
24
|
'And so, my fellow Americans, ask not what your country can do for you, ask what you can do for your country.'
|
|
@@ -29,12 +27,7 @@ const platform = os.platform()
|
|
|
29
27
|
const arch = os.arch()
|
|
30
28
|
const isDesktopGpu = (platform === 'linux' || platform === 'win32') && arch === 'x64'
|
|
31
29
|
const noGpu = proc.env && proc.env.NO_GPU === 'true'
|
|
32
|
-
// CI rows that pin the engine's GPU cascade (linux prebuilds bundle CUDA and
|
|
33
|
-
// Vulkan) export TTS_CPP_GPU_BACKEND; the desktop assertion expects the pinned
|
|
34
|
-
// backend and defaults to Vulkan when unset.
|
|
35
30
|
const pinnedGpuBackend = (proc.env && proc.env.TTS_CPP_GPU_BACKEND) || ''
|
|
36
|
-
const expectedDesktopBackend = pinnedGpuBackend === 'cuda' ? CUDA_BACKEND : VULKAN_BACKEND
|
|
37
|
-
const expectedDesktopBackendName = pinnedGpuBackend === 'cuda' ? 'CUDA' : 'Vulkan'
|
|
38
31
|
|
|
39
32
|
function resolveAudio8Files(models, useModelDir) {
|
|
40
33
|
if (useModelDir) return { modelDir: models.targetDir }
|
|
@@ -190,11 +183,12 @@ test(
|
|
|
190
183
|
const started = Date.now()
|
|
191
184
|
const result = await synthesize(model, 'Audio eight uses the GPU on this desktop.')
|
|
192
185
|
assertAudio(t, 'Audio8 GPU', result)
|
|
193
|
-
|
|
194
|
-
|
|
186
|
+
assertDesktopGpuBackend(
|
|
187
|
+
t,
|
|
188
|
+
'Audio8 GPU',
|
|
189
|
+
result.stats.backendDevice,
|
|
195
190
|
result.stats.backendId,
|
|
196
|
-
|
|
197
|
-
`Audio8 GPU reports the ${expectedDesktopBackendName} backend`
|
|
191
|
+
pinnedGpuBackend
|
|
198
192
|
)
|
|
199
193
|
t.is(result.stats.gpuUnsupported, 0, 'Audio8 GPU does not report a GPU fallback')
|
|
200
194
|
recordAudio8(t, 'audio8 gpu', result, Date.now() - started)
|
|
@@ -46,15 +46,11 @@ const {
|
|
|
46
46
|
} = require('../utils/downloadModel')
|
|
47
47
|
const { recordTtsStats } = require('../utils/perf-helper')
|
|
48
48
|
const { TTS_TEST_THREADS } = require('../utils/testThreads')
|
|
49
|
+
const { assertDesktopGpuBackend } = require('../utils/desktopGpuBackend')
|
|
49
50
|
|
|
50
51
|
const platform = os.platform()
|
|
51
52
|
const isMobile = platform === 'ios' || platform === 'android'
|
|
52
53
|
const isApple = platform === 'darwin' || platform === 'ios'
|
|
53
|
-
const GPU_BACKEND_IDS = { metal: 1, cuda: 2, vulkan: 3, opencl: 4 }
|
|
54
|
-
// CI rows whose prebuild bundles more than one usable GPU backend (the linux
|
|
55
|
-
// runners carry CUDA and Vulkan) pin the engine cascade through
|
|
56
|
-
// TTS_CPP_GPU_BACKEND; the assertions expect whatever the row pinned and fall
|
|
57
|
-
// back to the platform's cascade default when unset.
|
|
58
54
|
const PINNED_GPU_BACKEND = (proc.env && proc.env.TTS_CPP_GPU_BACKEND) || ''
|
|
59
55
|
// Parler GPU coverage is validated on Apple, the Android Device Farm, and the
|
|
60
56
|
// linux CUDA lane. Keep desktop Vulkan out until dedicated runs prove it there.
|
|
@@ -153,12 +149,7 @@ function assertGpuBackend(t, engineTag, stats, allowPolicyCpu = false) {
|
|
|
153
149
|
if (platform === 'darwin' || platform === 'ios') {
|
|
154
150
|
t.is(id, 1, `${engineTag}/${platform}: expected Metal backendId=1, got ${name}`)
|
|
155
151
|
} else if (platform === 'linux' || platform === 'win32') {
|
|
156
|
-
|
|
157
|
-
t.is(
|
|
158
|
-
id,
|
|
159
|
-
expectedId,
|
|
160
|
-
`${engineTag}/${platform}: expected ${backendIdToName(expectedId)} backendId=${expectedId}, got ${name}`
|
|
161
|
-
)
|
|
152
|
+
assertDesktopGpuBackend(t, `${engineTag}/${platform}`, dev, id, PINNED_GPU_BACKEND)
|
|
162
153
|
} else if (platform === 'android') {
|
|
163
154
|
t.ok(
|
|
164
155
|
id === 3 || id === 4,
|