@qvac/tts-ggml 0.9.2 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +317 -33
- package/docs/pocket-tts.md +201 -0
- package/examples/moss-dialogue-tts.js +113 -0
- package/examples/moss-tts.js +154 -0
- package/examples/pocket-tts.js +51 -0
- package/examples/supertonic-chunk-stream-tts.js +156 -0
- package/examples/supertonic-sentence-stream-tts.js +4 -4
- package/index.d.ts +352 -29
- package/index.js +698 -52
- package/lib/fit.d.ts +127 -0
- package/lib/fit.js +29 -0
- package/lib/pocketConfig.d.ts +10 -0
- package/lib/pocketConfig.js +145 -0
- package/package.json +14 -9
- package/test/integration/addon.test.js +5 -0
- package/test/integration/audio8.test.js +7 -0
- package/test/integration/chatterbox-mtl.test.js +6 -0
- package/test/integration/chatterbox-speed.test.js +13 -2
- package/test/integration/cosyvoice3-clone.test.js +16 -1
- package/test/integration/cosyvoice3-lavasr.test.js +6 -0
- package/test/integration/cosyvoice3.test.js +19 -0
- package/test/integration/fit.test.js +174 -0
- package/test/integration/gpu-smoke.test.js +13 -0
- package/test/integration/lavasr-enhancer.test.js +104 -3
- package/test/integration/moss.test.js +282 -0
- package/test/integration/multiple-runs.test.js +18 -2
- package/test/integration/output-sample-rate.test.js +6 -0
- package/test/integration/parler-wer.test.js +7 -1
- package/test/integration/parler.test.js +27 -3
- package/test/integration/pocket.integration.test.js +156 -0
- package/test/integration/supertonic-mtl.test.js +2 -0
- package/test/integration/supertonic.test.js +70 -0
- package/test/integration/supertonic3-quant.test.js +2 -0
- package/test/integration/voice-controls.test.js +56 -0
- package/test/mobile/integration.auto.cjs +16 -1
- package/test/mobile/pocket-worklet-host.cpp +92 -0
- package/test/mobile/pocket-worklet.cjs +47 -0
- package/test/mobile/test-groups.json +8 -4
- package/test/utils/runCosyvoiceTTS.js +1 -0
- package/test/utils/runTTS.js +6 -11
- package/tts.d.ts +13 -2
- package/tts.js +3 -2
|
@@ -9,10 +9,10 @@
|
|
|
9
9
|
* dispatches each `runJob` call to whichever engine the model was
|
|
10
10
|
* constructed with.
|
|
11
11
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
12
|
+
* Supertonic can also stream inside a sentence: `streamChunkTokens`
|
|
13
|
+
* makes the engine split the text into ~N-text-token chunks and emit
|
|
14
|
+
* each as it is synthesized. This sentence-level path is the one that
|
|
15
|
+
* also works with the LavaSR enhancer / denoiser.
|
|
16
16
|
*
|
|
17
17
|
* Usage:
|
|
18
18
|
* bare examples/supertonic-sentence-stream-tts.js [voice]
|
package/index.d.ts
CHANGED
|
@@ -1,12 +1,15 @@
|
|
|
1
1
|
import { type QvacResponse } from "@qvac/infer-base";
|
|
2
2
|
import * as errorModule from "./lib/error";
|
|
3
3
|
import { resolveBackendsDir as resolveBackendsDirImpl } from "./lib/backends";
|
|
4
|
+
import { assessFit as assessFitImpl, type Audio8FitRequest, type MossFitRequest, type ChatterboxFitRequest, type CosyvoiceFitRequest, type ParlerFitRequest, type SupertonicFitRequest, type TtsFitEngine, type TtsFitRequest, type TtsFitResult, type TtsFitStatus } from "./lib/fit";
|
|
4
5
|
import { type SentenceDelimiterPreset } from "./lib/textStreamAccumulator";
|
|
5
6
|
declare const ENGINE_CHATTERBOX = "chatterbox";
|
|
6
7
|
declare const ENGINE_SUPERTONIC = "supertonic";
|
|
7
8
|
declare const ENGINE_COSYVOICE3 = "cosyvoice3";
|
|
8
9
|
declare const ENGINE_PARLER = "parler";
|
|
9
10
|
declare const ENGINE_AUDIO8 = "audio8";
|
|
11
|
+
declare const ENGINE_MOSS = "moss";
|
|
12
|
+
declare const ENGINE_POCKET = "pocket";
|
|
10
13
|
declare const COSYVOICE_DIALECTS: {
|
|
11
14
|
readonly cantonese: "广东话";
|
|
12
15
|
readonly northeastern: "东北话";
|
|
@@ -54,7 +57,7 @@ declare const EMOTIONS: readonly ["command", "anger", "narration", "conversation
|
|
|
54
57
|
declare const PACES: readonly ["slow", "moderate", "fast"];
|
|
55
58
|
type Emotion = (typeof EMOTIONS)[number];
|
|
56
59
|
type Pace = (typeof PACES)[number];
|
|
57
|
-
type EngineType = typeof ENGINE_CHATTERBOX | typeof ENGINE_SUPERTONIC | typeof ENGINE_COSYVOICE3 | typeof ENGINE_PARLER | typeof ENGINE_AUDIO8;
|
|
60
|
+
type EngineType = typeof ENGINE_CHATTERBOX | typeof ENGINE_SUPERTONIC | typeof ENGINE_COSYVOICE3 | typeof ENGINE_PARLER | typeof ENGINE_AUDIO8 | typeof ENGINE_MOSS | typeof ENGINE_POCKET;
|
|
58
61
|
/**
|
|
59
62
|
* Model file paths for the GGML TTS backend. Engine is auto-detected
|
|
60
63
|
* from these fields (Chatterbox vs Supertonic) unless overridden via
|
|
@@ -62,6 +65,10 @@ type EngineType = typeof ENGINE_CHATTERBOX | typeof ENGINE_SUPERTONIC | typeof E
|
|
|
62
65
|
* through to the native layer as-is.
|
|
63
66
|
*/
|
|
64
67
|
interface TTSGgmlFiles {
|
|
68
|
+
pocketFlowModel?: string;
|
|
69
|
+
pocketMimiModel?: string;
|
|
70
|
+
pocketFrontend?: string;
|
|
71
|
+
pocketVoice?: string;
|
|
65
72
|
/**
|
|
66
73
|
* Bundle root. For Chatterbox, expected to contain
|
|
67
74
|
* `chatterbox-t3-turbo.gguf` + `chatterbox-s3gen.gguf` (turbo) or
|
|
@@ -97,6 +104,22 @@ interface TTSGgmlFiles {
|
|
|
97
104
|
*/
|
|
98
105
|
audio8CodecEncoder?: string;
|
|
99
106
|
audio8CodecEncoderPath?: string;
|
|
107
|
+
/**
|
|
108
|
+
* MOSS Delay backbone GGUF path: MOSS-TTS (`moss-tts-delay-*.gguf`) or the
|
|
109
|
+
* MOSS-TTSD dialogue checkpoint (`moss-ttsd-*.gguf`). Overrides `modelDir`.
|
|
110
|
+
*/
|
|
111
|
+
mossBackbone?: string;
|
|
112
|
+
mossBackbonePath?: string;
|
|
113
|
+
/** MOSS codec synthesis half (codes to 24 kHz wav). Overrides `modelDir`. */
|
|
114
|
+
mossCodecDecoder?: string;
|
|
115
|
+
mossCodecDecoderPath?: string;
|
|
116
|
+
/**
|
|
117
|
+
* MOSS codec analysis half (wav to codes). Only needed to clone a voice
|
|
118
|
+
* from `referenceAudio` or for `dialogueReferences`; a text-only deployment
|
|
119
|
+
* can leave it out.
|
|
120
|
+
*/
|
|
121
|
+
mossCodecEncoder?: string;
|
|
122
|
+
mossCodecEncoderPath?: string;
|
|
100
123
|
/**
|
|
101
124
|
* CosyVoice3 model directory holding the sub-model GGUFs
|
|
102
125
|
* (`cosyvoice3-{llm,flow,hift}-*.gguf`) plus `voice.gguf`, `vocab.json` and
|
|
@@ -122,6 +145,18 @@ interface TTSGgmlFiles {
|
|
|
122
145
|
cosyvoiceS3tokModelPath?: string;
|
|
123
146
|
cosyvoiceCampplusModel?: string;
|
|
124
147
|
cosyvoiceCampplusModelPath?: string;
|
|
148
|
+
/**
|
|
149
|
+
* CosyVoice3 Qwen2 BPE text frontend (`vocab.json` + `merges.txt`) and the
|
|
150
|
+
* baked default-voice GGUF (`voice.gguf`). Each is resolved under the model
|
|
151
|
+
* dir when unset; `cosyvoiceVoiceModel` is how a different baked voice is
|
|
152
|
+
* selected without a separate model directory.
|
|
153
|
+
*/
|
|
154
|
+
cosyvoiceVocab?: string;
|
|
155
|
+
cosyvoiceVocabPath?: string;
|
|
156
|
+
cosyvoiceMerges?: string;
|
|
157
|
+
cosyvoiceMergesPath?: string;
|
|
158
|
+
cosyvoiceVoiceModel?: string;
|
|
159
|
+
cosyvoiceVoiceModelPath?: string;
|
|
125
160
|
/**
|
|
126
161
|
* LavaSR enhancer GGUF: single-file Vocos bandwidth extension produced by
|
|
127
162
|
* tts-cpp/scripts/convert-lavasr-enhancer-to-gguf.py. When supplied, output
|
|
@@ -182,6 +217,8 @@ interface TTSGgmlRuntimeConfig {
|
|
|
182
217
|
* CosyVoice3 native chunk streaming emits at 24 kHz: a different rate is
|
|
183
218
|
* only accepted there when the LavaSR enhancer is active, because the
|
|
184
219
|
* enhancer's overlap-reprocess window resamples without chunk seams.
|
|
220
|
+
*
|
|
221
|
+
* MOSS emits its native 24 kHz only and rejects any other rate.
|
|
185
222
|
*/
|
|
186
223
|
outputSampleRate?: number;
|
|
187
224
|
backendsDir?: string;
|
|
@@ -266,13 +303,27 @@ interface Audio8VoiceFields {
|
|
|
266
303
|
* re-bakes the same recording (it is forwarded to the new addon instance)
|
|
267
304
|
* but cannot switch to a different one, so changing voices means a new
|
|
268
305
|
* instance. Audio8: the recording to clone, with `referenceText`
|
|
269
|
-
* alongside it.
|
|
306
|
+
* alongside it. MOSS: the recording to clone, sampled at 24 kHz and
|
|
307
|
+
* fixed at construction; needs `files.mossCodecEncoder`.
|
|
270
308
|
*/
|
|
271
309
|
referenceAudio?: string;
|
|
272
310
|
/** Audio8: what `referenceAudio` says. Required when cloning. */
|
|
273
311
|
referenceText?: string;
|
|
274
312
|
}
|
|
313
|
+
/**
|
|
314
|
+
* CosyVoice3 per-call instruction. Same values as the constructor's
|
|
315
|
+
* `instruct`; a per-call conditioning control replaces the configured one for
|
|
316
|
+
* that synthesis, and one control takes effect per synthesis.
|
|
317
|
+
*/
|
|
318
|
+
interface CosyvoiceJobFields {
|
|
319
|
+
instruct?: string | CosyvoiceInstruct;
|
|
320
|
+
}
|
|
275
321
|
interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields {
|
|
322
|
+
/** Pocket: generation/context controls; native sampling uses a portable RNG. */
|
|
323
|
+
maxTokens?: number;
|
|
324
|
+
noiseClamp?: number;
|
|
325
|
+
eosThreshold?: number;
|
|
326
|
+
framesAfterEos?: number;
|
|
276
327
|
files?: TTSGgmlFiles;
|
|
277
328
|
config?: TTSGgmlRuntimeConfig;
|
|
278
329
|
logger?: object;
|
|
@@ -281,22 +332,28 @@ interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTS
|
|
|
281
332
|
engine?: EngineType;
|
|
282
333
|
/** Chatterbox: directory of baked voice-conditioning tensors. */
|
|
283
334
|
voiceDir?: string;
|
|
284
|
-
/**
|
|
335
|
+
/**
|
|
336
|
+
* RNG seed for Chatterbox CFM/SineGen, Supertonic latent generation, or
|
|
337
|
+
* Pocket's portable sampling RNG. Pocket accepts integers from 0 to
|
|
338
|
+
* 4294967295 (inclusive).
|
|
339
|
+
*/
|
|
285
340
|
seed?: number;
|
|
286
341
|
/**
|
|
287
342
|
* Move N layers to the GPU backend. Chatterbox: pass 99 to move everything.
|
|
288
343
|
* Supertonic: pass 99 to offload on GPU-capable hosts, including Android.
|
|
289
344
|
* Audio8: pass 99 to use Vulkan on Linux and Windows.
|
|
345
|
+
* MOSS: any non-zero value selects the GPU backend.
|
|
290
346
|
* CosyVoice3: pass 99 to offload on Metal (macOS / iOS), Vulkan (desktop
|
|
291
347
|
* Linux / Windows), or OpenCL/Adreno (Android); other hosts fall back to
|
|
292
348
|
* CPU by policy.
|
|
293
349
|
*/
|
|
294
350
|
nGpuLayers?: number;
|
|
295
351
|
/**
|
|
296
|
-
* Chatterbox
|
|
352
|
+
* Chatterbox: cap on the T3 context length (prompt + generated speech
|
|
297
353
|
* tokens, 25 tokens ~= 1 second of audio). The KV cache is allocated up
|
|
298
354
|
* front at this length, so the cap directly bounds memory. Pass 0 to use
|
|
299
355
|
* the GGUF's full context; negative values are rejected.
|
|
356
|
+
* Pocket: FlowLM context capacity; accepts integers from 1 to 8192.
|
|
300
357
|
*/
|
|
301
358
|
nCtx?: number;
|
|
302
359
|
/**
|
|
@@ -309,27 +366,80 @@ interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTS
|
|
|
309
366
|
/** Override `std::thread::hardware_concurrency()`. */
|
|
310
367
|
threads?: number;
|
|
311
368
|
/**
|
|
312
|
-
*
|
|
313
|
-
*
|
|
369
|
+
* Native streaming chunk size; 0 disables. Chatterbox / CosyVoice3: speech
|
|
370
|
+
* tokens per chunk. MOSS: codec frames per chunk (12.5 per second).
|
|
371
|
+
* Supertonic: text tokens (Unicode code points) per chunk, about 50 for
|
|
372
|
+
* English (CJK is denser, 25-30). Supertonic native streaming cannot be
|
|
373
|
+
* combined with the LavaSR enhancer or denoiser.
|
|
314
374
|
*/
|
|
315
375
|
streamChunkTokens?: number;
|
|
316
376
|
/**
|
|
317
|
-
* Chatterbox / CosyVoice3 smaller first chunk for low
|
|
318
|
-
* latency.
|
|
377
|
+
* Chatterbox / CosyVoice3 / Supertonic smaller first chunk for low
|
|
378
|
+
* first-audio-out latency.
|
|
319
379
|
*/
|
|
320
380
|
streamFirstChunkTokens?: number;
|
|
321
381
|
/**
|
|
322
|
-
*
|
|
323
|
-
*
|
|
324
|
-
|
|
325
|
-
|
|
382
|
+
* Supertonic-only: boundary-snap window for clause and whitespace chunk
|
|
383
|
+
* boundaries, as a percentage of the chunk target (engine default 20).
|
|
384
|
+
*/
|
|
385
|
+
streamChunkTolerancePct?: number;
|
|
386
|
+
/**
|
|
387
|
+
* Supertonic-only: floor on every streamed chunk's size in text tokens
|
|
388
|
+
* (engine default 30); shorter trailing chunks merge into the previous one.
|
|
389
|
+
*/
|
|
390
|
+
streamMinChunkTokens?: number;
|
|
391
|
+
/**
|
|
392
|
+
* Chatterbox: sliding-window S3Gen for native streaming — keep only this many
|
|
393
|
+
* already-emitted speech tokens of left context per chunk, which bounds the
|
|
394
|
+
* per-chunk cost on long utterances (typical 25-50; 0 = cumulative prefix).
|
|
395
|
+
* CosyVoice3: reserved / not yet effective — the pinned tts-cpp engine
|
|
396
|
+
* accepts the value but does not read it.
|
|
326
397
|
*/
|
|
327
398
|
streamLeftContextTokens?: number;
|
|
328
399
|
/**
|
|
329
|
-
* Chatterbox
|
|
330
|
-
* effective — the engine runs a fixed 10-step schedule
|
|
400
|
+
* Chatterbox CFM Euler step count for native streaming chunks. CosyVoice3:
|
|
401
|
+
* reserved / not yet effective — the engine runs a fixed 10-step schedule
|
|
402
|
+
* and ignores this.
|
|
331
403
|
*/
|
|
332
404
|
cfmSteps?: number;
|
|
405
|
+
/**
|
|
406
|
+
* Chatterbox-only CFM Euler step count for batch synthesis; 0 = library
|
|
407
|
+
* default (2-step meanflow on Turbo, the model's full schedule on
|
|
408
|
+
* multilingual). The multilingual engine does not floor this, so values well
|
|
409
|
+
* below its schedule under-integrate.
|
|
410
|
+
*/
|
|
411
|
+
batchCfmSteps?: number;
|
|
412
|
+
/**
|
|
413
|
+
* Chatterbox-only cap on speech tokens T3 decodes per synthesis (25 tokens ~=
|
|
414
|
+
* 1 second); engine default 1000 (~40 s). Longer single-call audio also needs
|
|
415
|
+
* `nCtx` to cover prompt plus generated tokens, or use `maxSentenceChars`.
|
|
416
|
+
*/
|
|
417
|
+
nPredict?: number;
|
|
418
|
+
/**
|
|
419
|
+
* Chatterbox-only sentence auto-split: run T3 + S3Gen per segment of at most
|
|
420
|
+
* this many bytes (the reference CLI uses 180; 0 = off). Bounds each T3
|
|
421
|
+
* sequence, so `nPredict` applies per segment.
|
|
422
|
+
*/
|
|
423
|
+
maxSentenceChars?: number;
|
|
424
|
+
/**
|
|
425
|
+
* Chatterbox-only raised-cosine crossfade between auto-split batch segments,
|
|
426
|
+
* in milliseconds (engine default 30). Native streaming stays gapless.
|
|
427
|
+
*/
|
|
428
|
+
crossfadeMs?: number;
|
|
429
|
+
/** Chatterbox-only T3 repetition penalty (engine default 1.2; 1 = off). */
|
|
430
|
+
repeatPenalty?: number;
|
|
431
|
+
/**
|
|
432
|
+
* Chatterbox multilingual only: prosody intensity, above 0 more expressive,
|
|
433
|
+
* below 0 flatter (engine default 0.5). The Turbo variant ignores it.
|
|
434
|
+
*/
|
|
435
|
+
exaggeration?: number;
|
|
436
|
+
/**
|
|
437
|
+
* Chatterbox multilingual only: T3 classifier-free guidance scale (engine
|
|
438
|
+
* default 0.5; 0 disables). Separate from `cfgRate`, which steers S3Gen.
|
|
439
|
+
*/
|
|
440
|
+
cfgWeight?: number;
|
|
441
|
+
/** Chatterbox multilingual only: min-p sampling threshold (0 = off). */
|
|
442
|
+
minP?: number;
|
|
333
443
|
/**
|
|
334
444
|
* Chatterbox-only S3Gen classifier-free-guidance rate. The diffusion loop
|
|
335
445
|
* normally runs a batched conditioned + unconditioned pass combined by this
|
|
@@ -362,7 +472,32 @@ interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTS
|
|
|
362
472
|
voice?: string;
|
|
363
473
|
/** Alias for `voice` for compatibility with `@qvac/tts-onnx`. */
|
|
364
474
|
voiceName?: string;
|
|
365
|
-
/**
|
|
475
|
+
/**
|
|
476
|
+
* Supertonic-only: external voice JSON (`{ style_ttl, style_dp }`, e.g. a
|
|
477
|
+
* cloned voice) that overrides the baked `voice` preset. The engine checks
|
|
478
|
+
* the tensor sizes against the loaded model.
|
|
479
|
+
*/
|
|
480
|
+
voiceJsonPath?: string;
|
|
481
|
+
/**
|
|
482
|
+
* Supertonic-only: text synthesized once at load so GPU pipelines (and a
|
|
483
|
+
* Core ML vocoder sidecar) are ready before the first `run()`; skipped on a
|
|
484
|
+
* plain CPU run. Use a sentence of representative length. Wins over the
|
|
485
|
+
* `vulkanCacheDir` default pre-warm.
|
|
486
|
+
*/
|
|
487
|
+
prewarmText?: string;
|
|
488
|
+
/**
|
|
489
|
+
* Supertonic / CosyVoice3 Vulkan adapter index, also used by their LavaSR
|
|
490
|
+
* enhancer: 0 (default) = first adapter, N = the Nth, -1 = auto-pick by free
|
|
491
|
+
* VRAM preferring a discrete adapter. Only consulted on Vulkan.
|
|
492
|
+
*/
|
|
493
|
+
vulkanDevice?: number;
|
|
494
|
+
/**
|
|
495
|
+
* CosyVoice3-only: treat the baked-voice prompt frames as attention-only
|
|
496
|
+
* conditioning in the flow, for a faster flow whose output deviates slightly
|
|
497
|
+
* from the reference (engine default off).
|
|
498
|
+
*/
|
|
499
|
+
flowCutPrompt?: boolean;
|
|
500
|
+
/** Supertonic CFM steps (0 uses GGUF default); Pocket sampling steps (1–64, default 1). */
|
|
366
501
|
steps?: number;
|
|
367
502
|
/** Alias for `steps` for compatibility with `@qvac/tts-onnx`. */
|
|
368
503
|
numInferenceSteps?: number;
|
|
@@ -405,10 +540,13 @@ interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTS
|
|
|
405
540
|
/** Chatterbox MTL Cangjie TSV path for Chinese. */
|
|
406
541
|
cangjieTsvPath?: string;
|
|
407
542
|
/**
|
|
408
|
-
* Parler and
|
|
409
|
-
* defaults (Parler: temperature 1.0, top-k 50; Audio8:
|
|
410
|
-
* top-k 50, top-p 0.9
|
|
543
|
+
* Parler, Audio8 and Chatterbox sampling knobs; each unset defers to the
|
|
544
|
+
* engine's own defaults (Parler: temperature 1.0, top-k 50; Audio8:
|
|
545
|
+
* temperature 0.7, top-k 50, top-p 0.9; Chatterbox: temperature 0.8,
|
|
546
|
+
* top-k 1000, top-p 0.95, where temperature 0 decodes greedily and top-k 0
|
|
547
|
+
* disables the cutoff). Audio8 filters by top-k/top-p on the raw logits and
|
|
411
548
|
* only then applies the temperature, following its reference.
|
|
549
|
+
* Pocket: sampling temperature; accepts finite values from 0 to 10.
|
|
412
550
|
*/
|
|
413
551
|
temperature?: number;
|
|
414
552
|
topK?: number;
|
|
@@ -420,6 +558,22 @@ interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTS
|
|
|
420
558
|
maxFrames?: number;
|
|
421
559
|
/** Audio8: take the argmax instead of sampling. */
|
|
422
560
|
greedy?: boolean;
|
|
561
|
+
/**
|
|
562
|
+
* MOSS: target length in codec frames (12.5 per second), from 0 to 2015
|
|
563
|
+
* (about 161 s); 0 or unset keeps the length free. Set at construction or
|
|
564
|
+
* with `reload()`, not per call.
|
|
565
|
+
*/
|
|
566
|
+
durationTokens?: number;
|
|
567
|
+
/**
|
|
568
|
+
* MOSS-TTSD dialogue: one 24 kHz reference recording per speaker, in the
|
|
569
|
+
* order the text tags them (`[S1]`, `[S2]`, ...). The model continues the
|
|
570
|
+
* references, so the input text must open with each reference's transcript
|
|
571
|
+
* under its tag, followed by the lines to generate; sentence streaming is
|
|
572
|
+
* therefore rejected (use `run()` or `streamChunkTokens`). Needs
|
|
573
|
+
* `files.mossCodecEncoder`, excludes `referenceAudio`, and is fixed for the
|
|
574
|
+
* instance. With a `modelDir`, requires a `moss-ttsd-*.gguf` backbone.
|
|
575
|
+
*/
|
|
576
|
+
dialogueReferences?: string[];
|
|
423
577
|
minNewTokens?: number;
|
|
424
578
|
/** Parler prompt digit expansion (engine default: enabled). */
|
|
425
579
|
normalizeNumbers?: boolean;
|
|
@@ -432,16 +586,19 @@ interface InferenceState {
|
|
|
432
586
|
destroyed: boolean;
|
|
433
587
|
}
|
|
434
588
|
interface TTSOutputChunk {
|
|
589
|
+
chunkIndex?: number;
|
|
590
|
+
isLast?: boolean;
|
|
435
591
|
/** Signed 16-bit mono PCM audio payload. */
|
|
436
592
|
outputArray: Int16Array;
|
|
437
593
|
/**
|
|
438
|
-
* Output sample rate. The native engine rate (24000 for Chatterbox
|
|
439
|
-
* CosyVoice3; 44100 for Supertonic, Parler, and Audio8), or 48000
|
|
440
|
-
* LavaSR enhancer is active.
|
|
594
|
+
* Output sample rate. The native engine rate (24000 for Chatterbox,
|
|
595
|
+
* CosyVoice3 and MOSS; 44100 for Supertonic, Parler, and Audio8), or 48000
|
|
596
|
+
* when the LavaSR enhancer is active.
|
|
441
597
|
*/
|
|
442
598
|
sampleRate?: number;
|
|
443
599
|
}
|
|
444
600
|
interface RuntimeStats {
|
|
601
|
+
firstAudioMs?: number;
|
|
445
602
|
totalTime: number;
|
|
446
603
|
tokensPerSecond: number;
|
|
447
604
|
realTimeFactor: number;
|
|
@@ -455,13 +612,87 @@ interface RuntimeStats {
|
|
|
455
612
|
enhancerBackendDevice?: number;
|
|
456
613
|
/** LavaSR enhancer backend code, using the same values as `backendId`. */
|
|
457
614
|
enhancerBackendId?: number;
|
|
615
|
+
/** LavaSR denoiser compute device. -1 = not loaded, 0 = CPU, 1 = GPU. */
|
|
616
|
+
denoiserBackendDevice?: number;
|
|
617
|
+
/** LavaSR denoiser backend code, using the same values as `backendId`. */
|
|
618
|
+
denoiserBackendId?: number;
|
|
458
619
|
/** 1 when a present GPU is unsupported by engine policy; 0 otherwise. */
|
|
459
620
|
gpuUnsupported?: number;
|
|
460
621
|
/**
|
|
461
|
-
* Audio8 only: codec frames generated
|
|
462
|
-
*
|
|
622
|
+
* Audio8 and MOSS only: codec frames generated (Audio8 on a fixed 46 ms
|
|
623
|
+
* grid, MOSS at 12.5 per second). This is the unit their `tokensPerSecond`
|
|
624
|
+
* counts, in batch and in streaming alike.
|
|
463
625
|
*/
|
|
464
626
|
generatedFrames?: number;
|
|
627
|
+
/**
|
|
628
|
+
* Audio8 only, macOS / iOS: 1 while an Apple Core ML sidecar for the codec's
|
|
629
|
+
* synthesis stack is attached (a compiled `audio8-codec-decoder.mlmodelc`
|
|
630
|
+
* next to the decoder GGUF); 0 without one, or once a failing sidecar has
|
|
631
|
+
* been retired. Streams report the last chunk that supplied this field.
|
|
632
|
+
*/
|
|
633
|
+
codecSidecarLoaded?: number;
|
|
634
|
+
/**
|
|
635
|
+
* Audio8 only: 1 when this synthesis ran the codec's synthesis stack on the
|
|
636
|
+
* Apple Core ML sidecar -- a compiled `audio8-codec-decoder.mlmodelc` next
|
|
637
|
+
* to the decoder GGUF on macOS / iOS -- 0 when it ran on the ggml backend
|
|
638
|
+
* `backendId` reports (which the language model always uses). A loaded
|
|
639
|
+
* sidecar that cannot serve a call falls back to ggml and reports 0 for it.
|
|
640
|
+
* Streams report the last chunk that supplied this field, not a sum.
|
|
641
|
+
*/
|
|
642
|
+
codecOnCoreml?: number;
|
|
643
|
+
/** Chatterbox only: T3 decode wall time of the last synthesis, in ms. */
|
|
644
|
+
t3Ms?: number;
|
|
645
|
+
/** Chatterbox only: S3Gen + HiFT wall time of the last synthesis, in ms. */
|
|
646
|
+
s3genMs?: number;
|
|
647
|
+
/** Chatterbox only: speech tokens T3 emitted in the last synthesis. */
|
|
648
|
+
t3Tokens?: number;
|
|
649
|
+
/**
|
|
650
|
+
* CosyVoice3 / Audio8: the engine's own wall time for the last synthesis, in
|
|
651
|
+
* ms (the sum of its stages), excluding the addon's post-processing.
|
|
652
|
+
*/
|
|
653
|
+
stageTotalMs?: number;
|
|
654
|
+
/** CosyVoice3 stage wall times of the last synthesis, in ms. */
|
|
655
|
+
lmPrefillMs?: number;
|
|
656
|
+
lmDecodeMs?: number;
|
|
657
|
+
flowFrontendMs?: number;
|
|
658
|
+
ditEulerMs?: number;
|
|
659
|
+
hiftF0Ms?: number;
|
|
660
|
+
hiftSourceMs?: number;
|
|
661
|
+
hiftStftMs?: number;
|
|
662
|
+
hiftDecodeMs?: number;
|
|
663
|
+
/**
|
|
664
|
+
* CosyVoice3 work counters of the last synthesis: LM decode steps, speech
|
|
665
|
+
* tokens generated, flow frames in (prompt + generated), mel frames out,
|
|
666
|
+
* text ids in the LM prefill, and prompt speech tokens the LM continued from.
|
|
667
|
+
*/
|
|
668
|
+
decodeSteps?: number;
|
|
669
|
+
speechTokens?: number;
|
|
670
|
+
flowFrames?: number;
|
|
671
|
+
melFrames?: number;
|
|
672
|
+
textIds?: number;
|
|
673
|
+
promptSpeechTokens?: number;
|
|
674
|
+
/** Audio8 stage wall times of the last synthesis, in ms. */
|
|
675
|
+
voiceEncodeMs?: number;
|
|
676
|
+
promptMs?: number;
|
|
677
|
+
prefillMs?: number;
|
|
678
|
+
sampleMs?: number;
|
|
679
|
+
fastDecodeMs?: number;
|
|
680
|
+
slowDecodeMs?: number;
|
|
681
|
+
codecLatentMs?: number;
|
|
682
|
+
codecSynthMs?: number;
|
|
683
|
+
resampleMs?: number;
|
|
684
|
+
}
|
|
685
|
+
/**
|
|
686
|
+
* tts-cpp's canonical conditioning vocabulary and each engine's supported
|
|
687
|
+
* subset, keyed by engine type (`TTSGgml.getVoiceControls()`).
|
|
688
|
+
*/
|
|
689
|
+
interface VoiceControlsCatalog {
|
|
690
|
+
emotions: string[];
|
|
691
|
+
paces: string[];
|
|
692
|
+
engines: Record<string, {
|
|
693
|
+
emotions: string[];
|
|
694
|
+
paces: string[];
|
|
695
|
+
}>;
|
|
465
696
|
}
|
|
466
697
|
interface SentenceStreamChunkMeta {
|
|
467
698
|
chunkIndex?: number;
|
|
@@ -472,13 +703,13 @@ interface SentenceStreamChunkMeta {
|
|
|
472
703
|
*/
|
|
473
704
|
isLast?: boolean;
|
|
474
705
|
}
|
|
475
|
-
interface SentenceStreamOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields {
|
|
706
|
+
interface SentenceStreamOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields, CosyvoiceJobFields {
|
|
476
707
|
/** BCP-47 locale for `Intl.Segmenter` when available. */
|
|
477
708
|
locale?: string;
|
|
478
709
|
/** Maximum graphemes per chunk; defaults to 300, or 120 for Korean. */
|
|
479
710
|
maxChunkScalars?: number;
|
|
480
711
|
}
|
|
481
|
-
interface RunStreamingOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields {
|
|
712
|
+
interface RunStreamingOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields, CosyvoiceJobFields {
|
|
482
713
|
accumulateSentences?: boolean;
|
|
483
714
|
sentenceDelimiter?: RegExp;
|
|
484
715
|
sentenceDelimiterPreset?: SentenceDelimiterPreset;
|
|
@@ -487,7 +718,7 @@ interface RunStreamingOptions extends ParlerDescriptionFields, Audio8VoiceFields
|
|
|
487
718
|
}
|
|
488
719
|
/** Input accepted by `runStreaming`. */
|
|
489
720
|
type TextStreamInput = string | string[] | Iterable<string> | AsyncIterable<string>;
|
|
490
|
-
interface TTSRunInput extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields {
|
|
721
|
+
interface TTSRunInput extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields, CosyvoiceJobFields {
|
|
491
722
|
type?: string;
|
|
492
723
|
input: string;
|
|
493
724
|
streamOutput?: boolean;
|
|
@@ -501,7 +732,7 @@ interface TTSRunInput extends ParlerDescriptionFields, Audio8VoiceFields, TTSCon
|
|
|
501
732
|
}
|
|
502
733
|
/**
|
|
503
734
|
* GGML-backed TTS via the `tts-cpp` library. Wraps the chatterbox,
|
|
504
|
-
* supertonic, parler, cosyvoice3 and
|
|
735
|
+
* supertonic, parler, cosyvoice3, audio8 and moss engines behind a single
|
|
505
736
|
* engine-agnostic JavaScript surface. Engine type is auto-detected from
|
|
506
737
|
* `files` or selected explicitly with `engine`.
|
|
507
738
|
*
|
|
@@ -518,6 +749,8 @@ declare class TTSGgml {
|
|
|
518
749
|
static readonly ENGINE_COSYVOICE3 = "cosyvoice3";
|
|
519
750
|
static readonly ENGINE_PARLER = "parler";
|
|
520
751
|
static readonly ENGINE_AUDIO8 = "audio8";
|
|
752
|
+
static readonly ENGINE_MOSS = "moss";
|
|
753
|
+
static readonly ENGINE_POCKET = "pocket";
|
|
521
754
|
opts: object;
|
|
522
755
|
exclusiveRun: boolean;
|
|
523
756
|
logger: object;
|
|
@@ -528,6 +761,12 @@ declare class TTSGgml {
|
|
|
528
761
|
private _ttsInferenceQueueWaiter;
|
|
529
762
|
private _sentenceStreamCtx;
|
|
530
763
|
private _config;
|
|
764
|
+
private _pocketJobPending;
|
|
765
|
+
private _pocketCancelPromise;
|
|
766
|
+
private _pocketLifecycleInProgress;
|
|
767
|
+
private _pocketParams;
|
|
768
|
+
private _pocketOptions;
|
|
769
|
+
private _pocketFiles;
|
|
531
770
|
private _lazySessionLoading;
|
|
532
771
|
private _outputSampleRate;
|
|
533
772
|
private _engineType;
|
|
@@ -555,6 +794,23 @@ declare class TTSGgml {
|
|
|
555
794
|
private _streamLeftContextTokens?;
|
|
556
795
|
private _cfmSteps?;
|
|
557
796
|
private _cfgRate?;
|
|
797
|
+
private _batchCfmSteps?;
|
|
798
|
+
private _nPredict?;
|
|
799
|
+
private _maxSentenceChars?;
|
|
800
|
+
private _crossfadeMs?;
|
|
801
|
+
private _repeatPenalty?;
|
|
802
|
+
private _exaggeration?;
|
|
803
|
+
private _cfgWeight?;
|
|
804
|
+
private _minP?;
|
|
805
|
+
private _streamChunkTolerancePct?;
|
|
806
|
+
private _streamMinChunkTokens?;
|
|
807
|
+
private _voiceJsonPath?;
|
|
808
|
+
private _prewarmText?;
|
|
809
|
+
private _vulkanDevice?;
|
|
810
|
+
private _flowCutPrompt?;
|
|
811
|
+
private _cosyvoiceVocabPath?;
|
|
812
|
+
private _cosyvoiceMergesPath?;
|
|
813
|
+
private _cosyvoiceVoiceModelPath?;
|
|
558
814
|
private _promptText?;
|
|
559
815
|
private _instruct?;
|
|
560
816
|
private _voice?;
|
|
@@ -570,6 +826,11 @@ declare class TTSGgml {
|
|
|
570
826
|
private _audio8LmPath?;
|
|
571
827
|
private _audio8CodecDecoderPath?;
|
|
572
828
|
private _audio8CodecEncoderPath?;
|
|
829
|
+
private _mossBackbonePath?;
|
|
830
|
+
private _mossCodecDecoderPath?;
|
|
831
|
+
private _mossCodecEncoderPath?;
|
|
832
|
+
private _dialogueReferences?;
|
|
833
|
+
private _durationTokens?;
|
|
573
834
|
private _referenceText?;
|
|
574
835
|
private _greedy?;
|
|
575
836
|
private _description?;
|
|
@@ -589,12 +850,27 @@ declare class TTSGgml {
|
|
|
589
850
|
constructor(options?: TTSGgmlOptions);
|
|
590
851
|
private _resolveEngineAndModelPaths;
|
|
591
852
|
private _resolveAudio8ModelPaths;
|
|
853
|
+
private _mossBackbonePatterns;
|
|
854
|
+
private _findMossBackbone;
|
|
855
|
+
private _resolveMossModelPaths;
|
|
856
|
+
private _assignMossVoiceOptions;
|
|
592
857
|
private _assignSynthesisOptions;
|
|
593
858
|
private _assertEngineStreamingSupport;
|
|
594
859
|
private _requestsChunkStreaming;
|
|
595
860
|
private _assertParlerOptionConsistency;
|
|
596
861
|
private _assertSamplerOptionSupport;
|
|
862
|
+
/**
|
|
863
|
+
* Options only some engines read. Set on any other engine they would be
|
|
864
|
+
* dropped without effect, so they are rejected by name instead.
|
|
865
|
+
*/
|
|
866
|
+
private _assertEngineScopedOptions;
|
|
597
867
|
private _assertAudio8OptionConsistency;
|
|
868
|
+
private _assertMossOptionConsistency;
|
|
869
|
+
private _assertNoMossOnlyOptions;
|
|
870
|
+
private _assertMossDialogueReferences;
|
|
871
|
+
private _assertSentenceStreamingAllowed;
|
|
872
|
+
private _assertMossOutputRate;
|
|
873
|
+
private _assertMossVoiceConsistent;
|
|
598
874
|
/**
|
|
599
875
|
* A recording without its transcript is accepted by the model but degrades
|
|
600
876
|
* the clone silently, and a transcript alone has nothing to attach to, so
|
|
@@ -618,6 +894,12 @@ declare class TTSGgml {
|
|
|
618
894
|
* emotion/pace it declares support for, never Parler's template fields.
|
|
619
895
|
*/
|
|
620
896
|
private _resolveConditioningJobFields;
|
|
897
|
+
/**
|
|
898
|
+
* CosyVoice3's per-call instruction, rendered like the constructor's. The
|
|
899
|
+
* native layer reads it next to emotion/pace, and any of the three replaces
|
|
900
|
+
* the configured conditioning for that synthesis.
|
|
901
|
+
*/
|
|
902
|
+
private _resolveJobInstruct;
|
|
621
903
|
/**
|
|
622
904
|
* Parler's per-call surface: description/template fields, where a per-call
|
|
623
905
|
* template cannot be merged with a constructor-level free-text description.
|
|
@@ -645,6 +927,7 @@ declare class TTSGgml {
|
|
|
645
927
|
getApiDefinition(): string;
|
|
646
928
|
getState(): InferenceState;
|
|
647
929
|
load(..._args: unknown[]): Promise<void>;
|
|
930
|
+
private _loadModel;
|
|
648
931
|
/**
|
|
649
932
|
* Run text-to-speech. With `{ streamOutput: true }`, splits `input` into
|
|
650
933
|
* chunks and emits PCM through `response.onUpdate` for each chunk.
|
|
@@ -672,8 +955,8 @@ declare class TTSGgml {
|
|
|
672
955
|
private _runTextStreamOrchestrator;
|
|
673
956
|
private _sentenceStreamTextIterableDrive;
|
|
674
957
|
/**
|
|
675
|
-
* Audio8
|
|
676
|
-
* streaming aggregate has to count frames too rather than characters.
|
|
958
|
+
* Audio8 and MOSS report `tokensPerSecond` as codec frames per second, so
|
|
959
|
+
* the streaming aggregate has to count frames too rather than characters.
|
|
677
960
|
*/
|
|
678
961
|
private _pacesOnFrames;
|
|
679
962
|
private _runStreamOrchestrator;
|
|
@@ -685,6 +968,7 @@ declare class TTSGgml {
|
|
|
685
968
|
private _buildSupertonicParams;
|
|
686
969
|
private _buildParlerParams;
|
|
687
970
|
private _buildAudio8Params;
|
|
971
|
+
private _buildMossParams;
|
|
688
972
|
/** The knobs every engine in SAMPLING_ENGINES reads. */
|
|
689
973
|
private _assignSamplingParams;
|
|
690
974
|
/**
|
|
@@ -697,6 +981,7 @@ declare class TTSGgml {
|
|
|
697
981
|
private _assignLavasrParams;
|
|
698
982
|
private _createAddon;
|
|
699
983
|
unload(): Promise<void>;
|
|
984
|
+
private _unloadModel;
|
|
700
985
|
destroy(): Promise<void>;
|
|
701
986
|
private _runInternal;
|
|
702
987
|
private _mergeSentenceStreamStats;
|
|
@@ -707,12 +992,19 @@ declare class TTSGgml {
|
|
|
707
992
|
private _handleAddonOutput;
|
|
708
993
|
private _enrichStreamChunk;
|
|
709
994
|
private _handleAddonStats;
|
|
995
|
+
private _waitPocketCancel;
|
|
996
|
+
private _withPocketLifecycle;
|
|
997
|
+
private _checkPocketReload;
|
|
998
|
+
private _checkPocketRequest;
|
|
999
|
+
private _dispatchJob;
|
|
710
1000
|
cancel(): Promise<void>;
|
|
711
1001
|
private _failAndClearActiveResponse;
|
|
712
1002
|
/** Everything reload() may overwrite, so a rejected reload can undo itself. */
|
|
713
1003
|
private _captureReloadableState;
|
|
714
1004
|
private _restoreReloadableState;
|
|
715
1005
|
private _applyReloadableRuntimeConfig;
|
|
1006
|
+
private _assertMossReloadKeepsVoice;
|
|
1007
|
+
private _applyMossReload;
|
|
716
1008
|
private _applyReloadableConditioning;
|
|
717
1009
|
private _applyReloadableParlerConfig;
|
|
718
1010
|
/**
|
|
@@ -721,6 +1013,7 @@ declare class TTSGgml {
|
|
|
721
1013
|
* reload is not validated against state the caller never accepted.
|
|
722
1014
|
*/
|
|
723
1015
|
private _applyReloadableConfig;
|
|
1016
|
+
private _reloadPocket;
|
|
724
1017
|
reload(newConfig?: Record<string, unknown>): Promise<void>;
|
|
725
1018
|
/**
|
|
726
1019
|
* The voice a reload lands on. Same rule as _mergeAudio8Voice and
|
|
@@ -744,6 +1037,13 @@ declare class TTSGgml {
|
|
|
744
1037
|
*/
|
|
745
1038
|
private _applyAudio8Reload;
|
|
746
1039
|
static getModelKey(_params?: unknown): string;
|
|
1040
|
+
/**
|
|
1041
|
+
* tts-cpp's canonical emotion / pace vocabulary and each engine's supported
|
|
1042
|
+
* subset, read from the native library without loading a model. Engines are
|
|
1043
|
+
* keyed by engine type, so tts-cpp's `cosyvoice` appears as `cosyvoice3`;
|
|
1044
|
+
* an engine this package does not wrap keeps its tts-cpp name.
|
|
1045
|
+
*/
|
|
1046
|
+
static getVoiceControls(): VoiceControlsCatalog;
|
|
747
1047
|
private _requireAddon;
|
|
748
1048
|
private _optionalAddon;
|
|
749
1049
|
private _getLogger;
|
|
@@ -762,6 +1062,17 @@ type NamespaceTextStreamInput = TextStreamInput;
|
|
|
762
1062
|
type NamespaceRunInput = TTSRunInput;
|
|
763
1063
|
type NamespaceInferenceState = InferenceState;
|
|
764
1064
|
type NamespaceCosyvoiceInstruct = CosyvoiceInstruct;
|
|
1065
|
+
type NamespaceVoiceControlsCatalog = VoiceControlsCatalog;
|
|
1066
|
+
type NamespaceFitEngine = TtsFitEngine;
|
|
1067
|
+
type NamespaceFitRequest = TtsFitRequest;
|
|
1068
|
+
type NamespaceFitResult = TtsFitResult;
|
|
1069
|
+
type NamespaceFitStatus = TtsFitStatus;
|
|
1070
|
+
type NamespaceSupertonicFit = SupertonicFitRequest;
|
|
1071
|
+
type NamespaceParlerFit = ParlerFitRequest;
|
|
1072
|
+
type NamespaceChatterboxFit = ChatterboxFitRequest;
|
|
1073
|
+
type NamespaceAudio8Fit = Audio8FitRequest;
|
|
1074
|
+
type NamespaceMossFit = MossFitRequest;
|
|
1075
|
+
type NamespaceCosyvoiceFit = CosyvoiceFitRequest;
|
|
765
1076
|
declare namespace TTSGgml {
|
|
766
1077
|
export import QvacErrorAddonTTSGgml = errorModule.QvacErrorAddonTTSGgml;
|
|
767
1078
|
export import ERR_CODES = errorModule.ERR_CODES;
|
|
@@ -779,6 +1090,18 @@ declare namespace TTSGgml {
|
|
|
779
1090
|
type TTSRunInput = NamespaceRunInput;
|
|
780
1091
|
type InferenceState = NamespaceInferenceState;
|
|
781
1092
|
type CosyvoiceInstruct = NamespaceCosyvoiceInstruct;
|
|
1093
|
+
type VoiceControlsCatalog = NamespaceVoiceControlsCatalog;
|
|
1094
|
+
type TtsFitEngine = NamespaceFitEngine;
|
|
1095
|
+
type TtsFitRequest = NamespaceFitRequest;
|
|
1096
|
+
type TtsFitResult = NamespaceFitResult;
|
|
1097
|
+
type TtsFitStatus = NamespaceFitStatus;
|
|
1098
|
+
type SupertonicFitRequest = NamespaceSupertonicFit;
|
|
1099
|
+
type ParlerFitRequest = NamespaceParlerFit;
|
|
1100
|
+
type ChatterboxFitRequest = NamespaceChatterboxFit;
|
|
1101
|
+
type Audio8FitRequest = NamespaceAudio8Fit;
|
|
1102
|
+
type MossFitRequest = NamespaceMossFit;
|
|
1103
|
+
type CosyvoiceFitRequest = NamespaceCosyvoiceFit;
|
|
782
1104
|
const resolveBackendsDir: typeof resolveBackendsDirImpl;
|
|
1105
|
+
const assessFit: typeof assessFitImpl;
|
|
783
1106
|
}
|
|
784
1107
|
export = TTSGgml;
|