@qvac/tts-ggml 0.9.2 → 0.10.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/README.md +317 -33
  2. package/docs/pocket-tts.md +201 -0
  3. package/examples/moss-dialogue-tts.js +113 -0
  4. package/examples/moss-tts.js +154 -0
  5. package/examples/pocket-tts.js +51 -0
  6. package/examples/supertonic-chunk-stream-tts.js +156 -0
  7. package/examples/supertonic-sentence-stream-tts.js +4 -4
  8. package/index.d.ts +352 -29
  9. package/index.js +698 -52
  10. package/lib/fit.d.ts +127 -0
  11. package/lib/fit.js +29 -0
  12. package/lib/pocketConfig.d.ts +10 -0
  13. package/lib/pocketConfig.js +145 -0
  14. package/package.json +14 -9
  15. package/test/integration/addon.test.js +5 -0
  16. package/test/integration/audio8.test.js +7 -0
  17. package/test/integration/chatterbox-mtl.test.js +6 -0
  18. package/test/integration/chatterbox-speed.test.js +13 -2
  19. package/test/integration/cosyvoice3-clone.test.js +16 -1
  20. package/test/integration/cosyvoice3-lavasr.test.js +6 -0
  21. package/test/integration/cosyvoice3.test.js +19 -0
  22. package/test/integration/fit.test.js +174 -0
  23. package/test/integration/gpu-smoke.test.js +13 -0
  24. package/test/integration/lavasr-enhancer.test.js +104 -3
  25. package/test/integration/moss.test.js +282 -0
  26. package/test/integration/multiple-runs.test.js +18 -2
  27. package/test/integration/output-sample-rate.test.js +6 -0
  28. package/test/integration/parler-wer.test.js +7 -1
  29. package/test/integration/parler.test.js +27 -3
  30. package/test/integration/pocket.integration.test.js +156 -0
  31. package/test/integration/supertonic-mtl.test.js +2 -0
  32. package/test/integration/supertonic.test.js +70 -0
  33. package/test/integration/supertonic3-quant.test.js +2 -0
  34. package/test/integration/voice-controls.test.js +56 -0
  35. package/test/mobile/integration.auto.cjs +16 -1
  36. package/test/mobile/pocket-worklet-host.cpp +92 -0
  37. package/test/mobile/pocket-worklet.cjs +47 -0
  38. package/test/mobile/test-groups.json +8 -4
  39. package/test/utils/runCosyvoiceTTS.js +1 -0
  40. package/test/utils/runTTS.js +6 -11
  41. package/tts.d.ts +13 -2
  42. package/tts.js +3 -2
@@ -9,10 +9,10 @@
9
9
  * dispatches each `runJob` call to whichever engine the model was
10
10
  * constructed with.
11
11
  *
12
- * Sub-sentence native streaming (`streamChunkTokens`) is Chatterbox-
13
- * only at the C++ engine level; the constructor rejects those knobs
14
- * for Supertonic with a clear error. Use this sentence-level path
15
- * for low-latency Supertonic streaming.
12
+ * Supertonic can also stream inside a sentence: `streamChunkTokens`
13
+ * makes the engine split the text into ~N-text-token chunks and emit
14
+ * each as it is synthesized. This sentence-level path is the one that
15
+ * also works with the LavaSR enhancer / denoiser.
16
16
  *
17
17
  * Usage:
18
18
  * bare examples/supertonic-sentence-stream-tts.js [voice]
package/index.d.ts CHANGED
@@ -1,12 +1,15 @@
1
1
  import { type QvacResponse } from "@qvac/infer-base";
2
2
  import * as errorModule from "./lib/error";
3
3
  import { resolveBackendsDir as resolveBackendsDirImpl } from "./lib/backends";
4
+ import { assessFit as assessFitImpl, type Audio8FitRequest, type MossFitRequest, type ChatterboxFitRequest, type CosyvoiceFitRequest, type ParlerFitRequest, type SupertonicFitRequest, type TtsFitEngine, type TtsFitRequest, type TtsFitResult, type TtsFitStatus } from "./lib/fit";
4
5
  import { type SentenceDelimiterPreset } from "./lib/textStreamAccumulator";
5
6
  declare const ENGINE_CHATTERBOX = "chatterbox";
6
7
  declare const ENGINE_SUPERTONIC = "supertonic";
7
8
  declare const ENGINE_COSYVOICE3 = "cosyvoice3";
8
9
  declare const ENGINE_PARLER = "parler";
9
10
  declare const ENGINE_AUDIO8 = "audio8";
11
+ declare const ENGINE_MOSS = "moss";
12
+ declare const ENGINE_POCKET = "pocket";
10
13
  declare const COSYVOICE_DIALECTS: {
11
14
  readonly cantonese: "广东话";
12
15
  readonly northeastern: "东北话";
@@ -54,7 +57,7 @@ declare const EMOTIONS: readonly ["command", "anger", "narration", "conversation
54
57
  declare const PACES: readonly ["slow", "moderate", "fast"];
55
58
  type Emotion = (typeof EMOTIONS)[number];
56
59
  type Pace = (typeof PACES)[number];
57
- type EngineType = typeof ENGINE_CHATTERBOX | typeof ENGINE_SUPERTONIC | typeof ENGINE_COSYVOICE3 | typeof ENGINE_PARLER | typeof ENGINE_AUDIO8;
60
+ type EngineType = typeof ENGINE_CHATTERBOX | typeof ENGINE_SUPERTONIC | typeof ENGINE_COSYVOICE3 | typeof ENGINE_PARLER | typeof ENGINE_AUDIO8 | typeof ENGINE_MOSS | typeof ENGINE_POCKET;
58
61
  /**
59
62
  * Model file paths for the GGML TTS backend. Engine is auto-detected
60
63
  * from these fields (Chatterbox vs Supertonic) unless overridden via
@@ -62,6 +65,10 @@ type EngineType = typeof ENGINE_CHATTERBOX | typeof ENGINE_SUPERTONIC | typeof E
62
65
  * through to the native layer as-is.
63
66
  */
64
67
  interface TTSGgmlFiles {
68
+ pocketFlowModel?: string;
69
+ pocketMimiModel?: string;
70
+ pocketFrontend?: string;
71
+ pocketVoice?: string;
65
72
  /**
66
73
  * Bundle root. For Chatterbox, expected to contain
67
74
  * `chatterbox-t3-turbo.gguf` + `chatterbox-s3gen.gguf` (turbo) or
@@ -97,6 +104,22 @@ interface TTSGgmlFiles {
97
104
  */
98
105
  audio8CodecEncoder?: string;
99
106
  audio8CodecEncoderPath?: string;
107
+ /**
108
+ * MOSS Delay backbone GGUF path: MOSS-TTS (`moss-tts-delay-*.gguf`) or the
109
+ * MOSS-TTSD dialogue checkpoint (`moss-ttsd-*.gguf`). Overrides `modelDir`.
110
+ */
111
+ mossBackbone?: string;
112
+ mossBackbonePath?: string;
113
+ /** MOSS codec synthesis half (codes to 24 kHz wav). Overrides `modelDir`. */
114
+ mossCodecDecoder?: string;
115
+ mossCodecDecoderPath?: string;
116
+ /**
117
+ * MOSS codec analysis half (wav to codes). Only needed to clone a voice
118
+ * from `referenceAudio` or for `dialogueReferences`; a text-only deployment
119
+ * can leave it out.
120
+ */
121
+ mossCodecEncoder?: string;
122
+ mossCodecEncoderPath?: string;
100
123
  /**
101
124
  * CosyVoice3 model directory holding the sub-model GGUFs
102
125
  * (`cosyvoice3-{llm,flow,hift}-*.gguf`) plus `voice.gguf`, `vocab.json` and
@@ -122,6 +145,18 @@ interface TTSGgmlFiles {
122
145
  cosyvoiceS3tokModelPath?: string;
123
146
  cosyvoiceCampplusModel?: string;
124
147
  cosyvoiceCampplusModelPath?: string;
148
+ /**
149
+ * CosyVoice3 Qwen2 BPE text frontend (`vocab.json` + `merges.txt`) and the
150
+ * baked default-voice GGUF (`voice.gguf`). Each is resolved under the model
151
+ * dir when unset; `cosyvoiceVoiceModel` is how a different baked voice is
152
+ * selected without a separate model directory.
153
+ */
154
+ cosyvoiceVocab?: string;
155
+ cosyvoiceVocabPath?: string;
156
+ cosyvoiceMerges?: string;
157
+ cosyvoiceMergesPath?: string;
158
+ cosyvoiceVoiceModel?: string;
159
+ cosyvoiceVoiceModelPath?: string;
125
160
  /**
126
161
  * LavaSR enhancer GGUF: single-file Vocos bandwidth extension produced by
127
162
  * tts-cpp/scripts/convert-lavasr-enhancer-to-gguf.py. When supplied, output
@@ -182,6 +217,8 @@ interface TTSGgmlRuntimeConfig {
182
217
  * CosyVoice3 native chunk streaming emits at 24 kHz: a different rate is
183
218
  * only accepted there when the LavaSR enhancer is active, because the
184
219
  * enhancer's overlap-reprocess window resamples without chunk seams.
220
+ *
221
+ * MOSS emits its native 24 kHz only and rejects any other rate.
185
222
  */
186
223
  outputSampleRate?: number;
187
224
  backendsDir?: string;
@@ -266,13 +303,27 @@ interface Audio8VoiceFields {
266
303
  * re-bakes the same recording (it is forwarded to the new addon instance)
267
304
  * but cannot switch to a different one, so changing voices means a new
268
305
  * instance. Audio8: the recording to clone, with `referenceText`
269
- * alongside it.
306
+ * alongside it. MOSS: the recording to clone, sampled at 24 kHz and
307
+ * fixed at construction; needs `files.mossCodecEncoder`.
270
308
  */
271
309
  referenceAudio?: string;
272
310
  /** Audio8: what `referenceAudio` says. Required when cloning. */
273
311
  referenceText?: string;
274
312
  }
313
+ /**
314
+ * CosyVoice3 per-call instruction. Same values as the constructor's
315
+ * `instruct`; a per-call conditioning control replaces the configured one for
316
+ * that synthesis, and one control takes effect per synthesis.
317
+ */
318
+ interface CosyvoiceJobFields {
319
+ instruct?: string | CosyvoiceInstruct;
320
+ }
275
321
  interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields {
322
+ /** Pocket: generation/context controls; native sampling uses a portable RNG. */
323
+ maxTokens?: number;
324
+ noiseClamp?: number;
325
+ eosThreshold?: number;
326
+ framesAfterEos?: number;
276
327
  files?: TTSGgmlFiles;
277
328
  config?: TTSGgmlRuntimeConfig;
278
329
  logger?: object;
@@ -281,22 +332,28 @@ interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTS
281
332
  engine?: EngineType;
282
333
  /** Chatterbox: directory of baked voice-conditioning tensors. */
283
334
  voiceDir?: string;
284
- /** RNG seed for Chatterbox CFM/SineGen or Supertonic latent generation. */
335
+ /**
336
+ * RNG seed for Chatterbox CFM/SineGen, Supertonic latent generation, or
337
+ * Pocket's portable sampling RNG. Pocket accepts integers from 0 to
338
+ * 4294967295 (inclusive).
339
+ */
285
340
  seed?: number;
286
341
  /**
287
342
  * Move N layers to the GPU backend. Chatterbox: pass 99 to move everything.
288
343
  * Supertonic: pass 99 to offload on GPU-capable hosts, including Android.
289
344
  * Audio8: pass 99 to use Vulkan on Linux and Windows.
345
+ * MOSS: any non-zero value selects the GPU backend.
290
346
  * CosyVoice3: pass 99 to offload on Metal (macOS / iOS), Vulkan (desktop
291
347
  * Linux / Windows), or OpenCL/Adreno (Android); other hosts fall back to
292
348
  * CPU by policy.
293
349
  */
294
350
  nGpuLayers?: number;
295
351
  /**
296
- * Chatterbox-only cap on the T3 context length (prompt + generated speech
352
+ * Chatterbox: cap on the T3 context length (prompt + generated speech
297
353
  * tokens, 25 tokens ~= 1 second of audio). The KV cache is allocated up
298
354
  * front at this length, so the cap directly bounds memory. Pass 0 to use
299
355
  * the GGUF's full context; negative values are rejected.
356
+ * Pocket: FlowLM context capacity; accepts integers from 1 to 8192.
300
357
  */
301
358
  nCtx?: number;
302
359
  /**
@@ -309,27 +366,80 @@ interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTS
309
366
  /** Override `std::thread::hardware_concurrency()`. */
310
367
  threads?: number;
311
368
  /**
312
- * Chatterbox / CosyVoice3 speech tokens per native streaming chunk.
313
- * 0 disables.
369
+ * Native streaming chunk size; 0 disables. Chatterbox / CosyVoice3: speech
370
+ * tokens per chunk. MOSS: codec frames per chunk (12.5 per second).
371
+ * Supertonic: text tokens (Unicode code points) per chunk, about 50 for
372
+ * English (CJK is denser, 25-30). Supertonic native streaming cannot be
373
+ * combined with the LavaSR enhancer or denoiser.
314
374
  */
315
375
  streamChunkTokens?: number;
316
376
  /**
317
- * Chatterbox / CosyVoice3 smaller first chunk for low first-audio-out
318
- * latency.
377
+ * Chatterbox / CosyVoice3 / Supertonic smaller first chunk for low
378
+ * first-audio-out latency.
319
379
  */
320
380
  streamFirstChunkTokens?: number;
321
381
  /**
322
- * CosyVoice3-only: left-context speech tokens intended to be carried into each
323
- * streaming chunk. Reserved / not yet effective — the pinned tts-cpp engine
324
- * accepts the value but does not read it (true low-latency token2wav streaming
325
- * is still reserved), so setting it currently has no effect.
382
+ * Supertonic-only: boundary-snap window for clause and whitespace chunk
383
+ * boundaries, as a percentage of the chunk target (engine default 20).
384
+ */
385
+ streamChunkTolerancePct?: number;
386
+ /**
387
+ * Supertonic-only: floor on every streamed chunk's size in text tokens
388
+ * (engine default 30); shorter trailing chunks merge into the previous one.
389
+ */
390
+ streamMinChunkTokens?: number;
391
+ /**
392
+ * Chatterbox: sliding-window S3Gen for native streaming — keep only this many
393
+ * already-emitted speech tokens of left context per chunk, which bounds the
394
+ * per-chunk cost on long utterances (typical 25-50; 0 = cumulative prefix).
395
+ * CosyVoice3: reserved / not yet effective — the pinned tts-cpp engine
396
+ * accepts the value but does not read it.
326
397
  */
327
398
  streamLeftContextTokens?: number;
328
399
  /**
329
- * Chatterbox-only CFM Euler step count. CosyVoice3: reserved / not yet
330
- * effective — the engine runs a fixed 10-step schedule and ignores this.
400
+ * Chatterbox CFM Euler step count for native streaming chunks. CosyVoice3:
401
+ * reserved / not yet effective — the engine runs a fixed 10-step schedule
402
+ * and ignores this.
331
403
  */
332
404
  cfmSteps?: number;
405
+ /**
406
+ * Chatterbox-only CFM Euler step count for batch synthesis; 0 = library
407
+ * default (2-step meanflow on Turbo, the model's full schedule on
408
+ * multilingual). The multilingual engine does not floor this, so values well
409
+ * below its schedule under-integrate.
410
+ */
411
+ batchCfmSteps?: number;
412
+ /**
413
+ * Chatterbox-only cap on speech tokens T3 decodes per synthesis (25 tokens ~=
414
+ * 1 second); engine default 1000 (~40 s). Longer single-call audio also needs
415
+ * `nCtx` to cover prompt plus generated tokens, or use `maxSentenceChars`.
416
+ */
417
+ nPredict?: number;
418
+ /**
419
+ * Chatterbox-only sentence auto-split: run T3 + S3Gen per segment of at most
420
+ * this many bytes (the reference CLI uses 180; 0 = off). Bounds each T3
421
+ * sequence, so `nPredict` applies per segment.
422
+ */
423
+ maxSentenceChars?: number;
424
+ /**
425
+ * Chatterbox-only raised-cosine crossfade between auto-split batch segments,
426
+ * in milliseconds (engine default 30). Native streaming stays gapless.
427
+ */
428
+ crossfadeMs?: number;
429
+ /** Chatterbox-only T3 repetition penalty (engine default 1.2; 1 = off). */
430
+ repeatPenalty?: number;
431
+ /**
432
+ * Chatterbox multilingual only: prosody intensity, above 0 more expressive,
433
+ * below 0 flatter (engine default 0.5). The Turbo variant ignores it.
434
+ */
435
+ exaggeration?: number;
436
+ /**
437
+ * Chatterbox multilingual only: T3 classifier-free guidance scale (engine
438
+ * default 0.5; 0 disables). Separate from `cfgRate`, which steers S3Gen.
439
+ */
440
+ cfgWeight?: number;
441
+ /** Chatterbox multilingual only: min-p sampling threshold (0 = off). */
442
+ minP?: number;
333
443
  /**
334
444
  * Chatterbox-only S3Gen classifier-free-guidance rate. The diffusion loop
335
445
  * normally runs a batched conditioned + unconditioned pass combined by this
@@ -362,7 +472,32 @@ interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTS
362
472
  voice?: string;
363
473
  /** Alias for `voice` for compatibility with `@qvac/tts-onnx`. */
364
474
  voiceName?: string;
365
- /** Supertonic vector-estimator CFM steps. 0 uses the GGUF default. */
475
+ /**
476
+ * Supertonic-only: external voice JSON (`{ style_ttl, style_dp }`, e.g. a
477
+ * cloned voice) that overrides the baked `voice` preset. The engine checks
478
+ * the tensor sizes against the loaded model.
479
+ */
480
+ voiceJsonPath?: string;
481
+ /**
482
+ * Supertonic-only: text synthesized once at load so GPU pipelines (and a
483
+ * Core ML vocoder sidecar) are ready before the first `run()`; skipped on a
484
+ * plain CPU run. Use a sentence of representative length. Wins over the
485
+ * `vulkanCacheDir` default pre-warm.
486
+ */
487
+ prewarmText?: string;
488
+ /**
489
+ * Supertonic / CosyVoice3 Vulkan adapter index, also used by their LavaSR
490
+ * enhancer: 0 (default) = first adapter, N = the Nth, -1 = auto-pick by free
491
+ * VRAM preferring a discrete adapter. Only consulted on Vulkan.
492
+ */
493
+ vulkanDevice?: number;
494
+ /**
495
+ * CosyVoice3-only: treat the baked-voice prompt frames as attention-only
496
+ * conditioning in the flow, for a faster flow whose output deviates slightly
497
+ * from the reference (engine default off).
498
+ */
499
+ flowCutPrompt?: boolean;
500
+ /** Supertonic CFM steps (0 uses GGUF default); Pocket sampling steps (1–64, default 1). */
366
501
  steps?: number;
367
502
  /** Alias for `steps` for compatibility with `@qvac/tts-onnx`. */
368
503
  numInferenceSteps?: number;
@@ -405,10 +540,13 @@ interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTS
405
540
  /** Chatterbox MTL Cangjie TSV path for Chinese. */
406
541
  cangjieTsvPath?: string;
407
542
  /**
408
- * Parler and Audio8 sampling knobs; each unset defers to the engine's own
409
- * defaults (Parler: temperature 1.0, top-k 50; Audio8: temperature 0.7,
410
- * top-k 50, top-p 0.9). Audio8 filters by top-k/top-p on the raw logits and
543
+ * Parler, Audio8 and Chatterbox sampling knobs; each unset defers to the
544
+ * engine's own defaults (Parler: temperature 1.0, top-k 50; Audio8:
545
+ * temperature 0.7, top-k 50, top-p 0.9; Chatterbox: temperature 0.8,
546
+ * top-k 1000, top-p 0.95, where temperature 0 decodes greedily and top-k 0
547
+ * disables the cutoff). Audio8 filters by top-k/top-p on the raw logits and
411
548
  * only then applies the temperature, following its reference.
549
+ * Pocket: sampling temperature; accepts finite values from 0 to 10.
412
550
  */
413
551
  temperature?: number;
414
552
  topK?: number;
@@ -420,6 +558,22 @@ interface TTSGgmlOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTS
420
558
  maxFrames?: number;
421
559
  /** Audio8: take the argmax instead of sampling. */
422
560
  greedy?: boolean;
561
+ /**
562
+ * MOSS: target length in codec frames (12.5 per second), from 0 to 2015
563
+ * (about 161 s); 0 or unset keeps the length free. Set at construction or
564
+ * with `reload()`, not per call.
565
+ */
566
+ durationTokens?: number;
567
+ /**
568
+ * MOSS-TTSD dialogue: one 24 kHz reference recording per speaker, in the
569
+ * order the text tags them (`[S1]`, `[S2]`, ...). The model continues the
570
+ * references, so the input text must open with each reference's transcript
571
+ * under its tag, followed by the lines to generate; sentence streaming is
572
+ * therefore rejected (use `run()` or `streamChunkTokens`). Needs
573
+ * `files.mossCodecEncoder`, excludes `referenceAudio`, and is fixed for the
574
+ * instance. With a `modelDir`, requires a `moss-ttsd-*.gguf` backbone.
575
+ */
576
+ dialogueReferences?: string[];
423
577
  minNewTokens?: number;
424
578
  /** Parler prompt digit expansion (engine default: enabled). */
425
579
  normalizeNumbers?: boolean;
@@ -432,16 +586,19 @@ interface InferenceState {
432
586
  destroyed: boolean;
433
587
  }
434
588
  interface TTSOutputChunk {
589
+ chunkIndex?: number;
590
+ isLast?: boolean;
435
591
  /** Signed 16-bit mono PCM audio payload. */
436
592
  outputArray: Int16Array;
437
593
  /**
438
- * Output sample rate. The native engine rate (24000 for Chatterbox and
439
- * CosyVoice3; 44100 for Supertonic, Parler, and Audio8), or 48000 when the
440
- * LavaSR enhancer is active.
594
+ * Output sample rate. The native engine rate (24000 for Chatterbox,
595
+ * CosyVoice3 and MOSS; 44100 for Supertonic, Parler, and Audio8), or 48000
596
+ * when the LavaSR enhancer is active.
441
597
  */
442
598
  sampleRate?: number;
443
599
  }
444
600
  interface RuntimeStats {
601
+ firstAudioMs?: number;
445
602
  totalTime: number;
446
603
  tokensPerSecond: number;
447
604
  realTimeFactor: number;
@@ -455,13 +612,87 @@ interface RuntimeStats {
455
612
  enhancerBackendDevice?: number;
456
613
  /** LavaSR enhancer backend code, using the same values as `backendId`. */
457
614
  enhancerBackendId?: number;
615
+ /** LavaSR denoiser compute device. -1 = not loaded, 0 = CPU, 1 = GPU. */
616
+ denoiserBackendDevice?: number;
617
+ /** LavaSR denoiser backend code, using the same values as `backendId`. */
618
+ denoiserBackendId?: number;
458
619
  /** 1 when a present GPU is unsupported by engine policy; 0 otherwise. */
459
620
  gpuUnsupported?: number;
460
621
  /**
461
- * Audio8 only: codec frames generated, on a fixed 46 ms grid. This is the
462
- * unit its `tokensPerSecond` counts, in batch and in streaming alike.
622
+ * Audio8 and MOSS only: codec frames generated (Audio8 on a fixed 46 ms
623
+ * grid, MOSS at 12.5 per second). This is the unit their `tokensPerSecond`
624
+ * counts, in batch and in streaming alike.
463
625
  */
464
626
  generatedFrames?: number;
627
+ /**
628
+ * Audio8 only, macOS / iOS: 1 while an Apple Core ML sidecar for the codec's
629
+ * synthesis stack is attached (a compiled `audio8-codec-decoder.mlmodelc`
630
+ * next to the decoder GGUF); 0 without one, or once a failing sidecar has
631
+ * been retired. Streams report the last chunk that supplied this field.
632
+ */
633
+ codecSidecarLoaded?: number;
634
+ /**
635
+ * Audio8 only: 1 when this synthesis ran the codec's synthesis stack on the
636
+ * Apple Core ML sidecar -- a compiled `audio8-codec-decoder.mlmodelc` next
637
+ * to the decoder GGUF on macOS / iOS -- 0 when it ran on the ggml backend
638
+ * `backendId` reports (which the language model always uses). A loaded
639
+ * sidecar that cannot serve a call falls back to ggml and reports 0 for it.
640
+ * Streams report the last chunk that supplied this field, not a sum.
641
+ */
642
+ codecOnCoreml?: number;
643
+ /** Chatterbox only: T3 decode wall time of the last synthesis, in ms. */
644
+ t3Ms?: number;
645
+ /** Chatterbox only: S3Gen + HiFT wall time of the last synthesis, in ms. */
646
+ s3genMs?: number;
647
+ /** Chatterbox only: speech tokens T3 emitted in the last synthesis. */
648
+ t3Tokens?: number;
649
+ /**
650
+ * CosyVoice3 / Audio8: the engine's own wall time for the last synthesis, in
651
+ * ms (the sum of its stages), excluding the addon's post-processing.
652
+ */
653
+ stageTotalMs?: number;
654
+ /** CosyVoice3 stage wall times of the last synthesis, in ms. */
655
+ lmPrefillMs?: number;
656
+ lmDecodeMs?: number;
657
+ flowFrontendMs?: number;
658
+ ditEulerMs?: number;
659
+ hiftF0Ms?: number;
660
+ hiftSourceMs?: number;
661
+ hiftStftMs?: number;
662
+ hiftDecodeMs?: number;
663
+ /**
664
+ * CosyVoice3 work counters of the last synthesis: LM decode steps, speech
665
+ * tokens generated, flow frames in (prompt + generated), mel frames out,
666
+ * text ids in the LM prefill, and prompt speech tokens the LM continued from.
667
+ */
668
+ decodeSteps?: number;
669
+ speechTokens?: number;
670
+ flowFrames?: number;
671
+ melFrames?: number;
672
+ textIds?: number;
673
+ promptSpeechTokens?: number;
674
+ /** Audio8 stage wall times of the last synthesis, in ms. */
675
+ voiceEncodeMs?: number;
676
+ promptMs?: number;
677
+ prefillMs?: number;
678
+ sampleMs?: number;
679
+ fastDecodeMs?: number;
680
+ slowDecodeMs?: number;
681
+ codecLatentMs?: number;
682
+ codecSynthMs?: number;
683
+ resampleMs?: number;
684
+ }
685
+ /**
686
+ * tts-cpp's canonical conditioning vocabulary and each engine's supported
687
+ * subset, keyed by engine type (`TTSGgml.getVoiceControls()`).
688
+ */
689
+ interface VoiceControlsCatalog {
690
+ emotions: string[];
691
+ paces: string[];
692
+ engines: Record<string, {
693
+ emotions: string[];
694
+ paces: string[];
695
+ }>;
465
696
  }
466
697
  interface SentenceStreamChunkMeta {
467
698
  chunkIndex?: number;
@@ -472,13 +703,13 @@ interface SentenceStreamChunkMeta {
472
703
  */
473
704
  isLast?: boolean;
474
705
  }
475
- interface SentenceStreamOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields {
706
+ interface SentenceStreamOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields, CosyvoiceJobFields {
476
707
  /** BCP-47 locale for `Intl.Segmenter` when available. */
477
708
  locale?: string;
478
709
  /** Maximum graphemes per chunk; defaults to 300, or 120 for Korean. */
479
710
  maxChunkScalars?: number;
480
711
  }
481
- interface RunStreamingOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields {
712
+ interface RunStreamingOptions extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields, CosyvoiceJobFields {
482
713
  accumulateSentences?: boolean;
483
714
  sentenceDelimiter?: RegExp;
484
715
  sentenceDelimiterPreset?: SentenceDelimiterPreset;
@@ -487,7 +718,7 @@ interface RunStreamingOptions extends ParlerDescriptionFields, Audio8VoiceFields
487
718
  }
488
719
  /** Input accepted by `runStreaming`. */
489
720
  type TextStreamInput = string | string[] | Iterable<string> | AsyncIterable<string>;
490
- interface TTSRunInput extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields {
721
+ interface TTSRunInput extends ParlerDescriptionFields, Audio8VoiceFields, TTSConditioningFields, CosyvoiceJobFields {
491
722
  type?: string;
492
723
  input: string;
493
724
  streamOutput?: boolean;
@@ -501,7 +732,7 @@ interface TTSRunInput extends ParlerDescriptionFields, Audio8VoiceFields, TTSCon
501
732
  }
502
733
  /**
503
734
  * GGML-backed TTS via the `tts-cpp` library. Wraps the chatterbox,
504
- * supertonic, parler, cosyvoice3 and audio8 engines behind a single
735
+ * supertonic, parler, cosyvoice3, audio8 and moss engines behind a single
505
736
  * engine-agnostic JavaScript surface. Engine type is auto-detected from
506
737
  * `files` or selected explicitly with `engine`.
507
738
  *
@@ -518,6 +749,8 @@ declare class TTSGgml {
518
749
  static readonly ENGINE_COSYVOICE3 = "cosyvoice3";
519
750
  static readonly ENGINE_PARLER = "parler";
520
751
  static readonly ENGINE_AUDIO8 = "audio8";
752
+ static readonly ENGINE_MOSS = "moss";
753
+ static readonly ENGINE_POCKET = "pocket";
521
754
  opts: object;
522
755
  exclusiveRun: boolean;
523
756
  logger: object;
@@ -528,6 +761,12 @@ declare class TTSGgml {
528
761
  private _ttsInferenceQueueWaiter;
529
762
  private _sentenceStreamCtx;
530
763
  private _config;
764
+ private _pocketJobPending;
765
+ private _pocketCancelPromise;
766
+ private _pocketLifecycleInProgress;
767
+ private _pocketParams;
768
+ private _pocketOptions;
769
+ private _pocketFiles;
531
770
  private _lazySessionLoading;
532
771
  private _outputSampleRate;
533
772
  private _engineType;
@@ -555,6 +794,23 @@ declare class TTSGgml {
555
794
  private _streamLeftContextTokens?;
556
795
  private _cfmSteps?;
557
796
  private _cfgRate?;
797
+ private _batchCfmSteps?;
798
+ private _nPredict?;
799
+ private _maxSentenceChars?;
800
+ private _crossfadeMs?;
801
+ private _repeatPenalty?;
802
+ private _exaggeration?;
803
+ private _cfgWeight?;
804
+ private _minP?;
805
+ private _streamChunkTolerancePct?;
806
+ private _streamMinChunkTokens?;
807
+ private _voiceJsonPath?;
808
+ private _prewarmText?;
809
+ private _vulkanDevice?;
810
+ private _flowCutPrompt?;
811
+ private _cosyvoiceVocabPath?;
812
+ private _cosyvoiceMergesPath?;
813
+ private _cosyvoiceVoiceModelPath?;
558
814
  private _promptText?;
559
815
  private _instruct?;
560
816
  private _voice?;
@@ -570,6 +826,11 @@ declare class TTSGgml {
570
826
  private _audio8LmPath?;
571
827
  private _audio8CodecDecoderPath?;
572
828
  private _audio8CodecEncoderPath?;
829
+ private _mossBackbonePath?;
830
+ private _mossCodecDecoderPath?;
831
+ private _mossCodecEncoderPath?;
832
+ private _dialogueReferences?;
833
+ private _durationTokens?;
573
834
  private _referenceText?;
574
835
  private _greedy?;
575
836
  private _description?;
@@ -589,12 +850,27 @@ declare class TTSGgml {
589
850
  constructor(options?: TTSGgmlOptions);
590
851
  private _resolveEngineAndModelPaths;
591
852
  private _resolveAudio8ModelPaths;
853
+ private _mossBackbonePatterns;
854
+ private _findMossBackbone;
855
+ private _resolveMossModelPaths;
856
+ private _assignMossVoiceOptions;
592
857
  private _assignSynthesisOptions;
593
858
  private _assertEngineStreamingSupport;
594
859
  private _requestsChunkStreaming;
595
860
  private _assertParlerOptionConsistency;
596
861
  private _assertSamplerOptionSupport;
862
+ /**
863
+ * Options only some engines read. Set on any other engine they would be
864
+ * dropped without effect, so they are rejected by name instead.
865
+ */
866
+ private _assertEngineScopedOptions;
597
867
  private _assertAudio8OptionConsistency;
868
+ private _assertMossOptionConsistency;
869
+ private _assertNoMossOnlyOptions;
870
+ private _assertMossDialogueReferences;
871
+ private _assertSentenceStreamingAllowed;
872
+ private _assertMossOutputRate;
873
+ private _assertMossVoiceConsistent;
598
874
  /**
599
875
  * A recording without its transcript is accepted by the model but degrades
600
876
  * the clone silently, and a transcript alone has nothing to attach to, so
@@ -618,6 +894,12 @@ declare class TTSGgml {
618
894
  * emotion/pace it declares support for, never Parler's template fields.
619
895
  */
620
896
  private _resolveConditioningJobFields;
897
+ /**
898
+ * CosyVoice3's per-call instruction, rendered like the constructor's. The
899
+ * native layer reads it next to emotion/pace, and any of the three replaces
900
+ * the configured conditioning for that synthesis.
901
+ */
902
+ private _resolveJobInstruct;
621
903
  /**
622
904
  * Parler's per-call surface: description/template fields, where a per-call
623
905
  * template cannot be merged with a constructor-level free-text description.
@@ -645,6 +927,7 @@ declare class TTSGgml {
645
927
  getApiDefinition(): string;
646
928
  getState(): InferenceState;
647
929
  load(..._args: unknown[]): Promise<void>;
930
+ private _loadModel;
648
931
  /**
649
932
  * Run text-to-speech. With `{ streamOutput: true }`, splits `input` into
650
933
  * chunks and emits PCM through `response.onUpdate` for each chunk.
@@ -672,8 +955,8 @@ declare class TTSGgml {
672
955
  private _runTextStreamOrchestrator;
673
956
  private _sentenceStreamTextIterableDrive;
674
957
  /**
675
- * Audio8 reports `tokensPerSecond` as codec frames per second, so the
676
- * streaming aggregate has to count frames too rather than characters.
958
+ * Audio8 and MOSS report `tokensPerSecond` as codec frames per second, so
959
+ * the streaming aggregate has to count frames too rather than characters.
677
960
  */
678
961
  private _pacesOnFrames;
679
962
  private _runStreamOrchestrator;
@@ -685,6 +968,7 @@ declare class TTSGgml {
685
968
  private _buildSupertonicParams;
686
969
  private _buildParlerParams;
687
970
  private _buildAudio8Params;
971
+ private _buildMossParams;
688
972
  /** The knobs every engine in SAMPLING_ENGINES reads. */
689
973
  private _assignSamplingParams;
690
974
  /**
@@ -697,6 +981,7 @@ declare class TTSGgml {
697
981
  private _assignLavasrParams;
698
982
  private _createAddon;
699
983
  unload(): Promise<void>;
984
+ private _unloadModel;
700
985
  destroy(): Promise<void>;
701
986
  private _runInternal;
702
987
  private _mergeSentenceStreamStats;
@@ -707,12 +992,19 @@ declare class TTSGgml {
707
992
  private _handleAddonOutput;
708
993
  private _enrichStreamChunk;
709
994
  private _handleAddonStats;
995
+ private _waitPocketCancel;
996
+ private _withPocketLifecycle;
997
+ private _checkPocketReload;
998
+ private _checkPocketRequest;
999
+ private _dispatchJob;
710
1000
  cancel(): Promise<void>;
711
1001
  private _failAndClearActiveResponse;
712
1002
  /** Everything reload() may overwrite, so a rejected reload can undo itself. */
713
1003
  private _captureReloadableState;
714
1004
  private _restoreReloadableState;
715
1005
  private _applyReloadableRuntimeConfig;
1006
+ private _assertMossReloadKeepsVoice;
1007
+ private _applyMossReload;
716
1008
  private _applyReloadableConditioning;
717
1009
  private _applyReloadableParlerConfig;
718
1010
  /**
@@ -721,6 +1013,7 @@ declare class TTSGgml {
721
1013
  * reload is not validated against state the caller never accepted.
722
1014
  */
723
1015
  private _applyReloadableConfig;
1016
+ private _reloadPocket;
724
1017
  reload(newConfig?: Record<string, unknown>): Promise<void>;
725
1018
  /**
726
1019
  * The voice a reload lands on. Same rule as _mergeAudio8Voice and
@@ -744,6 +1037,13 @@ declare class TTSGgml {
744
1037
  */
745
1038
  private _applyAudio8Reload;
746
1039
  static getModelKey(_params?: unknown): string;
1040
+ /**
1041
+ * tts-cpp's canonical emotion / pace vocabulary and each engine's supported
1042
+ * subset, read from the native library without loading a model. Engines are
1043
+ * keyed by engine type, so tts-cpp's `cosyvoice` appears as `cosyvoice3`;
1044
+ * an engine this package does not wrap keeps its tts-cpp name.
1045
+ */
1046
+ static getVoiceControls(): VoiceControlsCatalog;
747
1047
  private _requireAddon;
748
1048
  private _optionalAddon;
749
1049
  private _getLogger;
@@ -762,6 +1062,17 @@ type NamespaceTextStreamInput = TextStreamInput;
762
1062
  type NamespaceRunInput = TTSRunInput;
763
1063
  type NamespaceInferenceState = InferenceState;
764
1064
  type NamespaceCosyvoiceInstruct = CosyvoiceInstruct;
1065
+ type NamespaceVoiceControlsCatalog = VoiceControlsCatalog;
1066
+ type NamespaceFitEngine = TtsFitEngine;
1067
+ type NamespaceFitRequest = TtsFitRequest;
1068
+ type NamespaceFitResult = TtsFitResult;
1069
+ type NamespaceFitStatus = TtsFitStatus;
1070
+ type NamespaceSupertonicFit = SupertonicFitRequest;
1071
+ type NamespaceParlerFit = ParlerFitRequest;
1072
+ type NamespaceChatterboxFit = ChatterboxFitRequest;
1073
+ type NamespaceAudio8Fit = Audio8FitRequest;
1074
+ type NamespaceMossFit = MossFitRequest;
1075
+ type NamespaceCosyvoiceFit = CosyvoiceFitRequest;
765
1076
  declare namespace TTSGgml {
766
1077
  export import QvacErrorAddonTTSGgml = errorModule.QvacErrorAddonTTSGgml;
767
1078
  export import ERR_CODES = errorModule.ERR_CODES;
@@ -779,6 +1090,18 @@ declare namespace TTSGgml {
779
1090
  type TTSRunInput = NamespaceRunInput;
780
1091
  type InferenceState = NamespaceInferenceState;
781
1092
  type CosyvoiceInstruct = NamespaceCosyvoiceInstruct;
1093
+ type VoiceControlsCatalog = NamespaceVoiceControlsCatalog;
1094
+ type TtsFitEngine = NamespaceFitEngine;
1095
+ type TtsFitRequest = NamespaceFitRequest;
1096
+ type TtsFitResult = NamespaceFitResult;
1097
+ type TtsFitStatus = NamespaceFitStatus;
1098
+ type SupertonicFitRequest = NamespaceSupertonicFit;
1099
+ type ParlerFitRequest = NamespaceParlerFit;
1100
+ type ChatterboxFitRequest = NamespaceChatterboxFit;
1101
+ type Audio8FitRequest = NamespaceAudio8Fit;
1102
+ type MossFitRequest = NamespaceMossFit;
1103
+ type CosyvoiceFitRequest = NamespaceCosyvoiceFit;
782
1104
  const resolveBackendsDir: typeof resolveBackendsDirImpl;
1105
+ const assessFit: typeof assessFitImpl;
783
1106
  }
784
1107
  export = TTSGgml;