@qvac/tts-ggml 0.12.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +127 -6
- package/docs/pocket-tts.md +12 -10
- package/examples/moss-sfx-fit.js +14 -7
- package/index.d.ts +43 -1130
- package/index.js +127 -3042
- package/lib/addonEvents.d.ts +26 -0
- package/lib/addonEvents.js +51 -0
- package/lib/conditioning.d.ts +50 -0
- package/lib/conditioning.js +184 -0
- package/lib/engineDetection.d.ts +14 -0
- package/lib/engineDetection.js +110 -0
- package/lib/engineIds.d.ts +15 -0
- package/lib/engineIds.js +39 -0
- package/lib/engineParams.d.ts +3 -0
- package/lib/engineParams.js +34 -0
- package/lib/engineSetup.d.ts +153 -0
- package/lib/engineSetup.js +2 -0
- package/lib/engines/audio8.d.ts +83 -0
- package/lib/engines/audio8.js +262 -0
- package/lib/engines/chatterbox.d.ts +10 -0
- package/lib/engines/chatterbox.js +112 -0
- package/lib/engines/cosyvoice.d.ts +87 -0
- package/lib/engines/cosyvoice.js +267 -0
- package/lib/engines/moss.d.ts +26 -0
- package/lib/engines/moss.js +177 -0
- package/lib/engines/mossSoundEffect.d.ts +41 -0
- package/lib/engines/mossSoundEffect.js +117 -0
- package/lib/engines/mossSpeech.d.ts +86 -0
- package/lib/engines/mossSpeech.js +228 -0
- package/lib/engines/parler.d.ts +71 -0
- package/lib/engines/parler.js +287 -0
- package/lib/engines/supertonic.d.ts +11 -0
- package/lib/engines/supertonic.js +115 -0
- package/lib/exclusiveQueue.d.ts +16 -0
- package/lib/exclusiveQueue.js +45 -0
- package/lib/fit.d.ts +3 -3
- package/lib/fit.js +4 -4
- package/lib/jobFields.d.ts +20 -0
- package/lib/jobFields.js +73 -0
- package/lib/modelFiles.d.ts +207 -0
- package/lib/modelFiles.js +117 -0
- package/lib/nativeParams.d.ts +12 -0
- package/lib/nativeParams.js +73 -0
- package/lib/optionValidation.d.ts +2 -0
- package/lib/optionValidation.js +103 -0
- package/lib/options.d.ts +4 -0
- package/lib/options.js +17 -0
- package/lib/pocketLifecycle.d.ts +67 -0
- package/lib/pocketLifecycle.js +170 -0
- package/lib/reload.d.ts +12 -0
- package/lib/reload.js +76 -0
- package/lib/sentenceStream.d.ts +85 -0
- package/lib/sentenceStream.js +367 -0
- package/lib/setup.d.ts +7 -0
- package/lib/setup.js +169 -0
- package/lib/streamingSupport.d.ts +8 -0
- package/lib/streamingSupport.js +77 -0
- package/lib/types.d.ts +465 -0
- package/lib/types.js +2 -0
- package/lib/voiceControls.d.ts +12 -0
- package/lib/voiceControls.js +23 -0
- package/package.json +17 -11
- package/test/integration/audio8.test.js +6 -12
- package/test/integration/fit.test.js +115 -18
- package/test/integration/gpu-smoke.test.js +2 -11
- package/test/integration/lavasr-enhancer.test.js +2 -15
- package/test/integration/moss-fit.test.js +13 -13
- package/test/integration/moss-sfx-fit.test.js +14 -14
- package/test/integration/moss-sfx.test.js +29 -4
- package/test/integration/moss-speech.test.js +19 -5
- package/test/integration/moss.test.js +93 -56
- package/test/mobile/integration.auto.cjs +10 -0
- package/test/mobile/test-groups.json +6 -2
- package/test/utils/desktopGpuBackend.js +21 -0
- package/test/utils/gpuOnly.js +15 -0
- package/test/utils/mossBackend.js +34 -0
- package/test/utils/mossGpuValidation.js +70 -0
- package/test/utils/perf-helper.js +275 -0
- package/test/utils/runTTS.js +1 -1
- package/test/utils/testThreads.js +5 -0
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The state the wrapper resolves from its constructor options, grouped by
|
|
3
|
+
* responsibility. Everything that builds native parameters, validates options
|
|
4
|
+
* or applies a reload reads (and, for reloadable fields, writes) one of these
|
|
5
|
+
* groups instead of the wrapper instance itself.
|
|
6
|
+
*/
|
|
7
|
+
import type { EngineType } from "./engineIds";
|
|
8
|
+
import type { NormalizedFiles } from "./modelFiles";
|
|
9
|
+
import type { TTSConfigurationParams } from "../tts";
|
|
10
|
+
export interface TTSGgmlRuntimeConfig {
|
|
11
|
+
/**
|
|
12
|
+
* Language code; default "en". Chatterbox MTL accepts
|
|
13
|
+
* es/fr/de/pt/it/zh/ja/ko/... CosyVoice3: reserved / not yet effective — the
|
|
14
|
+
* text-normalization frontend is not yet integrated, so it is accepted but
|
|
15
|
+
* not acted on.
|
|
16
|
+
*/
|
|
17
|
+
language?: string;
|
|
18
|
+
/**
|
|
19
|
+
* Route inference through a GPU backend (Metal / Vulkan / OpenCL) if
|
|
20
|
+
* available. Defaults to `false`. Audio8 uses Vulkan on Linux and Windows;
|
|
21
|
+
* CosyVoice3 selects Metal on Apple, Vulkan on desktop Linux / Windows, and
|
|
22
|
+
* OpenCL/Adreno on Android (Mali / Xclipse decline to CPU); the other
|
|
23
|
+
* GPU-capable engines select a backend for the host platform.
|
|
24
|
+
*/
|
|
25
|
+
useGPU?: boolean;
|
|
26
|
+
/**
|
|
27
|
+
* Desired output sample rate in Hz (8000-192000); omit to keep the engine's
|
|
28
|
+
* native rate. Resamples the native output (24 kHz for Chatterbox and
|
|
29
|
+
* CosyVoice3; 44.1 kHz for Supertonic, Parler, and Audio8), or the 48 kHz
|
|
30
|
+
* LavaSR-enhanced signal, before emitting. `TTSOutputChunk.sampleRate`
|
|
31
|
+
* reports the resulting rate.
|
|
32
|
+
*
|
|
33
|
+
* CosyVoice3 native chunk streaming emits at 24 kHz: a different rate is
|
|
34
|
+
* only accepted there when the LavaSR enhancer is active, because the
|
|
35
|
+
* enhancer's overlap-reprocess window resamples without chunk seams.
|
|
36
|
+
*
|
|
37
|
+
* MOSS emits its native 24 kHz only and rejects any other rate.
|
|
38
|
+
*/
|
|
39
|
+
outputSampleRate?: number;
|
|
40
|
+
backendsDir?: string;
|
|
41
|
+
openclCacheDir?: string;
|
|
42
|
+
vulkanCacheDir?: string;
|
|
43
|
+
}
|
|
44
|
+
export interface ModelPaths {
|
|
45
|
+
voicesDir?: string;
|
|
46
|
+
supertonicModelPath?: string;
|
|
47
|
+
t3ModelPath?: string;
|
|
48
|
+
s3genModelPath?: string;
|
|
49
|
+
cosyvoiceModelDir?: string;
|
|
50
|
+
cosyvoiceLlmModelPath?: string;
|
|
51
|
+
cosyvoiceFlowModelPath?: string;
|
|
52
|
+
cosyvoiceHiftModelPath?: string;
|
|
53
|
+
cosyvoiceS3tokModelPath?: string;
|
|
54
|
+
cosyvoiceCampplusModelPath?: string;
|
|
55
|
+
cosyvoiceVocabPath?: string;
|
|
56
|
+
cosyvoiceMergesPath?: string;
|
|
57
|
+
cosyvoiceVoiceModelPath?: string;
|
|
58
|
+
parlerModelPath?: string;
|
|
59
|
+
audio8LmPath?: string;
|
|
60
|
+
audio8CodecDecoderPath?: string;
|
|
61
|
+
audio8CodecEncoderPath?: string;
|
|
62
|
+
mossBackbonePath?: string;
|
|
63
|
+
mossCodecDecoderPath?: string;
|
|
64
|
+
mossCodecEncoderPath?: string;
|
|
65
|
+
mossSoundEffectPath?: string;
|
|
66
|
+
mossSpeechModelPath?: string;
|
|
67
|
+
mossSpeechCodecPath?: string;
|
|
68
|
+
mecabDictPath?: string;
|
|
69
|
+
cangjieTsvPath?: string;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Synthesis options as the wrapper keeps them: aliases collapsed, the
|
|
73
|
+
* CosyVoice3 instruct rendered to its string. Reload rewrites the reloadable
|
|
74
|
+
* subset in place and rolls the whole object back when a reload is rejected.
|
|
75
|
+
*/
|
|
76
|
+
export interface SynthesisSettings {
|
|
77
|
+
referenceAudio?: string;
|
|
78
|
+
referenceText?: string;
|
|
79
|
+
greedy?: boolean;
|
|
80
|
+
voiceDir?: string;
|
|
81
|
+
seed?: number;
|
|
82
|
+
nGpuLayers?: number;
|
|
83
|
+
nCtx?: number;
|
|
84
|
+
kvCacheType?: "f32" | "f16" | "q8_0";
|
|
85
|
+
threads?: number;
|
|
86
|
+
streamChunkTokens?: number;
|
|
87
|
+
streamFirstChunkTokens?: number;
|
|
88
|
+
streamChunkTolerancePct?: number;
|
|
89
|
+
streamMinChunkTokens?: number;
|
|
90
|
+
streamLeftContextTokens?: number;
|
|
91
|
+
cfmSteps?: number;
|
|
92
|
+
cfgRate?: number;
|
|
93
|
+
batchCfmSteps?: number;
|
|
94
|
+
nPredict?: number;
|
|
95
|
+
maxSentenceChars?: number;
|
|
96
|
+
crossfadeMs?: number;
|
|
97
|
+
repeatPenalty?: number;
|
|
98
|
+
exaggeration?: number;
|
|
99
|
+
cfgWeight?: number;
|
|
100
|
+
minP?: number;
|
|
101
|
+
voiceJsonPath?: string;
|
|
102
|
+
prewarmText?: string;
|
|
103
|
+
vulkanDevice?: number;
|
|
104
|
+
flowCutPrompt?: boolean;
|
|
105
|
+
/** CosyVoice3-only: transcript of the reference audio for zero-shot cloning. */
|
|
106
|
+
promptText?: string;
|
|
107
|
+
/** CosyVoice3-only: the structured/raw instruct2 control, rendered to its string. */
|
|
108
|
+
instruct?: string;
|
|
109
|
+
voice?: string;
|
|
110
|
+
steps?: number;
|
|
111
|
+
speed?: number;
|
|
112
|
+
noiseNpyPath?: string;
|
|
113
|
+
/** Parler voice-description surface; the all-defaults render is the models' fallback caption. */
|
|
114
|
+
description?: string;
|
|
115
|
+
emotion?: string;
|
|
116
|
+
pitch?: string;
|
|
117
|
+
pace?: string;
|
|
118
|
+
expressivity?: string;
|
|
119
|
+
noise?: string;
|
|
120
|
+
reverb?: string;
|
|
121
|
+
quality?: string;
|
|
122
|
+
temperature?: number;
|
|
123
|
+
topK?: number;
|
|
124
|
+
topP?: number;
|
|
125
|
+
maxFrames?: number;
|
|
126
|
+
minNewTokens?: number;
|
|
127
|
+
normalizeNumbers?: boolean;
|
|
128
|
+
/** MOSS: fixed for the instance (reload rejects a change). */
|
|
129
|
+
dialogueReferences?: string[];
|
|
130
|
+
durationTokens?: number;
|
|
131
|
+
}
|
|
132
|
+
export interface BackendPaths {
|
|
133
|
+
enhancerGgufPath?: string;
|
|
134
|
+
denoiserGgufPath?: string;
|
|
135
|
+
backendsDir?: string;
|
|
136
|
+
openclCacheDir?: string;
|
|
137
|
+
vulkanCacheDir?: string;
|
|
138
|
+
}
|
|
139
|
+
export interface PocketSetup {
|
|
140
|
+
files: NormalizedFiles;
|
|
141
|
+
options: Record<string, unknown>;
|
|
142
|
+
params: TTSConfigurationParams;
|
|
143
|
+
}
|
|
144
|
+
export interface EngineSetup {
|
|
145
|
+
readonly engineType: EngineType;
|
|
146
|
+
readonly config: TTSGgmlRuntimeConfig;
|
|
147
|
+
/** Validated `config.outputSampleRate`; null keeps the engine's native rate. */
|
|
148
|
+
outputSampleRate: number | null;
|
|
149
|
+
readonly paths: ModelPaths;
|
|
150
|
+
readonly settings: SynthesisSettings;
|
|
151
|
+
readonly backend: BackendPaths;
|
|
152
|
+
pocket: PocketSetup | null;
|
|
153
|
+
}
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import type { TTSConfigurationParams } from "../../tts";
|
|
2
|
+
import type { EngineSetup, ModelPaths } from "../engineSetup";
|
|
3
|
+
import { type NormalizedFiles } from "../modelFiles";
|
|
4
|
+
export declare const AUDIO8_LM_RE: RegExp;
|
|
5
|
+
export declare const AUDIO8_DECODER_RE: RegExp;
|
|
6
|
+
export declare const AUDIO8_ENCODER_RE: RegExp;
|
|
7
|
+
export declare const AUDIO8_VOICE_KEYS: readonly ["referenceAudio", "referenceText"];
|
|
8
|
+
export type Audio8VoiceKey = (typeof AUDIO8_VOICE_KEYS)[number];
|
|
9
|
+
export type Audio8VoiceFieldsResolved = Partial<Record<Audio8VoiceKey, string>>;
|
|
10
|
+
/**
|
|
11
|
+
* Voice cloning reference. Audio8's codec encoder turns `referenceAudio` into
|
|
12
|
+
* codes and they are prepended to the prompt as the speaker's own history, so
|
|
13
|
+
* no speaker encoder and no enrolment step are involved. `referenceText` is
|
|
14
|
+
* required alongside it there: the model conditions on it as the turn the
|
|
15
|
+
* recording answers, and a wrong one degrades the clone. Accepted at
|
|
16
|
+
* construction, on `reload()`, and per call (Audio8 only).
|
|
17
|
+
*/
|
|
18
|
+
export interface Audio8VoiceFields {
|
|
19
|
+
/**
|
|
20
|
+
* Chatterbox: voice-cloning reference audio path (wav). CosyVoice3:
|
|
21
|
+
* zero-shot / cross-lingual cloning reference (0.5-30 s hard limits,
|
|
22
|
+
* 5-15 s of clean speech recommended; multichannel input is downmixed to
|
|
23
|
+
* mono) — the native front-end tokenizes it (speech_tokenizer_v3),
|
|
24
|
+
* extracts the CAM++ speaker embedding and prompt mel at load, replacing
|
|
25
|
+
* the baked voice; requires the `cosyvoiceS3tokModel` +
|
|
26
|
+
* `cosyvoiceCampplusModel` files and fails the load (never silently falls
|
|
27
|
+
* back) when they are missing or the audio is unusable. Pair with
|
|
28
|
+
* `promptText` (the verbatim transcript) for zero-shot or omit it for
|
|
29
|
+
* cross-lingual. The reference is fixed at construction: `reload()`
|
|
30
|
+
* re-bakes the same recording (it is forwarded to the new addon instance)
|
|
31
|
+
* but cannot switch to a different one, so changing voices means a new
|
|
32
|
+
* instance. Audio8: the recording to clone, with `referenceText`
|
|
33
|
+
* alongside it. MOSS: the recording to clone, sampled at 24 kHz and
|
|
34
|
+
* fixed at construction; needs `files.mossCodecEncoder`.
|
|
35
|
+
*/
|
|
36
|
+
referenceAudio?: string;
|
|
37
|
+
/** Audio8: what `referenceAudio` says. Required when cloning. */
|
|
38
|
+
referenceText?: string;
|
|
39
|
+
}
|
|
40
|
+
/** The Audio8 sampler knobs a `reload()` may carry. */
|
|
41
|
+
export interface Audio8SamplingFields {
|
|
42
|
+
greedy?: boolean;
|
|
43
|
+
temperature?: number;
|
|
44
|
+
topK?: number;
|
|
45
|
+
topP?: number;
|
|
46
|
+
maxFrames?: number;
|
|
47
|
+
seed?: number;
|
|
48
|
+
threads?: number;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Collect the Audio8 voice-cloning properties present on `source` (a run
|
|
52
|
+
* input, streaming options, or constructor options). Returns undefined when
|
|
53
|
+
* none are set.
|
|
54
|
+
*/
|
|
55
|
+
export declare function pickAudio8VoiceFields(source: Audio8VoiceFields | null | undefined): Audio8VoiceFieldsResolved | undefined;
|
|
56
|
+
/**
|
|
57
|
+
* Mirrors Audio8Model::validateSampling, which stays the source of truth.
|
|
58
|
+
* Reload checks the merged knobs here because native only sees them when the
|
|
59
|
+
* instance is rebuilt: without this the refused value is already on the JS
|
|
60
|
+
* object by then, and the two would describe different samplers.
|
|
61
|
+
*/
|
|
62
|
+
export declare function assertAudio8SamplingValid(sampling: Audio8SamplingFields, where: string): void;
|
|
63
|
+
export declare function resolveAudio8ModelPaths(files: NormalizedFiles): Pick<ModelPaths, "audio8LmPath" | "audio8CodecDecoderPath" | "audio8CodecEncoderPath">;
|
|
64
|
+
/**
|
|
65
|
+
* A recording without its transcript is accepted by the model but degrades
|
|
66
|
+
* the clone silently, and a transcript alone has nothing to attach to, so
|
|
67
|
+
* both halves have to arrive together.
|
|
68
|
+
*/
|
|
69
|
+
export declare function assertAudio8VoiceConsistent(voice: Audio8VoiceFields, codecEncoderPath: string | undefined, where: string): void;
|
|
70
|
+
export declare function assertAudio8Options(setup: EngineSetup): void;
|
|
71
|
+
/**
|
|
72
|
+
* Extract + validate the per-call Audio8 voice fields from a run input or
|
|
73
|
+
* streaming options. Returns undefined when none are present.
|
|
74
|
+
*/
|
|
75
|
+
export declare function resolveAudio8JobFields(setup: EngineSetup, source: Audio8VoiceFields | null | undefined, where: string): Audio8VoiceFieldsResolved | undefined;
|
|
76
|
+
export declare function buildAudio8Params(setup: EngineSetup): TTSConfigurationParams;
|
|
77
|
+
/**
|
|
78
|
+
* Audio8 voice + sampling knobs are reloadable; they rebuild the engine's
|
|
79
|
+
* sampler and speaker history. Both merges are checked before either is
|
|
80
|
+
* written, so a rejected reload leaves the instance exactly as it was
|
|
81
|
+
* rather than half-moved onto the configuration that was refused.
|
|
82
|
+
*/
|
|
83
|
+
export declare function applyAudio8Reload(setup: EngineSetup, newConfig: Record<string, unknown>): void;
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.AUDIO8_VOICE_KEYS = exports.AUDIO8_ENCODER_RE = exports.AUDIO8_DECODER_RE = exports.AUDIO8_LM_RE = void 0;
|
|
4
|
+
exports.pickAudio8VoiceFields = pickAudio8VoiceFields;
|
|
5
|
+
exports.assertAudio8SamplingValid = assertAudio8SamplingValid;
|
|
6
|
+
exports.resolveAudio8ModelPaths = resolveAudio8ModelPaths;
|
|
7
|
+
exports.assertAudio8VoiceConsistent = assertAudio8VoiceConsistent;
|
|
8
|
+
exports.assertAudio8Options = assertAudio8Options;
|
|
9
|
+
exports.resolveAudio8JobFields = resolveAudio8JobFields;
|
|
10
|
+
exports.buildAudio8Params = buildAudio8Params;
|
|
11
|
+
exports.applyAudio8Reload = applyAudio8Reload;
|
|
12
|
+
const engineIds_1 = require("../engineIds");
|
|
13
|
+
const modelFiles_1 = require("../modelFiles");
|
|
14
|
+
const nativeParams_1 = require("../nativeParams");
|
|
15
|
+
const options_1 = require("../options");
|
|
16
|
+
// Audio8 ships three GGUFs per quant tier, with the tier in the filename
|
|
17
|
+
// (`audio8-lm-q8_0.gguf`); a bare `audio8-lm.gguf` matches too, as
|
|
18
|
+
// forward-compat with a single-tier release.
|
|
19
|
+
exports.AUDIO8_LM_RE = /^audio8-lm(-[a-z0-9_]+)?\.gguf$/i;
|
|
20
|
+
exports.AUDIO8_DECODER_RE = /^audio8-codec-decoder(-[a-z0-9_]+)?\.gguf$/i;
|
|
21
|
+
exports.AUDIO8_ENCODER_RE = /^audio8-codec-encoder(-[a-z0-9_]+)?\.gguf$/i;
|
|
22
|
+
exports.AUDIO8_VOICE_KEYS = ["referenceAudio", "referenceText"];
|
|
23
|
+
/**
|
|
24
|
+
* Collect the Audio8 voice-cloning properties present on `source` (a run
|
|
25
|
+
* input, streaming options, or constructor options). Returns undefined when
|
|
26
|
+
* none are set.
|
|
27
|
+
*/
|
|
28
|
+
function pickAudio8VoiceFields(source) {
|
|
29
|
+
if (source == null || typeof source !== "object")
|
|
30
|
+
return undefined;
|
|
31
|
+
const out = {};
|
|
32
|
+
let any = false;
|
|
33
|
+
for (const key of exports.AUDIO8_VOICE_KEYS) {
|
|
34
|
+
const value = source[key];
|
|
35
|
+
if (value != null && value !== "") {
|
|
36
|
+
out[key] = String(value);
|
|
37
|
+
any = true;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return any ? out : undefined;
|
|
41
|
+
}
|
|
42
|
+
const AUDIO8_NUMERIC_SAMPLING_FIELDS = [
|
|
43
|
+
"temperature",
|
|
44
|
+
"topK",
|
|
45
|
+
"topP",
|
|
46
|
+
"maxFrames",
|
|
47
|
+
"seed",
|
|
48
|
+
"threads",
|
|
49
|
+
];
|
|
50
|
+
function assertAudio8SamplingFinite(sampling, where) {
|
|
51
|
+
for (const field of AUDIO8_NUMERIC_SAMPLING_FIELDS) {
|
|
52
|
+
const value = sampling[field];
|
|
53
|
+
if (value === undefined || Number.isFinite(value))
|
|
54
|
+
continue;
|
|
55
|
+
throw new Error(`tts-ggml: ${where}: ${field} must be a finite number`);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
function assertNotNegative(value, name, zeroMeans, where) {
|
|
59
|
+
if (value === undefined || value >= 0)
|
|
60
|
+
return;
|
|
61
|
+
throw new Error(`tts-ggml: ${where}: ${name} must be >= 0 (0 = ${zeroMeans})`);
|
|
62
|
+
}
|
|
63
|
+
function assertAudio8TopP(value, where) {
|
|
64
|
+
if (value === undefined || (value > 0 && value <= 1))
|
|
65
|
+
return;
|
|
66
|
+
throw new Error(`tts-ggml: ${where}: topP must be in (0, 1]`);
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Mirrors Audio8Model::validateSampling, which stays the source of truth.
|
|
70
|
+
* Reload checks the merged knobs here because native only sees them when the
|
|
71
|
+
* instance is rebuilt: without this the refused value is already on the JS
|
|
72
|
+
* object by then, and the two would describe different samplers.
|
|
73
|
+
*/
|
|
74
|
+
function assertAudio8SamplingValid(sampling, where) {
|
|
75
|
+
assertAudio8SamplingFinite(sampling, where);
|
|
76
|
+
assertNotNegative(sampling.temperature, "temperature", "greedy", where);
|
|
77
|
+
assertNotNegative(sampling.topK, "topK", "no top-k cutoff", where);
|
|
78
|
+
assertNotNegative(sampling.maxFrames, "maxFrames", "engine default", where);
|
|
79
|
+
assertAudio8TopP(sampling.topP, where);
|
|
80
|
+
}
|
|
81
|
+
function resolveAudio8ModelPaths(files) {
|
|
82
|
+
return {
|
|
83
|
+
audio8LmPath: (0, modelFiles_1.firstNonEmpty)(files.audio8Lm, (0, modelFiles_1.findQuantRankedGguf)(files.modelDir, exports.AUDIO8_LM_RE)),
|
|
84
|
+
audio8CodecDecoderPath: (0, modelFiles_1.firstNonEmpty)(files.audio8CodecDecoder, (0, modelFiles_1.findQuantRankedGguf)(files.modelDir, exports.AUDIO8_DECODER_RE)),
|
|
85
|
+
audio8CodecEncoderPath: (0, modelFiles_1.firstNonEmpty)(files.audio8CodecEncoder, (0, modelFiles_1.findQuantRankedGguf)(files.modelDir, exports.AUDIO8_ENCODER_RE)),
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* A recording without its transcript is accepted by the model but degrades
|
|
90
|
+
* the clone silently, and a transcript alone has nothing to attach to, so
|
|
91
|
+
* both halves have to arrive together.
|
|
92
|
+
*/
|
|
93
|
+
function assertAudio8VoiceConsistent(voice, codecEncoderPath, where) {
|
|
94
|
+
if (!voice.referenceAudio && !voice.referenceText)
|
|
95
|
+
return;
|
|
96
|
+
if (!voice.referenceAudio) {
|
|
97
|
+
throw new Error(`tts-ggml: ${where}: referenceText needs a referenceAudio to ` +
|
|
98
|
+
"transcribe");
|
|
99
|
+
}
|
|
100
|
+
if (!voice.referenceText) {
|
|
101
|
+
throw new Error(`tts-ggml: ${where}: referenceAudio needs a referenceText saying ` +
|
|
102
|
+
"what the recording says");
|
|
103
|
+
}
|
|
104
|
+
if (!codecEncoderPath) {
|
|
105
|
+
throw new Error(`tts-ggml: ${where}: voice cloning needs the audio8 codec encoder ` +
|
|
106
|
+
"GGUF (files.audio8CodecEncoder)");
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
function assertAudio8Options(setup) {
|
|
110
|
+
if (setup.engineType === engineIds_1.ENGINE_AUDIO8) {
|
|
111
|
+
if (setup.backend.enhancerGgufPath || setup.backend.denoiserGgufPath) {
|
|
112
|
+
throw new Error("tts-ggml: the LavaSR enhancer/denoiser are not supported with " +
|
|
113
|
+
"the audio8 engine (native 44.1 kHz output needs no bandwidth " +
|
|
114
|
+
"extension). Drop lavasrEnhancer / lavasrDenoiser.");
|
|
115
|
+
}
|
|
116
|
+
assertAudio8VoiceConsistent({
|
|
117
|
+
referenceAudio: setup.settings.referenceAudio,
|
|
118
|
+
referenceText: setup.settings.referenceText,
|
|
119
|
+
}, setup.paths.audio8CodecEncoderPath, "constructor");
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
const audio8Only = (0, options_1.setOptionNames)({
|
|
123
|
+
referenceText: setup.settings.referenceText,
|
|
124
|
+
greedy: setup.settings.greedy,
|
|
125
|
+
});
|
|
126
|
+
if (audio8Only.length > 0) {
|
|
127
|
+
throw new Error(`tts-ggml: ${audio8Only.join(", ")} are audio8-only options ` +
|
|
128
|
+
`(engine is ${setup.engineType})`);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* The voice a per-call override actually synthesizes with. Mirrors
|
|
133
|
+
* Audio8Model::resolveVoice: a per-call recording replaces both halves, so
|
|
134
|
+
* it cannot inherit the configured transcript, which describes a different
|
|
135
|
+
* recording; a per-call transcript alone corrects the configured one.
|
|
136
|
+
*/
|
|
137
|
+
function mergeAudio8Voice(setup, fields) {
|
|
138
|
+
if (fields.referenceAudio) {
|
|
139
|
+
return {
|
|
140
|
+
referenceAudio: fields.referenceAudio,
|
|
141
|
+
referenceText: fields.referenceText,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
return {
|
|
145
|
+
referenceAudio: setup.settings.referenceAudio,
|
|
146
|
+
referenceText: fields.referenceText ?? setup.settings.referenceText,
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Extract + validate the per-call Audio8 voice fields from a run input or
|
|
151
|
+
* streaming options. Returns undefined when none are present.
|
|
152
|
+
*/
|
|
153
|
+
function resolveAudio8JobFields(setup, source, where) {
|
|
154
|
+
const fields = pickAudio8VoiceFields(source);
|
|
155
|
+
if (!fields)
|
|
156
|
+
return undefined;
|
|
157
|
+
if (setup.engineType !== engineIds_1.ENGINE_AUDIO8) {
|
|
158
|
+
throw new Error(`tts-ggml: ${where}: per-call referenceAudio/referenceText are ` +
|
|
159
|
+
`audio8-only (engine is ${setup.engineType})`);
|
|
160
|
+
}
|
|
161
|
+
assertAudio8VoiceConsistent(mergeAudio8Voice(setup, fields), setup.paths.audio8CodecEncoderPath, where);
|
|
162
|
+
return fields;
|
|
163
|
+
}
|
|
164
|
+
function buildAudio8Params(setup) {
|
|
165
|
+
// Backstop for callers that reach the params build by another route than
|
|
166
|
+
// the constructor or applyAudio8Reload, both of which check first.
|
|
167
|
+
assertAudio8VoiceConsistent({
|
|
168
|
+
referenceAudio: setup.settings.referenceAudio,
|
|
169
|
+
referenceText: setup.settings.referenceText,
|
|
170
|
+
}, setup.paths.audio8CodecEncoderPath, "reload");
|
|
171
|
+
const parameters = {
|
|
172
|
+
engineType: engineIds_1.ENGINE_AUDIO8,
|
|
173
|
+
audio8LmPath: setup.paths.audio8LmPath || "",
|
|
174
|
+
audio8CodecDecoderPath: setup.paths.audio8CodecDecoderPath || "",
|
|
175
|
+
};
|
|
176
|
+
if (setup.paths.audio8CodecEncoderPath) {
|
|
177
|
+
parameters.audio8CodecEncoderPath = setup.paths.audio8CodecEncoderPath;
|
|
178
|
+
}
|
|
179
|
+
if (setup.settings.referenceAudio != null) {
|
|
180
|
+
parameters.referenceAudio = setup.settings.referenceAudio;
|
|
181
|
+
}
|
|
182
|
+
if (setup.settings.referenceText != null) {
|
|
183
|
+
parameters.referenceText = setup.settings.referenceText;
|
|
184
|
+
}
|
|
185
|
+
if (setup.settings.greedy != null)
|
|
186
|
+
parameters.greedy = !!setup.settings.greedy;
|
|
187
|
+
(0, nativeParams_1.assignSamplingParams)(setup, parameters);
|
|
188
|
+
(0, nativeParams_1.assignBackendParams)(setup, parameters);
|
|
189
|
+
return parameters;
|
|
190
|
+
}
|
|
191
|
+
/**
|
|
192
|
+
* The voice a reload lands on. Same rule as mergeAudio8Voice and
|
|
193
|
+
* Audio8Model::resolveVoice: a new recording replaces both halves, because
|
|
194
|
+
* the configured transcript describes the recording being replaced. Reload
|
|
195
|
+
* reads `undefined` as "not supplied", so an explicit empty string reaches
|
|
196
|
+
* the guard instead of being ignored.
|
|
197
|
+
*/
|
|
198
|
+
function mergeAudio8ReloadVoice(setup, config) {
|
|
199
|
+
if (config.referenceAudio !== undefined) {
|
|
200
|
+
return {
|
|
201
|
+
referenceAudio: config.referenceAudio,
|
|
202
|
+
referenceText: config.referenceText,
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
return {
|
|
206
|
+
referenceAudio: setup.settings.referenceAudio,
|
|
207
|
+
referenceText: config.referenceText !== undefined
|
|
208
|
+
? config.referenceText
|
|
209
|
+
: setup.settings.referenceText,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
/**
|
|
213
|
+
* The knobs a reload lands on, merged the same way as the voice so the
|
|
214
|
+
* whole set can be checked before any of it is written.
|
|
215
|
+
*/
|
|
216
|
+
function mergeAudio8ReloadSampling(setup, config) {
|
|
217
|
+
return {
|
|
218
|
+
greedy: config.greedy ?? setup.settings.greedy,
|
|
219
|
+
temperature: config.temperature ?? setup.settings.temperature,
|
|
220
|
+
topK: config.topK ?? setup.settings.topK,
|
|
221
|
+
topP: config.topP ?? setup.settings.topP,
|
|
222
|
+
maxFrames: config.maxFrames ?? setup.settings.maxFrames,
|
|
223
|
+
seed: config.seed ?? setup.settings.seed,
|
|
224
|
+
threads: config.threads ?? setup.settings.threads,
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
function applyAudio8Sampling(setup, config) {
|
|
228
|
+
if (config.greedy !== undefined)
|
|
229
|
+
setup.settings.greedy = config.greedy;
|
|
230
|
+
if (config.temperature !== undefined) {
|
|
231
|
+
setup.settings.temperature = config.temperature;
|
|
232
|
+
}
|
|
233
|
+
if (config.topK !== undefined)
|
|
234
|
+
setup.settings.topK = config.topK;
|
|
235
|
+
if (config.topP !== undefined)
|
|
236
|
+
setup.settings.topP = config.topP;
|
|
237
|
+
if (config.maxFrames !== undefined) {
|
|
238
|
+
setup.settings.maxFrames = config.maxFrames;
|
|
239
|
+
}
|
|
240
|
+
if (config.seed !== undefined)
|
|
241
|
+
setup.settings.seed = config.seed;
|
|
242
|
+
if (config.threads !== undefined)
|
|
243
|
+
setup.settings.threads = config.threads;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Audio8 voice + sampling knobs are reloadable; they rebuild the engine's
|
|
247
|
+
* sampler and speaker history. Both merges are checked before either is
|
|
248
|
+
* written, so a rejected reload leaves the instance exactly as it was
|
|
249
|
+
* rather than half-moved onto the configuration that was refused.
|
|
250
|
+
*/
|
|
251
|
+
function applyAudio8Reload(setup, newConfig) {
|
|
252
|
+
if (setup.engineType !== engineIds_1.ENGINE_AUDIO8)
|
|
253
|
+
return;
|
|
254
|
+
const config = newConfig;
|
|
255
|
+
const voice = mergeAudio8ReloadVoice(setup, config);
|
|
256
|
+
const sampling = mergeAudio8ReloadSampling(setup, config);
|
|
257
|
+
assertAudio8VoiceConsistent(voice, setup.paths.audio8CodecEncoderPath, "reload");
|
|
258
|
+
assertAudio8SamplingValid(sampling, "reload");
|
|
259
|
+
setup.settings.referenceAudio = voice.referenceAudio;
|
|
260
|
+
setup.settings.referenceText = voice.referenceText;
|
|
261
|
+
applyAudio8Sampling(setup, sampling);
|
|
262
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { TTSConfigurationParams } from "../../tts";
|
|
2
|
+
import type { EngineSetup, ModelPaths } from "../engineSetup";
|
|
3
|
+
import { type NormalizedFiles } from "../modelFiles";
|
|
4
|
+
export declare const CHATTERBOX_T3_TURBO = "chatterbox-t3-turbo.gguf";
|
|
5
|
+
export declare const CHATTERBOX_T3_MTL = "chatterbox-t3-mtl.gguf";
|
|
6
|
+
export declare const CHATTERBOX_S3GEN_DEFAULT = "chatterbox-s3gen.gguf";
|
|
7
|
+
export declare const CHATTERBOX_S3GEN_MTL = "chatterbox-s3gen-mtl.gguf";
|
|
8
|
+
export declare function hasChatterboxModelDir(modelDir: string): boolean;
|
|
9
|
+
export declare function resolveChatterboxModelPaths(files: NormalizedFiles): Pick<ModelPaths, "t3ModelPath" | "s3genModelPath">;
|
|
10
|
+
export declare function buildChatterboxParams(setup: EngineSetup): TTSConfigurationParams;
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CHATTERBOX_S3GEN_MTL = exports.CHATTERBOX_S3GEN_DEFAULT = exports.CHATTERBOX_T3_MTL = exports.CHATTERBOX_T3_TURBO = void 0;
|
|
4
|
+
exports.hasChatterboxModelDir = hasChatterboxModelDir;
|
|
5
|
+
exports.resolveChatterboxModelPaths = resolveChatterboxModelPaths;
|
|
6
|
+
exports.buildChatterboxParams = buildChatterboxParams;
|
|
7
|
+
/* eslint-disable @typescript-eslint/no-require-imports -- Bare modules expose CommonJS export shapes. */
|
|
8
|
+
const path = require("bare-path");
|
|
9
|
+
const engineIds_1 = require("../engineIds");
|
|
10
|
+
const modelFiles_1 = require("../modelFiles");
|
|
11
|
+
const nativeParams_1 = require("../nativeParams");
|
|
12
|
+
exports.CHATTERBOX_T3_TURBO = "chatterbox-t3-turbo.gguf";
|
|
13
|
+
exports.CHATTERBOX_T3_MTL = "chatterbox-t3-mtl.gguf";
|
|
14
|
+
exports.CHATTERBOX_S3GEN_DEFAULT = "chatterbox-s3gen.gguf";
|
|
15
|
+
exports.CHATTERBOX_S3GEN_MTL = "chatterbox-s3gen-mtl.gguf";
|
|
16
|
+
function hasChatterboxModelDir(modelDir) {
|
|
17
|
+
return ((0, modelFiles_1.fileExistsSafe)(path.join(modelDir, exports.CHATTERBOX_T3_TURBO)) ||
|
|
18
|
+
(0, modelFiles_1.fileExistsSafe)(path.join(modelDir, exports.CHATTERBOX_T3_MTL)));
|
|
19
|
+
}
|
|
20
|
+
function resolveChatterboxModelDirPaths(modelDir) {
|
|
21
|
+
const turboT3 = path.join(modelDir, exports.CHATTERBOX_T3_TURBO);
|
|
22
|
+
const multilingualT3 = path.join(modelDir, exports.CHATTERBOX_T3_MTL);
|
|
23
|
+
const defaultS3 = path.join(modelDir, exports.CHATTERBOX_S3GEN_DEFAULT);
|
|
24
|
+
const multilingualS3 = path.join(modelDir, exports.CHATTERBOX_S3GEN_MTL);
|
|
25
|
+
if ((0, modelFiles_1.fileExistsSafe)(multilingualT3) &&
|
|
26
|
+
!(0, modelFiles_1.fileExistsSafe)(turboT3)) {
|
|
27
|
+
return {
|
|
28
|
+
t3: multilingualT3,
|
|
29
|
+
s3: (0, modelFiles_1.fileExistsSafe)(multilingualS3)
|
|
30
|
+
? multilingualS3
|
|
31
|
+
: defaultS3,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
return { t3: turboT3, s3: defaultS3 };
|
|
35
|
+
}
|
|
36
|
+
function resolveChatterboxModelPaths(files) {
|
|
37
|
+
if (!files.modelDir) {
|
|
38
|
+
return { t3ModelPath: files.t3Model, s3genModelPath: files.s3genModel };
|
|
39
|
+
}
|
|
40
|
+
const resolved = resolveChatterboxModelDirPaths(files.modelDir);
|
|
41
|
+
return {
|
|
42
|
+
t3ModelPath: (0, modelFiles_1.firstNonEmpty)(files.t3Model, resolved.t3),
|
|
43
|
+
s3genModelPath: (0, modelFiles_1.firstNonEmpty)(files.s3genModel, resolved.s3),
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function buildChatterboxParams(setup) {
|
|
47
|
+
const parameters = {
|
|
48
|
+
engineType: engineIds_1.ENGINE_CHATTERBOX,
|
|
49
|
+
t3ModelPath: setup.paths.t3ModelPath || "",
|
|
50
|
+
s3genModelPath: setup.paths.s3genModelPath || "",
|
|
51
|
+
language: setup.config.language || "en",
|
|
52
|
+
};
|
|
53
|
+
(0, nativeParams_1.assignCommonNativeParams)(setup, parameters);
|
|
54
|
+
if (setup.settings.referenceAudio != null) {
|
|
55
|
+
parameters.referenceAudio = setup.settings.referenceAudio;
|
|
56
|
+
}
|
|
57
|
+
if (setup.settings.voiceDir != null)
|
|
58
|
+
parameters.voiceDir = setup.settings.voiceDir;
|
|
59
|
+
if (setup.settings.nCtx != null)
|
|
60
|
+
parameters.nCtx = setup.settings.nCtx | 0;
|
|
61
|
+
if (setup.settings.kvCacheType != null) {
|
|
62
|
+
parameters.kvCacheType = String(setup.settings.kvCacheType);
|
|
63
|
+
}
|
|
64
|
+
if (setup.settings.streamChunkTokens != null) {
|
|
65
|
+
parameters.streamChunkTokens = setup.settings.streamChunkTokens | 0;
|
|
66
|
+
}
|
|
67
|
+
if (setup.settings.streamFirstChunkTokens != null) {
|
|
68
|
+
parameters.streamFirstChunkTokens =
|
|
69
|
+
setup.settings.streamFirstChunkTokens | 0;
|
|
70
|
+
}
|
|
71
|
+
if (setup.settings.cfmSteps != null) {
|
|
72
|
+
parameters.cfmSteps = setup.settings.cfmSteps | 0;
|
|
73
|
+
}
|
|
74
|
+
if (setup.settings.streamLeftContextTokens != null) {
|
|
75
|
+
parameters.streamLeftContextTokens = setup.settings.streamLeftContextTokens | 0;
|
|
76
|
+
}
|
|
77
|
+
if (setup.settings.batchCfmSteps != null) {
|
|
78
|
+
parameters.batchCfmSteps = setup.settings.batchCfmSteps | 0;
|
|
79
|
+
}
|
|
80
|
+
if (setup.settings.cfgRate != null) {
|
|
81
|
+
parameters.cfgRate = Number(setup.settings.cfgRate);
|
|
82
|
+
}
|
|
83
|
+
if (setup.settings.nPredict != null)
|
|
84
|
+
parameters.nPredict = setup.settings.nPredict | 0;
|
|
85
|
+
if (setup.settings.maxSentenceChars != null) {
|
|
86
|
+
parameters.maxSentenceChars = setup.settings.maxSentenceChars | 0;
|
|
87
|
+
}
|
|
88
|
+
if (setup.settings.crossfadeMs != null) {
|
|
89
|
+
parameters.crossfadeMs = setup.settings.crossfadeMs | 0;
|
|
90
|
+
}
|
|
91
|
+
(0, nativeParams_1.assignSamplingParams)(setup, parameters);
|
|
92
|
+
if (setup.settings.repeatPenalty != null) {
|
|
93
|
+
parameters.repeatPenalty = Number(setup.settings.repeatPenalty);
|
|
94
|
+
}
|
|
95
|
+
if (setup.settings.exaggeration != null) {
|
|
96
|
+
parameters.exaggeration = Number(setup.settings.exaggeration);
|
|
97
|
+
}
|
|
98
|
+
if (setup.settings.cfgWeight != null) {
|
|
99
|
+
parameters.cfgWeight = Number(setup.settings.cfgWeight);
|
|
100
|
+
}
|
|
101
|
+
if (setup.settings.minP != null)
|
|
102
|
+
parameters.minP = Number(setup.settings.minP);
|
|
103
|
+
if (setup.settings.speed != null)
|
|
104
|
+
parameters.speed = Number(setup.settings.speed);
|
|
105
|
+
if (setup.paths.mecabDictPath) {
|
|
106
|
+
parameters.mecabDictPath = setup.paths.mecabDictPath;
|
|
107
|
+
}
|
|
108
|
+
if (setup.paths.cangjieTsvPath) {
|
|
109
|
+
parameters.cangjieTsvPath = setup.paths.cangjieTsvPath;
|
|
110
|
+
}
|
|
111
|
+
return parameters;
|
|
112
|
+
}
|