@qvac/tts-ggml 0.13.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -11
- package/docs/pocket-tts.md +12 -10
- package/examples/audio8-streaming.js +74 -0
- package/index.d.ts +43 -1130
- package/index.js +129 -3041
- package/lib/addonEvents.d.ts +26 -0
- package/lib/addonEvents.js +51 -0
- package/lib/conditioning.d.ts +50 -0
- package/lib/conditioning.js +184 -0
- package/lib/engineDetection.d.ts +14 -0
- package/lib/engineDetection.js +110 -0
- package/lib/engineIds.d.ts +15 -0
- package/lib/engineIds.js +39 -0
- package/lib/engineParams.d.ts +3 -0
- package/lib/engineParams.js +34 -0
- package/lib/engineSetup.d.ts +153 -0
- package/lib/engineSetup.js +2 -0
- package/lib/engines/audio8.d.ts +83 -0
- package/lib/engines/audio8.js +262 -0
- package/lib/engines/chatterbox.d.ts +10 -0
- package/lib/engines/chatterbox.js +112 -0
- package/lib/engines/cosyvoice.d.ts +87 -0
- package/lib/engines/cosyvoice.js +267 -0
- package/lib/engines/moss.d.ts +26 -0
- package/lib/engines/moss.js +177 -0
- package/lib/engines/mossSoundEffect.d.ts +41 -0
- package/lib/engines/mossSoundEffect.js +117 -0
- package/lib/engines/mossSpeech.d.ts +86 -0
- package/lib/engines/mossSpeech.js +228 -0
- package/lib/engines/parler.d.ts +71 -0
- package/lib/engines/parler.js +287 -0
- package/lib/engines/supertonic.d.ts +11 -0
- package/lib/engines/supertonic.js +115 -0
- package/lib/exclusiveQueue.d.ts +16 -0
- package/lib/exclusiveQueue.js +45 -0
- package/lib/jobFields.d.ts +20 -0
- package/lib/jobFields.js +73 -0
- package/lib/modelFiles.d.ts +207 -0
- package/lib/modelFiles.js +117 -0
- package/lib/nativeParams.d.ts +12 -0
- package/lib/nativeParams.js +73 -0
- package/lib/optionValidation.d.ts +2 -0
- package/lib/optionValidation.js +103 -0
- package/lib/options.d.ts +4 -0
- package/lib/options.js +17 -0
- package/lib/pocketLifecycle.d.ts +67 -0
- package/lib/pocketLifecycle.js +170 -0
- package/lib/reload.d.ts +12 -0
- package/lib/reload.js +76 -0
- package/lib/sentenceStream.d.ts +85 -0
- package/lib/sentenceStream.js +367 -0
- package/lib/setup.d.ts +7 -0
- package/lib/setup.js +169 -0
- package/lib/streamingSupport.d.ts +8 -0
- package/lib/streamingSupport.js +77 -0
- package/lib/types.d.ts +465 -0
- package/lib/types.js +2 -0
- package/lib/voiceControls.d.ts +12 -0
- package/lib/voiceControls.js +23 -0
- package/package.json +11 -10
- package/test/integration/audio8-streaming.test.js +66 -0
- package/test/integration/audio8.test.js +6 -12
- package/test/integration/gpu-smoke.test.js +2 -11
- package/test/integration/lavasr-enhancer.test.js +2 -15
- package/test/integration/moss-sfx.test.js +29 -4
- package/test/integration/moss.test.js +1 -1
- package/test/utils/desktopGpuBackend.js +21 -0
- package/test/utils/runTTS.js +1 -1
- package/tts.d.ts +1 -0
package/index.js
CHANGED
|
@@ -34,8 +34,6 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
34
34
|
})();
|
|
35
35
|
/* eslint-disable @typescript-eslint/no-require-imports -- Bare modules and @qvac/logging expose CommonJS export shapes. */
|
|
36
36
|
const bareOs = require("bare-os");
|
|
37
|
-
const path = require("bare-path");
|
|
38
|
-
const fs = require("bare-fs");
|
|
39
37
|
const loggingModule = require("@qvac/logging");
|
|
40
38
|
// Published logging releases expose a CJS constructor; the workspace exposes
|
|
41
39
|
// an ESM default. Bare require(ESM) returns a namespace without __esModule.
|
|
@@ -44,1092 +42,24 @@ const QvacLogger = typeof loggingExport === "function" ? loggingExport : logging
|
|
|
44
42
|
/* eslint-enable @typescript-eslint/no-require-imports */
|
|
45
43
|
const infer_base_1 = require("@qvac/infer-base");
|
|
46
44
|
const tts_1 = require("./tts");
|
|
45
|
+
const addonEvents_1 = require("./lib/addonEvents");
|
|
47
46
|
const errorModule = __importStar(require("./lib/error"));
|
|
48
|
-
const pocketConfig_1 = require("./lib/pocketConfig");
|
|
49
47
|
const fit_1 = require("./lib/fit");
|
|
50
|
-
const textChunker_1 = require("./lib/textChunker");
|
|
51
48
|
const textStreamAccumulator_1 = require("./lib/textStreamAccumulator");
|
|
49
|
+
const engineIds_1 = require("./lib/engineIds");
|
|
50
|
+
const engineParams_1 = require("./lib/engineParams");
|
|
51
|
+
const audio8_1 = require("./lib/engines/audio8");
|
|
52
|
+
const parler_1 = require("./lib/engines/parler");
|
|
53
|
+
const exclusiveQueue_1 = require("./lib/exclusiveQueue");
|
|
54
|
+
const jobFields_1 = require("./lib/jobFields");
|
|
55
|
+
const pocketLifecycle_1 = require("./lib/pocketLifecycle");
|
|
56
|
+
const reload_1 = require("./lib/reload");
|
|
57
|
+
const sentenceStream_1 = require("./lib/sentenceStream");
|
|
58
|
+
const setup_1 = require("./lib/setup");
|
|
59
|
+
const streamingSupport_1 = require("./lib/streamingSupport");
|
|
60
|
+
const voiceControls_1 = require("./lib/voiceControls");
|
|
52
61
|
const { platform } = bareOs;
|
|
53
62
|
const { ERR_CODES, QvacErrorAddonTTSGgml } = errorModule;
|
|
54
|
-
const ENGINE_CHATTERBOX = "chatterbox";
|
|
55
|
-
const ENGINE_SUPERTONIC = "supertonic";
|
|
56
|
-
// CosyVoice3 (Fun-CosyVoice3-0.5B / 1.5B). Ships as a small set of GGUFs
|
|
57
|
-
// (cosyvoice3-{llm,flow,hift}-*.gguf) plus voice.gguf, vocab.json and
|
|
58
|
-
// merges.txt; a modelDir holding them (or an explicit
|
|
59
|
-
// `files.cosyvoiceModelDir`) routes here.
|
|
60
|
-
const ENGINE_COSYVOICE3 = "cosyvoice3";
|
|
61
|
-
const ENGINE_PARLER = "parler";
|
|
62
|
-
const ENGINE_AUDIO8 = "audio8";
|
|
63
|
-
const ENGINE_MOSS = "moss";
|
|
64
|
-
const ENGINE_MOSS_SFX = "moss-sfx";
|
|
65
|
-
const ENGINE_MOSS_SPEECH = "moss-speech";
|
|
66
|
-
const ENGINE_POCKET = "pocket";
|
|
67
|
-
const MIN_OUTPUT_SAMPLE_RATE = 8000;
|
|
68
|
-
const MAX_OUTPUT_SAMPLE_RATE = 192000;
|
|
69
|
-
const CHATTERBOX_T3_TURBO = "chatterbox-t3-turbo.gguf";
|
|
70
|
-
const CHATTERBOX_T3_MTL = "chatterbox-t3-mtl.gguf";
|
|
71
|
-
const CHATTERBOX_S3GEN_DEFAULT = "chatterbox-s3gen.gguf";
|
|
72
|
-
const CHATTERBOX_S3GEN_MTL = "chatterbox-s3gen-mtl.gguf";
|
|
73
|
-
const SUPERTONIC_DEFAULT = "supertonic.gguf";
|
|
74
|
-
const SUPERTONIC_MTL = "supertonic2.gguf";
|
|
75
|
-
const SUPERTONIC_V3_RE = /^supertonic3(-[a-z0-9_]+)?\.gguf$/i;
|
|
76
|
-
const SUPERTONIC_V3_QUANT_ORDER = [
|
|
77
|
-
"f16",
|
|
78
|
-
"f32",
|
|
79
|
-
"q8_0",
|
|
80
|
-
"q4_0",
|
|
81
|
-
];
|
|
82
|
-
// The LLM sub-model is the tell for CosyVoice3 modelDir auto-detection.
|
|
83
|
-
const COSYVOICE3_LLM_RE = /^cosyvoice3-llm(-[a-z0-9_]+)?\.gguf$/i;
|
|
84
|
-
// CosyVoice3 instruct2 control vocabulary (cosyvoice/utils/common.py
|
|
85
|
-
// instruct_list). The structured `instruct` option renders to the exact
|
|
86
|
-
// trained instruction string; the native engine wraps it as
|
|
87
|
-
// "You are a helpful assistant. " + <instruction> + "<|endofprompt|>" and
|
|
88
|
-
// drops the LM prompt speech tokens. One control applies per synthesis
|
|
89
|
-
// (the model is trained on single instructions).
|
|
90
|
-
const COSYVOICE_DIALECTS = {
|
|
91
|
-
cantonese: "广东话",
|
|
92
|
-
northeastern: "东北话",
|
|
93
|
-
gansu: "甘肃话",
|
|
94
|
-
guizhou: "贵州话",
|
|
95
|
-
henan: "河南话",
|
|
96
|
-
hubei: "湖北话",
|
|
97
|
-
hunan: "湖南话",
|
|
98
|
-
jiangxi: "江西话",
|
|
99
|
-
minnan: "闽南话",
|
|
100
|
-
ningxia: "宁夏话",
|
|
101
|
-
shanxi: "山西话",
|
|
102
|
-
shaanxi: "陕西话",
|
|
103
|
-
shandong: "山东话",
|
|
104
|
-
shanghai: "上海话",
|
|
105
|
-
sichuan: "四川话",
|
|
106
|
-
tianjin: "天津话",
|
|
107
|
-
yunnan: "云南话",
|
|
108
|
-
};
|
|
109
|
-
const COSYVOICE_VOLUMES = {
|
|
110
|
-
loud: "Please say a sentence as loudly as possible.",
|
|
111
|
-
soft: "Please say a sentence in a very soft voice.",
|
|
112
|
-
};
|
|
113
|
-
const COSYVOICE_STYLES = {
|
|
114
|
-
peppa: "我想体验一下小猪佩奇风格,可以吗?",
|
|
115
|
-
robot: "你可以尝试用机器人的方式解答吗?",
|
|
116
|
-
};
|
|
117
|
-
/**
|
|
118
|
-
* Look up a structured-instruct control value, throwing a clear error for an
|
|
119
|
-
* invalid key instead of letting an `undefined` render into the instruction
|
|
120
|
-
* string.
|
|
121
|
-
*/
|
|
122
|
-
function cosyvoiceInstructValue(map, key, kind) {
|
|
123
|
-
const value = map[key];
|
|
124
|
-
if (value == null) {
|
|
125
|
-
throw new Error(`Invalid CosyVoice instruct ${kind} "${key}". Valid ${kind}s: ${Object.keys(map).join(", ")}.`);
|
|
126
|
-
}
|
|
127
|
-
return value;
|
|
128
|
-
}
|
|
129
|
-
/**
|
|
130
|
-
* Render a CosyVoice3 `instruct` option to the trained instruction string.
|
|
131
|
-
* A raw string passes through (trimmed); the structured form emits exactly one
|
|
132
|
-
* control by precedence dialect > volume > style. Returns "" for no
|
|
133
|
-
* instruction. An invalid structured key throws.
|
|
134
|
-
*/
|
|
135
|
-
function renderCosyvoiceInstruct(instruct) {
|
|
136
|
-
// Only `undefined` means "omitted" (zero-shot). An explicit `null` is a
|
|
137
|
-
// malformed value, not an omission, so it is rejected below like any other
|
|
138
|
-
// non-object.
|
|
139
|
-
if (instruct === undefined)
|
|
140
|
-
return "";
|
|
141
|
-
if (typeof instruct === "string")
|
|
142
|
-
return instruct.trim();
|
|
143
|
-
// A JS caller can pass values the type forbids (null, a number, an array, a
|
|
144
|
-
// Date, a class instance). Check defensively as `unknown`: require an ordinary
|
|
145
|
-
// object literal so exotic objects can't slip past the presence checks below
|
|
146
|
-
// and degrade to zero-shot, and so control fields are read from the object's
|
|
147
|
-
// own properties rather than an inherited prototype.
|
|
148
|
-
const control = instruct;
|
|
149
|
-
if (control === null || typeof control !== "object" || Array.isArray(control)) {
|
|
150
|
-
throw new Error("Invalid CosyVoice instruct: expected a string or a plain control " +
|
|
151
|
-
"object such as { dialect: 'cantonese' }.");
|
|
152
|
-
}
|
|
153
|
-
const proto = Object.getPrototypeOf(control);
|
|
154
|
-
if (proto !== Object.prototype && proto !== null) {
|
|
155
|
-
throw new Error("Invalid CosyVoice instruct: expected a plain control object such as " +
|
|
156
|
-
"{ dialect: 'cantonese' }, not an instance of another type.");
|
|
157
|
-
}
|
|
158
|
-
// Reject unknown structured keys (typos like `{ dialekt: 'cantonese' }`)
|
|
159
|
-
// before the precedence chain, which would otherwise fall through to "".
|
|
160
|
-
const supportedControls = ["dialect", "volume", "style"];
|
|
161
|
-
const unknownKeys = Object.keys(instruct).filter((key) => !supportedControls.includes(key));
|
|
162
|
-
if (unknownKeys.length > 0) {
|
|
163
|
-
const hint = unknownKeys.some((key) => key === "emotion" || key === "speed")
|
|
164
|
-
? " Emotion and speaking rate moved to the top-level `emotion` / `pace` options."
|
|
165
|
-
: "";
|
|
166
|
-
throw new Error(`Invalid CosyVoice instruct key(s): ${unknownKeys.join(", ")}. ` +
|
|
167
|
-
`Valid keys: dialect, volume, style.${hint}`);
|
|
168
|
-
}
|
|
169
|
-
// Presence, not truthiness: a control that is set to a malformed value
|
|
170
|
-
// (`{ dialect: '' }`, `{ dialect: null }`) must raise via
|
|
171
|
-
// cosyvoiceInstructValue rather than fall through to zero-shot. `undefined`
|
|
172
|
-
// means the field was not set, so it is skipped. Precedence: dialect >
|
|
173
|
-
// volume > style.
|
|
174
|
-
if (instruct.dialect !== undefined) {
|
|
175
|
-
return `请用${cosyvoiceInstructValue(COSYVOICE_DIALECTS, instruct.dialect, "dialect")}表达。`;
|
|
176
|
-
}
|
|
177
|
-
if (instruct.volume !== undefined) {
|
|
178
|
-
return cosyvoiceInstructValue(COSYVOICE_VOLUMES, instruct.volume, "volume");
|
|
179
|
-
}
|
|
180
|
-
if (instruct.style !== undefined) {
|
|
181
|
-
return cosyvoiceInstructValue(COSYVOICE_STYLES, instruct.style, "style");
|
|
182
|
-
}
|
|
183
|
-
return "";
|
|
184
|
-
}
|
|
185
|
-
// Parler GGUFs ship per quant tier with the quant in the filename
|
|
186
|
-
// (`parler-mini-v1-q8_0.gguf`); a bare `parler.gguf` deliberately does not
|
|
187
|
-
// match, keeping ambiguous files out of the modelDir auto-detect path.
|
|
188
|
-
const PARLER_RE = /^parler-(mini|large|indic)(-v\d+)?(-[a-z0-9_]+)?\.gguf$/i;
|
|
189
|
-
const PARLER_VARIANT_ORDER = ["mini", "large", "indic"];
|
|
190
|
-
const PARLER_QUANT_ORDER = ["q8_0", "q6_k", "f16", "f32"];
|
|
191
|
-
const PARLER_DESCRIPTION_KEYS = ["description", "voiceDescription"];
|
|
192
|
-
// What native Parler reads as template fields. `emotion` / `pace` stay listed
|
|
193
|
-
// here because build_description() still renders them, even though they are
|
|
194
|
-
// cross-engine options at the JS surface.
|
|
195
|
-
const PARLER_TEMPLATE_KEYS = [
|
|
196
|
-
"voice",
|
|
197
|
-
"emotion",
|
|
198
|
-
"pitch",
|
|
199
|
-
"pace",
|
|
200
|
-
"expressivity",
|
|
201
|
-
"noise",
|
|
202
|
-
"reverb",
|
|
203
|
-
"quality",
|
|
204
|
-
];
|
|
205
|
-
// The subset that stays Parler-only now that emotion/pace are cross-engine.
|
|
206
|
-
const PARLER_ONLY_TEMPLATE_KEYS = [
|
|
207
|
-
"voice",
|
|
208
|
-
"pitch",
|
|
209
|
-
"expressivity",
|
|
210
|
-
"noise",
|
|
211
|
-
"reverb",
|
|
212
|
-
"quality",
|
|
213
|
-
];
|
|
214
|
-
const PARLER_FIELD_KEYS = [
|
|
215
|
-
...PARLER_DESCRIPTION_KEYS,
|
|
216
|
-
...PARLER_TEMPLATE_KEYS,
|
|
217
|
-
];
|
|
218
|
-
const PARLER_ONLY_KEYS = [
|
|
219
|
-
...PARLER_DESCRIPTION_KEYS,
|
|
220
|
-
...PARLER_ONLY_TEMPLATE_KEYS,
|
|
221
|
-
];
|
|
222
|
-
// Canonical cross-engine conditioning vocabulary. This is a MIRROR, not the
|
|
223
|
-
// source of truth: tts-cpp owns it in include/tts-cpp/voice_controls.h, and
|
|
224
|
-
// test/unit/voice-controls.test.js pins this copy against the native lists.
|
|
225
|
-
const EMOTIONS = [
|
|
226
|
-
"command",
|
|
227
|
-
"anger",
|
|
228
|
-
"narration",
|
|
229
|
-
"conversation",
|
|
230
|
-
"disgust",
|
|
231
|
-
"fear",
|
|
232
|
-
"happy",
|
|
233
|
-
"neutral",
|
|
234
|
-
"proper noun",
|
|
235
|
-
"news",
|
|
236
|
-
"sad",
|
|
237
|
-
"surprise",
|
|
238
|
-
];
|
|
239
|
-
const PACES = ["slow", "moderate", "fast"];
|
|
240
|
-
const CONDITIONING_KEYS = ["emotion", "pace"];
|
|
241
|
-
// Audio8 ships three GGUFs per quant tier, with the tier in the filename
|
|
242
|
-
// (`audio8-lm-q8_0.gguf`); a bare `audio8-lm.gguf` matches too, as
|
|
243
|
-
// forward-compat with a single-tier release.
|
|
244
|
-
const AUDIO8_LM_RE = /^audio8-lm(-[a-z0-9_]+)?\.gguf$/i;
|
|
245
|
-
const AUDIO8_DECODER_RE = /^audio8-codec-decoder(-[a-z0-9_]+)?\.gguf$/i;
|
|
246
|
-
const AUDIO8_ENCODER_RE = /^audio8-codec-encoder(-[a-z0-9_]+)?\.gguf$/i;
|
|
247
|
-
const AUDIO8_QUANT_ORDER = ["q8_0", "f16", "q4_0", "f32"];
|
|
248
|
-
const AUDIO8_VOICE_KEYS = ["referenceAudio", "referenceText"];
|
|
249
|
-
const MOSS_BACKBONE_RE = /^moss-tts-delay(-[a-z0-9_]+)?\.gguf$/i;
|
|
250
|
-
const MOSS_DIALOGUE_BACKBONE_RE = /^moss-ttsd(-[a-z0-9_]+)?\.gguf$/i;
|
|
251
|
-
const MOSS_DECODER_RE = /^moss-codec-decoder(-[a-z0-9_]+)?\.gguf$/i;
|
|
252
|
-
const MOSS_ENCODER_RE = /^moss-codec-encoder(-[a-z0-9_]+)?\.gguf$/i;
|
|
253
|
-
const MOSS_NATIVE_SAMPLE_RATE = 24000;
|
|
254
|
-
const MOSS_FRAMES_PER_SECOND = 12.5;
|
|
255
|
-
const MOSS_MAX_NEW_TOKENS = 2048;
|
|
256
|
-
const MOSS_MAX_CHANNELS = 32;
|
|
257
|
-
const MOSS_TERMINATION_ROWS = 2;
|
|
258
|
-
const MOSS_MAX_DURATION_TOKENS = MOSS_MAX_NEW_TOKENS - (MOSS_MAX_CHANNELS - 1) - MOSS_TERMINATION_ROWS;
|
|
259
|
-
const MOSS_INSTANCE_VOICE_KEYS = ["referenceAudio", "dialogueReferences"];
|
|
260
|
-
const MOSS_SFX_RE = /^moss-sfx(-[a-z0-9_]+)*\.gguf$/i;
|
|
261
|
-
const MOSS_SFX_NATIVE_SAMPLE_RATE = 48000;
|
|
262
|
-
const MOSS_SFX_MAX_SECONDS = 30;
|
|
263
|
-
const MOSS_SFX_MAX_STEPS = 1000;
|
|
264
|
-
const MOSS_SFX_MAX_GUIDANCE = 50;
|
|
265
|
-
const MOSS_SFX_MAX_SHIFT = 100;
|
|
266
|
-
const MOSS_SFX_CALL_KEYS = [
|
|
267
|
-
"seconds",
|
|
268
|
-
"negativePrompt",
|
|
269
|
-
"steps",
|
|
270
|
-
"guidance",
|
|
271
|
-
"shift",
|
|
272
|
-
];
|
|
273
|
-
const MOSS_SPEECH_RE = /^moss-speech(-(?!codec)[a-z0-9_]+)?\.gguf$/i;
|
|
274
|
-
const MOSS_SPEECH_CODEC_RE = /^moss-speech-codec(-[a-z0-9_]+)?\.gguf$/i;
|
|
275
|
-
const MOSS_SPEECH_NATIVE_SAMPLE_RATE = 24000;
|
|
276
|
-
const MOSS_SPEECH_MIN_SAMPLE_RATE = 8000;
|
|
277
|
-
const MOSS_SPEECH_MAX_SAMPLE_RATE = 192000;
|
|
278
|
-
const MOSS_SPEECH_MAX_NEW_TOKENS = 4096;
|
|
279
|
-
const MOSS_SPEECH_MAX_REPLY_SECONDS = 3600;
|
|
280
|
-
const MOSS_SPEECH_MAX_TEMPERATURE = 10;
|
|
281
|
-
const MOSS_SPEECH_MAX_TOP_P = 1;
|
|
282
|
-
const MOSS_SPEECH_ROLES = ["system", "user", "assistant"];
|
|
283
|
-
const MOSS_SPEECH_ONLY_KEYS = [
|
|
284
|
-
"audio",
|
|
285
|
-
"sampleRate",
|
|
286
|
-
"messages",
|
|
287
|
-
"systemPrompt",
|
|
288
|
-
"replyVoice",
|
|
289
|
-
"replyVoiceSampleRate",
|
|
290
|
-
"textReply",
|
|
291
|
-
"maxReplySeconds",
|
|
292
|
-
"maxNewTokens",
|
|
293
|
-
];
|
|
294
|
-
const MOSS_SPEECH_CONTROL_KEYS = [
|
|
295
|
-
"textReply",
|
|
296
|
-
"maxReplySeconds",
|
|
297
|
-
"maxNewTokens",
|
|
298
|
-
"greedy",
|
|
299
|
-
"temperature",
|
|
300
|
-
"topP",
|
|
301
|
-
"topK",
|
|
302
|
-
];
|
|
303
|
-
/** Engines that draw tokens, and so accept temperature / topK / topP / maxFrames. */
|
|
304
|
-
const SAMPLING_ENGINES = [ENGINE_PARLER, ENGINE_AUDIO8];
|
|
305
|
-
// Per-engine supported subsets, mirroring controls::supported_emotions() /
|
|
306
|
-
// supported_paces(). An empty list means the engine has no such control.
|
|
307
|
-
const ENGINE_EMOTIONS = {
|
|
308
|
-
[ENGINE_PARLER]: EMOTIONS,
|
|
309
|
-
// Only the emotions Fun-CosyVoice3 has a trained instruction for; the rest,
|
|
310
|
-
// and the upstream spelling "angry", are rejected rather than paraphrased.
|
|
311
|
-
[ENGINE_COSYVOICE3]: ["anger", "happy", "neutral", "sad"],
|
|
312
|
-
[ENGINE_SUPERTONIC]: [],
|
|
313
|
-
[ENGINE_CHATTERBOX]: [],
|
|
314
|
-
[ENGINE_POCKET]: [],
|
|
315
|
-
[ENGINE_AUDIO8]: [],
|
|
316
|
-
[ENGINE_MOSS]: [],
|
|
317
|
-
[ENGINE_MOSS_SFX]: [],
|
|
318
|
-
[ENGINE_MOSS_SPEECH]: [],
|
|
319
|
-
};
|
|
320
|
-
const ENGINE_PACES = {
|
|
321
|
-
[ENGINE_PARLER]: PACES, // rendered into the training caption
|
|
322
|
-
[ENGINE_COSYVOICE3]: PACES, // slow/fast -> instruct; moderate -> none
|
|
323
|
-
[ENGINE_SUPERTONIC]: PACES, // mapped onto the duration multiplier
|
|
324
|
-
[ENGINE_CHATTERBOX]: [], // time-stretch only; use `speed`
|
|
325
|
-
[ENGINE_POCKET]: [],
|
|
326
|
-
[ENGINE_AUDIO8]: [], // no rate control at all
|
|
327
|
-
[ENGINE_MOSS]: [],
|
|
328
|
-
[ENGINE_MOSS_SFX]: [],
|
|
329
|
-
[ENGINE_MOSS_SPEECH]: [],
|
|
330
|
-
};
|
|
331
|
-
// Which channels an engine can change per call. Supertonic takes its pace
|
|
332
|
-
// through EngineOptions when the engine is built, and tts-cpp exposes no
|
|
333
|
-
// per-call surface for it, so it only moves at construction or reload().
|
|
334
|
-
// tts-cpp engine names (tts-cpp/voice_controls.h engine_name) that differ from
|
|
335
|
-
// this package's engine types.
|
|
336
|
-
const FABRIC_ENGINE_TYPES = {
|
|
337
|
-
cosyvoice: ENGINE_COSYVOICE3,
|
|
338
|
-
};
|
|
339
|
-
const ENGINE_PER_CALL_CONDITIONING = {
|
|
340
|
-
[ENGINE_PARLER]: CONDITIONING_KEYS,
|
|
341
|
-
[ENGINE_COSYVOICE3]: CONDITIONING_KEYS,
|
|
342
|
-
[ENGINE_SUPERTONIC]: [],
|
|
343
|
-
[ENGINE_CHATTERBOX]: [],
|
|
344
|
-
[ENGINE_POCKET]: [],
|
|
345
|
-
[ENGINE_AUDIO8]: [],
|
|
346
|
-
[ENGINE_MOSS]: [],
|
|
347
|
-
[ENGINE_MOSS_SFX]: [],
|
|
348
|
-
[ENGINE_MOSS_SPEECH]: [],
|
|
349
|
-
};
|
|
350
|
-
function normalizeError(error) {
|
|
351
|
-
return typeof error === "string"
|
|
352
|
-
? error
|
|
353
|
-
: error instanceof Error
|
|
354
|
-
? error
|
|
355
|
-
: new Error("Unknown TTS error");
|
|
356
|
-
}
|
|
357
|
-
function firstNonEmpty(...candidates) {
|
|
358
|
-
for (const value of candidates) {
|
|
359
|
-
if (value != null && value !== "")
|
|
360
|
-
return value;
|
|
361
|
-
}
|
|
362
|
-
return undefined;
|
|
363
|
-
}
|
|
364
|
-
function fileExistsSafe(filePath) {
|
|
365
|
-
if (!filePath)
|
|
366
|
-
return false;
|
|
367
|
-
try {
|
|
368
|
-
return fs.existsSync(filePath);
|
|
369
|
-
}
|
|
370
|
-
catch {
|
|
371
|
-
return false;
|
|
372
|
-
}
|
|
373
|
-
}
|
|
374
|
-
function findSupertonicV3InDir(modelDir) {
|
|
375
|
-
if (!modelDir)
|
|
376
|
-
return undefined;
|
|
377
|
-
let entries;
|
|
378
|
-
try {
|
|
379
|
-
entries = fs.readdirSync(modelDir);
|
|
380
|
-
}
|
|
381
|
-
catch {
|
|
382
|
-
return undefined;
|
|
383
|
-
}
|
|
384
|
-
const matches = entries.filter((name) => SUPERTONIC_V3_RE.test(name));
|
|
385
|
-
if (matches.length === 0)
|
|
386
|
-
return undefined;
|
|
387
|
-
function rank(name) {
|
|
388
|
-
if (/^supertonic3\.gguf$/i.test(name))
|
|
389
|
-
return 0;
|
|
390
|
-
const match = name.match(/^supertonic3-(.+)\.gguf$/i);
|
|
391
|
-
const index = match
|
|
392
|
-
? SUPERTONIC_V3_QUANT_ORDER.indexOf(match[1].toLowerCase())
|
|
393
|
-
: -1;
|
|
394
|
-
return index === -1
|
|
395
|
-
? SUPERTONIC_V3_QUANT_ORDER.length + 1
|
|
396
|
-
: index + 1;
|
|
397
|
-
}
|
|
398
|
-
matches.sort((left, right) => rank(left) - rank(right));
|
|
399
|
-
return path.join(modelDir, matches[0]);
|
|
400
|
-
}
|
|
401
|
-
/**
|
|
402
|
-
* True when `modelDir` contains a CosyVoice3 LLM GGUF (the sub-model that
|
|
403
|
-
* unambiguously identifies a CosyVoice3 model directory).
|
|
404
|
-
*/
|
|
405
|
-
function dirHasCosyvoice3(modelDir) {
|
|
406
|
-
if (!modelDir)
|
|
407
|
-
return false;
|
|
408
|
-
let entries;
|
|
409
|
-
try {
|
|
410
|
-
entries = fs.readdirSync(modelDir);
|
|
411
|
-
}
|
|
412
|
-
catch {
|
|
413
|
-
return false;
|
|
414
|
-
}
|
|
415
|
-
return entries.some((name) => COSYVOICE3_LLM_RE.test(name));
|
|
416
|
-
}
|
|
417
|
-
/**
|
|
418
|
-
* Find a Parler GGUF in `modelDir`, ranked by variant (mini before large
|
|
419
|
-
* before indic) and, within a variant, by quant tier (q8_0 > q6_k > f16 > f32;
|
|
420
|
-
* a bare `parler-<variant>.gguf` wins as forward-compat).
|
|
421
|
-
*/
|
|
422
|
-
function findParlerInDir(modelDir) {
|
|
423
|
-
if (!modelDir)
|
|
424
|
-
return undefined;
|
|
425
|
-
let entries;
|
|
426
|
-
try {
|
|
427
|
-
entries = fs.readdirSync(modelDir);
|
|
428
|
-
}
|
|
429
|
-
catch {
|
|
430
|
-
return undefined;
|
|
431
|
-
}
|
|
432
|
-
const matches = entries.filter((name) => PARLER_RE.test(name));
|
|
433
|
-
if (matches.length === 0)
|
|
434
|
-
return undefined;
|
|
435
|
-
function rank(name) {
|
|
436
|
-
const match = name.match(PARLER_RE);
|
|
437
|
-
if (!match)
|
|
438
|
-
return Number.MAX_SAFE_INTEGER;
|
|
439
|
-
const variant = PARLER_VARIANT_ORDER.indexOf(match[1].toLowerCase());
|
|
440
|
-
let quantRank = 0;
|
|
441
|
-
if (match[3]) {
|
|
442
|
-
const index = PARLER_QUANT_ORDER.indexOf(match[3].slice(1).toLowerCase());
|
|
443
|
-
quantRank =
|
|
444
|
-
index === -1 ? PARLER_QUANT_ORDER.length + 2 : index + 1;
|
|
445
|
-
}
|
|
446
|
-
return variant * 100 + quantRank;
|
|
447
|
-
}
|
|
448
|
-
matches.sort((left, right) => rank(left) - rank(right));
|
|
449
|
-
return path.join(modelDir, matches[0]);
|
|
450
|
-
}
|
|
451
|
-
/** Names of the entries that carry a value, for "wrong engine" messages. */
|
|
452
|
-
function setOptionNames(options) {
|
|
453
|
-
return Object.entries(options)
|
|
454
|
-
.filter(([, value]) => value != null)
|
|
455
|
-
.map(([name]) => name);
|
|
456
|
-
}
|
|
457
|
-
function readDirSafe(modelDir) {
|
|
458
|
-
if (!modelDir)
|
|
459
|
-
return [];
|
|
460
|
-
try {
|
|
461
|
-
return fs.readdirSync(modelDir);
|
|
462
|
-
}
|
|
463
|
-
catch {
|
|
464
|
-
return [];
|
|
465
|
-
}
|
|
466
|
-
}
|
|
467
|
-
function ggufQuantRank(name, pattern) {
|
|
468
|
-
const match = name.match(pattern);
|
|
469
|
-
if (!match)
|
|
470
|
-
return Number.MAX_SAFE_INTEGER;
|
|
471
|
-
if (!match[1])
|
|
472
|
-
return 0;
|
|
473
|
-
const index = AUDIO8_QUANT_ORDER.indexOf(match[1].slice(1).toLowerCase());
|
|
474
|
-
return index === -1 ? AUDIO8_QUANT_ORDER.length + 1 : index + 1;
|
|
475
|
-
}
|
|
476
|
-
/**
|
|
477
|
-
* Find one Audio8 or MOSS GGUF in `modelDir`, preferring the quant tier that
|
|
478
|
-
* reads best for its size (q8_0 > f16 > q4_0 > f32).
|
|
479
|
-
*/
|
|
480
|
-
function findQuantRankedGguf(modelDir, pattern) {
|
|
481
|
-
if (!modelDir)
|
|
482
|
-
return undefined;
|
|
483
|
-
const matches = readDirSafe(modelDir).filter((name) => pattern.test(name));
|
|
484
|
-
if (matches.length === 0)
|
|
485
|
-
return undefined;
|
|
486
|
-
matches.sort((left, right) => ggufQuantRank(left, pattern) - ggufQuantRank(right, pattern));
|
|
487
|
-
return path.join(modelDir, matches[0]);
|
|
488
|
-
}
|
|
489
|
-
function assertSoundEffectRange(name, value, accepts, range) {
|
|
490
|
-
if (typeof value === "number" && Number.isFinite(value) && accepts(value))
|
|
491
|
-
return;
|
|
492
|
-
throw new Error(`tts-ggml: moss-sfx ${name} must be ${range} (got ${String(value)})`);
|
|
493
|
-
}
|
|
494
|
-
function assertSoundEffectFields(fields) {
|
|
495
|
-
if (fields.seconds !== undefined) {
|
|
496
|
-
assertSoundEffectRange("seconds", fields.seconds, (n) => n > 0 && n <= MOSS_SFX_MAX_SECONDS, `in (0, ${MOSS_SFX_MAX_SECONDS}]`);
|
|
497
|
-
}
|
|
498
|
-
if (fields.steps !== undefined) {
|
|
499
|
-
assertSoundEffectRange("steps", fields.steps, (n) => Number.isInteger(n) && n >= 1 && n <= MOSS_SFX_MAX_STEPS, `an integer in [1, ${MOSS_SFX_MAX_STEPS}]`);
|
|
500
|
-
}
|
|
501
|
-
if (fields.guidance !== undefined) {
|
|
502
|
-
assertSoundEffectRange("guidance", fields.guidance, (n) => n >= 1 && n <= MOSS_SFX_MAX_GUIDANCE, `in [1, ${MOSS_SFX_MAX_GUIDANCE}]`);
|
|
503
|
-
}
|
|
504
|
-
if (fields.shift !== undefined) {
|
|
505
|
-
assertSoundEffectRange("shift", fields.shift, (n) => n > 0 && n <= MOSS_SFX_MAX_SHIFT, `in (0, ${MOSS_SFX_MAX_SHIFT}]`);
|
|
506
|
-
}
|
|
507
|
-
if (fields.negativePrompt !== undefined && typeof fields.negativePrompt !== "string") {
|
|
508
|
-
throw new Error("tts-ggml: moss-sfx negativePrompt must be a string");
|
|
509
|
-
}
|
|
510
|
-
}
|
|
511
|
-
function isSpeechPcm(value) {
|
|
512
|
-
return value instanceof Int16Array || value instanceof Float32Array;
|
|
513
|
-
}
|
|
514
|
-
function assertSpeechSampleRate(value, name, where) {
|
|
515
|
-
if (typeof value === "number" &&
|
|
516
|
-
Number.isInteger(value) &&
|
|
517
|
-
value >= MOSS_SPEECH_MIN_SAMPLE_RATE &&
|
|
518
|
-
value <= MOSS_SPEECH_MAX_SAMPLE_RATE) {
|
|
519
|
-
return;
|
|
520
|
-
}
|
|
521
|
-
throw new Error(`tts-ggml: ${where}: moss-speech ${name} must be an integer in ` +
|
|
522
|
-
`[${MOSS_SPEECH_MIN_SAMPLE_RATE}, ${MOSS_SPEECH_MAX_SAMPLE_RATE}] (got ${String(value)})`);
|
|
523
|
-
}
|
|
524
|
-
function assertSpeechAudio(audio, rate, names, where) {
|
|
525
|
-
if (!isSpeechPcm(audio) || audio.length === 0) {
|
|
526
|
-
throw new Error(`tts-ggml: ${where}: moss-speech ${names[0]} must be a non-empty Int16Array or Float32Array`);
|
|
527
|
-
}
|
|
528
|
-
assertSpeechSampleRate(rate, names[1], where);
|
|
529
|
-
}
|
|
530
|
-
function assertSpeechMessage(message, index, where) {
|
|
531
|
-
const label = `messages[${index}]`;
|
|
532
|
-
if (message == null || typeof message !== "object") {
|
|
533
|
-
throw new Error(`tts-ggml: ${where}: moss-speech ${label} must be an object`);
|
|
534
|
-
}
|
|
535
|
-
const turn = message;
|
|
536
|
-
if (!MOSS_SPEECH_ROLES.includes(turn.role)) {
|
|
537
|
-
throw new Error(`tts-ggml: ${where}: moss-speech ${label}.role must be one of ${MOSS_SPEECH_ROLES.join(", ")}`);
|
|
538
|
-
}
|
|
539
|
-
const hasText = typeof turn.text === "string" && turn.text.length > 0;
|
|
540
|
-
const hasAudio = turn.audio !== undefined;
|
|
541
|
-
if (hasText === hasAudio) {
|
|
542
|
-
throw new Error(`tts-ggml: ${where}: moss-speech ${label} carries exactly one of text or audio`);
|
|
543
|
-
}
|
|
544
|
-
if (hasAudio) {
|
|
545
|
-
assertSpeechAudio(turn.audio, turn.sampleRate, [`${label}.audio`, `${label}.sampleRate`], where);
|
|
546
|
-
}
|
|
547
|
-
}
|
|
548
|
-
function assertSpeechMessages(messages, where) {
|
|
549
|
-
if (messages === undefined)
|
|
550
|
-
return;
|
|
551
|
-
if (!Array.isArray(messages)) {
|
|
552
|
-
throw new Error(`tts-ggml: ${where}: moss-speech messages must be an array`);
|
|
553
|
-
}
|
|
554
|
-
assertEachSpeechMessage(messages, where);
|
|
555
|
-
}
|
|
556
|
-
function assertEachSpeechMessage(messages, where) {
|
|
557
|
-
messages.forEach((message, index) => assertSpeechMessage(message, index, where));
|
|
558
|
-
}
|
|
559
|
-
function assertSpeechUserTurn(fields, where) {
|
|
560
|
-
const hasText = typeof fields.input === "string" && fields.input.trim().length > 0;
|
|
561
|
-
const hasAudio = fields.audio !== undefined;
|
|
562
|
-
if (hasText && hasAudio) {
|
|
563
|
-
throw new Error(`tts-ggml: ${where}: moss-speech takes the user turn as audio or input text, not both`);
|
|
564
|
-
}
|
|
565
|
-
if (hasAudio) {
|
|
566
|
-
assertSpeechAudio(fields.audio, fields.sampleRate, ["audio", "sampleRate"], where);
|
|
567
|
-
return;
|
|
568
|
-
}
|
|
569
|
-
if (!hasText) {
|
|
570
|
-
throw new Error(`tts-ggml: ${where}: moss-speech needs the user turn as audio (with sampleRate) or input text`);
|
|
571
|
-
}
|
|
572
|
-
}
|
|
573
|
-
function assertSpeechRange(name, value, accepts, range, where) {
|
|
574
|
-
if (value === undefined)
|
|
575
|
-
return;
|
|
576
|
-
if (typeof value === "number" && Number.isFinite(value) && accepts(value))
|
|
577
|
-
return;
|
|
578
|
-
const got = typeof value === "number" ? String(value) : typeof value;
|
|
579
|
-
throw new Error(`tts-ggml: ${where}: moss-speech ${name} must be ${range} (got ${got})`);
|
|
580
|
-
}
|
|
581
|
-
function assertSpeechControls(fields, where) {
|
|
582
|
-
assertSpeechRange("maxReplySeconds", fields.maxReplySeconds, (n) => n >= 0 && n <= MOSS_SPEECH_MAX_REPLY_SECONDS, `in [0, ${MOSS_SPEECH_MAX_REPLY_SECONDS}]`, where);
|
|
583
|
-
assertSpeechRange("maxNewTokens", fields.maxNewTokens, (n) => Number.isInteger(n) && n >= 1 && n <= MOSS_SPEECH_MAX_NEW_TOKENS, `an integer in [1, ${MOSS_SPEECH_MAX_NEW_TOKENS}]`, where);
|
|
584
|
-
assertSpeechRange("temperature", fields.temperature, (n) => n > 0 && n <= MOSS_SPEECH_MAX_TEMPERATURE, `in (0, ${MOSS_SPEECH_MAX_TEMPERATURE}]`, where);
|
|
585
|
-
assertSpeechRange("topP", fields.topP, (n) => n > 0 && n <= MOSS_SPEECH_MAX_TOP_P, `in (0, ${MOSS_SPEECH_MAX_TOP_P}]`, where);
|
|
586
|
-
assertSpeechRange("topK", fields.topK, (n) => Number.isInteger(n) && n >= 0, "an integer >= 0", where);
|
|
587
|
-
assertSpeechBooleans(fields, where);
|
|
588
|
-
if (fields.systemPrompt !== undefined && typeof fields.systemPrompt !== "string") {
|
|
589
|
-
throw new Error(`tts-ggml: ${where}: moss-speech systemPrompt must be a string`);
|
|
590
|
-
}
|
|
591
|
-
}
|
|
592
|
-
function assertSpeechBooleans(fields, where) {
|
|
593
|
-
for (const flag of ["textReply", "greedy"]) {
|
|
594
|
-
if (fields[flag] !== undefined && typeof fields[flag] !== "boolean") {
|
|
595
|
-
throw new Error(`tts-ggml: ${where}: moss-speech ${flag} must be a boolean`);
|
|
596
|
-
}
|
|
597
|
-
}
|
|
598
|
-
}
|
|
599
|
-
function assertSpeechRateHasAudio(audio, rate, names, where) {
|
|
600
|
-
if (audio !== undefined || rate === undefined)
|
|
601
|
-
return;
|
|
602
|
-
throw new Error(`tts-ggml: ${where}: moss-speech ${names[1]} needs ${names[0]}`);
|
|
603
|
-
}
|
|
604
|
-
function assertSpeechCall(fields, where) {
|
|
605
|
-
assertSpeechUserTurn(fields, where);
|
|
606
|
-
assertSpeechRateHasAudio(fields.audio, fields.sampleRate, ["audio", "sampleRate"], where);
|
|
607
|
-
assertSpeechRateHasAudio(fields.replyVoice, fields.replyVoiceSampleRate, ["replyVoice", "replyVoiceSampleRate"], where);
|
|
608
|
-
assertSpeechMessages(fields.messages, where);
|
|
609
|
-
if (fields.replyVoice !== undefined) {
|
|
610
|
-
assertSpeechAudio(fields.replyVoice, fields.replyVoiceSampleRate, ["replyVoice", "replyVoiceSampleRate"], where);
|
|
611
|
-
}
|
|
612
|
-
assertSpeechControls(fields, where);
|
|
613
|
-
}
|
|
614
|
-
function speechUserTurn(fields) {
|
|
615
|
-
if (fields.audio !== undefined) {
|
|
616
|
-
return { role: "user", audio: fields.audio, sampleRate: fields.sampleRate };
|
|
617
|
-
}
|
|
618
|
-
return { role: "user", text: fields.input };
|
|
619
|
-
}
|
|
620
|
-
function speechMessages(fields) {
|
|
621
|
-
const system = fields.systemPrompt ? [{ role: "system", text: fields.systemPrompt }] : [];
|
|
622
|
-
return [...system, ...(fields.messages ?? []), speechUserTurn(fields)];
|
|
623
|
-
}
|
|
624
|
-
function speechReplyVoice(fields) {
|
|
625
|
-
if (fields.replyVoice === undefined)
|
|
626
|
-
return {};
|
|
627
|
-
return { replyVoice: fields.replyVoice, replyVoiceSampleRate: fields.replyVoiceSampleRate };
|
|
628
|
-
}
|
|
629
|
-
function speechControls(fields) {
|
|
630
|
-
const present = MOSS_SPEECH_CONTROL_KEYS.filter((key) => fields[key] !== undefined);
|
|
631
|
-
return Object.fromEntries(present.map((key) => [key, fields[key]]));
|
|
632
|
-
}
|
|
633
|
-
/** Collect the fields only MOSS-Speech reads that are present on `source`; undefined when none are set. */
|
|
634
|
-
function pickSpeechFields(source) {
|
|
635
|
-
if (source == null || typeof source !== "object")
|
|
636
|
-
return undefined;
|
|
637
|
-
const present = MOSS_SPEECH_ONLY_KEYS.filter((key) => source[key] !== undefined);
|
|
638
|
-
if (present.length === 0)
|
|
639
|
-
return undefined;
|
|
640
|
-
return Object.fromEntries(present.map((key) => [key, source[key]]));
|
|
641
|
-
}
|
|
642
|
-
/**
|
|
643
|
-
* Collect the MOSS-SoundEffect generation controls present on `source`.
|
|
644
|
-
* Returns undefined when none are set.
|
|
645
|
-
*/
|
|
646
|
-
function pickSoundEffectFields(source) {
|
|
647
|
-
if (source == null || typeof source !== "object")
|
|
648
|
-
return undefined;
|
|
649
|
-
const present = MOSS_SFX_CALL_KEYS.filter((key) => source[key] !== undefined);
|
|
650
|
-
if (present.length === 0)
|
|
651
|
-
return undefined;
|
|
652
|
-
return Object.fromEntries(present.map((key) => [key, source[key]]));
|
|
653
|
-
}
|
|
654
|
-
/**
|
|
655
|
-
* Collect the Audio8 voice-cloning properties present on `source` (a run
|
|
656
|
-
* input, streaming options, or constructor options). Returns undefined when
|
|
657
|
-
* none are set.
|
|
658
|
-
*/
|
|
659
|
-
function pickAudio8VoiceFields(source) {
|
|
660
|
-
if (source == null || typeof source !== "object")
|
|
661
|
-
return undefined;
|
|
662
|
-
const out = {};
|
|
663
|
-
let any = false;
|
|
664
|
-
for (const key of AUDIO8_VOICE_KEYS) {
|
|
665
|
-
const value = source[key];
|
|
666
|
-
if (value != null && value !== "") {
|
|
667
|
-
out[key] = String(value);
|
|
668
|
-
any = true;
|
|
669
|
-
}
|
|
670
|
-
}
|
|
671
|
-
return any ? out : undefined;
|
|
672
|
-
}
|
|
673
|
-
/**
|
|
674
|
-
* Collect the Parler description/template properties present on `source`
|
|
675
|
-
* (a run input, streaming options, or constructor options). Returns undefined
|
|
676
|
-
* when none are set.
|
|
677
|
-
*/
|
|
678
|
-
function pickParlerDescFields(source) {
|
|
679
|
-
if (source == null || typeof source !== "object")
|
|
680
|
-
return undefined;
|
|
681
|
-
const out = {};
|
|
682
|
-
let any = false;
|
|
683
|
-
for (const key of PARLER_FIELD_KEYS) {
|
|
684
|
-
const value = source[key];
|
|
685
|
-
if (value != null && value !== "") {
|
|
686
|
-
out[key] = String(value);
|
|
687
|
-
any = true;
|
|
688
|
-
}
|
|
689
|
-
}
|
|
690
|
-
return any ? out : undefined;
|
|
691
|
-
}
|
|
692
|
-
/**
|
|
693
|
-
* Same-level conflict check: a free-text description cannot be merged with
|
|
694
|
-
* template fields, so setting both together is an error.
|
|
695
|
-
*/
|
|
696
|
-
function assertParlerDescFieldsConsistent(fields, where) {
|
|
697
|
-
if (!fields)
|
|
698
|
-
return;
|
|
699
|
-
const hasDescription = PARLER_DESCRIPTION_KEYS.some((key) => fields[key] != null);
|
|
700
|
-
const templateKeys = PARLER_TEMPLATE_KEYS.filter((key) => fields[key] != null);
|
|
701
|
-
if (hasDescription && templateKeys.length > 0) {
|
|
702
|
-
throw new Error(`tts-ggml: ${where}: 'description' is mutually exclusive with the ` +
|
|
703
|
-
`voice-template options (got ${templateKeys.join(", ")})`);
|
|
704
|
-
}
|
|
705
|
-
}
|
|
706
|
-
/** Which of `keys` are set on `source`. Used to name stray options in errors. */
|
|
707
|
-
function keysPresent(source, keys) {
|
|
708
|
-
if (source == null || typeof source !== "object")
|
|
709
|
-
return [];
|
|
710
|
-
const record = source;
|
|
711
|
-
return keys.filter((key) => record[key] != null && record[key] !== "");
|
|
712
|
-
}
|
|
713
|
-
/** Collect the cross-engine conditioning properties present on `source`. */
|
|
714
|
-
function pickConditioningFields(source) {
|
|
715
|
-
if (source == null || typeof source !== "object")
|
|
716
|
-
return undefined;
|
|
717
|
-
const out = {};
|
|
718
|
-
let any = false;
|
|
719
|
-
for (const key of CONDITIONING_KEYS) {
|
|
720
|
-
const value = source[key];
|
|
721
|
-
if (value != null && value !== "") {
|
|
722
|
-
out[key] = value;
|
|
723
|
-
any = true;
|
|
724
|
-
}
|
|
725
|
-
}
|
|
726
|
-
return any ? out : undefined;
|
|
727
|
-
}
|
|
728
|
-
function assertCanonicalValue(value, canonical, kind) {
|
|
729
|
-
if (canonical.includes(value.toLowerCase()))
|
|
730
|
-
return;
|
|
731
|
-
throw new Error(`tts-ggml: invalid ${kind} "${value}". ` +
|
|
732
|
-
`Valid ${kind}s: ${canonical.join(", ")}.`);
|
|
733
|
-
}
|
|
734
|
-
function supportedValues(engine, kind) {
|
|
735
|
-
return kind === "emotion" ? ENGINE_EMOTIONS[engine] : ENGINE_PACES[engine];
|
|
736
|
-
}
|
|
737
|
-
function assertEngineSupports(engine, kind, value) {
|
|
738
|
-
const supported = supportedValues(engine, kind);
|
|
739
|
-
if (supported.length === 0) {
|
|
740
|
-
const hint = kind === "pace"
|
|
741
|
-
? "; use `speed` (an exact multiplier) instead"
|
|
742
|
-
: "";
|
|
743
|
-
throw new Error(`tts-ggml: the ${engine} engine does not support \`${kind}\`${hint}`);
|
|
744
|
-
}
|
|
745
|
-
if (supported.includes(value.toLowerCase()))
|
|
746
|
-
return;
|
|
747
|
-
throw new Error(`tts-ggml: ${kind} "${value}" is not supported by the ${engine} engine. ` +
|
|
748
|
-
`${engine} supports: ${supported.join(", ")}.`);
|
|
749
|
-
}
|
|
750
|
-
/**
|
|
751
|
-
* Validate emotion/pace against the canonical vocabulary first (so a typo reads
|
|
752
|
-
* clearly), then against what this engine actually supports.
|
|
753
|
-
*/
|
|
754
|
-
function validateConditioning(engine, fields) {
|
|
755
|
-
if (!fields)
|
|
756
|
-
return;
|
|
757
|
-
if (fields.emotion != null) {
|
|
758
|
-
assertCanonicalValue(fields.emotion, EMOTIONS, "emotion");
|
|
759
|
-
assertEngineSupports(engine, "emotion", fields.emotion);
|
|
760
|
-
}
|
|
761
|
-
if (fields.pace != null) {
|
|
762
|
-
assertCanonicalValue(fields.pace, PACES, "pace");
|
|
763
|
-
assertEngineSupports(engine, "pace", fields.pace);
|
|
764
|
-
}
|
|
765
|
-
}
|
|
766
|
-
/**
|
|
767
|
-
* Reject a channel the engine supports but cannot change per call, so it is
|
|
768
|
-
* never accepted here and then dropped on the way to the engine. Runs after
|
|
769
|
-
* validateConditioning so an engine with no such control keeps its own message.
|
|
770
|
-
*/
|
|
771
|
-
function assertPerCallConditioning(engine, fields, where) {
|
|
772
|
-
if (!fields)
|
|
773
|
-
return;
|
|
774
|
-
const perCall = ENGINE_PER_CALL_CONDITIONING[engine];
|
|
775
|
-
const fixed = CONDITIONING_KEYS.filter((key) => fields[key] != null && !perCall.includes(key));
|
|
776
|
-
if (fixed.length === 0)
|
|
777
|
-
return;
|
|
778
|
-
throw new Error(`tts-ggml: ${where}: ${engine} applies ` +
|
|
779
|
-
`${fixed.map((key) => `\`${key}\``).join(", ")} when the engine is ` +
|
|
780
|
-
`built, so it cannot change per call; set it in the constructor or ` +
|
|
781
|
-
`reload({ ${fixed.join(", ")} })`);
|
|
782
|
-
}
|
|
783
|
-
// CosyVoice3 is trained on one instruction per synthesis. Only `pace: moderate`
|
|
784
|
-
// resolves to no instruction, so it is the one value that never conflicts; every
|
|
785
|
-
// emotion carries a trained instruction, `neutral` included.
|
|
786
|
-
const COSYVOICE_DISENGAGED = {
|
|
787
|
-
pace: "moderate",
|
|
788
|
-
};
|
|
789
|
-
function engagedCosyvoiceControls(fields, instruct) {
|
|
790
|
-
const engaged = [];
|
|
791
|
-
for (const key of CONDITIONING_KEYS) {
|
|
792
|
-
const value = fields?.[key];
|
|
793
|
-
if (value != null && value.toLowerCase() !== COSYVOICE_DISENGAGED[key]) {
|
|
794
|
-
engaged.push(`${key}="${value}"`);
|
|
795
|
-
}
|
|
796
|
-
}
|
|
797
|
-
if (instruct != null && instruct !== "")
|
|
798
|
-
engaged.push("instruct");
|
|
799
|
-
return engaged;
|
|
800
|
-
}
|
|
801
|
-
function assertSingleCosyvoiceControl(fields, instruct, where) {
|
|
802
|
-
const engaged = engagedCosyvoiceControls(fields, instruct);
|
|
803
|
-
if (engaged.length <= 1)
|
|
804
|
-
return;
|
|
805
|
-
throw new Error(`tts-ggml: ${where}: conflicting conditioning controls ` +
|
|
806
|
-
`(${engaged.join(", ")}); cosyvoice3 is trained on one instruction per ` +
|
|
807
|
-
`synthesis -- set exactly one (pace="moderate" disengages a channel)`);
|
|
808
|
-
}
|
|
809
|
-
function normalizeGgmlFiles(files) {
|
|
810
|
-
if (files == null || typeof files !== "object")
|
|
811
|
-
return {};
|
|
812
|
-
return {
|
|
813
|
-
modelDir: firstNonEmpty(files.modelDir),
|
|
814
|
-
pocketFlowModel: firstNonEmpty(files.pocketFlowModel),
|
|
815
|
-
pocketMimiModel: firstNonEmpty(files.pocketMimiModel),
|
|
816
|
-
pocketFrontend: firstNonEmpty(files.pocketFrontend),
|
|
817
|
-
pocketVoice: firstNonEmpty(files.pocketVoice),
|
|
818
|
-
t3Model: firstNonEmpty(files.t3Model, files.t3ModelPath, files.t3),
|
|
819
|
-
s3genModel: firstNonEmpty(files.s3genModel, files.s3genModelPath, files.s3gen),
|
|
820
|
-
supertonicModel: firstNonEmpty(files.supertonicModel, files.supertonicModelPath, files.supertonic),
|
|
821
|
-
// CosyVoice3: either a dedicated modelDir of cosyvoice3-*.gguf files, or
|
|
822
|
-
// explicit per-component paths. Falls back to the shared `modelDir`.
|
|
823
|
-
cosyvoiceModelDir: firstNonEmpty(files.cosyvoiceModelDir),
|
|
824
|
-
cosyvoiceLlmModel: firstNonEmpty(files.cosyvoiceLlmModel, files.cosyvoiceLlmModelPath),
|
|
825
|
-
cosyvoiceFlowModel: firstNonEmpty(files.cosyvoiceFlowModel, files.cosyvoiceFlowModelPath),
|
|
826
|
-
cosyvoiceHiftModel: firstNonEmpty(files.cosyvoiceHiftModel, files.cosyvoiceHiftModelPath),
|
|
827
|
-
cosyvoiceS3tokModel: firstNonEmpty(files.cosyvoiceS3tokModel, files.cosyvoiceS3tokModelPath),
|
|
828
|
-
cosyvoiceCampplusModel: firstNonEmpty(files.cosyvoiceCampplusModel, files.cosyvoiceCampplusModelPath),
|
|
829
|
-
cosyvoiceVocab: firstNonEmpty(files.cosyvoiceVocab, files.cosyvoiceVocabPath),
|
|
830
|
-
cosyvoiceMerges: firstNonEmpty(files.cosyvoiceMerges, files.cosyvoiceMergesPath),
|
|
831
|
-
cosyvoiceVoiceModel: firstNonEmpty(files.cosyvoiceVoiceModel, files.cosyvoiceVoiceModelPath),
|
|
832
|
-
parlerModel: firstNonEmpty(files.parlerModel, files.parlerModelPath, files.parler),
|
|
833
|
-
audio8Lm: firstNonEmpty(files.audio8Lm, files.audio8LmPath),
|
|
834
|
-
audio8CodecDecoder: firstNonEmpty(files.audio8CodecDecoder, files.audio8CodecDecoderPath),
|
|
835
|
-
audio8CodecEncoder: firstNonEmpty(files.audio8CodecEncoder, files.audio8CodecEncoderPath),
|
|
836
|
-
mossBackbone: firstNonEmpty(files.mossBackbone, files.mossBackbonePath),
|
|
837
|
-
mossCodecDecoder: firstNonEmpty(files.mossCodecDecoder, files.mossCodecDecoderPath),
|
|
838
|
-
mossCodecEncoder: firstNonEmpty(files.mossCodecEncoder, files.mossCodecEncoderPath),
|
|
839
|
-
mossSoundEffect: firstNonEmpty(files.mossSoundEffect, files.mossSoundEffectPath),
|
|
840
|
-
mossSpeechModel: firstNonEmpty(files.mossSpeechModel, files.mossSpeechModelPath),
|
|
841
|
-
mossSpeechCodec: firstNonEmpty(files.mossSpeechCodec, files.mossSpeechCodecPath),
|
|
842
|
-
voicesDir: firstNonEmpty(files.voicesDir),
|
|
843
|
-
lavasrEnhancer: firstNonEmpty(files.lavasrEnhancer),
|
|
844
|
-
lavasrDenoiser: firstNonEmpty(files.lavasrDenoiser),
|
|
845
|
-
mecabDictDir: firstNonEmpty(files.mecabDictDir, files.mecabDictPath),
|
|
846
|
-
cangjieTsvPath: firstNonEmpty(files.cangjieTsvPath, files.cangjieTsv),
|
|
847
|
-
};
|
|
848
|
-
}
|
|
849
|
-
function detectEngineType(engine, files) {
|
|
850
|
-
if (engine === ENGINE_CHATTERBOX ||
|
|
851
|
-
engine === ENGINE_SUPERTONIC ||
|
|
852
|
-
engine === ENGINE_COSYVOICE3 ||
|
|
853
|
-
engine === ENGINE_PARLER ||
|
|
854
|
-
engine === ENGINE_AUDIO8 ||
|
|
855
|
-
engine === ENGINE_MOSS ||
|
|
856
|
-
engine === ENGINE_MOSS_SFX ||
|
|
857
|
-
engine === ENGINE_MOSS_SPEECH ||
|
|
858
|
-
engine === ENGINE_POCKET) {
|
|
859
|
-
return engine;
|
|
860
|
-
}
|
|
861
|
-
if (engine != null && engine !== "") {
|
|
862
|
-
throw new Error("tts-ggml: 'engine' option must be 'chatterbox', 'supertonic', " +
|
|
863
|
-
`'cosyvoice3', 'parler', 'audio8', 'moss', 'moss-sfx', 'moss-speech' or 'pocket' (got '${String(engine)}')`);
|
|
864
|
-
}
|
|
865
|
-
if (files.pocketFlowModel || files.pocketMimiModel)
|
|
866
|
-
return ENGINE_POCKET;
|
|
867
|
-
// Explicit CosyVoice3 files/dir take precedence over shared-modelDir sniffing.
|
|
868
|
-
if (files.cosyvoiceModelDir || files.cosyvoiceLlmModel) {
|
|
869
|
-
return ENGINE_COSYVOICE3;
|
|
870
|
-
}
|
|
871
|
-
if (files.t3Model || files.s3genModel)
|
|
872
|
-
return ENGINE_CHATTERBOX;
|
|
873
|
-
if (files.supertonicModel)
|
|
874
|
-
return ENGINE_SUPERTONIC;
|
|
875
|
-
if (files.parlerModel)
|
|
876
|
-
return ENGINE_PARLER;
|
|
877
|
-
if (files.audio8Lm || files.audio8CodecDecoder)
|
|
878
|
-
return ENGINE_AUDIO8;
|
|
879
|
-
if (files.mossBackbone || files.mossCodecDecoder)
|
|
880
|
-
return ENGINE_MOSS;
|
|
881
|
-
if (files.mossSoundEffect)
|
|
882
|
-
return ENGINE_MOSS_SFX;
|
|
883
|
-
if (files.mossSpeechModel || files.mossSpeechCodec)
|
|
884
|
-
return ENGINE_MOSS_SPEECH;
|
|
885
|
-
if (files.modelDir) {
|
|
886
|
-
if (dirHasCosyvoice3(files.modelDir))
|
|
887
|
-
return ENGINE_COSYVOICE3;
|
|
888
|
-
const hasChatterbox = fileExistsSafe(path.join(files.modelDir, CHATTERBOX_T3_TURBO)) ||
|
|
889
|
-
fileExistsSafe(path.join(files.modelDir, CHATTERBOX_T3_MTL));
|
|
890
|
-
const hasSupertonic = fileExistsSafe(path.join(files.modelDir, SUPERTONIC_DEFAULT)) ||
|
|
891
|
-
fileExistsSafe(path.join(files.modelDir, SUPERTONIC_MTL)) ||
|
|
892
|
-
!!findSupertonicV3InDir(files.modelDir);
|
|
893
|
-
if (hasChatterbox)
|
|
894
|
-
return ENGINE_CHATTERBOX;
|
|
895
|
-
if (hasSupertonic)
|
|
896
|
-
return ENGINE_SUPERTONIC;
|
|
897
|
-
if (findParlerInDir(files.modelDir))
|
|
898
|
-
return ENGINE_PARLER;
|
|
899
|
-
if (findQuantRankedGguf(files.modelDir, AUDIO8_LM_RE))
|
|
900
|
-
return ENGINE_AUDIO8;
|
|
901
|
-
if (findQuantRankedGguf(files.modelDir, MOSS_BACKBONE_RE))
|
|
902
|
-
return ENGINE_MOSS;
|
|
903
|
-
if (findQuantRankedGguf(files.modelDir, MOSS_DIALOGUE_BACKBONE_RE)) {
|
|
904
|
-
return ENGINE_MOSS;
|
|
905
|
-
}
|
|
906
|
-
if (findQuantRankedGguf(files.modelDir, MOSS_SFX_RE))
|
|
907
|
-
return ENGINE_MOSS_SFX;
|
|
908
|
-
if (findQuantRankedGguf(files.modelDir, MOSS_SPEECH_RE))
|
|
909
|
-
return ENGINE_MOSS_SPEECH;
|
|
910
|
-
if (fileExistsSafe(path.join(files.modelDir, "flow-lm.gguf")))
|
|
911
|
-
return ENGINE_POCKET;
|
|
912
|
-
}
|
|
913
|
-
return ENGINE_CHATTERBOX;
|
|
914
|
-
}
|
|
915
|
-
function resolveSupertonicModelDirPath(modelDir) {
|
|
916
|
-
const english = path.join(modelDir, SUPERTONIC_DEFAULT);
|
|
917
|
-
const multilingual = path.join(modelDir, SUPERTONIC_MTL);
|
|
918
|
-
const versionThree = findSupertonicV3InDir(modelDir);
|
|
919
|
-
if (fileExistsSafe(english))
|
|
920
|
-
return english;
|
|
921
|
-
if (fileExistsSafe(multilingual))
|
|
922
|
-
return multilingual;
|
|
923
|
-
if (versionThree)
|
|
924
|
-
return versionThree;
|
|
925
|
-
return english;
|
|
926
|
-
}
|
|
927
|
-
function resolveChatterboxModelDirPaths(modelDir) {
|
|
928
|
-
const turboT3 = path.join(modelDir, CHATTERBOX_T3_TURBO);
|
|
929
|
-
const multilingualT3 = path.join(modelDir, CHATTERBOX_T3_MTL);
|
|
930
|
-
const defaultS3 = path.join(modelDir, CHATTERBOX_S3GEN_DEFAULT);
|
|
931
|
-
const multilingualS3 = path.join(modelDir, CHATTERBOX_S3GEN_MTL);
|
|
932
|
-
if (fileExistsSafe(multilingualT3) &&
|
|
933
|
-
!fileExistsSafe(turboT3)) {
|
|
934
|
-
return {
|
|
935
|
-
t3: multilingualT3,
|
|
936
|
-
s3: fileExistsSafe(multilingualS3)
|
|
937
|
-
? multilingualS3
|
|
938
|
-
: defaultS3,
|
|
939
|
-
};
|
|
940
|
-
}
|
|
941
|
-
return { t3: turboT3, s3: defaultS3 };
|
|
942
|
-
}
|
|
943
|
-
function defaultAccumulateSentencesForStreamInput(textStream) {
|
|
944
|
-
if (textStream == null ||
|
|
945
|
-
typeof textStream === "string" ||
|
|
946
|
-
Array.isArray(textStream)) {
|
|
947
|
-
return false;
|
|
948
|
-
}
|
|
949
|
-
return (typeof textStream[Symbol.asyncIterator] === "function");
|
|
950
|
-
}
|
|
951
|
-
function ttsOutputDebugString(data) {
|
|
952
|
-
if (!data)
|
|
953
|
-
return "";
|
|
954
|
-
if (typeof data === "string")
|
|
955
|
-
return data;
|
|
956
|
-
if (typeof data === "number" ||
|
|
957
|
-
typeof data === "boolean" ||
|
|
958
|
-
typeof data === "bigint") {
|
|
959
|
-
return data.toString();
|
|
960
|
-
}
|
|
961
|
-
if (typeof data === "symbol")
|
|
962
|
-
return data.description || "";
|
|
963
|
-
if (typeof data === "function")
|
|
964
|
-
return data.name;
|
|
965
|
-
const value = data;
|
|
966
|
-
const summary = {};
|
|
967
|
-
if (value.sampleRate != null)
|
|
968
|
-
summary.sampleRate = value.sampleRate;
|
|
969
|
-
if (value.chunkIndex != null)
|
|
970
|
-
summary.chunkIndex = value.chunkIndex;
|
|
971
|
-
if (value.isLast != null)
|
|
972
|
-
summary.isLast = value.isLast;
|
|
973
|
-
if (value.sentenceChunk != null) {
|
|
974
|
-
summary.sentenceChunk = value.sentenceChunk;
|
|
975
|
-
}
|
|
976
|
-
if (value.outputArray &&
|
|
977
|
-
typeof value.outputArray.length === "number") {
|
|
978
|
-
summary.outputArrayLen = value.outputArray.length;
|
|
979
|
-
}
|
|
980
|
-
return JSON.stringify(summary);
|
|
981
|
-
}
|
|
982
|
-
function resolveDefaultLazySessionLoading(lazySessionLoading) {
|
|
983
|
-
if (lazySessionLoading != null)
|
|
984
|
-
return lazySessionLoading;
|
|
985
|
-
return platform() === "ios" || platform() === "android";
|
|
986
|
-
}
|
|
987
|
-
function validateOutputSampleRate(outputSampleRate) {
|
|
988
|
-
if (outputSampleRate == null)
|
|
989
|
-
return null;
|
|
990
|
-
if (outputSampleRate < MIN_OUTPUT_SAMPLE_RATE ||
|
|
991
|
-
outputSampleRate > MAX_OUTPUT_SAMPLE_RATE) {
|
|
992
|
-
throw new Error(`outputSampleRate must be between ${MIN_OUTPUT_SAMPLE_RATE} and ` +
|
|
993
|
-
`${MAX_OUTPUT_SAMPLE_RATE}, got ${outputSampleRate}`);
|
|
994
|
-
}
|
|
995
|
-
return outputSampleRate;
|
|
996
|
-
}
|
|
997
|
-
function resolveEnhancerGgufPath(files, enhancer) {
|
|
998
|
-
if (enhancer != null && enhancer.type !== "lavasr") {
|
|
999
|
-
throw new Error(`tts-ggml: unknown enhancer.type '${String(enhancer.type)}', expected 'lavasr'.`);
|
|
1000
|
-
}
|
|
1001
|
-
return firstNonEmpty(files.lavasrEnhancer, enhancer?.enhancerPath);
|
|
1002
|
-
}
|
|
1003
|
-
function resolveDenoiserGgufPath(files, denoiser) {
|
|
1004
|
-
if (denoiser != null && denoiser.type !== "lavasr") {
|
|
1005
|
-
throw new Error(`tts-ggml: unknown denoiser.type '${String(denoiser.type)}', expected 'lavasr'.`);
|
|
1006
|
-
}
|
|
1007
|
-
return firstNonEmpty(files.lavasrDenoiser, denoiser?.denoiserPath);
|
|
1008
|
-
}
|
|
1009
|
-
function assertGpuIntentConsistent(useGPU, nGpuLayers) {
|
|
1010
|
-
if (typeof useGPU !== "boolean" || nGpuLayers == null)
|
|
1011
|
-
return;
|
|
1012
|
-
if (useGPU === (nGpuLayers !== 0))
|
|
1013
|
-
return;
|
|
1014
|
-
throw new Error(`tts-ggml: useGPU=${String(useGPU)} conflicts with ` +
|
|
1015
|
-
`nGpuLayers=${nGpuLayers}. Either drop one of the two, or make ` +
|
|
1016
|
-
"them agree (useGPU:true + nGpuLayers!=0, or " +
|
|
1017
|
-
"useGPU:false + nGpuLayers=0).");
|
|
1018
|
-
}
|
|
1019
|
-
const AUDIO8_NUMERIC_SAMPLING_FIELDS = [
|
|
1020
|
-
"temperature",
|
|
1021
|
-
"topK",
|
|
1022
|
-
"topP",
|
|
1023
|
-
"maxFrames",
|
|
1024
|
-
"seed",
|
|
1025
|
-
"threads",
|
|
1026
|
-
];
|
|
1027
|
-
function assertAudio8SamplingFinite(sampling, where) {
|
|
1028
|
-
for (const field of AUDIO8_NUMERIC_SAMPLING_FIELDS) {
|
|
1029
|
-
const value = sampling[field];
|
|
1030
|
-
if (value === undefined || Number.isFinite(value))
|
|
1031
|
-
continue;
|
|
1032
|
-
throw new Error(`tts-ggml: ${where}: ${field} must be a finite number`);
|
|
1033
|
-
}
|
|
1034
|
-
}
|
|
1035
|
-
function isValidDurationTokens(value) {
|
|
1036
|
-
return Number.isInteger(value) && value >= 0 && value <= MOSS_MAX_DURATION_TOKENS;
|
|
1037
|
-
}
|
|
1038
|
-
function assertMossDurationTokens(value, where) {
|
|
1039
|
-
if (value == null || isValidDurationTokens(value))
|
|
1040
|
-
return;
|
|
1041
|
-
throw new Error(`tts-ggml: ${where}: durationTokens must be an integer from 0 to ` +
|
|
1042
|
-
`${MOSS_MAX_DURATION_TOKENS} (0 = free length)`);
|
|
1043
|
-
}
|
|
1044
|
-
function copyDialogueReferences(references) {
|
|
1045
|
-
if (references == null)
|
|
1046
|
-
return undefined;
|
|
1047
|
-
return Array.isArray(references)
|
|
1048
|
-
? [...references]
|
|
1049
|
-
: references;
|
|
1050
|
-
}
|
|
1051
|
-
function isNonEmptyString(value) {
|
|
1052
|
-
return typeof value === "string" && value !== "";
|
|
1053
|
-
}
|
|
1054
|
-
function assertDialogueReferenceList(references) {
|
|
1055
|
-
if (Array.isArray(references) &&
|
|
1056
|
-
references.length > 0 &&
|
|
1057
|
-
references.every(isNonEmptyString)) {
|
|
1058
|
-
return;
|
|
1059
|
-
}
|
|
1060
|
-
throw new Error("tts-ggml: dialogueReferences must be a non-empty array of WAV paths, " +
|
|
1061
|
-
"one per speaker");
|
|
1062
|
-
}
|
|
1063
|
-
function assertNotNegative(value, name, zeroMeans, where) {
|
|
1064
|
-
if (value === undefined || value >= 0)
|
|
1065
|
-
return;
|
|
1066
|
-
throw new Error(`tts-ggml: ${where}: ${name} must be >= 0 (0 = ${zeroMeans})`);
|
|
1067
|
-
}
|
|
1068
|
-
function assertAudio8TopP(value, where) {
|
|
1069
|
-
if (value === undefined || (value > 0 && value <= 1))
|
|
1070
|
-
return;
|
|
1071
|
-
throw new Error(`tts-ggml: ${where}: topP must be in (0, 1]`);
|
|
1072
|
-
}
|
|
1073
|
-
/**
|
|
1074
|
-
* Mirrors Audio8Model::validateSampling, which stays the source of truth.
|
|
1075
|
-
* Reload checks the merged knobs here because native only sees them when the
|
|
1076
|
-
* instance is rebuilt: without this the refused value is already on the JS
|
|
1077
|
-
* object by then, and the two would describe different samplers.
|
|
1078
|
-
*/
|
|
1079
|
-
function assertAudio8SamplingValid(sampling, where) {
|
|
1080
|
-
assertAudio8SamplingFinite(sampling, where);
|
|
1081
|
-
assertNotNegative(sampling.temperature, "temperature", "greedy", where);
|
|
1082
|
-
assertNotNegative(sampling.topK, "topK", "no top-k cutoff", where);
|
|
1083
|
-
assertNotNegative(sampling.maxFrames, "maxFrames", "engine default", where);
|
|
1084
|
-
assertAudio8TopP(sampling.topP, where);
|
|
1085
|
-
}
|
|
1086
|
-
function isAudioOutputEvent(data) {
|
|
1087
|
-
return (data != null &&
|
|
1088
|
-
typeof data === "object" &&
|
|
1089
|
-
"outputArray" in data &&
|
|
1090
|
-
data.outputArray != null);
|
|
1091
|
-
}
|
|
1092
|
-
function isStatsEvent(data) {
|
|
1093
|
-
return (data != null &&
|
|
1094
|
-
typeof data === "object" &&
|
|
1095
|
-
("totalTime" in data ||
|
|
1096
|
-
"audioDurationMs" in data ||
|
|
1097
|
-
"totalSamples" in data));
|
|
1098
|
-
}
|
|
1099
|
-
/**
|
|
1100
|
-
* What `tokensPerSecond` counts. Audio8 paces on the codec frame grid and
|
|
1101
|
-
* reports frames per second in batch synthesis; the text-paced engines report
|
|
1102
|
-
* characters per second. Streaming has to keep the same unit as `run()` or the
|
|
1103
|
-
* number silently changes meaning between the two entry points.
|
|
1104
|
-
*/
|
|
1105
|
-
function emptyStreamAccumulator() {
|
|
1106
|
-
return {
|
|
1107
|
-
totalTime: 0,
|
|
1108
|
-
audioDurationMs: 0,
|
|
1109
|
-
totalSamples: 0,
|
|
1110
|
-
generatedFrames: 0,
|
|
1111
|
-
};
|
|
1112
|
-
}
|
|
1113
|
-
function streamedTokenCount(chunks, accumulator, countsFrames) {
|
|
1114
|
-
return countsFrames
|
|
1115
|
-
? accumulator.generatedFrames
|
|
1116
|
-
: chunks.join("").length;
|
|
1117
|
-
}
|
|
1118
|
-
function computeSentenceStreamStats(chunks, accumulator, countsFrames) {
|
|
1119
|
-
const { generatedFrames, ...totals } = accumulator;
|
|
1120
|
-
const produced = streamedTokenCount(chunks, accumulator, countsFrames);
|
|
1121
|
-
const stats = {
|
|
1122
|
-
...totals,
|
|
1123
|
-
tokensPerSecond: accumulator.totalTime > 0 ? produced / accumulator.totalTime : 0,
|
|
1124
|
-
realTimeFactor: accumulator.audioDurationMs > 0
|
|
1125
|
-
? (accumulator.totalTime * 1000) /
|
|
1126
|
-
accumulator.audioDurationMs
|
|
1127
|
-
: 0,
|
|
1128
|
-
};
|
|
1129
|
-
if (countsFrames)
|
|
1130
|
-
stats.generatedFrames = generatedFrames;
|
|
1131
|
-
return stats;
|
|
1132
|
-
}
|
|
1133
63
|
/**
|
|
1134
64
|
* GGML-backed TTS via the `tts-cpp` library. Wraps the chatterbox,
|
|
1135
65
|
* supertonic, parler, cosyvoice3, audio8 and moss engines behind a single
|
|
@@ -1144,15 +74,15 @@ class TTSGgml {
|
|
|
1144
74
|
static inferenceManagerConfig = {
|
|
1145
75
|
noAdditionalDownload: true,
|
|
1146
76
|
};
|
|
1147
|
-
static ENGINE_CHATTERBOX = ENGINE_CHATTERBOX;
|
|
1148
|
-
static ENGINE_SUPERTONIC = ENGINE_SUPERTONIC;
|
|
1149
|
-
static ENGINE_COSYVOICE3 = ENGINE_COSYVOICE3;
|
|
1150
|
-
static ENGINE_PARLER = ENGINE_PARLER;
|
|
1151
|
-
static ENGINE_AUDIO8 = ENGINE_AUDIO8;
|
|
1152
|
-
static ENGINE_MOSS = ENGINE_MOSS;
|
|
1153
|
-
static ENGINE_MOSS_SFX = ENGINE_MOSS_SFX;
|
|
1154
|
-
static ENGINE_MOSS_SPEECH = ENGINE_MOSS_SPEECH;
|
|
1155
|
-
static ENGINE_POCKET = ENGINE_POCKET;
|
|
77
|
+
static ENGINE_CHATTERBOX = engineIds_1.ENGINE_CHATTERBOX;
|
|
78
|
+
static ENGINE_SUPERTONIC = engineIds_1.ENGINE_SUPERTONIC;
|
|
79
|
+
static ENGINE_COSYVOICE3 = engineIds_1.ENGINE_COSYVOICE3;
|
|
80
|
+
static ENGINE_PARLER = engineIds_1.ENGINE_PARLER;
|
|
81
|
+
static ENGINE_AUDIO8 = engineIds_1.ENGINE_AUDIO8;
|
|
82
|
+
static ENGINE_MOSS = engineIds_1.ENGINE_MOSS;
|
|
83
|
+
static ENGINE_MOSS_SFX = engineIds_1.ENGINE_MOSS_SFX;
|
|
84
|
+
static ENGINE_MOSS_SPEECH = engineIds_1.ENGINE_MOSS_SPEECH;
|
|
85
|
+
static ENGINE_POCKET = engineIds_1.ENGINE_POCKET;
|
|
1156
86
|
opts;
|
|
1157
87
|
exclusiveRun;
|
|
1158
88
|
logger;
|
|
@@ -1160,98 +90,14 @@ class TTSGgml {
|
|
|
1160
90
|
addon;
|
|
1161
91
|
_job;
|
|
1162
92
|
_runExclusive;
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
_pocketOptions = {};
|
|
1171
|
-
_pocketFiles = {};
|
|
93
|
+
_exclusiveStreams = new exclusiveQueue_1.ExclusiveResponseQueue();
|
|
94
|
+
/** Everything resolved from the constructor options; reload rewrites parts of it. */
|
|
95
|
+
_setup;
|
|
96
|
+
_sentenceStream;
|
|
97
|
+
_events;
|
|
98
|
+
/** The Pocket execution rules, or null for every other engine. */
|
|
99
|
+
_pocket;
|
|
1172
100
|
_lazySessionLoading;
|
|
1173
|
-
_outputSampleRate;
|
|
1174
|
-
_engineType;
|
|
1175
|
-
_voicesDir;
|
|
1176
|
-
_supertonicModelPath;
|
|
1177
|
-
_t3ModelPath;
|
|
1178
|
-
_s3genModelPath;
|
|
1179
|
-
_cosyvoiceModelDir;
|
|
1180
|
-
_cosyvoiceLlmModelPath;
|
|
1181
|
-
_cosyvoiceFlowModelPath;
|
|
1182
|
-
_cosyvoiceHiftModelPath;
|
|
1183
|
-
_cosyvoiceS3tokModelPath;
|
|
1184
|
-
_cosyvoiceCampplusModelPath;
|
|
1185
|
-
_mecabDictPath;
|
|
1186
|
-
_cangjieTsvPath;
|
|
1187
|
-
_referenceAudio;
|
|
1188
|
-
_voiceDir;
|
|
1189
|
-
_seed;
|
|
1190
|
-
_nGpuLayers;
|
|
1191
|
-
_nCtx;
|
|
1192
|
-
_kvCacheType;
|
|
1193
|
-
_threads;
|
|
1194
|
-
_streamChunkTokens;
|
|
1195
|
-
_streamFirstChunkTokens;
|
|
1196
|
-
_streamLeftContextTokens;
|
|
1197
|
-
_cfmSteps;
|
|
1198
|
-
_cfgRate;
|
|
1199
|
-
_batchCfmSteps;
|
|
1200
|
-
_nPredict;
|
|
1201
|
-
_maxSentenceChars;
|
|
1202
|
-
_crossfadeMs;
|
|
1203
|
-
_repeatPenalty;
|
|
1204
|
-
_exaggeration;
|
|
1205
|
-
_cfgWeight;
|
|
1206
|
-
_minP;
|
|
1207
|
-
_streamChunkTolerancePct;
|
|
1208
|
-
_streamMinChunkTokens;
|
|
1209
|
-
_voiceJsonPath;
|
|
1210
|
-
_prewarmText;
|
|
1211
|
-
_vulkanDevice;
|
|
1212
|
-
_flowCutPrompt;
|
|
1213
|
-
_cosyvoiceVocabPath;
|
|
1214
|
-
_cosyvoiceMergesPath;
|
|
1215
|
-
_cosyvoiceVoiceModelPath;
|
|
1216
|
-
_promptText;
|
|
1217
|
-
_instruct;
|
|
1218
|
-
_voice;
|
|
1219
|
-
_steps;
|
|
1220
|
-
_speed;
|
|
1221
|
-
_noiseNpyPath;
|
|
1222
|
-
_enhancerGgufPath;
|
|
1223
|
-
_denoiserGgufPath;
|
|
1224
|
-
_backendsDir;
|
|
1225
|
-
_openclCacheDir;
|
|
1226
|
-
_vulkanCacheDir;
|
|
1227
|
-
_parlerModelPath;
|
|
1228
|
-
_audio8LmPath;
|
|
1229
|
-
_audio8CodecDecoderPath;
|
|
1230
|
-
_audio8CodecEncoderPath;
|
|
1231
|
-
_mossBackbonePath;
|
|
1232
|
-
_mossCodecDecoderPath;
|
|
1233
|
-
_mossCodecEncoderPath;
|
|
1234
|
-
_mossSoundEffectPath;
|
|
1235
|
-
_mossSpeechModelPath;
|
|
1236
|
-
_mossSpeechCodecPath;
|
|
1237
|
-
_dialogueReferences;
|
|
1238
|
-
_durationTokens;
|
|
1239
|
-
_referenceText;
|
|
1240
|
-
_greedy;
|
|
1241
|
-
_description;
|
|
1242
|
-
_emotion;
|
|
1243
|
-
_pitch;
|
|
1244
|
-
_pace;
|
|
1245
|
-
_expressivity;
|
|
1246
|
-
_noise;
|
|
1247
|
-
_reverb;
|
|
1248
|
-
_quality;
|
|
1249
|
-
_temperature;
|
|
1250
|
-
_topK;
|
|
1251
|
-
_topP;
|
|
1252
|
-
_maxFrames;
|
|
1253
|
-
_minNewTokens;
|
|
1254
|
-
_normalizeNumbers;
|
|
1255
101
|
constructor(options = {}) {
|
|
1256
102
|
this.opts = options.opts || {};
|
|
1257
103
|
this.exclusiveRun = !!options.exclusiveRun;
|
|
@@ -1262,720 +108,47 @@ class TTSGgml {
|
|
|
1262
108
|
destroyed: false,
|
|
1263
109
|
};
|
|
1264
110
|
this.addon = null;
|
|
1265
|
-
this._sentenceStreamCtx = null;
|
|
1266
|
-
this._ttsInferenceQueueWaiter = Promise.resolve();
|
|
1267
111
|
this._job = (0, infer_base_1.createJobHandler)({
|
|
1268
|
-
cancel: () => this.
|
|
112
|
+
cancel: () => this._pocket ? this.cancel() : this._optionalAddon()?.cancel(),
|
|
1269
113
|
});
|
|
1270
|
-
this._runExclusive = this.exclusiveRun
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1282
|
-
if (normalizedFiles.lavasrEnhancer || normalizedFiles.lavasrDenoiser) {
|
|
1283
|
-
throw new Error("Pocket does not support LavaSR enhancement or denoising");
|
|
1284
|
-
}
|
|
1285
|
-
this._pocketFiles = normalizedFiles;
|
|
1286
|
-
this._pocketOptions = { ...options };
|
|
1287
|
-
this._pocketParams = (0, pocketConfig_1.buildPocketParams)(normalizedFiles, options, this._config);
|
|
1288
|
-
}
|
|
1289
|
-
this._resolveEngineAndModelPaths(normalizedFiles);
|
|
1290
|
-
this._mecabDictPath = firstNonEmpty(options.mecabDictPath, options.mecabDictDir, normalizedFiles.mecabDictDir);
|
|
1291
|
-
this._cangjieTsvPath = firstNonEmpty(options.cangjieTsvPath, normalizedFiles.cangjieTsvPath);
|
|
1292
|
-
this._assignSynthesisOptions(options);
|
|
1293
|
-
this._enhancerGgufPath = resolveEnhancerGgufPath(normalizedFiles, options.enhancer);
|
|
1294
|
-
this._denoiserGgufPath = resolveDenoiserGgufPath(normalizedFiles, options.denoiser);
|
|
1295
|
-
this._backendsDir = firstNonEmpty(options.backendsDir, this._config.backendsDir);
|
|
1296
|
-
this._openclCacheDir = firstNonEmpty(options.openclCacheDir, this._config.openclCacheDir);
|
|
1297
|
-
this._vulkanCacheDir = firstNonEmpty(options.vulkanCacheDir, this._config.vulkanCacheDir);
|
|
1298
|
-
assertGpuIntentConsistent(this._config.useGPU, this._nGpuLayers);
|
|
1299
|
-
this._assertEngineStreamingSupport();
|
|
1300
|
-
if (this._config.useGPU === undefined &&
|
|
1301
|
-
this._nGpuLayers == null) {
|
|
1302
|
-
this._config.useGPU = false;
|
|
1303
|
-
}
|
|
1304
|
-
}
|
|
1305
|
-
_resolveEngineAndModelPaths(files) {
|
|
1306
|
-
this._voicesDir = files.voicesDir;
|
|
1307
|
-
if (this._engineType === ENGINE_POCKET)
|
|
1308
|
-
return;
|
|
1309
|
-
if (this._engineType === ENGINE_COSYVOICE3) {
|
|
1310
|
-
// CosyVoice3 discovers its sub-model GGUFs from a model directory; the
|
|
1311
|
-
// native engine resolves the individual components. Explicit
|
|
1312
|
-
// per-component paths win over the directory.
|
|
1313
|
-
this._cosyvoiceModelDir = firstNonEmpty(files.cosyvoiceModelDir, files.modelDir);
|
|
1314
|
-
this._cosyvoiceLlmModelPath = files.cosyvoiceLlmModel;
|
|
1315
|
-
this._cosyvoiceFlowModelPath = files.cosyvoiceFlowModel;
|
|
1316
|
-
this._cosyvoiceHiftModelPath = files.cosyvoiceHiftModel;
|
|
1317
|
-
this._cosyvoiceS3tokModelPath = files.cosyvoiceS3tokModel;
|
|
1318
|
-
this._cosyvoiceCampplusModelPath = files.cosyvoiceCampplusModel;
|
|
1319
|
-
this._cosyvoiceVocabPath = files.cosyvoiceVocab;
|
|
1320
|
-
this._cosyvoiceMergesPath = files.cosyvoiceMerges;
|
|
1321
|
-
this._cosyvoiceVoiceModelPath = files.cosyvoiceVoiceModel;
|
|
1322
|
-
return;
|
|
1323
|
-
}
|
|
1324
|
-
if (this._engineType === ENGINE_SUPERTONIC) {
|
|
1325
|
-
this._supertonicModelPath = firstNonEmpty(files.supertonicModel, files.modelDir
|
|
1326
|
-
? resolveSupertonicModelDirPath(files.modelDir)
|
|
1327
|
-
: undefined);
|
|
1328
|
-
return;
|
|
1329
|
-
}
|
|
1330
|
-
if (this._engineType === ENGINE_PARLER) {
|
|
1331
|
-
this._parlerModelPath = firstNonEmpty(files.parlerModel, files.modelDir ? findParlerInDir(files.modelDir) : undefined);
|
|
1332
|
-
return;
|
|
1333
|
-
}
|
|
1334
|
-
if (this._engineType === ENGINE_AUDIO8) {
|
|
1335
|
-
this._resolveAudio8ModelPaths(files);
|
|
1336
|
-
return;
|
|
1337
|
-
}
|
|
1338
|
-
if (this._engineType === ENGINE_MOSS) {
|
|
1339
|
-
this._resolveMossModelPaths(files);
|
|
1340
|
-
return;
|
|
1341
|
-
}
|
|
1342
|
-
if (this._engineType === ENGINE_MOSS_SFX) {
|
|
1343
|
-
this._mossSoundEffectPath = firstNonEmpty(files.mossSoundEffect, findQuantRankedGguf(files.modelDir, MOSS_SFX_RE));
|
|
1344
|
-
return;
|
|
1345
|
-
}
|
|
1346
|
-
if (this._engineType === ENGINE_MOSS_SPEECH) {
|
|
1347
|
-
this._mossSpeechModelPath = firstNonEmpty(files.mossSpeechModel, findQuantRankedGguf(files.modelDir, MOSS_SPEECH_RE));
|
|
1348
|
-
this._mossSpeechCodecPath = firstNonEmpty(files.mossSpeechCodec, findQuantRankedGguf(files.modelDir, MOSS_SPEECH_CODEC_RE));
|
|
1349
|
-
return;
|
|
1350
|
-
}
|
|
1351
|
-
if (files.modelDir) {
|
|
1352
|
-
const resolved = resolveChatterboxModelDirPaths(files.modelDir);
|
|
1353
|
-
this._t3ModelPath = firstNonEmpty(files.t3Model, resolved.t3);
|
|
1354
|
-
this._s3genModelPath = firstNonEmpty(files.s3genModel, resolved.s3);
|
|
1355
|
-
}
|
|
1356
|
-
else {
|
|
1357
|
-
this._t3ModelPath = files.t3Model;
|
|
1358
|
-
this._s3genModelPath = files.s3genModel;
|
|
1359
|
-
}
|
|
1360
|
-
}
|
|
1361
|
-
_resolveAudio8ModelPaths(files) {
|
|
1362
|
-
this._audio8LmPath = firstNonEmpty(files.audio8Lm, findQuantRankedGguf(files.modelDir, AUDIO8_LM_RE));
|
|
1363
|
-
this._audio8CodecDecoderPath = firstNonEmpty(files.audio8CodecDecoder, findQuantRankedGguf(files.modelDir, AUDIO8_DECODER_RE));
|
|
1364
|
-
this._audio8CodecEncoderPath = firstNonEmpty(files.audio8CodecEncoder, findQuantRankedGguf(files.modelDir, AUDIO8_ENCODER_RE));
|
|
1365
|
-
}
|
|
1366
|
-
_mossBackbonePatterns() {
|
|
1367
|
-
return this._dialogueReferences
|
|
1368
|
-
? [MOSS_DIALOGUE_BACKBONE_RE]
|
|
1369
|
-
: [MOSS_BACKBONE_RE, MOSS_DIALOGUE_BACKBONE_RE];
|
|
1370
|
-
}
|
|
1371
|
-
_findMossBackbone(modelDir) {
|
|
1372
|
-
return firstNonEmpty(...this._mossBackbonePatterns().map((pattern) => findQuantRankedGguf(modelDir, pattern)));
|
|
1373
|
-
}
|
|
1374
|
-
_resolveMossModelPaths(files) {
|
|
1375
|
-
this._mossBackbonePath = firstNonEmpty(files.mossBackbone, this._findMossBackbone(files.modelDir));
|
|
1376
|
-
this._mossCodecDecoderPath = firstNonEmpty(files.mossCodecDecoder, findQuantRankedGguf(files.modelDir, MOSS_DECODER_RE));
|
|
1377
|
-
this._mossCodecEncoderPath = firstNonEmpty(files.mossCodecEncoder, findQuantRankedGguf(files.modelDir, MOSS_ENCODER_RE));
|
|
1378
|
-
}
|
|
1379
|
-
_assignMossVoiceOptions(options) {
|
|
1380
|
-
this._dialogueReferences = copyDialogueReferences(options.dialogueReferences);
|
|
1381
|
-
this._durationTokens = options.durationTokens ?? undefined;
|
|
1382
|
-
}
|
|
1383
|
-
_assignSynthesisOptions(options) {
|
|
1384
|
-
this._referenceAudio = options.referenceAudio;
|
|
1385
|
-
this._referenceText = options.referenceText;
|
|
1386
|
-
this._greedy = options.greedy;
|
|
1387
|
-
this._voiceDir = options.voiceDir;
|
|
1388
|
-
this._seed = options.seed;
|
|
1389
|
-
this._nGpuLayers = options.nGpuLayers;
|
|
1390
|
-
this._nCtx = options.nCtx;
|
|
1391
|
-
this._kvCacheType = options.kvCacheType;
|
|
1392
|
-
this._threads = options.threads;
|
|
1393
|
-
this._streamChunkTokens = options.streamChunkTokens;
|
|
1394
|
-
this._streamFirstChunkTokens = options.streamFirstChunkTokens;
|
|
1395
|
-
this._streamChunkTolerancePct = options.streamChunkTolerancePct;
|
|
1396
|
-
this._streamMinChunkTokens = options.streamMinChunkTokens;
|
|
1397
|
-
this._streamLeftContextTokens = options.streamLeftContextTokens;
|
|
1398
|
-
this._cfmSteps = options.cfmSteps;
|
|
1399
|
-
this._cfgRate = options.cfgRate;
|
|
1400
|
-
this._batchCfmSteps = options.batchCfmSteps;
|
|
1401
|
-
this._nPredict = options.nPredict;
|
|
1402
|
-
this._maxSentenceChars = options.maxSentenceChars;
|
|
1403
|
-
this._crossfadeMs = options.crossfadeMs;
|
|
1404
|
-
this._repeatPenalty = options.repeatPenalty;
|
|
1405
|
-
this._exaggeration = options.exaggeration;
|
|
1406
|
-
this._cfgWeight = options.cfgWeight;
|
|
1407
|
-
this._minP = options.minP;
|
|
1408
|
-
this._voiceJsonPath = options.voiceJsonPath;
|
|
1409
|
-
this._prewarmText = options.prewarmText;
|
|
1410
|
-
this._vulkanDevice = options.vulkanDevice;
|
|
1411
|
-
this._flowCutPrompt = options.flowCutPrompt;
|
|
1412
|
-
// CosyVoice3-only: transcript of the reference audio for zero-shot cloning.
|
|
1413
|
-
this._promptText = options.promptText;
|
|
1414
|
-
// CosyVoice3-only: render the structured/raw instruct2 control to its string.
|
|
1415
|
-
this._instruct = renderCosyvoiceInstruct(options.instruct) || undefined;
|
|
1416
|
-
this._voice = firstNonEmpty(options.voice, options.voiceName);
|
|
1417
|
-
this._steps = firstNonEmpty(options.steps, options.numInferenceSteps);
|
|
1418
|
-
this._speed = options.speed;
|
|
1419
|
-
this._noiseNpyPath = options.noiseNpyPath;
|
|
1420
|
-
// Parler voice-description surface (all optional; the all-defaults render
|
|
1421
|
-
// is the models' recommended fallback caption).
|
|
1422
|
-
this._description = firstNonEmpty(options.description, options.voiceDescription);
|
|
1423
|
-
this._emotion = options.emotion;
|
|
1424
|
-
this._pitch = options.pitch;
|
|
1425
|
-
this._pace = options.pace;
|
|
1426
|
-
this._expressivity = options.expressivity;
|
|
1427
|
-
this._noise = options.noise;
|
|
1428
|
-
this._reverb = options.reverb;
|
|
1429
|
-
this._quality = options.quality;
|
|
1430
|
-
this._temperature = options.temperature;
|
|
1431
|
-
this._topK = options.topK;
|
|
1432
|
-
this._topP = options.topP;
|
|
1433
|
-
this._maxFrames = options.maxFrames;
|
|
1434
|
-
this._minNewTokens = options.minNewTokens;
|
|
1435
|
-
this._normalizeNumbers = options.normalizeNumbers;
|
|
1436
|
-
}
|
|
1437
|
-
_assertEngineStreamingSupport() {
|
|
1438
|
-
// Checked ahead of the shared denoiser guard below: Supertonic rejects both
|
|
1439
|
-
// LavaSR stages for one reason (tts-cpp has only one-shot enhance /
|
|
1440
|
-
// denoise, and the engine reports no native rate before synthesis).
|
|
1441
|
-
if (this._engineType === ENGINE_SUPERTONIC &&
|
|
1442
|
-
(this._enhancerGgufPath || this._denoiserGgufPath) &&
|
|
1443
|
-
this._requestsChunkStreaming()) {
|
|
1444
|
-
throw new Error("tts-ggml: the LavaSR enhancer/denoiser are not supported with " +
|
|
1445
|
-
"supertonic native chunk streaming (streamChunkTokens > 0). Use " +
|
|
1446
|
-
"batch synthesis or sentence-level streaming via runStream() / " +
|
|
1447
|
-
"runStreaming() / run({ streamOutput: true }) for enhanced or " +
|
|
1448
|
-
"denoised output.");
|
|
1449
|
-
}
|
|
1450
|
-
if (this._engineType === ENGINE_AUDIO8 &&
|
|
1451
|
-
(this._streamChunkTokens != null ||
|
|
1452
|
-
this._streamFirstChunkTokens != null)) {
|
|
1453
|
-
throw new Error("tts-ggml: streamChunkTokens / streamFirstChunkTokens are not " +
|
|
1454
|
-
"supported by the audio8 engine, which synthesizes a whole " +
|
|
1455
|
-
"utterance at a time. Use sentence-level streaming via " +
|
|
1456
|
-
"runStream() / runStreaming() / run({ streamOutput: true }).");
|
|
1457
|
-
}
|
|
1458
|
-
if ((this._engineType === ENGINE_MOSS_SFX ||
|
|
1459
|
-
this._engineType === ENGINE_MOSS_SPEECH) &&
|
|
1460
|
-
(this._streamChunkTokens != null ||
|
|
1461
|
-
this._streamFirstChunkTokens != null)) {
|
|
1462
|
-
throw new Error("tts-ggml: streamChunkTokens / streamFirstChunkTokens are not " +
|
|
1463
|
-
`supported by the ${this._engineType} engine, which returns a ` +
|
|
1464
|
-
"whole result at a time. Use run().");
|
|
1465
|
-
}
|
|
1466
|
-
if (this._engineType === ENGINE_MOSS &&
|
|
1467
|
-
this._streamFirstChunkTokens != null) {
|
|
1468
|
-
throw new Error("tts-ggml: streamFirstChunkTokens is not supported by the moss " +
|
|
1469
|
-
"engine, which streams fixed-size chunks; set streamChunkTokens " +
|
|
1470
|
-
`(codec frames per chunk, ${MOSS_FRAMES_PER_SECOND} per second) only.`);
|
|
1471
|
-
}
|
|
1472
|
-
// Runs before the denoiser guard so a Parler description/template conflict
|
|
1473
|
-
// is reported ahead of the engine-agnostic streaming constraints.
|
|
1474
|
-
this._assertParlerOptionConsistency();
|
|
1475
|
-
this._assertEngineScopedOptions();
|
|
1476
|
-
this._assertCosyvoiceOptionConsistency();
|
|
1477
|
-
this._assertCosyvoiceCloneConsistent();
|
|
1478
|
-
this._assertAudio8OptionConsistency();
|
|
1479
|
-
this._assertMossOptionConsistency();
|
|
1480
|
-
this._assertMossSoundEffectOptionConsistency();
|
|
1481
|
-
this._assertMossSpeechOptionConsistency();
|
|
1482
|
-
this._assertConditioningConsistency("constructor");
|
|
1483
|
-
if (this._denoiserGgufPath && this._requestsChunkStreaming()) {
|
|
1484
|
-
throw new Error("tts-ggml: the LavaSR denoiser is not yet supported with " +
|
|
1485
|
-
"native chunk streaming (streamChunkTokens > 0). Use batch " +
|
|
1486
|
-
"synthesis, or drop the denoiser for streaming. Streaming " +
|
|
1487
|
-
"denoise is a planned follow-up (needs a stateful streaming " +
|
|
1488
|
-
"denoiser).");
|
|
1489
|
-
}
|
|
1490
|
-
}
|
|
1491
|
-
// Every streaming engine starts native chunk streaming on
|
|
1492
|
-
// streamChunkTokens > 0 alone: a count of 0 means batch, and
|
|
1493
|
-
// streamFirstChunkTokens only sizes the first chunk once streaming is on.
|
|
1494
|
-
_requestsChunkStreaming() {
|
|
1495
|
-
return (this._streamChunkTokens ?? 0) > 0;
|
|
1496
|
-
}
|
|
1497
|
-
_assertParlerOptionConsistency() {
|
|
1498
|
-
if (this._engineType === ENGINE_PARLER) {
|
|
1499
|
-
assertParlerDescFieldsConsistent(pickParlerDescFields({
|
|
1500
|
-
description: this._description,
|
|
1501
|
-
voice: this._voice,
|
|
1502
|
-
emotion: this._emotion,
|
|
1503
|
-
pitch: this._pitch,
|
|
1504
|
-
pace: this._pace,
|
|
1505
|
-
expressivity: this._expressivity,
|
|
1506
|
-
noise: this._noise,
|
|
1507
|
-
reverb: this._reverb,
|
|
1508
|
-
quality: this._quality,
|
|
1509
|
-
}), "constructor");
|
|
1510
|
-
return;
|
|
1511
|
-
}
|
|
1512
|
-
const parlerOnly = [];
|
|
1513
|
-
if (this._description != null) {
|
|
1514
|
-
parlerOnly.push("description/voiceDescription");
|
|
1515
|
-
}
|
|
1516
|
-
// emotion / pace are absent because they are cross-engine, validated by
|
|
1517
|
-
// _assertConditioningConsistency; temperature / topK / topP / maxFrames are
|
|
1518
|
-
// absent because audio8 samples too, so SAMPLING_ENGINES decides them.
|
|
1519
|
-
parlerOnly.push(...setOptionNames({
|
|
1520
|
-
pitch: this._pitch,
|
|
1521
|
-
expressivity: this._expressivity,
|
|
1522
|
-
noise: this._noise,
|
|
1523
|
-
reverb: this._reverb,
|
|
1524
|
-
quality: this._quality,
|
|
1525
|
-
minNewTokens: this._minNewTokens,
|
|
1526
|
-
normalizeNumbers: this._normalizeNumbers,
|
|
1527
|
-
}));
|
|
1528
|
-
if (parlerOnly.length > 0) {
|
|
1529
|
-
throw new Error(`tts-ggml: ${parlerOnly.join(", ")} are parler-only options ` +
|
|
1530
|
-
`(engine is ${this._engineType})`);
|
|
1531
|
-
}
|
|
1532
|
-
this._assertSamplerOptionSupport();
|
|
1533
|
-
}
|
|
1534
|
-
_assertSamplerOptionSupport() {
|
|
1535
|
-
if (this._engineType === ENGINE_POCKET || SAMPLING_ENGINES.includes(this._engineType))
|
|
1536
|
-
return;
|
|
1537
|
-
// Chatterbox's T3 samples too, but its length cap is nPredict, not maxFrames.
|
|
1538
|
-
const tokenSampling = this._engineType === ENGINE_CHATTERBOX
|
|
1539
|
-
? {}
|
|
1540
|
-
: {
|
|
1541
|
-
temperature: this._temperature,
|
|
1542
|
-
topK: this._topK,
|
|
1543
|
-
topP: this._topP,
|
|
1544
|
-
};
|
|
1545
|
-
const sampling = setOptionNames({
|
|
1546
|
-
...tokenSampling,
|
|
1547
|
-
maxFrames: this._maxFrames,
|
|
114
|
+
this._runExclusive = (0, exclusiveQueue_1.resolveExclusiveRunner)(this.exclusiveRun);
|
|
115
|
+
this._lazySessionLoading = (0, setup_1.resolveDefaultLazySessionLoading)(options.lazySessionLoading);
|
|
116
|
+
this._setup = (0, setup_1.resolveEngineSetup)(options);
|
|
117
|
+
this._sentenceStream = new sentenceStream_1.SentenceStreamer({
|
|
118
|
+
startJob: () => this._job.start(),
|
|
119
|
+
dispatchJob: (job) => this._dispatchJob(job),
|
|
120
|
+
endJob: (stats) => this._events.endJob(stats),
|
|
121
|
+
failJob: (error) => this._job.fail(error),
|
|
122
|
+
// Audio8 and MOSS report `tokensPerSecond` as codec frames per second,
|
|
123
|
+
// so the streaming aggregate counts frames too rather than characters.
|
|
124
|
+
countsFrames: () => this._setup.engineType === engineIds_1.ENGINE_AUDIO8 ||
|
|
125
|
+
this._setup.engineType === engineIds_1.ENGINE_MOSS,
|
|
1548
126
|
});
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
/**
|
|
1555
|
-
* Options only some engines read. Set on any other engine they would be
|
|
1556
|
-
* dropped without effect, so they are rejected by name instead.
|
|
1557
|
-
*/
|
|
1558
|
-
_assertEngineScopedOptions() {
|
|
1559
|
-
const scoped = [
|
|
1560
|
-
[
|
|
1561
|
-
[ENGINE_CHATTERBOX],
|
|
1562
|
-
{
|
|
1563
|
-
batchCfmSteps: this._batchCfmSteps,
|
|
1564
|
-
nPredict: this._nPredict,
|
|
1565
|
-
maxSentenceChars: this._maxSentenceChars,
|
|
1566
|
-
crossfadeMs: this._crossfadeMs,
|
|
1567
|
-
repeatPenalty: this._repeatPenalty,
|
|
1568
|
-
exaggeration: this._exaggeration,
|
|
1569
|
-
cfgWeight: this._cfgWeight,
|
|
1570
|
-
minP: this._minP,
|
|
1571
|
-
},
|
|
1572
|
-
],
|
|
1573
|
-
[
|
|
1574
|
-
[ENGINE_CHATTERBOX, ENGINE_COSYVOICE3],
|
|
1575
|
-
{ streamLeftContextTokens: this._streamLeftContextTokens },
|
|
1576
|
-
],
|
|
1577
|
-
[
|
|
1578
|
-
[ENGINE_SUPERTONIC],
|
|
1579
|
-
{
|
|
1580
|
-
voiceJsonPath: this._voiceJsonPath,
|
|
1581
|
-
prewarmText: this._prewarmText,
|
|
1582
|
-
streamChunkTolerancePct: this._streamChunkTolerancePct,
|
|
1583
|
-
streamMinChunkTokens: this._streamMinChunkTokens,
|
|
1584
|
-
},
|
|
1585
|
-
],
|
|
1586
|
-
[
|
|
1587
|
-
[ENGINE_SUPERTONIC, ENGINE_COSYVOICE3],
|
|
1588
|
-
{ vulkanDevice: this._vulkanDevice },
|
|
1589
|
-
],
|
|
1590
|
-
[[ENGINE_COSYVOICE3], { flowCutPrompt: this._flowCutPrompt }],
|
|
1591
|
-
];
|
|
1592
|
-
for (const [engines, fields] of scoped) {
|
|
1593
|
-
if (engines.includes(this._engineType))
|
|
1594
|
-
continue;
|
|
1595
|
-
const names = setOptionNames(fields);
|
|
1596
|
-
if (names.length === 0)
|
|
1597
|
-
continue;
|
|
1598
|
-
throw new Error(`tts-ggml: ${names.join(", ")} are ${engines.join("/")}-only ` +
|
|
1599
|
-
`options (engine is ${this._engineType})`);
|
|
1600
|
-
}
|
|
1601
|
-
}
|
|
1602
|
-
_assertAudio8OptionConsistency() {
|
|
1603
|
-
if (this._engineType === ENGINE_AUDIO8) {
|
|
1604
|
-
if (this._enhancerGgufPath || this._denoiserGgufPath) {
|
|
1605
|
-
throw new Error("tts-ggml: the LavaSR enhancer/denoiser are not supported with " +
|
|
1606
|
-
"the audio8 engine (native 44.1 kHz output needs no bandwidth " +
|
|
1607
|
-
"extension). Drop lavasrEnhancer / lavasrDenoiser.");
|
|
1608
|
-
}
|
|
1609
|
-
this._assertAudio8VoiceConsistent({
|
|
1610
|
-
referenceAudio: this._referenceAudio,
|
|
1611
|
-
referenceText: this._referenceText,
|
|
1612
|
-
}, "constructor");
|
|
1613
|
-
return;
|
|
1614
|
-
}
|
|
1615
|
-
const audio8Only = setOptionNames({
|
|
1616
|
-
referenceText: this._referenceText,
|
|
1617
|
-
greedy: this._greedy,
|
|
127
|
+
this._events = new addonEvents_1.AddonEventRouter({
|
|
128
|
+
job: this._job,
|
|
129
|
+
stream: this._sentenceStream,
|
|
130
|
+
logger: () => this._getLogger(),
|
|
131
|
+
reportStats: () => !!this.opts.stats,
|
|
1618
132
|
});
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
1630
|
-
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
if (this._steps != null) {
|
|
1636
|
-
throw new Error("tts-ggml: the moss-sfx engine takes steps per call " +
|
|
1637
|
-
"(run({ input, steps })), not in the constructor");
|
|
1638
|
-
}
|
|
1639
|
-
this._assertMossSoundEffectOutputRate();
|
|
1640
|
-
if (!this._mossSoundEffectPath) {
|
|
1641
|
-
throw new Error("tts-ggml: the moss-sfx engine needs its GGUF: stage " +
|
|
1642
|
-
"moss-sfx-*.gguf in modelDir or set files.mossSoundEffect");
|
|
1643
|
-
}
|
|
1644
|
-
}
|
|
1645
|
-
_assertMossSpeechOptionConsistency() {
|
|
1646
|
-
if (this._engineType !== ENGINE_MOSS_SPEECH)
|
|
1647
|
-
return;
|
|
1648
|
-
if (this._enhancerGgufPath || this._denoiserGgufPath) {
|
|
1649
|
-
throw new Error("tts-ggml: the LavaSR enhancer/denoiser are not supported with " +
|
|
1650
|
-
"the moss-speech engine. Drop lavasrEnhancer / lavasrDenoiser.");
|
|
1651
|
-
}
|
|
1652
|
-
if (this._referenceAudio) {
|
|
1653
|
-
throw new Error("tts-ggml: the moss-speech engine takes its reply voice per call " +
|
|
1654
|
-
"(run({ replyVoice, replyVoiceSampleRate })), not referenceAudio");
|
|
1655
|
-
}
|
|
1656
|
-
this._assertMossSpeechOutputRate();
|
|
1657
|
-
if (!this._mossSpeechModelPath || !this._mossSpeechCodecPath) {
|
|
1658
|
-
throw new Error("tts-ggml: the moss-speech engine needs its model and codec GGUFs: " +
|
|
1659
|
-
"stage moss-speech-*.gguf and moss-speech-codec-*.gguf in modelDir " +
|
|
1660
|
-
"or set files.mossSpeechModel and files.mossSpeechCodec");
|
|
1661
|
-
}
|
|
1662
|
-
}
|
|
1663
|
-
_assertMossSpeechOutputRate() {
|
|
1664
|
-
if (this._outputSampleRate == null ||
|
|
1665
|
-
this._outputSampleRate === MOSS_SPEECH_NATIVE_SAMPLE_RATE) {
|
|
1666
|
-
return;
|
|
1667
|
-
}
|
|
1668
|
-
throw new Error(`tts-ggml: the moss-speech engine outputs ${MOSS_SPEECH_NATIVE_SAMPLE_RATE} Hz ` +
|
|
1669
|
-
`audio and cannot resample (outputSampleRate=${this._outputSampleRate})`);
|
|
1670
|
-
}
|
|
1671
|
-
_assertMossSoundEffectOutputRate() {
|
|
1672
|
-
if (this._outputSampleRate == null ||
|
|
1673
|
-
this._outputSampleRate === MOSS_SFX_NATIVE_SAMPLE_RATE) {
|
|
1674
|
-
return;
|
|
1675
|
-
}
|
|
1676
|
-
throw new Error(`tts-ggml: the moss-sfx engine outputs ${MOSS_SFX_NATIVE_SAMPLE_RATE} Hz ` +
|
|
1677
|
-
`audio and cannot resample (outputSampleRate=${this._outputSampleRate})`);
|
|
1678
|
-
}
|
|
1679
|
-
_assertMossOptionConsistency() {
|
|
1680
|
-
if (this._engineType !== ENGINE_MOSS) {
|
|
1681
|
-
this._assertNoMossOnlyOptions();
|
|
1682
|
-
return;
|
|
1683
|
-
}
|
|
1684
|
-
if (this._enhancerGgufPath || this._denoiserGgufPath) {
|
|
1685
|
-
throw new Error("tts-ggml: the LavaSR enhancer/denoiser are not supported with " +
|
|
1686
|
-
"the moss engine. Drop lavasrEnhancer / lavasrDenoiser.");
|
|
1687
|
-
}
|
|
1688
|
-
this._assertMossOutputRate();
|
|
1689
|
-
this._assertMossVoiceConsistent();
|
|
1690
|
-
assertMossDurationTokens(this._durationTokens, "constructor");
|
|
1691
|
-
this._assertMossDialogueReferences();
|
|
1692
|
-
}
|
|
1693
|
-
_assertNoMossOnlyOptions() {
|
|
1694
|
-
const mossOnly = setOptionNames({
|
|
1695
|
-
durationTokens: this._durationTokens,
|
|
1696
|
-
dialogueReferences: this._dialogueReferences !== undefined || undefined,
|
|
1697
|
-
});
|
|
1698
|
-
if (mossOnly.length === 0)
|
|
1699
|
-
return;
|
|
1700
|
-
throw new Error(`tts-ggml: ${mossOnly.join(", ")} are moss-only options ` +
|
|
1701
|
-
`(engine is ${this._engineType})`);
|
|
1702
|
-
}
|
|
1703
|
-
_assertMossDialogueReferences() {
|
|
1704
|
-
if (this._dialogueReferences === undefined)
|
|
1705
|
-
return;
|
|
1706
|
-
assertDialogueReferenceList(this._dialogueReferences);
|
|
1707
|
-
if (this._referenceAudio) {
|
|
1708
|
-
throw new Error("tts-ggml: referenceAudio and dialogueReferences are exclusive; " +
|
|
1709
|
-
"pass one recording per speaker in dialogueReferences");
|
|
1710
|
-
}
|
|
1711
|
-
if (!this._mossCodecEncoderPath) {
|
|
1712
|
-
throw new Error("tts-ggml: dialogue synthesis with the moss engine needs the codec " +
|
|
1713
|
-
"encoder GGUF (files.mossCodecEncoder)");
|
|
1714
|
-
}
|
|
1715
|
-
if (!this._mossBackbonePath) {
|
|
1716
|
-
throw new Error("tts-ggml: dialogue synthesis with the moss engine needs the MOSS-TTSD " +
|
|
1717
|
-
"backbone: stage moss-ttsd-*.gguf in modelDir or set files.mossBackbone");
|
|
1718
|
-
}
|
|
1719
|
-
}
|
|
1720
|
-
_assertSentenceStreamingAllowed(where) {
|
|
1721
|
-
if (this._engineType === ENGINE_MOSS_SPEECH) {
|
|
1722
|
-
throw new Error(`tts-ggml: ${where}: the moss-speech engine answers one turn at a ` +
|
|
1723
|
-
"time and does not stream; use run()");
|
|
1724
|
-
}
|
|
1725
|
-
if (this._engineType === ENGINE_MOSS_SFX) {
|
|
1726
|
-
throw new Error(`tts-ggml: ${where}: the moss-sfx engine generates a whole sound ` +
|
|
1727
|
-
"effect from one prompt and does not stream; use run()");
|
|
1728
|
-
}
|
|
1729
|
-
if (this._dialogueReferences === undefined)
|
|
1730
|
-
return;
|
|
1731
|
-
throw new Error(`tts-ggml: ${where}: MOSS dialogue cannot be split into sentences, ` +
|
|
1732
|
-
"because every job must open with the reference transcripts; use " +
|
|
1733
|
-
"run() or native streaming (streamChunkTokens)");
|
|
1734
|
-
}
|
|
1735
|
-
_assertMossOutputRate() {
|
|
1736
|
-
if (this._outputSampleRate == null ||
|
|
1737
|
-
this._outputSampleRate === MOSS_NATIVE_SAMPLE_RATE) {
|
|
1738
|
-
return;
|
|
1739
|
-
}
|
|
1740
|
-
throw new Error(`tts-ggml: the moss engine emits ${MOSS_NATIVE_SAMPLE_RATE} Hz only; ` +
|
|
1741
|
-
`drop outputSampleRate (got ${this._outputSampleRate}).`);
|
|
1742
|
-
}
|
|
1743
|
-
_assertMossVoiceConsistent() {
|
|
1744
|
-
if (!this._referenceAudio || this._mossCodecEncoderPath)
|
|
1745
|
-
return;
|
|
1746
|
-
throw new Error("tts-ggml: voice cloning with the moss engine needs the codec " +
|
|
1747
|
-
"encoder GGUF (files.mossCodecEncoder)");
|
|
1748
|
-
}
|
|
1749
|
-
/**
|
|
1750
|
-
* A recording without its transcript is accepted by the model but degrades
|
|
1751
|
-
* the clone silently, and a transcript alone has nothing to attach to, so
|
|
1752
|
-
* both halves have to arrive together.
|
|
1753
|
-
*/
|
|
1754
|
-
_assertAudio8VoiceConsistent(voice, where) {
|
|
1755
|
-
if (!voice.referenceAudio && !voice.referenceText)
|
|
1756
|
-
return;
|
|
1757
|
-
if (!voice.referenceAudio) {
|
|
1758
|
-
throw new Error(`tts-ggml: ${where}: referenceText needs a referenceAudio to ` +
|
|
1759
|
-
"transcribe");
|
|
1760
|
-
}
|
|
1761
|
-
if (!voice.referenceText) {
|
|
1762
|
-
throw new Error(`tts-ggml: ${where}: referenceAudio needs a referenceText saying ` +
|
|
1763
|
-
"what the recording says");
|
|
1764
|
-
}
|
|
1765
|
-
if (!this._audio8CodecEncoderPath) {
|
|
1766
|
-
throw new Error(`tts-ggml: ${where}: voice cloning needs the audio8 codec encoder ` +
|
|
1767
|
-
"GGUF (files.audio8CodecEncoder)");
|
|
1768
|
-
}
|
|
1769
|
-
}
|
|
1770
|
-
/**
|
|
1771
|
-
* `promptText` is deliberately not required: its absence selects
|
|
1772
|
-
* cross-lingual mode. With a model dir present but no cloning GGUFs in it,
|
|
1773
|
-
* the native load fail-closes instead.
|
|
1774
|
-
*/
|
|
1775
|
-
_assertCosyvoiceCloneConsistent() {
|
|
1776
|
-
if (this._engineType !== ENGINE_COSYVOICE3)
|
|
1777
|
-
return;
|
|
1778
|
-
if (!this._referenceAudio)
|
|
1779
|
-
return;
|
|
1780
|
-
const haveExplicit = this._cosyvoiceS3tokModelPath && this._cosyvoiceCampplusModelPath;
|
|
1781
|
-
if (!haveExplicit && !this._cosyvoiceModelDir) {
|
|
1782
|
-
throw new Error("tts-ggml: referenceAudio (CosyVoice3 voice cloning) needs the " +
|
|
1783
|
-
"speech-tokenizer and speaker-encoder GGUFs: set " +
|
|
1784
|
-
"files.cosyvoiceS3tokModel + files.cosyvoiceCampplusModel, or a " +
|
|
1785
|
-
"files.cosyvoiceModelDir containing cosyvoice3-s3tok*.gguf and " +
|
|
1786
|
-
"cosyvoice3-campplus*.gguf");
|
|
1787
|
-
}
|
|
1788
|
-
}
|
|
1789
|
-
_assertCosyvoiceOptionConsistency() {
|
|
1790
|
-
if (this._engineType === ENGINE_COSYVOICE3)
|
|
1791
|
-
return;
|
|
1792
|
-
const cosyvoiceOnly = [];
|
|
1793
|
-
const cosyvoiceOnlyFields = {
|
|
1794
|
-
instruct: this._instruct,
|
|
1795
|
-
promptText: this._promptText,
|
|
1796
|
-
};
|
|
1797
|
-
for (const [key, value] of Object.entries(cosyvoiceOnlyFields)) {
|
|
1798
|
-
if (value != null)
|
|
1799
|
-
cosyvoiceOnly.push(key);
|
|
1800
|
-
}
|
|
1801
|
-
if (cosyvoiceOnly.length > 0) {
|
|
1802
|
-
throw new Error(`tts-ggml: ${cosyvoiceOnly.join(", ")} are cosyvoice3-only options ` +
|
|
1803
|
-
`(engine is ${this._engineType})`);
|
|
1804
|
-
}
|
|
1805
|
-
}
|
|
1806
|
-
/**
|
|
1807
|
-
* Validate the cross-engine emotion/pace surface against this engine, and
|
|
1808
|
-
* enforce CosyVoice3's one-instruction-per-synthesis rule.
|
|
1809
|
-
*/
|
|
1810
|
-
_assertConditioningConsistency(where) {
|
|
1811
|
-
const fields = pickConditioningFields({
|
|
1812
|
-
emotion: this._emotion,
|
|
1813
|
-
pace: this._pace,
|
|
1814
|
-
});
|
|
1815
|
-
validateConditioning(this._engineType, fields);
|
|
1816
|
-
if (this._engineType === ENGINE_COSYVOICE3) {
|
|
1817
|
-
assertSingleCosyvoiceControl(fields, this._instruct, where);
|
|
1818
|
-
}
|
|
1819
|
-
}
|
|
1820
|
-
/**
|
|
1821
|
-
* The per-call surface of a non-Parler engine: only the cross-engine
|
|
1822
|
-
* emotion/pace it declares support for, never Parler's template fields.
|
|
1823
|
-
*/
|
|
1824
|
-
_resolveConditioningJobFields(source, where, instruct) {
|
|
1825
|
-
const parlerOnly = keysPresent(source, PARLER_ONLY_KEYS);
|
|
1826
|
-
if (parlerOnly.length > 0) {
|
|
1827
|
-
throw new Error(`tts-ggml: ${where}: per-call description/voice-template options ` +
|
|
1828
|
-
`are parler-only (engine is ${this._engineType}; got ` +
|
|
1829
|
-
`${parlerOnly.join(", ")})`);
|
|
1830
|
-
}
|
|
1831
|
-
const fields = pickConditioningFields(source);
|
|
1832
|
-
if (!fields)
|
|
1833
|
-
return undefined;
|
|
1834
|
-
validateConditioning(this._engineType, fields);
|
|
1835
|
-
assertPerCallConditioning(this._engineType, fields, where);
|
|
1836
|
-
if (this._engineType === ENGINE_COSYVOICE3) {
|
|
1837
|
-
// A per-call control replaces the constructor's, so only the per-call
|
|
1838
|
-
// fields participate in the one-instruction count.
|
|
1839
|
-
assertSingleCosyvoiceControl(fields, instruct, where);
|
|
1840
|
-
}
|
|
1841
|
-
return fields;
|
|
1842
|
-
}
|
|
1843
|
-
/**
|
|
1844
|
-
* CosyVoice3's per-call instruction, rendered like the constructor's. The
|
|
1845
|
-
* native layer reads it next to emotion/pace, and any of the three replaces
|
|
1846
|
-
* the configured conditioning for that synthesis.
|
|
1847
|
-
*/
|
|
1848
|
-
_resolveJobInstruct(source, where) {
|
|
1849
|
-
if (source?.instruct === undefined)
|
|
1850
|
-
return undefined;
|
|
1851
|
-
if (this._engineType !== ENGINE_COSYVOICE3) {
|
|
1852
|
-
throw new Error(`tts-ggml: ${where}: per-call instruct is cosyvoice3-only ` +
|
|
1853
|
-
`(engine is ${this._engineType})`);
|
|
1854
|
-
}
|
|
1855
|
-
return renderCosyvoiceInstruct(source.instruct) || undefined;
|
|
1856
|
-
}
|
|
1857
|
-
/**
|
|
1858
|
-
* Parler's per-call surface: description/template fields, where a per-call
|
|
1859
|
-
* template cannot be merged with a constructor-level free-text description.
|
|
1860
|
-
*/
|
|
1861
|
-
_resolveParlerJobFields(source, where) {
|
|
1862
|
-
const fields = pickParlerDescFields(source);
|
|
1863
|
-
if (!fields)
|
|
1864
|
-
return undefined;
|
|
1865
|
-
assertParlerDescFieldsConsistent(fields, where);
|
|
1866
|
-
const conditioning = pickConditioningFields(source);
|
|
1867
|
-
validateConditioning(this._engineType, conditioning);
|
|
1868
|
-
assertPerCallConditioning(this._engineType, conditioning, where);
|
|
1869
|
-
const hasDescription = PARLER_DESCRIPTION_KEYS.some((key) => fields[key] != null);
|
|
1870
|
-
if (!hasDescription && this._description != null) {
|
|
1871
|
-
throw new Error(`tts-ggml: ${where}: per-call template options cannot be combined ` +
|
|
1872
|
-
"with a constructor-level description; pass a per-call " +
|
|
1873
|
-
"description instead");
|
|
1874
|
-
}
|
|
1875
|
-
return fields;
|
|
1876
|
-
}
|
|
1877
|
-
/**
|
|
1878
|
-
* The voice a per-call override actually synthesizes with. Mirrors
|
|
1879
|
-
* Audio8Model::resolveVoice: a per-call recording replaces both halves, so
|
|
1880
|
-
* it cannot inherit the configured transcript, which describes a different
|
|
1881
|
-
* recording; a per-call transcript alone corrects the configured one.
|
|
1882
|
-
*/
|
|
1883
|
-
_mergeAudio8Voice(fields) {
|
|
1884
|
-
if (fields.referenceAudio) {
|
|
1885
|
-
return {
|
|
1886
|
-
referenceAudio: fields.referenceAudio,
|
|
1887
|
-
referenceText: fields.referenceText,
|
|
1888
|
-
};
|
|
1889
|
-
}
|
|
1890
|
-
return {
|
|
1891
|
-
referenceAudio: this._referenceAudio,
|
|
1892
|
-
referenceText: fields.referenceText ?? this._referenceText,
|
|
1893
|
-
};
|
|
1894
|
-
}
|
|
1895
|
-
/**
|
|
1896
|
-
* Extract + validate the per-call Audio8 voice fields from a run input or
|
|
1897
|
-
* streaming options. Returns undefined when none are present.
|
|
1898
|
-
*/
|
|
1899
|
-
_resolveAudio8JobFields(source, where) {
|
|
1900
|
-
const fields = pickAudio8VoiceFields(source);
|
|
1901
|
-
if (!fields)
|
|
1902
|
-
return undefined;
|
|
1903
|
-
if (this._engineType !== ENGINE_AUDIO8) {
|
|
1904
|
-
throw new Error(`tts-ggml: ${where}: per-call referenceAudio/referenceText are ` +
|
|
1905
|
-
`audio8-only (engine is ${this._engineType})`);
|
|
1906
|
-
}
|
|
1907
|
-
this._assertAudio8VoiceConsistent(this._mergeAudio8Voice(fields), where);
|
|
1908
|
-
return fields;
|
|
1909
|
-
}
|
|
1910
|
-
_assertNoSpeechFields(source, where) {
|
|
1911
|
-
const fields = pickSpeechFields(source);
|
|
1912
|
-
if (!fields)
|
|
1913
|
-
return;
|
|
1914
|
-
throw new Error(`tts-ggml: ${where}: ${Object.keys(fields).join(", ")} are ` +
|
|
1915
|
-
`moss-speech-only options (engine is ${this._engineType})`);
|
|
1916
|
-
}
|
|
1917
|
-
_assertNoOtherEngineFields(source, where) {
|
|
1918
|
-
this._resolveConditioningJobFields(source, where, this._resolveJobInstruct(source, where));
|
|
1919
|
-
this._resolveAudio8JobFields(source, where);
|
|
1920
|
-
this._resolveSoundEffectJobFields(source, where);
|
|
1921
|
-
}
|
|
1922
|
-
/**
|
|
1923
|
-
* Validate the MOSS-Speech call and fold the new user turn (spoken `audio`
|
|
1924
|
-
* or text `input`) and `systemPrompt` into the native `messages` list.
|
|
1925
|
-
*/
|
|
1926
|
-
_resolveSpeechJobFields(source, where) {
|
|
1927
|
-
const fields = source ?? {};
|
|
1928
|
-
assertSpeechCall(fields, where);
|
|
1929
|
-
return {
|
|
1930
|
-
messages: speechMessages(fields),
|
|
1931
|
-
...speechReplyVoice(fields),
|
|
1932
|
-
...speechControls(fields),
|
|
1933
|
-
};
|
|
1934
|
-
}
|
|
1935
|
-
/**
|
|
1936
|
-
* Extract + validate the per-call MOSS-SoundEffect controls from a run
|
|
1937
|
-
* input. Returns undefined when none are present.
|
|
1938
|
-
*/
|
|
1939
|
-
_resolveSoundEffectJobFields(source, where) {
|
|
1940
|
-
const fields = pickSoundEffectFields(source);
|
|
1941
|
-
if (!fields)
|
|
1942
|
-
return undefined;
|
|
1943
|
-
if (this._engineType !== ENGINE_MOSS_SFX) {
|
|
1944
|
-
throw new Error(`tts-ggml: ${where}: ${Object.keys(fields).join(", ")} are ` +
|
|
1945
|
-
`moss-sfx-only options (engine is ${this._engineType})`);
|
|
1946
|
-
}
|
|
1947
|
-
assertSoundEffectFields(fields);
|
|
1948
|
-
return fields;
|
|
1949
|
-
}
|
|
1950
|
-
/**
|
|
1951
|
-
* The per-call fields of whichever engine is loaded, if any are set. Parler
|
|
1952
|
-
* takes the full description/template surface, Audio8 its voice override,
|
|
1953
|
-
* and every engine the cross-engine conditioning it supports.
|
|
1954
|
-
*/
|
|
1955
|
-
_resolveJobFields(source, where) {
|
|
1956
|
-
if (this._engineType === ENGINE_MOSS_SPEECH) {
|
|
1957
|
-
this._assertNoOtherEngineFields(source, where);
|
|
1958
|
-
return this._resolveSpeechJobFields(source, where);
|
|
1959
|
-
}
|
|
1960
|
-
const instruct = this._resolveJobInstruct(source, where);
|
|
1961
|
-
const engineFields = this._engineType === ENGINE_PARLER
|
|
1962
|
-
? this._resolveParlerJobFields(source, where)
|
|
1963
|
-
: this._resolveConditioningJobFields(source, where, instruct);
|
|
1964
|
-
const audio8 = this._resolveAudio8JobFields(source, where);
|
|
1965
|
-
const soundEffect = this._resolveSoundEffectJobFields(source, where);
|
|
1966
|
-
this._assertNoSpeechFields(source, where);
|
|
1967
|
-
if (!engineFields && !audio8 && !soundEffect && instruct === undefined) {
|
|
1968
|
-
return undefined;
|
|
1969
|
-
}
|
|
1970
|
-
return {
|
|
1971
|
-
...(engineFields ?? {}),
|
|
1972
|
-
...(audio8 ?? {}),
|
|
1973
|
-
...(soundEffect ?? {}),
|
|
1974
|
-
...(instruct === undefined ? {} : { instruct }),
|
|
1975
|
-
};
|
|
133
|
+
this._pocket =
|
|
134
|
+
this._setup.engineType === engineIds_1.ENGINE_POCKET
|
|
135
|
+
? new pocketLifecycle_1.PocketLifecycle({
|
|
136
|
+
setup: this._setup,
|
|
137
|
+
state: this.state,
|
|
138
|
+
logger: () => this._getLogger(),
|
|
139
|
+
optionalAddon: () => this._optionalAddon(),
|
|
140
|
+
setAddon: (addon) => {
|
|
141
|
+
this.addon = addon;
|
|
142
|
+
},
|
|
143
|
+
createAddon: (params) => this._spawnAddon(params),
|
|
144
|
+
unloadModel: () => this._unloadModel(),
|
|
145
|
+
activeJob: () => this._job.active,
|
|
146
|
+
failStream: (reason) => this._sentenceStream.fail(reason),
|
|
147
|
+
})
|
|
148
|
+
: null;
|
|
1976
149
|
}
|
|
1977
150
|
getEngineType() {
|
|
1978
|
-
return this.
|
|
151
|
+
return this._setup.engineType;
|
|
1979
152
|
}
|
|
1980
153
|
getApiDefinition() {
|
|
1981
154
|
const api = (0, infer_base_1.getApiDefinition)();
|
|
@@ -2005,25 +178,14 @@ class TTSGgml {
|
|
|
2005
178
|
this.state.weightsLoaded = true;
|
|
2006
179
|
}
|
|
2007
180
|
async run(input) {
|
|
2008
|
-
await this.
|
|
2009
|
-
this.
|
|
2010
|
-
if (input?.streamOutput === true && this.
|
|
2011
|
-
this.
|
|
2012
|
-
|
|
2013
|
-
|
|
2014
|
-
|
|
2015
|
-
|
|
2016
|
-
adds: "run with streamOutput: non-empty string `input` is required",
|
|
2017
|
-
});
|
|
2018
|
-
}
|
|
2019
|
-
const jobFields = this._resolveJobFields(input, "run");
|
|
2020
|
-
const runStream = () => this._runStreamOrchestrator(input.input, {
|
|
2021
|
-
locale: input.locale,
|
|
2022
|
-
maxChunkScalars: input.maxChunkScalars,
|
|
2023
|
-
}, jobFields);
|
|
2024
|
-
return this.exclusiveRun
|
|
2025
|
-
? this._enqueueExclusiveTtsResponse(runStream)
|
|
2026
|
-
: runStream();
|
|
181
|
+
await this._pocket?.waitCancel();
|
|
182
|
+
this._pocket?.checkReload();
|
|
183
|
+
if (input?.streamOutput === true && this._setup.engineType !== engineIds_1.ENGINE_POCKET) {
|
|
184
|
+
(0, streamingSupport_1.assertSentenceStreamingAllowed)(this._setup, "run with streamOutput");
|
|
185
|
+
const text = (0, sentenceStream_1.requireStreamOutputText)(input.input);
|
|
186
|
+
const jobFields = (0, jobFields_1.resolveJobFields)(this._setup, input, "run");
|
|
187
|
+
const { locale, maxChunkScalars } = input;
|
|
188
|
+
return this._startStream(() => this._runStreamOrchestrator(text, { locale, maxChunkScalars }, jobFields));
|
|
2027
189
|
}
|
|
2028
190
|
return this._runExclusive(() => this._runInternal(input));
|
|
2029
191
|
}
|
|
@@ -2032,20 +194,19 @@ class TTSGgml {
|
|
|
2032
194
|
* `run({ input: text, streamOutput: true, ...options })`.
|
|
2033
195
|
*/
|
|
2034
196
|
async runStream(text, options = {}) {
|
|
2035
|
-
await this.
|
|
2036
|
-
this.
|
|
197
|
+
await this._pocket?.waitCancel();
|
|
198
|
+
this._pocket?.checkReload();
|
|
2037
199
|
const normalized = options == null || typeof options !== "object" ? {} : options;
|
|
2038
|
-
if (this.
|
|
2039
|
-
|
|
2040
|
-
return this.exclusiveRun ? this._enqueueExclusiveTtsResponse(run) : run();
|
|
200
|
+
if (this._setup.engineType === engineIds_1.ENGINE_POCKET) {
|
|
201
|
+
return this._startStream(() => this._runStreamOrchestrator(text, normalized, (0, jobFields_1.resolveJobFields)(this._setup, normalized, "runStream")));
|
|
2041
202
|
}
|
|
2042
203
|
return this.run({
|
|
2043
204
|
input: text,
|
|
2044
205
|
streamOutput: true,
|
|
2045
206
|
locale: normalized.locale,
|
|
2046
207
|
maxChunkScalars: normalized.maxChunkScalars,
|
|
2047
|
-
...(pickParlerDescFields(normalized) ?? {}),
|
|
2048
|
-
...(pickAudio8VoiceFields(normalized) ?? {}),
|
|
208
|
+
...((0, parler_1.pickParlerDescFields)(normalized) ?? {}),
|
|
209
|
+
...((0, audio8_1.pickAudio8VoiceFields)(normalized) ?? {}),
|
|
2049
210
|
...(normalized.instruct === undefined
|
|
2050
211
|
? {}
|
|
2051
212
|
: { instruct: normalized.instruct }),
|
|
@@ -2059,239 +220,39 @@ class TTSGgml {
|
|
|
2059
220
|
* small streamed fragments are coalesced.
|
|
2060
221
|
*/
|
|
2061
222
|
async runStreaming(textStream, options = {}) {
|
|
2062
|
-
await this.
|
|
2063
|
-
this.
|
|
2064
|
-
this.
|
|
2065
|
-
const jobFields = this.
|
|
2066
|
-
const streamOptions =
|
|
2067
|
-
let normalized =
|
|
223
|
+
await this._pocket?.waitCancel();
|
|
224
|
+
this._pocket?.checkReload();
|
|
225
|
+
(0, streamingSupport_1.assertSentenceStreamingAllowed)(this._setup, "runStreaming");
|
|
226
|
+
const jobFields = (0, jobFields_1.resolveJobFields)(this._setup, options, "runStreaming");
|
|
227
|
+
const streamOptions = (0, sentenceStream_1.resolveRunStreamingOptions)(textStream, options);
|
|
228
|
+
let normalized = (0, sentenceStream_1.normalizeTextStream)(textStream);
|
|
2068
229
|
if (streamOptions.accumulateSentences) {
|
|
2069
230
|
normalized = (0, textStreamAccumulator_1.accumulateTextStream)(normalized, {
|
|
2070
231
|
sentenceDelimiterPreset: streamOptions.sentenceDelimiterPreset,
|
|
2071
232
|
maxBufferScalars: streamOptions.maxBufferScalars,
|
|
2072
233
|
flushAfterMs: streamOptions.flushAfterMs,
|
|
2073
234
|
sentenceDelimiter: streamOptions.sentenceDelimiter,
|
|
2074
|
-
language: this.
|
|
2075
|
-
});
|
|
2076
|
-
}
|
|
2077
|
-
const runStream = () => this._runTextStreamOrchestrator(normalized, jobFields);
|
|
2078
|
-
return this.exclusiveRun
|
|
2079
|
-
? this._enqueueExclusiveTtsResponse(runStream)
|
|
2080
|
-
: runStream();
|
|
2081
|
-
}
|
|
2082
|
-
async _enqueueExclusiveTtsResponse(run) {
|
|
2083
|
-
const previous = this._ttsInferenceQueueWaiter;
|
|
2084
|
-
let releaseSlot = () => { };
|
|
2085
|
-
this._ttsInferenceQueueWaiter = new Promise((resolve) => {
|
|
2086
|
-
releaseSlot = resolve;
|
|
2087
|
-
});
|
|
2088
|
-
await previous;
|
|
2089
|
-
let response;
|
|
2090
|
-
try {
|
|
2091
|
-
response = run();
|
|
2092
|
-
}
|
|
2093
|
-
catch (error) {
|
|
2094
|
-
releaseSlot();
|
|
2095
|
-
throw error;
|
|
2096
|
-
}
|
|
2097
|
-
void response
|
|
2098
|
-
.await()
|
|
2099
|
-
.finally(releaseSlot)
|
|
2100
|
-
.catch(() => { });
|
|
2101
|
-
return response;
|
|
2102
|
-
}
|
|
2103
|
-
_resolveRunStreamingOptions(textStream, options) {
|
|
2104
|
-
const normalized = options == null || typeof options !== "object" ? {} : options;
|
|
2105
|
-
let accumulateSentences = normalized.accumulateSentences;
|
|
2106
|
-
if (accumulateSentences === undefined) {
|
|
2107
|
-
accumulateSentences =
|
|
2108
|
-
defaultAccumulateSentencesForStreamInput(textStream);
|
|
2109
|
-
}
|
|
2110
|
-
return {
|
|
2111
|
-
accumulateSentences: !!accumulateSentences,
|
|
2112
|
-
sentenceDelimiterPreset: normalized.sentenceDelimiterPreset === "latin" ||
|
|
2113
|
-
normalized.sentenceDelimiterPreset === "cjk" ||
|
|
2114
|
-
normalized.sentenceDelimiterPreset === "multilingual"
|
|
2115
|
-
? normalized.sentenceDelimiterPreset
|
|
2116
|
-
: "multilingual",
|
|
2117
|
-
maxBufferScalars: normalized.maxBufferScalars,
|
|
2118
|
-
flushAfterMs: normalized.flushAfterMs ?? textStreamAccumulator_1.DEFAULT_FLUSH_AFTER_MS,
|
|
2119
|
-
sentenceDelimiter: normalized.sentenceDelimiter instanceof RegExp
|
|
2120
|
-
? normalized.sentenceDelimiter
|
|
2121
|
-
: undefined,
|
|
2122
|
-
};
|
|
2123
|
-
}
|
|
2124
|
-
_normalizeTextStream(textStream) {
|
|
2125
|
-
if (textStream == null) {
|
|
2126
|
-
throw new QvacErrorAddonTTSGgml({
|
|
2127
|
-
code: ERR_CODES.FAILED_TO_APPEND,
|
|
2128
|
-
adds: "runStreaming: text stream is required",
|
|
235
|
+
language: this._setup.config.language,
|
|
2129
236
|
});
|
|
2130
237
|
}
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
return (
|
|
2134
|
-
yield textStream;
|
|
2135
|
-
})();
|
|
2136
|
-
}
|
|
2137
|
-
if (typeof textStream[Symbol.asyncIterator] === "function") {
|
|
2138
|
-
return textStream;
|
|
2139
|
-
}
|
|
2140
|
-
if (Array.isArray(textStream) ||
|
|
2141
|
-
typeof textStream[Symbol.iterator] === "function") {
|
|
2142
|
-
// eslint-disable-next-line @typescript-eslint/require-await -- adapts synchronous iterables to the async iterable contract.
|
|
2143
|
-
return (async function* fromIterable() {
|
|
2144
|
-
for (const value of textStream) {
|
|
2145
|
-
yield value;
|
|
2146
|
-
}
|
|
2147
|
-
})();
|
|
2148
|
-
}
|
|
2149
|
-
throw new QvacErrorAddonTTSGgml({
|
|
2150
|
-
code: ERR_CODES.FAILED_TO_APPEND,
|
|
2151
|
-
adds: "runStreaming: expected string, array of strings, Iterable, or AsyncIterable",
|
|
238
|
+
return this._startStream(() => {
|
|
239
|
+
this._pocket?.checkRequest();
|
|
240
|
+
return this._sentenceStream.streamText(normalized, jobFields);
|
|
2152
241
|
});
|
|
2153
242
|
}
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
this._sentenceStreamCtx = {
|
|
2158
|
-
textStreamMode: true,
|
|
2159
|
-
asyncTextSource: source,
|
|
2160
|
-
chunks: [],
|
|
2161
|
-
chunkIdx: 0,
|
|
2162
|
-
acc: emptyStreamAccumulator(),
|
|
2163
|
-
chunkResolver: null,
|
|
2164
|
-
jobFields,
|
|
2165
|
-
};
|
|
2166
|
-
const context = this._sentenceStreamCtx;
|
|
2167
|
-
void this._sentenceStreamTextIterableDrive().catch((error) => {
|
|
2168
|
-
if (context !== this._sentenceStreamCtx)
|
|
2169
|
-
return;
|
|
2170
|
-
this._rejectActiveChunk(error);
|
|
2171
|
-
this._sentenceStreamCtx = null;
|
|
2172
|
-
this._job.fail(normalizeError(error));
|
|
2173
|
-
});
|
|
2174
|
-
return response;
|
|
2175
|
-
}
|
|
2176
|
-
async _sentenceStreamTextIterableDrive() {
|
|
2177
|
-
const context = this._sentenceStreamCtx;
|
|
2178
|
-
if (!context ||
|
|
2179
|
-
!context.textStreamMode ||
|
|
2180
|
-
!context.asyncTextSource) {
|
|
2181
|
-
return;
|
|
2182
|
-
}
|
|
2183
|
-
try {
|
|
2184
|
-
for await (const piece of context.asyncTextSource) {
|
|
2185
|
-
if (context !== this._sentenceStreamCtx)
|
|
2186
|
-
return;
|
|
2187
|
-
const text = String(piece).trim();
|
|
2188
|
-
if (text.length === 0)
|
|
2189
|
-
continue;
|
|
2190
|
-
context.chunks.push(text);
|
|
2191
|
-
context.chunkIdx = context.chunks.length - 1;
|
|
2192
|
-
const done = new Promise((resolve, reject) => {
|
|
2193
|
-
context.chunkResolver = { resolve, reject };
|
|
2194
|
-
});
|
|
2195
|
-
// Dispatch may fail before the completion promise is awaited.
|
|
2196
|
-
void done.catch(() => { });
|
|
2197
|
-
await this._dispatchJob({
|
|
2198
|
-
type: "text",
|
|
2199
|
-
input: text,
|
|
2200
|
-
...(context.jobFields ?? {}),
|
|
2201
|
-
});
|
|
2202
|
-
await done;
|
|
2203
|
-
}
|
|
2204
|
-
}
|
|
2205
|
-
catch (error) {
|
|
2206
|
-
if (context !== this._sentenceStreamCtx)
|
|
2207
|
-
return;
|
|
2208
|
-
this._rejectActiveChunk(error);
|
|
2209
|
-
this._sentenceStreamCtx = null;
|
|
2210
|
-
this._job.fail(normalizeError(error));
|
|
2211
|
-
return;
|
|
2212
|
-
}
|
|
2213
|
-
if (context !== this._sentenceStreamCtx)
|
|
2214
|
-
return;
|
|
2215
|
-
const current = this._sentenceStreamCtx;
|
|
2216
|
-
const chunks = current?.chunks || [];
|
|
2217
|
-
const accumulator = current?.acc || emptyStreamAccumulator();
|
|
2218
|
-
this._sentenceStreamCtx = null;
|
|
2219
|
-
this._endJobWithStats(chunks.length === 0
|
|
2220
|
-
? {
|
|
2221
|
-
totalTime: 0,
|
|
2222
|
-
tokensPerSecond: 0,
|
|
2223
|
-
realTimeFactor: 0,
|
|
2224
|
-
audioDurationMs: 0,
|
|
2225
|
-
totalSamples: 0,
|
|
2226
|
-
}
|
|
2227
|
-
: computeSentenceStreamStats(chunks, accumulator, this._pacesOnFrames()));
|
|
2228
|
-
}
|
|
2229
|
-
/**
|
|
2230
|
-
* Audio8 and MOSS report `tokensPerSecond` as codec frames per second, so
|
|
2231
|
-
* the streaming aggregate has to count frames too rather than characters.
|
|
2232
|
-
*/
|
|
2233
|
-
_pacesOnFrames() {
|
|
2234
|
-
return (this._engineType === ENGINE_AUDIO8 || this._engineType === ENGINE_MOSS);
|
|
243
|
+
/** Streams line up behind each other under `exclusiveRun`; otherwise they start at once. */
|
|
244
|
+
_startStream(run) {
|
|
245
|
+
return this.exclusiveRun ? this._exclusiveStreams.enqueue(run) : run();
|
|
2235
246
|
}
|
|
2236
247
|
_runStreamOrchestrator(text, options, jobFields) {
|
|
2237
|
-
this.
|
|
2238
|
-
const chunks = (0,
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
maxScalars: options.maxChunkScalars,
|
|
2242
|
-
});
|
|
2243
|
-
if (chunks.length === 0) {
|
|
2244
|
-
throw new QvacErrorAddonTTSGgml({
|
|
2245
|
-
code: ERR_CODES.FAILED_TO_APPEND,
|
|
2246
|
-
adds: "chunked synthesis: text produced no chunks after split",
|
|
2247
|
-
});
|
|
2248
|
-
}
|
|
2249
|
-
this._checkPocketRequest();
|
|
2250
|
-
const response = this._job.start();
|
|
2251
|
-
this._sentenceStreamCtx = {
|
|
2252
|
-
chunks,
|
|
2253
|
-
chunkIdx: 0,
|
|
2254
|
-
acc: emptyStreamAccumulator(),
|
|
2255
|
-
chunkResolver: null,
|
|
2256
|
-
jobFields,
|
|
2257
|
-
};
|
|
2258
|
-
const context = this._sentenceStreamCtx;
|
|
2259
|
-
void this._sentenceStreamDriveBody().catch((error) => {
|
|
2260
|
-
if (context !== this._sentenceStreamCtx)
|
|
2261
|
-
return;
|
|
2262
|
-
this._rejectActiveChunk(error);
|
|
2263
|
-
this._sentenceStreamCtx = null;
|
|
2264
|
-
this._job.fail(normalizeError(error));
|
|
2265
|
-
});
|
|
2266
|
-
return response;
|
|
2267
|
-
}
|
|
2268
|
-
async _sentenceStreamDriveBody() {
|
|
2269
|
-
const context = this._sentenceStreamCtx;
|
|
2270
|
-
if (!context || context.textStreamMode)
|
|
2271
|
-
return;
|
|
2272
|
-
for (let index = 0; index < context.chunks.length; index++) {
|
|
2273
|
-
if (context !== this._sentenceStreamCtx)
|
|
2274
|
-
return;
|
|
2275
|
-
context.chunkIdx = index;
|
|
2276
|
-
const done = new Promise((resolve, reject) => {
|
|
2277
|
-
context.chunkResolver = { resolve, reject };
|
|
2278
|
-
});
|
|
2279
|
-
// Dispatch may fail before the completion promise is awaited.
|
|
2280
|
-
void done.catch(() => { });
|
|
2281
|
-
await this._dispatchJob({
|
|
2282
|
-
type: "text",
|
|
2283
|
-
input: context.chunks[index],
|
|
2284
|
-
...(context.jobFields ?? {}),
|
|
2285
|
-
});
|
|
2286
|
-
await done;
|
|
2287
|
-
}
|
|
2288
|
-
if (context !== this._sentenceStreamCtx)
|
|
2289
|
-
return;
|
|
2290
|
-
this._sentenceStreamCtx = null;
|
|
248
|
+
this._pocket?.checkRequest();
|
|
249
|
+
const chunks = (0, sentenceStream_1.splitSentenceChunks)(text, options, this._setup.config.language);
|
|
250
|
+
this._pocket?.checkRequest();
|
|
251
|
+
return this._sentenceStream.streamChunks(chunks, jobFields);
|
|
2291
252
|
}
|
|
2292
253
|
async _load() {
|
|
2293
|
-
this._getLogger().info("[TTSGgml] Language:", this.
|
|
2294
|
-
const addon = this.
|
|
254
|
+
this._getLogger().info("[TTSGgml] Language:", this._setup.config.language || "en");
|
|
255
|
+
const addon = this._spawnAddon(this._buildTtsParams());
|
|
2295
256
|
this.addon = addon;
|
|
2296
257
|
try {
|
|
2297
258
|
await addon.activate();
|
|
@@ -2307,407 +268,11 @@ class TTSGgml {
|
|
|
2307
268
|
}
|
|
2308
269
|
}
|
|
2309
270
|
_buildTtsParams() {
|
|
2310
|
-
|
|
2311
|
-
return { ...this._pocketParams };
|
|
2312
|
-
if (this._engineType === ENGINE_SUPERTONIC) {
|
|
2313
|
-
return this._buildSupertonicParams();
|
|
2314
|
-
}
|
|
2315
|
-
if (this._engineType === ENGINE_COSYVOICE3) {
|
|
2316
|
-
return this._buildCosyvoiceParams();
|
|
2317
|
-
}
|
|
2318
|
-
if (this._engineType === ENGINE_PARLER) {
|
|
2319
|
-
return this._buildParlerParams();
|
|
2320
|
-
}
|
|
2321
|
-
if (this._engineType === ENGINE_AUDIO8) {
|
|
2322
|
-
return this._buildAudio8Params();
|
|
2323
|
-
}
|
|
2324
|
-
if (this._engineType === ENGINE_MOSS) {
|
|
2325
|
-
return this._buildMossParams();
|
|
2326
|
-
}
|
|
2327
|
-
if (this._engineType === ENGINE_MOSS_SFX) {
|
|
2328
|
-
return this._buildMossSoundEffectParams();
|
|
2329
|
-
}
|
|
2330
|
-
if (this._engineType === ENGINE_MOSS_SPEECH) {
|
|
2331
|
-
return this._buildMossSpeechParams();
|
|
2332
|
-
}
|
|
2333
|
-
return this._buildChatterboxParams();
|
|
2334
|
-
}
|
|
2335
|
-
_buildCosyvoiceParams() {
|
|
2336
|
-
const parameters = {
|
|
2337
|
-
engineType: ENGINE_COSYVOICE3,
|
|
2338
|
-
cosyvoiceModelDir: this._cosyvoiceModelDir || "",
|
|
2339
|
-
language: this._config.language || "en",
|
|
2340
|
-
};
|
|
2341
|
-
if (this._cosyvoiceLlmModelPath) {
|
|
2342
|
-
parameters.cosyvoiceLlmModelPath = this._cosyvoiceLlmModelPath;
|
|
2343
|
-
}
|
|
2344
|
-
if (this._cosyvoiceFlowModelPath) {
|
|
2345
|
-
parameters.cosyvoiceFlowModelPath = this._cosyvoiceFlowModelPath;
|
|
2346
|
-
}
|
|
2347
|
-
if (this._cosyvoiceHiftModelPath) {
|
|
2348
|
-
parameters.cosyvoiceHiftModelPath = this._cosyvoiceHiftModelPath;
|
|
2349
|
-
}
|
|
2350
|
-
if (this._cosyvoiceS3tokModelPath) {
|
|
2351
|
-
parameters.cosyvoiceS3tokModelPath = this._cosyvoiceS3tokModelPath;
|
|
2352
|
-
}
|
|
2353
|
-
if (this._cosyvoiceCampplusModelPath) {
|
|
2354
|
-
parameters.cosyvoiceCampplusModelPath = this._cosyvoiceCampplusModelPath;
|
|
2355
|
-
}
|
|
2356
|
-
if (this._cosyvoiceVocabPath) {
|
|
2357
|
-
parameters.cosyvoiceVocabPath = this._cosyvoiceVocabPath;
|
|
2358
|
-
}
|
|
2359
|
-
if (this._cosyvoiceMergesPath) {
|
|
2360
|
-
parameters.cosyvoiceMergesPath = this._cosyvoiceMergesPath;
|
|
2361
|
-
}
|
|
2362
|
-
if (this._cosyvoiceVoiceModelPath) {
|
|
2363
|
-
parameters.cosyvoiceVoiceModelPath = this._cosyvoiceVoiceModelPath;
|
|
2364
|
-
}
|
|
2365
|
-
if (this._vulkanDevice != null) {
|
|
2366
|
-
parameters.vulkanDevice = this._vulkanDevice | 0;
|
|
2367
|
-
}
|
|
2368
|
-
if (this._flowCutPrompt != null) {
|
|
2369
|
-
parameters.flowCutPrompt = !!this._flowCutPrompt;
|
|
2370
|
-
}
|
|
2371
|
-
if (this._referenceAudio != null) {
|
|
2372
|
-
parameters.referenceAudio = this._referenceAudio;
|
|
2373
|
-
}
|
|
2374
|
-
if (this._promptText != null) {
|
|
2375
|
-
parameters.promptText = String(this._promptText);
|
|
2376
|
-
}
|
|
2377
|
-
if (this._instruct)
|
|
2378
|
-
parameters.instruct = this._instruct;
|
|
2379
|
-
// Canonical values; the native layer maps them to the trained instructions.
|
|
2380
|
-
if (this._emotion != null)
|
|
2381
|
-
parameters.emotion = String(this._emotion);
|
|
2382
|
-
if (this._pace != null)
|
|
2383
|
-
parameters.pace = String(this._pace);
|
|
2384
|
-
if (this._voice)
|
|
2385
|
-
parameters.voice = this._voice;
|
|
2386
|
-
this._assignCommonNativeParams(parameters);
|
|
2387
|
-
if (this._cfmSteps != null)
|
|
2388
|
-
parameters.cfmSteps = this._cfmSteps | 0;
|
|
2389
|
-
if (this._streamChunkTokens != null) {
|
|
2390
|
-
parameters.streamChunkTokens = this._streamChunkTokens | 0;
|
|
2391
|
-
}
|
|
2392
|
-
if (this._streamFirstChunkTokens != null) {
|
|
2393
|
-
parameters.streamFirstChunkTokens = this._streamFirstChunkTokens | 0;
|
|
2394
|
-
}
|
|
2395
|
-
if (this._streamLeftContextTokens != null) {
|
|
2396
|
-
parameters.streamLeftContextTokens = this._streamLeftContextTokens | 0;
|
|
2397
|
-
}
|
|
2398
|
-
return parameters;
|
|
2399
|
-
}
|
|
2400
|
-
_buildChatterboxParams() {
|
|
2401
|
-
const parameters = {
|
|
2402
|
-
engineType: ENGINE_CHATTERBOX,
|
|
2403
|
-
t3ModelPath: this._t3ModelPath || "",
|
|
2404
|
-
s3genModelPath: this._s3genModelPath || "",
|
|
2405
|
-
language: this._config.language || "en",
|
|
2406
|
-
};
|
|
2407
|
-
this._assignCommonNativeParams(parameters);
|
|
2408
|
-
if (this._referenceAudio != null) {
|
|
2409
|
-
parameters.referenceAudio = this._referenceAudio;
|
|
2410
|
-
}
|
|
2411
|
-
if (this._voiceDir != null)
|
|
2412
|
-
parameters.voiceDir = this._voiceDir;
|
|
2413
|
-
if (this._nCtx != null)
|
|
2414
|
-
parameters.nCtx = this._nCtx | 0;
|
|
2415
|
-
if (this._kvCacheType != null) {
|
|
2416
|
-
parameters.kvCacheType = String(this._kvCacheType);
|
|
2417
|
-
}
|
|
2418
|
-
if (this._streamChunkTokens != null) {
|
|
2419
|
-
parameters.streamChunkTokens = this._streamChunkTokens | 0;
|
|
2420
|
-
}
|
|
2421
|
-
if (this._streamFirstChunkTokens != null) {
|
|
2422
|
-
parameters.streamFirstChunkTokens =
|
|
2423
|
-
this._streamFirstChunkTokens | 0;
|
|
2424
|
-
}
|
|
2425
|
-
if (this._cfmSteps != null) {
|
|
2426
|
-
parameters.cfmSteps = this._cfmSteps | 0;
|
|
2427
|
-
}
|
|
2428
|
-
if (this._streamLeftContextTokens != null) {
|
|
2429
|
-
parameters.streamLeftContextTokens = this._streamLeftContextTokens | 0;
|
|
2430
|
-
}
|
|
2431
|
-
if (this._batchCfmSteps != null) {
|
|
2432
|
-
parameters.batchCfmSteps = this._batchCfmSteps | 0;
|
|
2433
|
-
}
|
|
2434
|
-
if (this._cfgRate != null) {
|
|
2435
|
-
parameters.cfgRate = Number(this._cfgRate);
|
|
2436
|
-
}
|
|
2437
|
-
if (this._nPredict != null)
|
|
2438
|
-
parameters.nPredict = this._nPredict | 0;
|
|
2439
|
-
if (this._maxSentenceChars != null) {
|
|
2440
|
-
parameters.maxSentenceChars = this._maxSentenceChars | 0;
|
|
2441
|
-
}
|
|
2442
|
-
if (this._crossfadeMs != null) {
|
|
2443
|
-
parameters.crossfadeMs = this._crossfadeMs | 0;
|
|
2444
|
-
}
|
|
2445
|
-
this._assignSamplingParams(parameters);
|
|
2446
|
-
if (this._repeatPenalty != null) {
|
|
2447
|
-
parameters.repeatPenalty = Number(this._repeatPenalty);
|
|
2448
|
-
}
|
|
2449
|
-
if (this._exaggeration != null) {
|
|
2450
|
-
parameters.exaggeration = Number(this._exaggeration);
|
|
2451
|
-
}
|
|
2452
|
-
if (this._cfgWeight != null) {
|
|
2453
|
-
parameters.cfgWeight = Number(this._cfgWeight);
|
|
2454
|
-
}
|
|
2455
|
-
if (this._minP != null)
|
|
2456
|
-
parameters.minP = Number(this._minP);
|
|
2457
|
-
if (this._speed != null)
|
|
2458
|
-
parameters.speed = Number(this._speed);
|
|
2459
|
-
if (this._mecabDictPath) {
|
|
2460
|
-
parameters.mecabDictPath = this._mecabDictPath;
|
|
2461
|
-
}
|
|
2462
|
-
if (this._cangjieTsvPath) {
|
|
2463
|
-
parameters.cangjieTsvPath = this._cangjieTsvPath;
|
|
2464
|
-
}
|
|
2465
|
-
return parameters;
|
|
2466
|
-
}
|
|
2467
|
-
_buildSupertonicParams() {
|
|
2468
|
-
const parameters = {
|
|
2469
|
-
engineType: ENGINE_SUPERTONIC,
|
|
2470
|
-
supertonicModelPath: this._supertonicModelPath || "",
|
|
2471
|
-
language: this._config.language || "en",
|
|
2472
|
-
};
|
|
2473
|
-
this._assignCommonNativeParams(parameters);
|
|
2474
|
-
if (this._voice)
|
|
2475
|
-
parameters.voice = this._voice;
|
|
2476
|
-
if (this._steps != null)
|
|
2477
|
-
parameters.steps = this._steps | 0;
|
|
2478
|
-
if (this._speed != null)
|
|
2479
|
-
parameters.speed = Number(this._speed);
|
|
2480
|
-
// The native layer maps the step onto a multiplier relative to the GGUF's
|
|
2481
|
-
// own default_speed, and rejects pace + speed together.
|
|
2482
|
-
if (this._pace != null)
|
|
2483
|
-
parameters.pace = String(this._pace);
|
|
2484
|
-
if (this._noiseNpyPath) {
|
|
2485
|
-
parameters.noiseNpyPath = this._noiseNpyPath;
|
|
2486
|
-
}
|
|
2487
|
-
if (this._vulkanCacheDir) {
|
|
2488
|
-
parameters.vulkanCacheDir = this._vulkanCacheDir;
|
|
2489
|
-
}
|
|
2490
|
-
if (this._voiceJsonPath)
|
|
2491
|
-
parameters.voiceJsonPath = this._voiceJsonPath;
|
|
2492
|
-
if (this._prewarmText)
|
|
2493
|
-
parameters.prewarmText = this._prewarmText;
|
|
2494
|
-
if (this._vulkanDevice != null) {
|
|
2495
|
-
parameters.vulkanDevice = this._vulkanDevice | 0;
|
|
2496
|
-
}
|
|
2497
|
-
if (this._streamChunkTokens != null) {
|
|
2498
|
-
parameters.streamChunkTokens = this._streamChunkTokens | 0;
|
|
2499
|
-
}
|
|
2500
|
-
if (this._streamFirstChunkTokens != null) {
|
|
2501
|
-
parameters.streamFirstChunkTokens = this._streamFirstChunkTokens | 0;
|
|
2502
|
-
}
|
|
2503
|
-
if (this._streamChunkTolerancePct != null) {
|
|
2504
|
-
parameters.streamChunkTolerancePct = this._streamChunkTolerancePct | 0;
|
|
2505
|
-
}
|
|
2506
|
-
if (this._streamMinChunkTokens != null) {
|
|
2507
|
-
parameters.streamMinChunkTokens = this._streamMinChunkTokens | 0;
|
|
2508
|
-
}
|
|
2509
|
-
return parameters;
|
|
2510
|
-
}
|
|
2511
|
-
_buildParlerParams() {
|
|
2512
|
-
// Re-checked here (not only in the constructor) so reload({...}) cannot
|
|
2513
|
-
// smuggle in a description/template conflict.
|
|
2514
|
-
if (this._description != null) {
|
|
2515
|
-
const templateSet = [
|
|
2516
|
-
this._voice,
|
|
2517
|
-
this._emotion,
|
|
2518
|
-
this._pitch,
|
|
2519
|
-
this._pace,
|
|
2520
|
-
this._expressivity,
|
|
2521
|
-
this._noise,
|
|
2522
|
-
this._reverb,
|
|
2523
|
-
this._quality,
|
|
2524
|
-
].some((value) => value != null && value !== "");
|
|
2525
|
-
if (templateSet) {
|
|
2526
|
-
throw new Error("tts-ggml: 'description' is mutually exclusive with the " +
|
|
2527
|
-
"voice-template options");
|
|
2528
|
-
}
|
|
2529
|
-
}
|
|
2530
|
-
const parameters = {
|
|
2531
|
-
engineType: ENGINE_PARLER,
|
|
2532
|
-
parlerModelPath: this._parlerModelPath || "",
|
|
2533
|
-
};
|
|
2534
|
-
if (this._description != null) {
|
|
2535
|
-
parameters.description = String(this._description);
|
|
2536
|
-
}
|
|
2537
|
-
if (this._voice)
|
|
2538
|
-
parameters.voice = String(this._voice);
|
|
2539
|
-
if (this._emotion != null) {
|
|
2540
|
-
parameters.emotion = String(this._emotion);
|
|
2541
|
-
}
|
|
2542
|
-
if (this._pitch != null)
|
|
2543
|
-
parameters.pitch = String(this._pitch);
|
|
2544
|
-
if (this._pace != null)
|
|
2545
|
-
parameters.pace = String(this._pace);
|
|
2546
|
-
if (this._expressivity != null) {
|
|
2547
|
-
parameters.expressivity = String(this._expressivity);
|
|
2548
|
-
}
|
|
2549
|
-
if (this._noise != null)
|
|
2550
|
-
parameters.noise = String(this._noise);
|
|
2551
|
-
if (this._reverb != null)
|
|
2552
|
-
parameters.reverb = String(this._reverb);
|
|
2553
|
-
if (this._quality != null) {
|
|
2554
|
-
parameters.quality = String(this._quality);
|
|
2555
|
-
}
|
|
2556
|
-
this._assignSamplingParams(parameters);
|
|
2557
|
-
if (this._minNewTokens != null) {
|
|
2558
|
-
parameters.minNewTokens = this._minNewTokens | 0;
|
|
2559
|
-
}
|
|
2560
|
-
if (this._streamChunkTokens != null) {
|
|
2561
|
-
parameters.streamChunkTokens = this._streamChunkTokens | 0;
|
|
2562
|
-
}
|
|
2563
|
-
if (this._streamFirstChunkTokens != null) {
|
|
2564
|
-
parameters.streamFirstChunkTokens =
|
|
2565
|
-
this._streamFirstChunkTokens | 0;
|
|
2566
|
-
}
|
|
2567
|
-
if (this._normalizeNumbers != null) {
|
|
2568
|
-
parameters.normalizeNumbers = !!this._normalizeNumbers;
|
|
2569
|
-
}
|
|
2570
|
-
this._assignBackendParams(parameters);
|
|
2571
|
-
return parameters;
|
|
2572
|
-
}
|
|
2573
|
-
_buildAudio8Params() {
|
|
2574
|
-
// Backstop for callers that reach the params build by another route than
|
|
2575
|
-
// the constructor or _applyAudio8Reload, both of which check first.
|
|
2576
|
-
this._assertAudio8VoiceConsistent({
|
|
2577
|
-
referenceAudio: this._referenceAudio,
|
|
2578
|
-
referenceText: this._referenceText,
|
|
2579
|
-
}, "reload");
|
|
2580
|
-
const parameters = {
|
|
2581
|
-
engineType: ENGINE_AUDIO8,
|
|
2582
|
-
audio8LmPath: this._audio8LmPath || "",
|
|
2583
|
-
audio8CodecDecoderPath: this._audio8CodecDecoderPath || "",
|
|
2584
|
-
};
|
|
2585
|
-
if (this._audio8CodecEncoderPath) {
|
|
2586
|
-
parameters.audio8CodecEncoderPath = this._audio8CodecEncoderPath;
|
|
2587
|
-
}
|
|
2588
|
-
if (this._referenceAudio != null) {
|
|
2589
|
-
parameters.referenceAudio = this._referenceAudio;
|
|
2590
|
-
}
|
|
2591
|
-
if (this._referenceText != null) {
|
|
2592
|
-
parameters.referenceText = this._referenceText;
|
|
2593
|
-
}
|
|
2594
|
-
if (this._greedy != null)
|
|
2595
|
-
parameters.greedy = !!this._greedy;
|
|
2596
|
-
this._assignSamplingParams(parameters);
|
|
2597
|
-
this._assignBackendParams(parameters);
|
|
2598
|
-
return parameters;
|
|
2599
|
-
}
|
|
2600
|
-
_buildMossParams() {
|
|
2601
|
-
this._assertMossOutputRate();
|
|
2602
|
-
const parameters = {
|
|
2603
|
-
engineType: ENGINE_MOSS,
|
|
2604
|
-
mossBackbonePath: this._mossBackbonePath || "",
|
|
2605
|
-
mossCodecDecoderPath: this._mossCodecDecoderPath || "",
|
|
2606
|
-
language: this._config.language || "en",
|
|
2607
|
-
};
|
|
2608
|
-
if (this._mossCodecEncoderPath) {
|
|
2609
|
-
parameters.mossCodecEncoderPath = this._mossCodecEncoderPath;
|
|
2610
|
-
}
|
|
2611
|
-
if (this._referenceAudio != null) {
|
|
2612
|
-
parameters.referenceAudio = this._referenceAudio;
|
|
2613
|
-
}
|
|
2614
|
-
if (this._dialogueReferences) {
|
|
2615
|
-
parameters.dialogueReferences = [...this._dialogueReferences];
|
|
2616
|
-
}
|
|
2617
|
-
if (this._durationTokens != null) {
|
|
2618
|
-
parameters.durationTokens = this._durationTokens | 0;
|
|
2619
|
-
}
|
|
2620
|
-
if (this._streamChunkTokens != null) {
|
|
2621
|
-
parameters.streamChunkTokens = this._streamChunkTokens | 0;
|
|
2622
|
-
}
|
|
2623
|
-
this._assignBackendParams(parameters);
|
|
2624
|
-
return parameters;
|
|
2625
|
-
}
|
|
2626
|
-
_buildMossSoundEffectParams() {
|
|
2627
|
-
this._assertMossSoundEffectOutputRate();
|
|
2628
|
-
const parameters = {
|
|
2629
|
-
engineType: ENGINE_MOSS_SFX,
|
|
2630
|
-
mossSoundEffectPath: this._mossSoundEffectPath || "",
|
|
2631
|
-
};
|
|
2632
|
-
this._assignBackendParams(parameters);
|
|
2633
|
-
return parameters;
|
|
2634
|
-
}
|
|
2635
|
-
_buildMossSpeechParams() {
|
|
2636
|
-
this._assertMossSpeechOutputRate();
|
|
2637
|
-
const parameters = {
|
|
2638
|
-
engineType: ENGINE_MOSS_SPEECH,
|
|
2639
|
-
mossSpeechModelPath: this._mossSpeechModelPath || "",
|
|
2640
|
-
mossSpeechCodecPath: this._mossSpeechCodecPath || "",
|
|
2641
|
-
};
|
|
2642
|
-
this._assignBackendParams(parameters);
|
|
2643
|
-
return parameters;
|
|
2644
|
-
}
|
|
2645
|
-
/** The knobs every engine in SAMPLING_ENGINES reads. */
|
|
2646
|
-
_assignSamplingParams(parameters) {
|
|
2647
|
-
if (this._temperature != null) {
|
|
2648
|
-
parameters.temperature = Number(this._temperature);
|
|
2649
|
-
}
|
|
2650
|
-
if (this._topK != null)
|
|
2651
|
-
parameters.topK = this._topK | 0;
|
|
2652
|
-
if (this._topP != null)
|
|
2653
|
-
parameters.topP = Number(this._topP);
|
|
2654
|
-
if (this._maxFrames != null) {
|
|
2655
|
-
parameters.maxFrames = this._maxFrames | 0;
|
|
2656
|
-
}
|
|
2657
|
-
}
|
|
2658
|
-
/**
|
|
2659
|
-
* Backend and output plumbing for the engines that take no `language`,
|
|
2660
|
-
* where `_assignCommonNativeParams` would be the wrong shape.
|
|
2661
|
-
*/
|
|
2662
|
-
_assignBackendParams(parameters) {
|
|
2663
|
-
if (this._seed != null)
|
|
2664
|
-
parameters.seed = this._seed | 0;
|
|
2665
|
-
if (this._threads != null)
|
|
2666
|
-
parameters.threads = this._threads | 0;
|
|
2667
|
-
if (this._outputSampleRate != null) {
|
|
2668
|
-
parameters.outputSampleRate = this._outputSampleRate | 0;
|
|
2669
|
-
}
|
|
2670
|
-
if (this._nGpuLayers != null) {
|
|
2671
|
-
parameters.nGpuLayers = this._nGpuLayers | 0;
|
|
2672
|
-
}
|
|
2673
|
-
if (this._config.useGPU != null) {
|
|
2674
|
-
parameters.useGPU = !!this._config.useGPU;
|
|
2675
|
-
}
|
|
2676
|
-
this._assignLavasrParams(parameters);
|
|
2677
|
-
if (this._backendsDir) {
|
|
2678
|
-
parameters.backendsDir = this._backendsDir;
|
|
2679
|
-
}
|
|
2680
|
-
}
|
|
2681
|
-
_assignCommonNativeParams(parameters) {
|
|
2682
|
-
if (this._seed != null)
|
|
2683
|
-
parameters.seed = this._seed | 0;
|
|
2684
|
-
if (this._threads != null)
|
|
2685
|
-
parameters.threads = this._threads | 0;
|
|
2686
|
-
if (this._nGpuLayers != null) {
|
|
2687
|
-
parameters.nGpuLayers = this._nGpuLayers | 0;
|
|
2688
|
-
}
|
|
2689
|
-
if (this._outputSampleRate != null) {
|
|
2690
|
-
parameters.outputSampleRate = this._outputSampleRate | 0;
|
|
2691
|
-
}
|
|
2692
|
-
if (this._config.useGPU != null) {
|
|
2693
|
-
parameters.useGPU = !!this._config.useGPU;
|
|
2694
|
-
}
|
|
2695
|
-
this._assignLavasrParams(parameters);
|
|
2696
|
-
if (this._backendsDir) {
|
|
2697
|
-
parameters.backendsDir = this._backendsDir;
|
|
2698
|
-
}
|
|
2699
|
-
if (this._openclCacheDir) {
|
|
2700
|
-
parameters.openclCacheDir = this._openclCacheDir;
|
|
2701
|
-
}
|
|
271
|
+
return (0, engineParams_1.buildNativeParams)(this._setup);
|
|
2702
272
|
}
|
|
2703
|
-
/**
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
parameters.lavasrEnhancerPath = this._enhancerGgufPath;
|
|
2707
|
-
}
|
|
2708
|
-
if (this._denoiserGgufPath) {
|
|
2709
|
-
parameters.lavasrDenoiserPath = this._denoiserGgufPath;
|
|
2710
|
-
}
|
|
273
|
+
/** A native instance that reports through this wrapper's output callback. */
|
|
274
|
+
_spawnAddon(configuration) {
|
|
275
|
+
return this._createAddon(configuration, this._addonOutputCallback.bind(this));
|
|
2711
276
|
}
|
|
2712
277
|
_createAddon(configuration, outputCallback) {
|
|
2713
278
|
// eslint-disable-next-line @typescript-eslint/no-require-imports -- native binding is resolved lazily from package prebuilds.
|
|
@@ -2719,7 +284,7 @@ class TTSGgml {
|
|
|
2719
284
|
}
|
|
2720
285
|
async _unloadModel() {
|
|
2721
286
|
await this.cancel();
|
|
2722
|
-
this.
|
|
287
|
+
this._sentenceStream.fail("Model was unloaded");
|
|
2723
288
|
const addon = this._optionalAddon();
|
|
2724
289
|
try {
|
|
2725
290
|
if (addon)
|
|
@@ -2739,36 +304,24 @@ class TTSGgml {
|
|
|
2739
304
|
this.state.destroyed = true;
|
|
2740
305
|
});
|
|
2741
306
|
}
|
|
307
|
+
async _withPocketLifecycle(action) {
|
|
308
|
+
return this._pocket ? this._pocket.withLifecycle(action) : action();
|
|
309
|
+
}
|
|
2742
310
|
async _runInternal(input) {
|
|
2743
|
-
await this.
|
|
2744
|
-
this.
|
|
2745
|
-
const jobFields = this.
|
|
311
|
+
await this._pocket?.waitCancel();
|
|
312
|
+
this._pocket?.checkRequest();
|
|
313
|
+
const jobFields = (0, jobFields_1.resolveJobFields)(this._setup, input, "run");
|
|
2746
314
|
const signal = input?.signal;
|
|
2747
|
-
|
|
315
|
+
// Pocket keeps the signal to itself: the abort fails the response with
|
|
316
|
+
// the caller's reason and stops native work behind the completion barrier.
|
|
317
|
+
const pocketSignal = this._pocket && signal && !signal.aborted ? signal : undefined;
|
|
2748
318
|
const response = this._job.start({
|
|
2749
319
|
signal: pocketSignal ? undefined : signal,
|
|
2750
320
|
});
|
|
2751
321
|
if (signal?.aborted)
|
|
2752
322
|
return response;
|
|
2753
|
-
if (pocketSignal)
|
|
2754
|
-
|
|
2755
|
-
if (this._job.active !== response)
|
|
2756
|
-
return;
|
|
2757
|
-
const reason = pocketSignal.reason;
|
|
2758
|
-
const error = reason instanceof Error ? reason : new Error(typeof reason === "string" ? `Aborted: ${reason}` : "Aborted");
|
|
2759
|
-
// Preserve the caller's reason while stopping native work. Cancellation
|
|
2760
|
-
// retains the completion barrier until its terminal callback arrives.
|
|
2761
|
-
try {
|
|
2762
|
-
response.failed(error);
|
|
2763
|
-
}
|
|
2764
|
-
finally {
|
|
2765
|
-
void this.cancel().catch((error) => this._getLogger().error("Pocket abort cancellation failed", error));
|
|
2766
|
-
}
|
|
2767
|
-
};
|
|
2768
|
-
pocketSignal.addEventListener("abort", onAbort, { once: true });
|
|
2769
|
-
const detach = () => pocketSignal.removeEventListener("abort", onAbort);
|
|
2770
|
-
void response.await().then(detach, detach);
|
|
2771
|
-
}
|
|
323
|
+
if (pocketSignal)
|
|
324
|
+
this._pocket?.attachAbort(response, pocketSignal);
|
|
2772
325
|
try {
|
|
2773
326
|
await this._dispatchJob({
|
|
2774
327
|
type: input.type || "text",
|
|
@@ -2777,495 +330,42 @@ class TTSGgml {
|
|
|
2777
330
|
});
|
|
2778
331
|
}
|
|
2779
332
|
catch (error) {
|
|
2780
|
-
this._job.fail(normalizeError(error));
|
|
333
|
+
this._job.fail((0, sentenceStream_1.normalizeError)(error));
|
|
2781
334
|
throw error;
|
|
2782
335
|
}
|
|
2783
336
|
return response;
|
|
2784
337
|
}
|
|
2785
|
-
_mergeSentenceStreamStats(accumulator, data) {
|
|
2786
|
-
if (accumulator.firstAudioMs === undefined &&
|
|
2787
|
-
typeof data.firstAudioMs === "number" &&
|
|
2788
|
-
Number.isFinite(data.firstAudioMs) &&
|
|
2789
|
-
data.firstAudioMs >= 0) {
|
|
2790
|
-
accumulator.firstAudioMs = data.firstAudioMs;
|
|
2791
|
-
}
|
|
2792
|
-
accumulator.totalTime +=
|
|
2793
|
-
typeof data.totalTime === "number" ? data.totalTime : 0;
|
|
2794
|
-
accumulator.audioDurationMs +=
|
|
2795
|
-
typeof data.audioDurationMs === "number"
|
|
2796
|
-
? data.audioDurationMs
|
|
2797
|
-
: 0;
|
|
2798
|
-
accumulator.totalSamples +=
|
|
2799
|
-
typeof data.totalSamples === "number" ? data.totalSamples : 0;
|
|
2800
|
-
accumulator.generatedFrames +=
|
|
2801
|
-
typeof data.generatedFrames === "number" ? data.generatedFrames : 0;
|
|
2802
|
-
if (typeof data.codecSidecarLoaded === "number") {
|
|
2803
|
-
accumulator.codecSidecarLoaded = data.codecSidecarLoaded;
|
|
2804
|
-
}
|
|
2805
|
-
if (typeof data.codecOnCoreml === "number") {
|
|
2806
|
-
accumulator.codecOnCoreml = data.codecOnCoreml;
|
|
2807
|
-
}
|
|
2808
|
-
}
|
|
2809
|
-
_rejectActiveChunk(error) {
|
|
2810
|
-
const resolver = this._sentenceStreamCtx?.chunkResolver;
|
|
2811
|
-
if (!resolver)
|
|
2812
|
-
return;
|
|
2813
|
-
this._sentenceStreamCtx.chunkResolver = null;
|
|
2814
|
-
resolver.reject(error);
|
|
2815
|
-
}
|
|
2816
|
-
_endJobWithStats(stats) {
|
|
2817
|
-
if (this.opts.stats)
|
|
2818
|
-
this._job.end(stats);
|
|
2819
|
-
else
|
|
2820
|
-
this._job.end();
|
|
2821
|
-
}
|
|
2822
338
|
_addonOutputCallback(_addon, event, data, error) {
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
this._pocketJobPending = null;
|
|
2826
|
-
pending.resolve();
|
|
2827
|
-
}
|
|
2828
|
-
if (typeof error === "string" && error.length > 0) {
|
|
2829
|
-
this._handleAddonError(error);
|
|
2830
|
-
}
|
|
2831
|
-
else if (isAudioOutputEvent(data)) {
|
|
2832
|
-
this._handleAddonOutput(data);
|
|
2833
|
-
}
|
|
2834
|
-
else if (isStatsEvent(data)) {
|
|
2835
|
-
this._handleAddonStats(data);
|
|
2836
|
-
}
|
|
2837
|
-
else {
|
|
2838
|
-
this._getLogger().debug(`Received TTS event: ${String(event)}`);
|
|
2839
|
-
}
|
|
2840
|
-
}
|
|
2841
|
-
_handleAddonError(error) {
|
|
2842
|
-
this._getLogger().error(`TTS job failed with error: ${error}`);
|
|
2843
|
-
this._rejectActiveChunk(new Error(error));
|
|
2844
|
-
this._job.fail(error);
|
|
2845
|
-
}
|
|
2846
|
-
_handleAddonOutput(data) {
|
|
2847
|
-
try {
|
|
2848
|
-
this._getLogger().debug(`TTS job produced output: ${ttsOutputDebugString(data)}`);
|
|
2849
|
-
}
|
|
2850
|
-
catch (error) {
|
|
2851
|
-
if (error instanceof RangeError) {
|
|
2852
|
-
this._getLogger().debug("TTS job produced output: [data too large]");
|
|
2853
|
-
}
|
|
2854
|
-
else {
|
|
2855
|
-
throw error;
|
|
2856
|
-
}
|
|
2857
|
-
}
|
|
2858
|
-
this._job.output(this._sentenceStreamCtx
|
|
2859
|
-
? this._enrichStreamChunk(data)
|
|
2860
|
-
: data);
|
|
2861
|
-
}
|
|
2862
|
-
_enrichStreamChunk(data) {
|
|
2863
|
-
const context = this._sentenceStreamCtx;
|
|
2864
|
-
if (!context) {
|
|
2865
|
-
return data;
|
|
2866
|
-
}
|
|
2867
|
-
const index = context.chunkIdx;
|
|
2868
|
-
const enriched = {
|
|
2869
|
-
outputArray: data.outputArray,
|
|
2870
|
-
chunkIndex: index,
|
|
2871
|
-
sentenceChunk: context.chunks[index] || "",
|
|
2872
|
-
};
|
|
2873
|
-
if (data.sampleRate != null) {
|
|
2874
|
-
enriched.sampleRate = data.sampleRate;
|
|
2875
|
-
}
|
|
2876
|
-
if (!context.textStreamMode) {
|
|
2877
|
-
enriched.isLast = index >= context.chunks.length - 1 && data.isLast !== false;
|
|
2878
|
-
}
|
|
2879
|
-
return enriched;
|
|
2880
|
-
}
|
|
2881
|
-
_handleAddonStats(data) {
|
|
2882
|
-
this._getLogger().info(`TTS job completed. Stats: ${JSON.stringify(data)}`);
|
|
2883
|
-
const context = this._sentenceStreamCtx;
|
|
2884
|
-
if (!context) {
|
|
2885
|
-
this._endJobWithStats(data);
|
|
2886
|
-
return;
|
|
2887
|
-
}
|
|
2888
|
-
this._mergeSentenceStreamStats(context.acc, data);
|
|
2889
|
-
if (context.chunkResolver) {
|
|
2890
|
-
context.chunkResolver.resolve();
|
|
2891
|
-
context.chunkResolver = null;
|
|
2892
|
-
}
|
|
2893
|
-
if (!context.textStreamMode &&
|
|
2894
|
-
context.chunkIdx >= context.chunks.length - 1) {
|
|
2895
|
-
this._endJobWithStats(computeSentenceStreamStats(context.chunks, context.acc, this._pacesOnFrames()));
|
|
2896
|
-
}
|
|
2897
|
-
}
|
|
2898
|
-
async _waitPocketCancel() {
|
|
2899
|
-
if (this._engineType === ENGINE_POCKET && this._pocketCancelPromise)
|
|
2900
|
-
await this._pocketCancelPromise;
|
|
2901
|
-
}
|
|
2902
|
-
async _withPocketLifecycle(action) {
|
|
2903
|
-
if (this._engineType !== ENGINE_POCKET)
|
|
2904
|
-
return action();
|
|
2905
|
-
await this._waitPocketCancel();
|
|
2906
|
-
this._checkPocketReload();
|
|
2907
|
-
this._pocketLifecycleInProgress = true;
|
|
2908
|
-
try {
|
|
2909
|
-
await action();
|
|
2910
|
-
}
|
|
2911
|
-
finally {
|
|
2912
|
-
this._pocketLifecycleInProgress = false;
|
|
2913
|
-
}
|
|
2914
|
-
}
|
|
2915
|
-
_checkPocketReload() {
|
|
2916
|
-
if (this._pocketLifecycleInProgress)
|
|
2917
|
-
throw new Error("Pocket lifecycle operation is already in progress");
|
|
2918
|
-
}
|
|
2919
|
-
_checkPocketRequest() {
|
|
2920
|
-
if (this._engineType === ENGINE_POCKET) {
|
|
2921
|
-
if (this.state.destroyed)
|
|
2922
|
-
throw new Error("Pocket instance was destroyed");
|
|
2923
|
-
if (!this.state.weightsLoaded || !this._optionalAddon())
|
|
2924
|
-
throw new Error("Pocket model is not loaded");
|
|
2925
|
-
}
|
|
2926
|
-
if (this._engineType === ENGINE_POCKET && (this._job.active || this._pocketJobPending)) {
|
|
2927
|
-
throw new Error("Pocket synthesis is already in progress");
|
|
2928
|
-
}
|
|
2929
|
-
this._checkPocketReload();
|
|
339
|
+
this._pocket?.onNativeEvent(data, error);
|
|
340
|
+
this._events.route(event, data, error);
|
|
2930
341
|
}
|
|
2931
342
|
async _dispatchJob(input) {
|
|
2932
343
|
const addon = this._requireAddon();
|
|
2933
|
-
if (this.
|
|
2934
|
-
return addon.runJob(input);
|
|
2935
|
-
if (this._pocketJobPending)
|
|
2936
|
-
throw new Error("Pocket native job is already in progress");
|
|
2937
|
-
let resolve = () => { };
|
|
2938
|
-
const promise = new Promise((done) => { resolve = done; });
|
|
2939
|
-
const pending = { promise, resolve };
|
|
2940
|
-
this._pocketJobPending = pending;
|
|
2941
|
-
try {
|
|
2942
|
-
await addon.runJob(input);
|
|
2943
|
-
}
|
|
2944
|
-
catch (error) {
|
|
2945
|
-
if (this._pocketJobPending === pending)
|
|
2946
|
-
this._pocketJobPending = null;
|
|
2947
|
-
resolve();
|
|
2948
|
-
throw error;
|
|
344
|
+
if (this._setup.engineType === engineIds_1.ENGINE_AUDIO8 && this._sentenceStream.ctx) {
|
|
345
|
+
return addon.runJob({ ...input, streamOutput: true });
|
|
2949
346
|
}
|
|
347
|
+
return this._pocket ? this._pocket.runJob(addon, input) : addon.runJob(input);
|
|
2950
348
|
}
|
|
2951
349
|
async cancel() {
|
|
350
|
+
if (this._pocket)
|
|
351
|
+
return this._pocket.cancel();
|
|
2952
352
|
const addon = this._optionalAddon();
|
|
2953
|
-
if (
|
|
2954
|
-
|
|
2955
|
-
await addon.cancel();
|
|
2956
|
-
return;
|
|
2957
|
-
}
|
|
2958
|
-
if (this._pocketCancelPromise)
|
|
2959
|
-
return this._pocketCancelPromise;
|
|
2960
|
-
const pending = this._pocketJobPending;
|
|
2961
|
-
this._pocketCancelPromise = (async () => {
|
|
2962
|
-
this._failAndClearActiveResponse("Synthesis cancelled");
|
|
2963
|
-
if (addon?.cancel)
|
|
2964
|
-
await addon.cancel();
|
|
2965
|
-
// Joining the worker can precede delivery of its terminal callback.
|
|
2966
|
-
if (pending)
|
|
2967
|
-
await pending.promise;
|
|
2968
|
-
})();
|
|
2969
|
-
try {
|
|
2970
|
-
await this._pocketCancelPromise;
|
|
2971
|
-
}
|
|
2972
|
-
finally {
|
|
2973
|
-
this._pocketCancelPromise = null;
|
|
2974
|
-
}
|
|
2975
|
-
}
|
|
2976
|
-
_failAndClearActiveResponse(reason) {
|
|
2977
|
-
this._rejectActiveChunk(reason instanceof Error ? reason : new Error(reason));
|
|
2978
|
-
this._sentenceStreamCtx = null;
|
|
2979
|
-
this._job.fail(reason);
|
|
2980
|
-
}
|
|
2981
|
-
/** Everything reload() may overwrite, so a rejected reload can undo itself. */
|
|
2982
|
-
_captureReloadableState() {
|
|
2983
|
-
return {
|
|
2984
|
-
language: this._config.language,
|
|
2985
|
-
useGPU: this._config.useGPU,
|
|
2986
|
-
outputSampleRate: this._outputSampleRate,
|
|
2987
|
-
emotion: this._emotion,
|
|
2988
|
-
pace: this._pace,
|
|
2989
|
-
description: this._description,
|
|
2990
|
-
voice: this._voice,
|
|
2991
|
-
pitch: this._pitch,
|
|
2992
|
-
expressivity: this._expressivity,
|
|
2993
|
-
noise: this._noise,
|
|
2994
|
-
reverb: this._reverb,
|
|
2995
|
-
quality: this._quality,
|
|
2996
|
-
temperature: this._temperature,
|
|
2997
|
-
topK: this._topK,
|
|
2998
|
-
topP: this._topP,
|
|
2999
|
-
maxFrames: this._maxFrames,
|
|
3000
|
-
minNewTokens: this._minNewTokens,
|
|
3001
|
-
normalizeNumbers: this._normalizeNumbers,
|
|
3002
|
-
seed: this._seed,
|
|
3003
|
-
durationTokens: this._durationTokens,
|
|
3004
|
-
};
|
|
3005
|
-
}
|
|
3006
|
-
_restoreReloadableState(state) {
|
|
3007
|
-
this._config.language = state.language;
|
|
3008
|
-
this._config.useGPU = state.useGPU;
|
|
3009
|
-
this._outputSampleRate = state.outputSampleRate;
|
|
3010
|
-
this._emotion = state.emotion;
|
|
3011
|
-
this._pace = state.pace;
|
|
3012
|
-
this._description = state.description;
|
|
3013
|
-
this._voice = state.voice;
|
|
3014
|
-
this._pitch = state.pitch;
|
|
3015
|
-
this._expressivity = state.expressivity;
|
|
3016
|
-
this._noise = state.noise;
|
|
3017
|
-
this._reverb = state.reverb;
|
|
3018
|
-
this._quality = state.quality;
|
|
3019
|
-
this._temperature = state.temperature;
|
|
3020
|
-
this._topK = state.topK;
|
|
3021
|
-
this._topP = state.topP;
|
|
3022
|
-
this._maxFrames = state.maxFrames;
|
|
3023
|
-
this._minNewTokens = state.minNewTokens;
|
|
3024
|
-
this._normalizeNumbers = state.normalizeNumbers;
|
|
3025
|
-
this._seed = state.seed;
|
|
3026
|
-
this._durationTokens = state.durationTokens;
|
|
3027
|
-
}
|
|
3028
|
-
_applyReloadableRuntimeConfig(newConfig) {
|
|
3029
|
-
this._assertMossReloadKeepsVoice(newConfig);
|
|
3030
|
-
const runtimeConfig = newConfig;
|
|
3031
|
-
if (runtimeConfig.language !== undefined) {
|
|
3032
|
-
this._config.language = runtimeConfig.language;
|
|
3033
|
-
}
|
|
3034
|
-
if (runtimeConfig.useGPU !== undefined) {
|
|
3035
|
-
this._config.useGPU = runtimeConfig.useGPU;
|
|
3036
|
-
}
|
|
3037
|
-
if (runtimeConfig.outputSampleRate !== undefined) {
|
|
3038
|
-
this._outputSampleRate = validateOutputSampleRate(runtimeConfig.outputSampleRate);
|
|
3039
|
-
}
|
|
3040
|
-
this._applyAudio8Reload(newConfig);
|
|
3041
|
-
this._applyMossReload(newConfig);
|
|
3042
|
-
}
|
|
3043
|
-
_assertMossReloadKeepsVoice(newConfig) {
|
|
3044
|
-
if (this._engineType !== ENGINE_MOSS)
|
|
3045
|
-
return;
|
|
3046
|
-
const voiceKeys = MOSS_INSTANCE_VOICE_KEYS.filter((key) => newConfig[key] !== undefined);
|
|
3047
|
-
if (voiceKeys.length === 0)
|
|
3048
|
-
return;
|
|
3049
|
-
throw new Error(`tts-ggml: reload: the moss engine encodes ${voiceKeys.join(", ")} ` +
|
|
3050
|
-
"once per instance; create a new instance to clone different recordings");
|
|
3051
|
-
}
|
|
3052
|
-
_applyMossReload(newConfig) {
|
|
3053
|
-
if (this._engineType !== ENGINE_MOSS)
|
|
3054
|
-
return;
|
|
3055
|
-
const durationTokens = newConfig
|
|
3056
|
-
.durationTokens;
|
|
3057
|
-
if (durationTokens === undefined)
|
|
3058
|
-
return;
|
|
3059
|
-
assertMossDurationTokens(durationTokens, "reload");
|
|
3060
|
-
this._durationTokens = durationTokens ?? undefined;
|
|
3061
|
-
}
|
|
3062
|
-
// Cross-engine conditioning is reloadable on every engine that supports it,
|
|
3063
|
-
// so both families change emotion the same way.
|
|
3064
|
-
_applyReloadableConditioning(newConfig) {
|
|
3065
|
-
const conditioning = newConfig;
|
|
3066
|
-
if (conditioning.emotion === undefined &&
|
|
3067
|
-
conditioning.pace === undefined) {
|
|
3068
|
-
return;
|
|
3069
|
-
}
|
|
3070
|
-
if (conditioning.emotion !== undefined) {
|
|
3071
|
-
this._emotion = conditioning.emotion;
|
|
3072
|
-
}
|
|
3073
|
-
if (conditioning.pace !== undefined) {
|
|
3074
|
-
this._pace = conditioning.pace;
|
|
3075
|
-
}
|
|
3076
|
-
this._assertConditioningConsistency("reload");
|
|
3077
|
-
}
|
|
3078
|
-
// Parler description/template + sampling knobs are reloadable; they rebuild
|
|
3079
|
-
// the engine's default description / sampler. _buildParlerParams re-validates
|
|
3080
|
-
// so a wrong-engine reload still throws.
|
|
3081
|
-
_applyReloadableParlerConfig(newConfig) {
|
|
3082
|
-
if (this._engineType !== ENGINE_PARLER)
|
|
3083
|
-
return;
|
|
3084
|
-
const parlerConfig = newConfig;
|
|
3085
|
-
if (parlerConfig.description !== undefined ||
|
|
3086
|
-
parlerConfig.voiceDescription !== undefined) {
|
|
3087
|
-
this._description = firstNonEmpty(parlerConfig.description, parlerConfig.voiceDescription);
|
|
3088
|
-
}
|
|
3089
|
-
if (parlerConfig.voice !== undefined) {
|
|
3090
|
-
this._voice = parlerConfig.voice;
|
|
3091
|
-
}
|
|
3092
|
-
if (parlerConfig.pitch !== undefined) {
|
|
3093
|
-
this._pitch = parlerConfig.pitch;
|
|
3094
|
-
}
|
|
3095
|
-
if (parlerConfig.expressivity !== undefined) {
|
|
3096
|
-
this._expressivity = parlerConfig.expressivity;
|
|
3097
|
-
}
|
|
3098
|
-
if (parlerConfig.noise !== undefined) {
|
|
3099
|
-
this._noise = parlerConfig.noise;
|
|
3100
|
-
}
|
|
3101
|
-
if (parlerConfig.reverb !== undefined) {
|
|
3102
|
-
this._reverb = parlerConfig.reverb;
|
|
3103
|
-
}
|
|
3104
|
-
if (parlerConfig.quality !== undefined) {
|
|
3105
|
-
this._quality = parlerConfig.quality;
|
|
3106
|
-
}
|
|
3107
|
-
if (parlerConfig.temperature !== undefined) {
|
|
3108
|
-
this._temperature = parlerConfig.temperature;
|
|
3109
|
-
}
|
|
3110
|
-
if (parlerConfig.topK !== undefined) {
|
|
3111
|
-
this._topK = parlerConfig.topK;
|
|
3112
|
-
}
|
|
3113
|
-
if (parlerConfig.topP !== undefined) {
|
|
3114
|
-
this._topP = parlerConfig.topP;
|
|
3115
|
-
}
|
|
3116
|
-
if (parlerConfig.maxFrames !== undefined) {
|
|
3117
|
-
this._maxFrames = parlerConfig.maxFrames;
|
|
3118
|
-
}
|
|
3119
|
-
if (parlerConfig.minNewTokens !== undefined) {
|
|
3120
|
-
this._minNewTokens = parlerConfig.minNewTokens;
|
|
3121
|
-
}
|
|
3122
|
-
if (parlerConfig.normalizeNumbers !== undefined) {
|
|
3123
|
-
this._normalizeNumbers = parlerConfig.normalizeNumbers;
|
|
3124
|
-
}
|
|
3125
|
-
if (parlerConfig.seed !== undefined) {
|
|
3126
|
-
this._seed = parlerConfig.seed;
|
|
3127
|
-
}
|
|
3128
|
-
}
|
|
3129
|
-
/**
|
|
3130
|
-
* Apply the new configuration and build the native parameters from it. A
|
|
3131
|
-
* rejected value leaves the instance exactly as it was, so a later partial
|
|
3132
|
-
* reload is not validated against state the caller never accepted.
|
|
3133
|
-
*/
|
|
3134
|
-
_applyReloadableConfig(newConfig) {
|
|
3135
|
-
const snapshot = this._captureReloadableState();
|
|
3136
|
-
try {
|
|
3137
|
-
this._applyReloadableRuntimeConfig(newConfig);
|
|
3138
|
-
this._applyReloadableConditioning(newConfig);
|
|
3139
|
-
this._applyReloadableParlerConfig(newConfig);
|
|
3140
|
-
return this._buildTtsParams();
|
|
3141
|
-
}
|
|
3142
|
-
catch (error) {
|
|
3143
|
-
this._restoreReloadableState(snapshot);
|
|
3144
|
-
throw error;
|
|
3145
|
-
}
|
|
3146
|
-
}
|
|
3147
|
-
async _reloadPocket(newConfig) {
|
|
3148
|
-
await this._waitPocketCancel();
|
|
3149
|
-
this._checkPocketReload();
|
|
3150
|
-
if (this.state.destroyed)
|
|
3151
|
-
throw new Error("Pocket instance was destroyed");
|
|
3152
|
-
const options = { ...this._pocketOptions, ...newConfig };
|
|
3153
|
-
const config = { ...this._config, ...newConfig };
|
|
3154
|
-
const params = (0, pocketConfig_1.buildPocketParams)(this._pocketFiles, options, config);
|
|
3155
|
-
this._pocketLifecycleInProgress = true;
|
|
3156
|
-
let replacement = null;
|
|
3157
|
-
try {
|
|
3158
|
-
// Drain native work and release its model before allocating another one.
|
|
3159
|
-
// Failed activation leaves the instance unloaded with its last good config.
|
|
3160
|
-
await this._unloadModel();
|
|
3161
|
-
replacement = this._createAddon(params, this._addonOutputCallback.bind(this));
|
|
3162
|
-
await replacement.activate();
|
|
3163
|
-
this.addon = replacement;
|
|
3164
|
-
replacement = null;
|
|
3165
|
-
this._pocketParams = params;
|
|
3166
|
-
this._pocketOptions = options;
|
|
3167
|
-
this._config.language = "en";
|
|
3168
|
-
this._config.useGPU = false;
|
|
3169
|
-
this._outputSampleRate = typeof params.outputSampleRate === "number" ? params.outputSampleRate : null;
|
|
3170
|
-
this._config.outputSampleRate = this._outputSampleRate ?? undefined;
|
|
3171
|
-
this.state.configLoaded = true;
|
|
3172
|
-
this.state.weightsLoaded = true;
|
|
3173
|
-
}
|
|
3174
|
-
finally {
|
|
3175
|
-
try {
|
|
3176
|
-
if (replacement)
|
|
3177
|
-
await replacement.destroyInstance();
|
|
3178
|
-
}
|
|
3179
|
-
finally {
|
|
3180
|
-
this._pocketLifecycleInProgress = false;
|
|
3181
|
-
}
|
|
3182
|
-
}
|
|
353
|
+
if (addon?.cancel)
|
|
354
|
+
await addon.cancel();
|
|
3183
355
|
}
|
|
3184
356
|
async reload(newConfig = {}) {
|
|
3185
357
|
this._getLogger().debug("Reloading addon with new configuration", newConfig);
|
|
3186
|
-
if (this.
|
|
3187
|
-
return this.
|
|
3188
|
-
const parameters = this.
|
|
358
|
+
if (this._pocket)
|
|
359
|
+
return this._pocket.reload(newConfig);
|
|
360
|
+
const parameters = (0, reload_1.applyReloadableConfig)(this._setup, newConfig);
|
|
3189
361
|
await this.cancel();
|
|
3190
|
-
this.
|
|
362
|
+
this._sentenceStream.fail("Model was reloaded");
|
|
3191
363
|
const existingAddon = this._optionalAddon();
|
|
3192
364
|
if (existingAddon)
|
|
3193
365
|
await existingAddon.destroyInstance();
|
|
3194
|
-
this.addon = this.
|
|
366
|
+
this.addon = this._spawnAddon(parameters);
|
|
3195
367
|
await this._requireAddon().activate();
|
|
3196
368
|
}
|
|
3197
|
-
/**
|
|
3198
|
-
* The voice a reload lands on. Same rule as _mergeAudio8Voice and
|
|
3199
|
-
* Audio8Model::resolveVoice: a new recording replaces both halves, because
|
|
3200
|
-
* the configured transcript describes the recording being replaced. Reload
|
|
3201
|
-
* reads `undefined` as "not supplied", so an explicit empty string reaches
|
|
3202
|
-
* the guard instead of being ignored.
|
|
3203
|
-
*/
|
|
3204
|
-
_mergeAudio8ReloadVoice(config) {
|
|
3205
|
-
if (config.referenceAudio !== undefined) {
|
|
3206
|
-
return {
|
|
3207
|
-
referenceAudio: config.referenceAudio,
|
|
3208
|
-
referenceText: config.referenceText,
|
|
3209
|
-
};
|
|
3210
|
-
}
|
|
3211
|
-
return {
|
|
3212
|
-
referenceAudio: this._referenceAudio,
|
|
3213
|
-
referenceText: config.referenceText !== undefined
|
|
3214
|
-
? config.referenceText
|
|
3215
|
-
: this._referenceText,
|
|
3216
|
-
};
|
|
3217
|
-
}
|
|
3218
|
-
/**
|
|
3219
|
-
* The knobs a reload lands on, merged the same way as the voice so the
|
|
3220
|
-
* whole set can be checked before any of it is written.
|
|
3221
|
-
*/
|
|
3222
|
-
_mergeAudio8ReloadSampling(config) {
|
|
3223
|
-
return {
|
|
3224
|
-
greedy: config.greedy ?? this._greedy,
|
|
3225
|
-
temperature: config.temperature ?? this._temperature,
|
|
3226
|
-
topK: config.topK ?? this._topK,
|
|
3227
|
-
topP: config.topP ?? this._topP,
|
|
3228
|
-
maxFrames: config.maxFrames ?? this._maxFrames,
|
|
3229
|
-
seed: config.seed ?? this._seed,
|
|
3230
|
-
threads: config.threads ?? this._threads,
|
|
3231
|
-
};
|
|
3232
|
-
}
|
|
3233
|
-
_applyAudio8Sampling(config) {
|
|
3234
|
-
if (config.greedy !== undefined)
|
|
3235
|
-
this._greedy = config.greedy;
|
|
3236
|
-
if (config.temperature !== undefined) {
|
|
3237
|
-
this._temperature = config.temperature;
|
|
3238
|
-
}
|
|
3239
|
-
if (config.topK !== undefined)
|
|
3240
|
-
this._topK = config.topK;
|
|
3241
|
-
if (config.topP !== undefined)
|
|
3242
|
-
this._topP = config.topP;
|
|
3243
|
-
if (config.maxFrames !== undefined) {
|
|
3244
|
-
this._maxFrames = config.maxFrames;
|
|
3245
|
-
}
|
|
3246
|
-
if (config.seed !== undefined)
|
|
3247
|
-
this._seed = config.seed;
|
|
3248
|
-
if (config.threads !== undefined)
|
|
3249
|
-
this._threads = config.threads;
|
|
3250
|
-
}
|
|
3251
|
-
/**
|
|
3252
|
-
* Audio8 voice + sampling knobs are reloadable; they rebuild the engine's
|
|
3253
|
-
* sampler and speaker history. Both merges are checked before either is
|
|
3254
|
-
* written, so a rejected reload leaves the instance exactly as it was
|
|
3255
|
-
* rather than half-moved onto the configuration that was refused.
|
|
3256
|
-
*/
|
|
3257
|
-
_applyAudio8Reload(newConfig) {
|
|
3258
|
-
if (this._engineType !== ENGINE_AUDIO8)
|
|
3259
|
-
return;
|
|
3260
|
-
const config = newConfig;
|
|
3261
|
-
const voice = this._mergeAudio8ReloadVoice(config);
|
|
3262
|
-
const sampling = this._mergeAudio8ReloadSampling(config);
|
|
3263
|
-
this._assertAudio8VoiceConsistent(voice, "reload");
|
|
3264
|
-
assertAudio8SamplingValid(sampling, "reload");
|
|
3265
|
-
this._referenceAudio = voice.referenceAudio;
|
|
3266
|
-
this._referenceText = voice.referenceText;
|
|
3267
|
-
this._applyAudio8Sampling(sampling);
|
|
3268
|
-
}
|
|
3269
369
|
static getModelKey(_params) {
|
|
3270
370
|
void _params;
|
|
3271
371
|
return "tts-ggml";
|
|
@@ -3279,19 +379,7 @@ class TTSGgml {
|
|
|
3279
379
|
static getVoiceControls() {
|
|
3280
380
|
// eslint-disable-next-line @typescript-eslint/no-require-imports -- native binding is resolved lazily from package prebuilds.
|
|
3281
381
|
const binding = require("./binding");
|
|
3282
|
-
|
|
3283
|
-
const engines = {};
|
|
3284
|
-
for (const [name, controls] of Object.entries(native.engines)) {
|
|
3285
|
-
engines[FABRIC_ENGINE_TYPES[name] ?? name] = {
|
|
3286
|
-
emotions: [...controls.emotions],
|
|
3287
|
-
paces: [...controls.paces],
|
|
3288
|
-
};
|
|
3289
|
-
}
|
|
3290
|
-
return {
|
|
3291
|
-
emotions: [...native.emotions],
|
|
3292
|
-
paces: [...native.paces],
|
|
3293
|
-
engines,
|
|
3294
|
-
};
|
|
382
|
+
return (0, voiceControls_1.toVoiceControlsCatalog)(binding.getVoiceControls());
|
|
3295
383
|
}
|
|
3296
384
|
_requireAddon() {
|
|
3297
385
|
const addon = this._optionalAddon();
|