@chatpanel/gateway 0.6.51 → 0.6.53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/server.js +91 -8
- package/src/tts-engine.js +86 -15
- package/src/tts-models.js +64 -1
- package/src/tts-voices.js +99 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chatpanel/gateway",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.53",
|
|
4
4
|
"description": "Local privacy gateway — redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/src/server.js
CHANGED
|
@@ -42,6 +42,7 @@ import { STT_MODEL_CATALOG, isKnownSttModel, isValidCustomSttId, DEFAULT_STT_MOD
|
|
|
42
42
|
import * as ttsEngine from './tts-engine.js';
|
|
43
43
|
import { TTS_MODEL_CATALOG, TTS_VOICES, isKnownTtsModel, isValidCustomTtsId, isKnownVoice, isValidVoiceId, DEFAULT_TTS_MODEL, DEFAULT_TTS_VOICE, TTS_DTYPES, isValidTtsDtype, MAX_TTS_CHARS } from './tts-models.js';
|
|
44
44
|
import { ttsDestination, synthesizeRemote, isValidRemoteVoice } from './tts-remote.js';
|
|
45
|
+
import * as ttsVoices from './tts-voices.js';
|
|
45
46
|
import { resolvePro, checkQuota, consume, usage } from './freegate.js';
|
|
46
47
|
import { publicConfig, applyConfigPatch, applyNerModelSelection, persistConfig, configPath } from './configstore.js';
|
|
47
48
|
import { resolveDestination, aggregateModelsAsync, listDestinations } from './router.js';
|
|
@@ -52,7 +53,7 @@ import * as openai from './openai.js';
|
|
|
52
53
|
import * as responses from './responses.js';
|
|
53
54
|
import * as anthropic from './anthropic.js';
|
|
54
55
|
|
|
55
|
-
export const VERSION = '0.6.
|
|
56
|
+
export const VERSION = '0.6.53';
|
|
56
57
|
|
|
57
58
|
// WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
|
|
58
59
|
// store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
|
|
@@ -981,8 +982,14 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
981
982
|
progress: ttsEngine.progress(),
|
|
982
983
|
available,
|
|
983
984
|
voice: cfg.tts?.voice || DEFAULT_TTS_VOICE,
|
|
984
|
-
//
|
|
985
|
-
|
|
985
|
+
// Architecture decides whether voices mean anything: Kokoro picks one from
|
|
986
|
+
// a style bank, VITS/MMS is single-speaker. An empty list tells the UI to
|
|
987
|
+
// hide the picker rather than offer choices that cannot take effect.
|
|
988
|
+
arch: ttsEngine.arch(),
|
|
989
|
+
supportsVoices: ttsEngine.supportsVoices(),
|
|
990
|
+
supportsCustomVoices: ttsEngine.supportsCustomVoices(),
|
|
991
|
+
sampleRate: ttsEngine.sampleRate(),
|
|
992
|
+
voices: ttsEngine.arch() === 'vits' ? [] : TTS_VOICES.map((v) => ({ ...v, installed: ttsEngine.voiceOnDisk(v.id, active) })),
|
|
986
993
|
dtype: cfg.tts?.dtype || 'auto',
|
|
987
994
|
loadedDtype: ttsEngine.health().dtype,
|
|
988
995
|
runtime: ttsEngine.health().runtime,
|
|
@@ -996,8 +1003,14 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
996
1003
|
if (id && !(isKnownTtsModel(id) || isValidCustomTtsId(id))) return sendJson(res, 400, { error: { message: 'unknown or invalid model id', type: 'bad_model' } });
|
|
997
1004
|
// A voice id becomes a filename, so it is checked against the catalog AND
|
|
998
1005
|
// its shape before it is ever persisted.
|
|
1006
|
+
// A default voice may be a built-in Kokoro one OR a saved custom one; both
|
|
1007
|
+
// live in the same field, so both shapes are accepted and both validated.
|
|
999
1008
|
const voice = body && typeof body.voice === 'string' ? body.voice.trim() : null;
|
|
1000
|
-
if (voice
|
|
1009
|
+
if (voice) {
|
|
1010
|
+
const cid = ttsVoices.parseCustomVoice(voice);
|
|
1011
|
+
const okVoice = cid ? !!ttsVoices.getVoice(cid) : (isKnownVoice(voice) && isValidVoiceId(voice));
|
|
1012
|
+
if (!okVoice) return sendJson(res, 400, { error: { message: 'unknown or invalid voice', type: 'bad_voice' } });
|
|
1013
|
+
}
|
|
1001
1014
|
const dtype = body && typeof body.dtype === 'string' && isValidTtsDtype(body.dtype) ? body.dtype : undefined;
|
|
1002
1015
|
if (!cfg.tts) cfg.tts = { enabled: true, model: DEFAULT_TTS_MODEL, voice: DEFAULT_TTS_VOICE, allowDownload: true };
|
|
1003
1016
|
if (id) cfg.tts.model = id;
|
|
@@ -1009,6 +1022,59 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
1009
1022
|
}
|
|
1010
1023
|
}
|
|
1011
1024
|
|
|
1025
|
+
// --- Custom voices: a speaker embedding derived from a sample the user
|
|
1026
|
+
// recorded. The AUDIO is embedded in-process and then discarded; only the 512
|
|
1027
|
+
// floats are stored, under ~/.chatpanel, and they never leave this machine.
|
|
1028
|
+
// See src/tts-voices.js for why the rules here are tighter than elsewhere.
|
|
1029
|
+
if (pathname === '/tts/voices') {
|
|
1030
|
+
if (req.method === 'GET') {
|
|
1031
|
+
return sendJson(res, 200, {
|
|
1032
|
+
voices: ttsVoices.listVoices(),
|
|
1033
|
+
// Whether a saved voice can actually be USED right now depends on the
|
|
1034
|
+
// active model — only SpeechT5 takes an embedding. Saying so here stops
|
|
1035
|
+
// the UI offering voices that would be silently ignored.
|
|
1036
|
+
usable: ttsEngine.supportsCustomVoices(),
|
|
1037
|
+
embedder: diarizeEngine.DIARIZE_MODEL,
|
|
1038
|
+
embedderReady: diarizeEngine.isReady(),
|
|
1039
|
+
});
|
|
1040
|
+
}
|
|
1041
|
+
if (req.method === 'POST') {
|
|
1042
|
+
let body = null;
|
|
1043
|
+
try { body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')); } catch { body = null; }
|
|
1044
|
+
const name = body && typeof body.name === 'string' ? body.name.trim() : '';
|
|
1045
|
+
const pcm = body && Array.isArray(body.pcm) ? body.pcm : null;
|
|
1046
|
+
if (!name) return sendJson(res, 400, { error: { message: 'a name is required', type: 'bad_request' } });
|
|
1047
|
+
if (!pcm || pcm.length < 16000) {
|
|
1048
|
+
// Under a second of audio produces an embedding dominated by whatever
|
|
1049
|
+
// noise happened to be in it, and the resulting voice is arbitrary.
|
|
1050
|
+
return sendJson(res, 400, { error: { message: 'need at least 1 second of 16 kHz mono audio', type: 'sample_too_short' } });
|
|
1051
|
+
}
|
|
1052
|
+
try {
|
|
1053
|
+
// The embedder is the speaker model diarization already uses. If it is
|
|
1054
|
+
// not resident yet, start it and say so — a ~100 MB download is not
|
|
1055
|
+
// something to do silently while the user waits on a spinner.
|
|
1056
|
+
if (!diarizeEngine.isReady()) {
|
|
1057
|
+
diarizeEngine.download({ onLog: (m) => console.log(m) });
|
|
1058
|
+
return sendJson(res, 503, {
|
|
1059
|
+
error: { message: 'the speaker model is downloading (~100 MB) — try again in a moment', type: 'embedder_not_ready' },
|
|
1060
|
+
progress: diarizeEngine.progress(),
|
|
1061
|
+
});
|
|
1062
|
+
}
|
|
1063
|
+
const vec = await diarizeEngine.embed(Float32Array.from(pcm));
|
|
1064
|
+
const saved = ttsVoices.saveVoice({ name, vec });
|
|
1065
|
+
console.log(`[tts] saved custom voice "${saved.name}" (${saved.dim}-d, sample discarded)`);
|
|
1066
|
+
return sendJson(res, 201, { ...saved, usable: ttsEngine.supportsCustomVoices() });
|
|
1067
|
+
} catch (e) {
|
|
1068
|
+
return sendJson(res, 400, { error: { message: e.message, type: 'save_failed' } });
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
if (req.method === 'DELETE') {
|
|
1072
|
+
const id = url.searchParams.get('id') || '';
|
|
1073
|
+
// Deleting someone's voice print is not a soft delete — the file is gone.
|
|
1074
|
+
return sendJson(res, 200, { deleted: ttsVoices.deleteVoice(id) });
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1012
1078
|
// POST /tts — { text, voice?, speed? } → audio/wav — and its OpenAI-compatible
|
|
1013
1079
|
// twin POST /v1/audio/speech ({ input, voice, speed, response_format }), so any
|
|
1014
1080
|
// OpenAI client or tunnel can drive local speech with no ChatPanel-specific code.
|
|
@@ -1032,7 +1098,21 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
1032
1098
|
// not a Kokoro one), so the local catalog check would reject every valid id.
|
|
1033
1099
|
const rawVoice = body && typeof body.voice === 'string' && body.voice.trim() ? body.voice.trim() : null;
|
|
1034
1100
|
const voice = rawVoice || (dest ? dest.voice : (cfg.tts?.voice || DEFAULT_TTS_VOICE));
|
|
1035
|
-
|
|
1101
|
+
// `custom:<id>` names a saved voice. It is resolved to an embedding here so
|
|
1102
|
+
// the engine never has to know where voices are stored.
|
|
1103
|
+
const customId = dest ? null : ttsVoices.parseCustomVoice(voice);
|
|
1104
|
+
let speakerEmbedding = null;
|
|
1105
|
+
if (customId) {
|
|
1106
|
+
const rec = ttsVoices.getVoice(customId);
|
|
1107
|
+
if (!rec) return sendJson(res, 404, { error: { message: 'no such saved voice', type: 'bad_voice' } });
|
|
1108
|
+
if (!ttsEngine.supportsCustomVoices() && ttsEngine.isReady()) {
|
|
1109
|
+
return sendJson(res, 409, { error: { message: `the active model (${ttsEngine.arch()}) cannot use a recorded voice — switch to SpeechT5`, type: 'voice_unsupported' } });
|
|
1110
|
+
}
|
|
1111
|
+
speakerEmbedding = rec.vec;
|
|
1112
|
+
}
|
|
1113
|
+
const voiceOk = customId ? true
|
|
1114
|
+
: dest ? isValidRemoteVoice(voice)
|
|
1115
|
+
: (isKnownVoice(voice) && isValidVoiceId(voice));
|
|
1036
1116
|
if (!voiceOk) return sendJson(res, 400, { error: { message: 'unknown or invalid voice', type: 'bad_voice' } });
|
|
1037
1117
|
// We synthesize WAV only. Say so rather than returning WAV bytes under an mp3
|
|
1038
1118
|
// content-type — a client that trusts the header would play noise.
|
|
@@ -1068,13 +1148,16 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
1068
1148
|
dtype: cfg.tts?.dtype || 'auto',
|
|
1069
1149
|
});
|
|
1070
1150
|
if (!ok) return sendJson(res, 503, { error: { message: ttsEngine.health().error || 'tts model not ready', type: 'tts_unavailable' } });
|
|
1071
|
-
const pcm = await ttsEngine.synth(text, { voice, speed });
|
|
1072
|
-
|
|
1151
|
+
const pcm = await ttsEngine.synth(text, { voice, speed, speakerEmbedding });
|
|
1152
|
+
// The ACTIVE model's rate, not the constant: a VITS/MMS model emits 16 kHz
|
|
1153
|
+
// and writing it into a 24 kHz header plays it fast and chipmunked.
|
|
1154
|
+
const rate = ttsEngine.sampleRate();
|
|
1155
|
+
const out = fmt === 'pcm' ? Buffer.from(new Float32Array(pcm).buffer) : ttsEngine.toWav(pcm, rate);
|
|
1073
1156
|
res.writeHead(200, {
|
|
1074
1157
|
'Content-Type': fmt === 'pcm' ? 'application/octet-stream' : 'audio/wav',
|
|
1075
1158
|
'Content-Length': String(out.length),
|
|
1076
1159
|
'Cache-Control': 'no-store',
|
|
1077
|
-
'X-Tts-Sample-Rate': String(
|
|
1160
|
+
'X-Tts-Sample-Rate': String(rate),
|
|
1078
1161
|
});
|
|
1079
1162
|
return res.end(out);
|
|
1080
1163
|
} catch (e) {
|
package/src/tts-engine.js
CHANGED
|
@@ -27,7 +27,11 @@ import {
|
|
|
27
27
|
ttsModelDtype, isKnownTtsModel, isValidVoiceId, voiceLang,
|
|
28
28
|
} from './tts-models.js';
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
// Kokoro's rate. Kept as a named export because it is the default and several
|
|
31
|
+
// callers want a number before anything is loaded — but it is NOT universal: a
|
|
32
|
+
// VITS/MMS model outputs 16 kHz, and writing its samples into a 24 kHz WAV header
|
|
33
|
+
// plays it back fast and chipmunked. Use sampleRate() once a model is active.
|
|
34
|
+
export const SAMPLE_RATE = 24000;
|
|
31
35
|
|
|
32
36
|
let _state = 'off'; // 'off' | 'loading' | 'downloading' | 'ready' | 'error'
|
|
33
37
|
let _model = null;
|
|
@@ -37,14 +41,35 @@ let _dtype = null;
|
|
|
37
41
|
let _err = null;
|
|
38
42
|
let _progress = null;
|
|
39
43
|
let _initPromise = null;
|
|
44
|
+
let _arch = null; // 'style-tts2' (Kokoro) | 'vits' (MMS) | 'speecht5' (custom voices)
|
|
45
|
+
let _vocoder = null; // speecht5 only — mel → waveform
|
|
46
|
+
let _rate = SAMPLE_RATE; // the ACTIVE model's output rate
|
|
40
47
|
const _voices = new Map(); // voice id → Float32Array style bank
|
|
41
48
|
|
|
49
|
+
// The architectures this engine can actually drive. Anything else is refused at
|
|
50
|
+
// load with a message naming what it is, rather than failing later inside a
|
|
51
|
+
// forward pass with a shape error nobody can act on.
|
|
52
|
+
export const SUPPORTED_ARCH = { style_text_to_speech_2: 'style-tts2', vits: 'vits', speecht5: 'speecht5' };
|
|
53
|
+
|
|
54
|
+
// SpeechT5 is the only architecture here that takes a SPEAKER EMBEDDING, which is
|
|
55
|
+
// what makes a custom voice possible at all: Kokoro's voices are fixed style banks
|
|
56
|
+
// and VITS is single-speaker, so neither can be pointed at a person. It needs a
|
|
57
|
+
// separate vocoder (mel → waveform), hence the extra model id.
|
|
58
|
+
export const SPEECHT5_VOCODER = 'Xenova/speecht5_hifigan';
|
|
59
|
+
export function supportsCustomVoices() { return _arch === 'speecht5'; }
|
|
60
|
+
|
|
61
|
+
export function arch() { return _arch; }
|
|
62
|
+
export function sampleRate() { return _rate; }
|
|
63
|
+
// Kokoro picks a voice from a style bank; VITS is single-speaker and has none, so
|
|
64
|
+
// the UI must not offer a voice list that cannot do anything.
|
|
65
|
+
export function supportsVoices() { return _arch === 'style-tts2'; }
|
|
66
|
+
|
|
42
67
|
export function state() { return _state; }
|
|
43
68
|
export function isReady() { return _state === 'ready' && !!_net; }
|
|
44
69
|
export function progress() { return _progress; }
|
|
45
70
|
|
|
46
71
|
export function health() {
|
|
47
|
-
return { configured: _state !== 'off', ok: isReady(), state: _state, model: _model, dtype: _dtype, runtime: runtimeName(), error: _err };
|
|
72
|
+
return { configured: _state !== 'off', ok: isReady(), state: _state, model: _model, dtype: _dtype, runtime: runtimeName(), error: _err, arch: _arch, sampleRate: _rate, voices: supportsVoices(), customVoices: supportsCustomVoices() };
|
|
48
73
|
}
|
|
49
74
|
|
|
50
75
|
export function modelDir(modelId) {
|
|
@@ -119,18 +144,39 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true, dtype:
|
|
|
119
144
|
// this is the SAME transformers instance ensureLib already configured (env,
|
|
120
145
|
// cacheDir, wasm paths) — importing it here keeps ner-engine free of TTS.
|
|
121
146
|
const tf = await import('@huggingface/transformers');
|
|
147
|
+
|
|
148
|
+
// Which architecture is this? Read the config BEFORE choosing a class, so an
|
|
149
|
+
// unsupported model is refused by name instead of exploding inside a forward
|
|
150
|
+
// pass with a tensor-shape error.
|
|
151
|
+
let modelType = '';
|
|
152
|
+
try {
|
|
153
|
+
const conf = await tf.AutoConfig.from_pretrained(modelId);
|
|
154
|
+
modelType = String(conf?.model_type || '').toLowerCase();
|
|
155
|
+
} catch { /* no config we can read — fall through to the Kokoro default */ }
|
|
156
|
+
const kind = SUPPORTED_ARCH[modelType] || (modelType ? null : 'style-tts2');
|
|
157
|
+
if (!kind) throw new Error(`unsupported TTS architecture "${modelType}" — this engine drives Kokoro (style_text_to_speech_2) and VITS/MMS`);
|
|
158
|
+
|
|
159
|
+
const onProgress = (p) => {
|
|
160
|
+
if (p?.status === 'progress' && p.file) _progress = { model: modelId, file: p.file, pct: Math.round(p.progress || 0) };
|
|
161
|
+
};
|
|
162
|
+
const Klass = kind === 'vits' ? tf.VitsModel
|
|
163
|
+
: kind === 'speecht5' ? tf.SpeechT5ForTextToSpeech
|
|
164
|
+
: tf.StyleTextToSpeech2Model;
|
|
122
165
|
const [net, tok] = await Promise.all([
|
|
123
|
-
|
|
124
|
-
dtype,
|
|
125
|
-
progress_callback: (p) => {
|
|
126
|
-
if (p?.status === 'progress' && p.file) _progress = { model: modelId, file: p.file, pct: Math.round(p.progress || 0) };
|
|
127
|
-
},
|
|
128
|
-
}),
|
|
166
|
+
Klass.from_pretrained(modelId, { dtype, progress_callback: onProgress }),
|
|
129
167
|
tf.AutoTokenizer.from_pretrained(modelId),
|
|
130
168
|
]);
|
|
131
|
-
|
|
169
|
+
// The vocoder is a second download and a second failure point, so it is loaded
|
|
170
|
+
// only for the architecture that needs one.
|
|
171
|
+
_vocoder = kind === 'speecht5'
|
|
172
|
+
? await tf.SpeechT5HifiGan.from_pretrained(SPEECHT5_VOCODER, { dtype, progress_callback: onProgress })
|
|
173
|
+
: null;
|
|
174
|
+
_net = net; _tok = tok; _model = modelId; _dtype = dtype; _arch = kind;
|
|
175
|
+
// VITS/MMS emit 16 kHz; Kokoro 24 kHz. Take it from the model's own config
|
|
176
|
+
// where it says so, because guessing wrong plays the voice at the wrong pitch.
|
|
177
|
+
_rate = Number(net?.config?.sampling_rate) || (kind === 'style-tts2' ? SAMPLE_RATE : 16000);
|
|
132
178
|
_state = 'ready'; _err = null; _progress = null;
|
|
133
|
-
log(`[tts] ready — model ${modelId} @ ${dtype} (${runtimeName()}, offline) — local speech active`);
|
|
179
|
+
log(`[tts] ready — model ${modelId} @ ${dtype} (${kind}, ${_rate} Hz, ${runtimeName()}, offline) — local speech active`);
|
|
134
180
|
return true;
|
|
135
181
|
} catch (e) {
|
|
136
182
|
// A failed SWITCH keeps the previous working model, same as ner/stt.
|
|
@@ -229,11 +275,35 @@ export function splitSentences(text, maxChars = 300) {
|
|
|
229
275
|
* Synthesize ONE chunk. Returns 24 kHz mono Float32 PCM.
|
|
230
276
|
* @param {string} text @param {{voice?: string, speed?: number}} [opts]
|
|
231
277
|
*/
|
|
232
|
-
export async function synthChunk(text, { voice = DEFAULT_TTS_VOICE, speed = 1 } = {}) {
|
|
278
|
+
export async function synthChunk(text, { voice = DEFAULT_TTS_VOICE, speed = 1, speakerEmbedding = null } = {}) {
|
|
233
279
|
if (!isReady()) throw new Error('tts model not ready');
|
|
234
|
-
const { phonemize } = await import('phonemizer');
|
|
235
280
|
const tf = await import('@huggingface/transformers');
|
|
236
281
|
|
|
282
|
+
// VITS/MMS: single-speaker, tokenizes GRAPHEMES directly — no phonemizer, no
|
|
283
|
+
// style bank, no speed input. One language per model, which is the trade for
|
|
284
|
+
// ~40 MB and a thousand of them.
|
|
285
|
+
if (_arch === 'vits') {
|
|
286
|
+
const inputs = _tok(String(text));
|
|
287
|
+
const out = await _net(inputs);
|
|
288
|
+
return out.waveform.data;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
// SpeechT5: conditioned by a 512-d speaker embedding, which is the whole point —
|
|
292
|
+
// it is the one architecture here that can be pointed at a person's voice.
|
|
293
|
+
// Without an embedding there is no voice to speak in, so this refuses rather
|
|
294
|
+
// than inventing one.
|
|
295
|
+
if (_arch === 'speecht5') {
|
|
296
|
+
if (!speakerEmbedding || speakerEmbedding.length !== 512) {
|
|
297
|
+
throw new Error('this model needs a saved voice — record one in Settings → Text-to-speech');
|
|
298
|
+
}
|
|
299
|
+
const { input_ids } = _tok(String(text));
|
|
300
|
+
const emb = new tf.Tensor('float32', Float32Array.from(speakerEmbedding), [1, 512]);
|
|
301
|
+
const { waveform } = await _net.generate_speech(input_ids, emb, { vocoder: _vocoder });
|
|
302
|
+
return waveform.data;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
const { phonemize } = await import('phonemizer');
|
|
306
|
+
|
|
237
307
|
// G2P follows the VOICE, not the request — an American voice reading British
|
|
238
308
|
// phonemes is audibly wrong.
|
|
239
309
|
const phonemes = (await phonemize(String(text), voiceLang(voice))).join(' ');
|
|
@@ -253,11 +323,11 @@ export async function synthChunk(text, { voice = DEFAULT_TTS_VOICE, speed = 1 }
|
|
|
253
323
|
}
|
|
254
324
|
|
|
255
325
|
/** Synthesize arbitrary-length text, chunk by chunk. `onChunk` sees each as it lands. */
|
|
256
|
-
export async function synth(text, { voice = DEFAULT_TTS_VOICE, speed = 1, onChunk = null } = {}) {
|
|
326
|
+
export async function synth(text, { voice = DEFAULT_TTS_VOICE, speed = 1, speakerEmbedding = null, onChunk = null } = {}) {
|
|
257
327
|
const chunks = splitSentences(text);
|
|
258
328
|
const out = [];
|
|
259
329
|
for (const c of chunks) {
|
|
260
|
-
const pcm = await synthChunk(c, { voice, speed });
|
|
330
|
+
const pcm = await synthChunk(c, { voice, speed, speakerEmbedding });
|
|
261
331
|
out.push(pcm);
|
|
262
332
|
onChunk?.(pcm);
|
|
263
333
|
}
|
|
@@ -296,5 +366,6 @@ export function toWav(pcm, sampleRate = SAMPLE_RATE) {
|
|
|
296
366
|
|
|
297
367
|
export function _reset() {
|
|
298
368
|
_state = 'off'; _model = null; _net = null; _tok = null; _dtype = null;
|
|
299
|
-
_err = null; _progress = null; _initPromise = null;
|
|
369
|
+
_err = null; _progress = null; _initPromise = null; _arch = null; _rate = SAMPLE_RATE; _vocoder = null;
|
|
370
|
+
_voices.clear();
|
|
300
371
|
}
|
package/src/tts-models.js
CHANGED
|
@@ -40,8 +40,58 @@ export const TTS_MODEL_CATALOG = [
|
|
|
40
40
|
approxMB: 330, // fp32 on WASM; ~90 on native q8
|
|
41
41
|
ramMB: 600,
|
|
42
42
|
sampleRate: 24000,
|
|
43
|
+
voices: true,
|
|
43
44
|
note: 'Apache-2.0, 82M params. Natural voices at ~5× realtime on CPU. The default.',
|
|
44
45
|
},
|
|
46
|
+
{
|
|
47
|
+
id: 'onnx-community/Kokoro-82M-v1.1-zh-ONNX',
|
|
48
|
+
label: 'Kokoro 82M (v1.1, Chinese)',
|
|
49
|
+
lang: 'Chinese + English',
|
|
50
|
+
tier: 'balanced',
|
|
51
|
+
arch: 'style-tts2',
|
|
52
|
+
approxMB: 330,
|
|
53
|
+
ramMB: 600,
|
|
54
|
+
sampleRate: 24000,
|
|
55
|
+
voices: true,
|
|
56
|
+
note: 'The Mandarin-tuned Kokoro. Same engine and voice mechanism as v1.0.',
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
id: 'Xenova/speecht5_tts',
|
|
60
|
+
label: 'SpeechT5 — your own voice',
|
|
61
|
+
lang: 'English',
|
|
62
|
+
tier: 'custom',
|
|
63
|
+
arch: 'speecht5',
|
|
64
|
+
// PINNED to fp32, and not as a preference. At the runtime default (q8 on
|
|
65
|
+
// native) this model renders about half of all speaker embeddings as
|
|
66
|
+
// near-silence — measured: 91-92% of samples under the noise floor, where the
|
|
67
|
+
// same embeddings at fp32 produce clean speech at ~47% silence, which is just
|
|
68
|
+
// the pauses between words. The quantization is destroying the speaker
|
|
69
|
+
// conditioning, and the failure looks exactly like "voice cloning does not
|
|
70
|
+
// work" rather than "the precision is wrong".
|
|
71
|
+
dtype: 'fp32',
|
|
72
|
+
approxMB: 190, // plus the ~50 MB HiFi-GAN vocoder it needs
|
|
73
|
+
ramMB: 500,
|
|
74
|
+
sampleRate: 16000,
|
|
75
|
+
voices: false, // no built-in voices…
|
|
76
|
+
customVoices: true, // …but it is the ONE model here that can use yours
|
|
77
|
+
note: 'The only model here that speaks in a voice you record. Rougher than Kokoro, and the match is approximate — see the note under Your voices.',
|
|
78
|
+
},
|
|
79
|
+
{
|
|
80
|
+
// One MMS entry so the second architecture is DISCOVERABLE from the list rather
|
|
81
|
+
// than only findable by search. The rest of the family (~1000 languages) is
|
|
82
|
+
// exactly what the search box is for — listing them all here would be a menu,
|
|
83
|
+
// not a catalog.
|
|
84
|
+
id: 'Xenova/mms-tts-hin',
|
|
85
|
+
label: 'MMS TTS — Hindi',
|
|
86
|
+
lang: 'Hindi (हिन्दी)',
|
|
87
|
+
tier: 'light',
|
|
88
|
+
arch: 'vits',
|
|
89
|
+
approxMB: 40,
|
|
90
|
+
ramMB: 200,
|
|
91
|
+
sampleRate: 16000,
|
|
92
|
+
voices: false,
|
|
93
|
+
note: 'Tiny single-speaker VITS. Meta\u2019s MMS covers ~1000 languages — search "mms-tts" for yours.',
|
|
94
|
+
},
|
|
45
95
|
];
|
|
46
96
|
|
|
47
97
|
// Voice prefix → the language phonemizer must use for G2P. First letter = language,
|
|
@@ -84,7 +134,20 @@ export const TTS_VOICES = [
|
|
|
84
134
|
// THIRD-PARTY re-export of a gated checkpoint. That is a parakeet-engine-sized
|
|
85
135
|
// piece of work, not a catalog entry — so the seam is here and the engine is not.
|
|
86
136
|
export function ttsModelEngine(id) {
|
|
87
|
-
return ttsModel(id)?.
|
|
137
|
+
return ttsModel(id)?.arch || 'style-tts2';
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Does this catalog entry have selectable voices? Kokoro picks one from a style
|
|
141
|
+
// bank; VITS/MMS is single-speaker. Unknown (a searched model) resolves at load.
|
|
142
|
+
export function ttsModelHasVoices(id) {
|
|
143
|
+
const m = ttsModel(id);
|
|
144
|
+
return m ? m.voices !== false : true;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Can it speak in a voice the user recorded? Only SpeechT5 takes a speaker
|
|
148
|
+
// embedding — the other two are conditioned on something fixed.
|
|
149
|
+
export function ttsModelHasCustomVoices(id) {
|
|
150
|
+
return ttsModel(id)?.customVoices === true;
|
|
88
151
|
}
|
|
89
152
|
|
|
90
153
|
export function ttsModel(id) {
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
// Custom voices — a speaker embedding derived from a sample the user recorded.
|
|
2
|
+
//
|
|
3
|
+
// A voice print is biometric-adjacent data about a specific person, so the rules
|
|
4
|
+
// here are deliberately tighter than for anything else this gateway stores:
|
|
5
|
+
// • It NEVER leaves the machine. The embedding is computed in-process by the
|
|
6
|
+
// speaker model already on disk (the one diarization uses), written under
|
|
7
|
+
// ~/.chatpanel, and read back only to condition local synthesis.
|
|
8
|
+
// • The SAMPLE is never persisted. Only the 512 floats derived from it — audio
|
|
9
|
+
// of someone speaking is far more revealing than the vector, and keeping it
|
|
10
|
+
// would serve no purpose once the embedding exists.
|
|
11
|
+
// • Deleting means the file is gone, not flagged. There is no soft-delete for
|
|
12
|
+
// someone's voice.
|
|
13
|
+
//
|
|
14
|
+
// What this can and cannot do is stated plainly in the UI, and the reason lives
|
|
15
|
+
// here: SpeechT5 was trained against speechbrain x-vectors and we embed with
|
|
16
|
+
// wavlm-base-plus-sv, which is a different 512-d space. The result is a distinct,
|
|
17
|
+
// STABLE voice derived from the sample — the same sample always gives the same
|
|
18
|
+
// voice, and different speakers give clearly different ones — but it is not a
|
|
19
|
+
// faithful reproduction of the speaker. Calling that "cloning" without saying so
|
|
20
|
+
// would be a lie told by a feature name.
|
|
21
|
+
|
|
22
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync, rmSync } from 'node:fs';
|
|
23
|
+
import { join } from 'node:path';
|
|
24
|
+
import { randomUUID } from 'node:crypto';
|
|
25
|
+
import os from 'node:os';
|
|
26
|
+
|
|
27
|
+
export const EMBED_DIM = 512;
|
|
28
|
+
const MAX_VOICES = 20;
|
|
29
|
+
const MAX_NAME = 60;
|
|
30
|
+
|
|
31
|
+
export function voicesDir() {
|
|
32
|
+
return process.env.CHATPANEL_TTS_VOICES || join(os.homedir(), '.chatpanel', 'tts-voices');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// An id becomes a FILENAME and arrives from the wire, so it is shape-checked, not
|
|
36
|
+
// trusted. UUIDs are what we mint; the pattern is what we accept.
|
|
37
|
+
export function isValidVoiceRef(id) {
|
|
38
|
+
return /^[A-Za-z0-9_-]{1,64}$/.test(String(id || ''));
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// The wire form is `custom:<id>` so one `voice` field can name either a built-in
|
|
42
|
+
// Kokoro voice or a saved one, with no ambiguity between the two namespaces.
|
|
43
|
+
export function parseCustomVoice(voice) {
|
|
44
|
+
const s = String(voice || '');
|
|
45
|
+
if (!s.startsWith('custom:')) return null;
|
|
46
|
+
const id = s.slice(7);
|
|
47
|
+
return isValidVoiceRef(id) ? id : null;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function fileFor(id) {
|
|
51
|
+
return join(voicesDir(), `${id}.json`);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function listVoices() {
|
|
55
|
+
const dir = voicesDir();
|
|
56
|
+
if (!existsSync(dir)) return [];
|
|
57
|
+
const out = [];
|
|
58
|
+
for (const f of readdirSync(dir)) {
|
|
59
|
+
if (!f.endsWith('.json')) continue;
|
|
60
|
+
try {
|
|
61
|
+
const v = JSON.parse(readFileSync(join(dir, f), 'utf8'));
|
|
62
|
+
// The vector is deliberately NOT returned by the listing: the UI needs a
|
|
63
|
+
// name and an id, and 512 floats of someone's voice print have no business
|
|
64
|
+
// in a settings page's JSON.
|
|
65
|
+
if (v?.id && v?.name) out.push({ id: v.id, name: v.name, createdAt: v.createdAt || 0, dim: v.vec?.length || 0 });
|
|
66
|
+
} catch { /* a corrupt file must not break the list */ }
|
|
67
|
+
}
|
|
68
|
+
return out.sort((a, b) => b.createdAt - a.createdAt);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function getVoice(id) {
|
|
72
|
+
if (!isValidVoiceRef(id)) return null;
|
|
73
|
+
try {
|
|
74
|
+
const v = JSON.parse(readFileSync(fileFor(id), 'utf8'));
|
|
75
|
+
return Array.isArray(v?.vec) && v.vec.length === EMBED_DIM ? v : null;
|
|
76
|
+
} catch { return null; }
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Persist an embedding under a user-chosen name. Returns the stored record. */
|
|
80
|
+
export function saveVoice({ name, vec }) {
|
|
81
|
+
const clean = String(name || '').trim().slice(0, MAX_NAME);
|
|
82
|
+
if (!clean) throw new Error('a name is required');
|
|
83
|
+
if (!vec || vec.length !== EMBED_DIM) throw new Error(`expected a ${EMBED_DIM}-value embedding, got ${vec?.length || 0}`);
|
|
84
|
+
if (listVoices().length >= MAX_VOICES) throw new Error(`at most ${MAX_VOICES} saved voices — delete one first`);
|
|
85
|
+
const dir = voicesDir();
|
|
86
|
+
mkdirSync(dir, { recursive: true });
|
|
87
|
+
const rec = { id: randomUUID(), name: clean, createdAt: Date.now(), vec: Array.from(vec, (x) => Number(x) || 0) };
|
|
88
|
+
writeFileSync(fileFor(rec.id), JSON.stringify(rec));
|
|
89
|
+
return { id: rec.id, name: rec.name, createdAt: rec.createdAt, dim: rec.vec.length };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Remove one permanently. Returns whether there was anything to remove. */
|
|
93
|
+
export function deleteVoice(id) {
|
|
94
|
+
if (!isValidVoiceRef(id)) return false;
|
|
95
|
+
const p = fileFor(id);
|
|
96
|
+
if (!existsSync(p)) return false;
|
|
97
|
+
rmSync(p, { force: true });
|
|
98
|
+
return true;
|
|
99
|
+
}
|