@chatpanel/gateway 0.6.52 → 0.6.54
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/config.js +4 -1
- package/src/configstore.js +6 -1
- package/src/server.js +135 -5
- package/src/tts-engine.js +38 -9
- package/src/tts-models.js +27 -0
- package/src/tts-voices.js +99 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chatpanel/gateway",
|
|
3
|
-
"version": "0.6.
|
|
3
|
+
"version": "0.6.54",
|
|
4
4
|
"description": "Local privacy gateway — redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
package/src/config.js
CHANGED
|
@@ -9,7 +9,10 @@ import { readFileSync, existsSync } from 'node:fs';
|
|
|
9
9
|
import { join } from 'node:path';
|
|
10
10
|
import os from 'node:os';
|
|
11
11
|
|
|
12
|
-
|
|
12
|
+
// Exported so tests can assert that every section here survives persistConfig's
|
|
13
|
+
// allowlist — a new section that is not persisted reverts on restart, and that
|
|
14
|
+
// reads as a broken feature rather than an unsaved setting.
|
|
15
|
+
export const DEFAULTS = {
|
|
13
16
|
host: '127.0.0.1',
|
|
14
17
|
port: 4320,
|
|
15
18
|
|
package/src/configstore.js
CHANGED
|
@@ -19,13 +19,18 @@ export function persistConfig(cfg, path = configPath()) {
|
|
|
19
19
|
// detector key, and the entitlement/bridge tokens — same secret-at-rest posture
|
|
20
20
|
// as the history key/secret files, so it isn't left world-readable on a shared host.
|
|
21
21
|
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
|
22
|
+
// NOTE: this is an explicit allowlist, so a NEW config section does not persist
|
|
23
|
+
// until it is added here — and the symptom is a setting that silently reverts on
|
|
24
|
+
// restart, which reads as "the feature is broken" rather than "it was not saved".
|
|
25
|
+
// tests/configstore.test.js fails if a key in DEFAULTS is neither listed here nor
|
|
26
|
+
// deliberately excluded below.
|
|
22
27
|
const out = {
|
|
23
28
|
host: cfg.host, port: cfg.port, backend: cfg.backend,
|
|
24
29
|
// Destinations (the configured agents + API models) MUST persist — otherwise a
|
|
25
30
|
// restart drops them and every model falls back to the default OpenAI upstream.
|
|
26
31
|
destinations: cfg.destinations,
|
|
27
32
|
bridge: cfg.bridge, upstreams: cfg.upstreams, redaction: cfg.redaction,
|
|
28
|
-
ner: cfg.ner, stt: cfg.stt, allowedOrigins: cfg.allowedOrigins, maxBodyBytes: cfg.maxBodyBytes,
|
|
33
|
+
ner: cfg.ner, stt: cfg.stt, tts: cfg.tts, allowedOrigins: cfg.allowedOrigins, maxBodyBytes: cfg.maxBodyBytes,
|
|
29
34
|
pro: cfg.pro, logRequests: cfg.logRequests, logDetail: cfg.logDetail, tools: cfg.tools,
|
|
30
35
|
};
|
|
31
36
|
// mode on writeFileSync only applies when CREATING the file; chmod after covers an
|
package/src/server.js
CHANGED
|
@@ -42,6 +42,7 @@ import { STT_MODEL_CATALOG, isKnownSttModel, isValidCustomSttId, DEFAULT_STT_MOD
|
|
|
42
42
|
import * as ttsEngine from './tts-engine.js';
|
|
43
43
|
import { TTS_MODEL_CATALOG, TTS_VOICES, isKnownTtsModel, isValidCustomTtsId, isKnownVoice, isValidVoiceId, DEFAULT_TTS_MODEL, DEFAULT_TTS_VOICE, TTS_DTYPES, isValidTtsDtype, MAX_TTS_CHARS } from './tts-models.js';
|
|
44
44
|
import { ttsDestination, synthesizeRemote, isValidRemoteVoice } from './tts-remote.js';
|
|
45
|
+
import * as ttsVoices from './tts-voices.js';
|
|
45
46
|
import { resolvePro, checkQuota, consume, usage } from './freegate.js';
|
|
46
47
|
import { publicConfig, applyConfigPatch, applyNerModelSelection, persistConfig, configPath } from './configstore.js';
|
|
47
48
|
import { resolveDestination, aggregateModelsAsync, listDestinations } from './router.js';
|
|
@@ -52,7 +53,7 @@ import * as openai from './openai.js';
|
|
|
52
53
|
import * as responses from './responses.js';
|
|
53
54
|
import * as anthropic from './anthropic.js';
|
|
54
55
|
|
|
55
|
-
export const VERSION = '0.6.
|
|
56
|
+
export const VERSION = '0.6.54';
|
|
56
57
|
|
|
57
58
|
// WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
|
|
58
59
|
// store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
|
|
@@ -453,6 +454,16 @@ export function joinUpstream(base, pathname, search = '') {
|
|
|
453
454
|
return b + pathname + search;
|
|
454
455
|
}
|
|
455
456
|
|
|
457
|
+
// Paths this gateway serves ITSELF. Used only to tell "you asked for a local
|
|
458
|
+
// feature I do not have" apart from "you asked me to proxy something upstream" —
|
|
459
|
+
// without it, calling a route added in a newer version reports a provider failure.
|
|
460
|
+
// How long to wait for the speaker model before telling the caller to retry. Long
|
|
461
|
+
// enough to cover loading one already on disk (seconds) plus a slow first fetch,
|
|
462
|
+
// short enough that a stuck download does not hold a request open forever.
|
|
463
|
+
const EMBEDDER_WAIT_MS = 90_000;
|
|
464
|
+
|
|
465
|
+
const LOCAL_NAMESPACES = ['/tts', '/stt', '/ner', '/diarize', '/skills', '/config', '/logs', '/status', '/admin'];
|
|
466
|
+
|
|
456
467
|
async function handleApi(req, res, { adapter, kind, pathname, search, base, destKey, destProtocol, harness, trace }, outBody, vault) {
|
|
457
468
|
let upstream;
|
|
458
469
|
const up0 = trace ? trace.clock() : 0;
|
|
@@ -986,8 +997,14 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
986
997
|
// hide the picker rather than offer choices that cannot take effect.
|
|
987
998
|
arch: ttsEngine.arch(),
|
|
988
999
|
supportsVoices: ttsEngine.supportsVoices(),
|
|
1000
|
+
supportsCustomVoices: ttsEngine.supportsCustomVoices(),
|
|
989
1001
|
sampleRate: ttsEngine.sampleRate(),
|
|
990
|
-
|
|
1002
|
+
// Built-in voices belong to Kokoro alone. VITS is single-speaker and
|
|
1003
|
+
// SpeechT5 speaks only in a RECORDED voice, so offering Kokoro's list
|
|
1004
|
+
// for either would be offering choices that cannot take effect.
|
|
1005
|
+
voices: ttsEngine.arch() && ttsEngine.arch() !== 'style-tts2'
|
|
1006
|
+
? []
|
|
1007
|
+
: TTS_VOICES.map((v) => ({ ...v, installed: ttsEngine.voiceOnDisk(v.id, active) })),
|
|
991
1008
|
dtype: cfg.tts?.dtype || 'auto',
|
|
992
1009
|
loadedDtype: ttsEngine.health().dtype,
|
|
993
1010
|
runtime: ttsEngine.health().runtime,
|
|
@@ -1001,8 +1018,14 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
1001
1018
|
if (id && !(isKnownTtsModel(id) || isValidCustomTtsId(id))) return sendJson(res, 400, { error: { message: 'unknown or invalid model id', type: 'bad_model' } });
|
|
1002
1019
|
// A voice id becomes a filename, so it is checked against the catalog AND
|
|
1003
1020
|
// its shape before it is ever persisted.
|
|
1021
|
+
// A default voice may be a built-in Kokoro one OR a saved custom one; both
|
|
1022
|
+
// live in the same field, so both shapes are accepted and both validated.
|
|
1004
1023
|
const voice = body && typeof body.voice === 'string' ? body.voice.trim() : null;
|
|
1005
|
-
if (voice
|
|
1024
|
+
if (voice) {
|
|
1025
|
+
const cid = ttsVoices.parseCustomVoice(voice);
|
|
1026
|
+
const okVoice = cid ? !!ttsVoices.getVoice(cid) : (isKnownVoice(voice) && isValidVoiceId(voice));
|
|
1027
|
+
if (!okVoice) return sendJson(res, 400, { error: { message: 'unknown or invalid voice', type: 'bad_voice' } });
|
|
1028
|
+
}
|
|
1006
1029
|
const dtype = body && typeof body.dtype === 'string' && isValidTtsDtype(body.dtype) ? body.dtype : undefined;
|
|
1007
1030
|
if (!cfg.tts) cfg.tts = { enabled: true, model: DEFAULT_TTS_MODEL, voice: DEFAULT_TTS_VOICE, allowDownload: true };
|
|
1008
1031
|
if (id) cfg.tts.model = id;
|
|
@@ -1014,6 +1037,74 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
1014
1037
|
}
|
|
1015
1038
|
}
|
|
1016
1039
|
|
|
1040
|
+
// --- Custom voices: a speaker embedding derived from a sample the user
|
|
1041
|
+
// recorded. The AUDIO is embedded in-process and then discarded; only the 512
|
|
1042
|
+
// floats are stored, under ~/.chatpanel, and they never leave this machine.
|
|
1043
|
+
// See src/tts-voices.js for why the rules here are tighter than elsewhere.
|
|
1044
|
+
if (pathname === '/tts/voices') {
|
|
1045
|
+
if (req.method === 'GET') {
|
|
1046
|
+
return sendJson(res, 200, {
|
|
1047
|
+
voices: ttsVoices.listVoices(),
|
|
1048
|
+
// Whether a saved voice can actually be USED right now depends on the
|
|
1049
|
+
// active model — only SpeechT5 takes an embedding. Saying so here stops
|
|
1050
|
+
// the UI offering voices that would be silently ignored.
|
|
1051
|
+
usable: ttsEngine.supportsCustomVoices(),
|
|
1052
|
+
embedder: diarizeEngine.DIARIZE_MODEL,
|
|
1053
|
+
embedderReady: diarizeEngine.isReady(),
|
|
1054
|
+
});
|
|
1055
|
+
}
|
|
1056
|
+
if (req.method === 'POST') {
|
|
1057
|
+
let body = null;
|
|
1058
|
+
try { body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')); } catch { body = null; }
|
|
1059
|
+
const name = body && typeof body.name === 'string' ? body.name.trim() : '';
|
|
1060
|
+
const pcm = body && Array.isArray(body.pcm) ? body.pcm : null;
|
|
1061
|
+
if (!name) return sendJson(res, 400, { error: { message: 'a name is required', type: 'bad_request' } });
|
|
1062
|
+
if (!pcm || pcm.length < 16000) {
|
|
1063
|
+
// Under a second of audio produces an embedding dominated by whatever
|
|
1064
|
+
// noise happened to be in it, and the resulting voice is arbitrary.
|
|
1065
|
+
return sendJson(res, 400, { error: { message: 'need at least 1 second of 16 kHz mono audio', type: 'sample_too_short' } });
|
|
1066
|
+
}
|
|
1067
|
+
try {
|
|
1068
|
+
// The embedder is the speaker model diarization already uses. WAIT for it
|
|
1069
|
+
// rather than bailing: it is usually already on disk, where loading takes
|
|
1070
|
+
// a couple of seconds — and the caller is holding a recording someone
|
|
1071
|
+
// just made, so returning early means they lose it and record again.
|
|
1072
|
+
// Only a genuine first-time download can outlast the ceiling, and that is
|
|
1073
|
+
// the one case worth reporting as "come back in a moment".
|
|
1074
|
+
if (!diarizeEngine.isReady()) {
|
|
1075
|
+
const load = diarizeEngine.download({ onLog: (m) => console.log(m) });
|
|
1076
|
+
const timedOut = Symbol('timeout');
|
|
1077
|
+
const raced = await Promise.race([
|
|
1078
|
+
load.then(() => null).catch((e) => e),
|
|
1079
|
+
new Promise((r) => setTimeout(() => r(timedOut), EMBEDDER_WAIT_MS)),
|
|
1080
|
+
]);
|
|
1081
|
+
if (raced === timedOut || !diarizeEngine.isReady()) {
|
|
1082
|
+
return sendJson(res, 503, {
|
|
1083
|
+
error: {
|
|
1084
|
+
message: raced === timedOut
|
|
1085
|
+
? 'the speaker model is still downloading (~100 MB) — your recording was kept, press Save again shortly'
|
|
1086
|
+
: `the speaker model failed to load: ${diarizeEngine.health().error || 'unknown error'}`,
|
|
1087
|
+
type: 'embedder_not_ready',
|
|
1088
|
+
},
|
|
1089
|
+
progress: diarizeEngine.progress(),
|
|
1090
|
+
});
|
|
1091
|
+
}
|
|
1092
|
+
}
|
|
1093
|
+
const vec = await diarizeEngine.embed(Float32Array.from(pcm));
|
|
1094
|
+
const saved = ttsVoices.saveVoice({ name, vec });
|
|
1095
|
+
console.log(`[tts] saved custom voice "${saved.name}" (${saved.dim}-d, sample discarded)`);
|
|
1096
|
+
return sendJson(res, 201, { ...saved, usable: ttsEngine.supportsCustomVoices() });
|
|
1097
|
+
} catch (e) {
|
|
1098
|
+
return sendJson(res, 400, { error: { message: e.message, type: 'save_failed' } });
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1101
|
+
if (req.method === 'DELETE') {
|
|
1102
|
+
const id = url.searchParams.get('id') || '';
|
|
1103
|
+
// Deleting someone's voice print is not a soft delete — the file is gone.
|
|
1104
|
+
return sendJson(res, 200, { deleted: ttsVoices.deleteVoice(id) });
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
|
|
1017
1108
|
// POST /tts — { text, voice?, speed? } → audio/wav — and its OpenAI-compatible
|
|
1018
1109
|
// twin POST /v1/audio/speech ({ input, voice, speed, response_format }), so any
|
|
1019
1110
|
// OpenAI client or tunnel can drive local speech with no ChatPanel-specific code.
|
|
@@ -1037,7 +1128,31 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
1037
1128
|
// not a Kokoro one), so the local catalog check would reject every valid id.
|
|
1038
1129
|
const rawVoice = body && typeof body.voice === 'string' && body.voice.trim() ? body.voice.trim() : null;
|
|
1039
1130
|
const voice = rawVoice || (dest ? dest.voice : (cfg.tts?.voice || DEFAULT_TTS_VOICE));
|
|
1040
|
-
|
|
1131
|
+
// `custom:<id>` names a saved voice. It is resolved to an embedding here so
|
|
1132
|
+
// the engine never has to know where voices are stored.
|
|
1133
|
+
let customId = dest ? null : ttsVoices.parseCustomVoice(voice);
|
|
1134
|
+
// The active model may REQUIRE an embedding while the configured voice still
|
|
1135
|
+
// names a built-in one — switching to SpeechT5 does not rewrite `voice`, and
|
|
1136
|
+
// a client that never sends one inherits whatever was there. Rather than
|
|
1137
|
+
// failing with "record a voice" at someone who already has, fall back to the
|
|
1138
|
+
// most recent saved voice. Erroring is right only when there is genuinely
|
|
1139
|
+
// none to use.
|
|
1140
|
+
if (!dest && !customId && ttsEngine.supportsCustomVoices()) {
|
|
1141
|
+
const saved = ttsVoices.listVoices();
|
|
1142
|
+
if (saved.length) customId = saved[0].id;
|
|
1143
|
+
}
|
|
1144
|
+
let speakerEmbedding = null;
|
|
1145
|
+
if (customId) {
|
|
1146
|
+
const rec = ttsVoices.getVoice(customId);
|
|
1147
|
+
if (!rec) return sendJson(res, 404, { error: { message: 'no such saved voice', type: 'bad_voice' } });
|
|
1148
|
+
if (!ttsEngine.supportsCustomVoices() && ttsEngine.isReady()) {
|
|
1149
|
+
return sendJson(res, 409, { error: { message: `the active model (${ttsEngine.arch()}) cannot use a recorded voice — switch to SpeechT5`, type: 'voice_unsupported' } });
|
|
1150
|
+
}
|
|
1151
|
+
speakerEmbedding = rec.vec;
|
|
1152
|
+
}
|
|
1153
|
+
const voiceOk = customId ? true
|
|
1154
|
+
: dest ? isValidRemoteVoice(voice)
|
|
1155
|
+
: (isKnownVoice(voice) && isValidVoiceId(voice));
|
|
1041
1156
|
if (!voiceOk) return sendJson(res, 400, { error: { message: 'unknown or invalid voice', type: 'bad_voice' } });
|
|
1042
1157
|
// We synthesize WAV only. Say so rather than returning WAV bytes under an mp3
|
|
1043
1158
|
// content-type — a client that trusts the header would play noise.
|
|
@@ -1073,7 +1188,7 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
1073
1188
|
dtype: cfg.tts?.dtype || 'auto',
|
|
1074
1189
|
});
|
|
1075
1190
|
if (!ok) return sendJson(res, 503, { error: { message: ttsEngine.health().error || 'tts model not ready', type: 'tts_unavailable' } });
|
|
1076
|
-
const pcm = await ttsEngine.synth(text, { voice, speed });
|
|
1191
|
+
const pcm = await ttsEngine.synth(text, { voice, speed, speakerEmbedding });
|
|
1077
1192
|
// The ACTIVE model's rate, not the constant: a VITS/MMS model emits 16 kHz
|
|
1078
1193
|
// and writing it into a 24 kHz header plays it fast and chipmunked.
|
|
1079
1194
|
const rate = ttsEngine.sampleRate();
|
|
@@ -1083,6 +1198,7 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
1083
1198
|
'Content-Length': String(out.length),
|
|
1084
1199
|
'Cache-Control': 'no-store',
|
|
1085
1200
|
'X-Tts-Sample-Rate': String(rate),
|
|
1201
|
+
...(customId ? { 'X-Tts-Voice': `custom:${customId}` } : {}),
|
|
1086
1202
|
});
|
|
1087
1203
|
return res.end(out);
|
|
1088
1204
|
} catch (e) {
|
|
@@ -1214,6 +1330,20 @@ export function createGateway(cfg = loadConfig()) {
|
|
|
1214
1330
|
return sendJson(res, 200, await aggregateModelsAsync(cfg));
|
|
1215
1331
|
}
|
|
1216
1332
|
|
|
1333
|
+
// Anything under a LOCAL namespace that reached here matched no route, which
|
|
1334
|
+
// almost always means the caller is newer than this gateway. Falling through to
|
|
1335
|
+
// the model proxy makes that arrive as "upstream fetch failed", pointing the
|
|
1336
|
+
// user at their model provider for a feature their gateway simply does not
|
|
1337
|
+
// have yet — so these 404 with the actual reason instead.
|
|
1338
|
+
if (LOCAL_NAMESPACES.some((ns) => pathname === ns || pathname.startsWith(`${ns}/`))) {
|
|
1339
|
+
return sendJson(res, 404, {
|
|
1340
|
+
error: {
|
|
1341
|
+
message: `this gateway (${VERSION}) has no ${pathname} — update it to use this feature`,
|
|
1342
|
+
type: 'unknown_endpoint',
|
|
1343
|
+
},
|
|
1344
|
+
});
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1217
1347
|
const r = route(pathname, req.headers, cfg);
|
|
1218
1348
|
let raw;
|
|
1219
1349
|
try {
|
package/src/tts-engine.js
CHANGED
|
@@ -41,14 +41,22 @@ let _dtype = null;
|
|
|
41
41
|
let _err = null;
|
|
42
42
|
let _progress = null;
|
|
43
43
|
let _initPromise = null;
|
|
44
|
-
let _arch = null; // 'style-tts2' (Kokoro) | 'vits' (MMS
|
|
44
|
+
let _arch = null; // 'style-tts2' (Kokoro) | 'vits' (MMS) | 'speecht5' (custom voices)
|
|
45
|
+
let _vocoder = null; // speecht5 only — mel → waveform
|
|
45
46
|
let _rate = SAMPLE_RATE; // the ACTIVE model's output rate
|
|
46
47
|
const _voices = new Map(); // voice id → Float32Array style bank
|
|
47
48
|
|
|
48
49
|
// The architectures this engine can actually drive. Anything else is refused at
|
|
49
50
|
// load with a message naming what it is, rather than failing later inside a
|
|
50
51
|
// forward pass with a shape error nobody can act on.
|
|
51
|
-
export const SUPPORTED_ARCH = { style_text_to_speech_2: 'style-tts2', vits: 'vits' };
|
|
52
|
+
export const SUPPORTED_ARCH = { style_text_to_speech_2: 'style-tts2', vits: 'vits', speecht5: 'speecht5' };
|
|
53
|
+
|
|
54
|
+
// SpeechT5 is the only architecture here that takes a SPEAKER EMBEDDING, which is
|
|
55
|
+
// what makes a custom voice possible at all: Kokoro's voices are fixed style banks
|
|
56
|
+
// and VITS is single-speaker, so neither can be pointed at a person. It needs a
|
|
57
|
+
// separate vocoder (mel → waveform), hence the extra model id.
|
|
58
|
+
export const SPEECHT5_VOCODER = 'Xenova/speecht5_hifigan';
|
|
59
|
+
export function supportsCustomVoices() { return _arch === 'speecht5'; }
|
|
52
60
|
|
|
53
61
|
export function arch() { return _arch; }
|
|
54
62
|
export function sampleRate() { return _rate; }
|
|
@@ -61,7 +69,7 @@ export function isReady() { return _state === 'ready' && !!_net; }
|
|
|
61
69
|
export function progress() { return _progress; }
|
|
62
70
|
|
|
63
71
|
export function health() {
|
|
64
|
-
return { configured: _state !== 'off', ok: isReady(), state: _state, model: _model, dtype: _dtype, runtime: runtimeName(), error: _err, arch: _arch, sampleRate: _rate, voices: supportsVoices() };
|
|
72
|
+
return { configured: _state !== 'off', ok: isReady(), state: _state, model: _model, dtype: _dtype, runtime: runtimeName(), error: _err, arch: _arch, sampleRate: _rate, voices: supportsVoices(), customVoices: supportsCustomVoices() };
|
|
65
73
|
}
|
|
66
74
|
|
|
67
75
|
export function modelDir(modelId) {
|
|
@@ -151,15 +159,22 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true, dtype:
|
|
|
151
159
|
const onProgress = (p) => {
|
|
152
160
|
if (p?.status === 'progress' && p.file) _progress = { model: modelId, file: p.file, pct: Math.round(p.progress || 0) };
|
|
153
161
|
};
|
|
154
|
-
const Klass = kind === 'vits' ? tf.VitsModel
|
|
162
|
+
const Klass = kind === 'vits' ? tf.VitsModel
|
|
163
|
+
: kind === 'speecht5' ? tf.SpeechT5ForTextToSpeech
|
|
164
|
+
: tf.StyleTextToSpeech2Model;
|
|
155
165
|
const [net, tok] = await Promise.all([
|
|
156
166
|
Klass.from_pretrained(modelId, { dtype, progress_callback: onProgress }),
|
|
157
167
|
tf.AutoTokenizer.from_pretrained(modelId),
|
|
158
168
|
]);
|
|
169
|
+
// The vocoder is a second download and a second failure point, so it is loaded
|
|
170
|
+
// only for the architecture that needs one.
|
|
171
|
+
_vocoder = kind === 'speecht5'
|
|
172
|
+
? await tf.SpeechT5HifiGan.from_pretrained(SPEECHT5_VOCODER, { dtype, progress_callback: onProgress })
|
|
173
|
+
: null;
|
|
159
174
|
_net = net; _tok = tok; _model = modelId; _dtype = dtype; _arch = kind;
|
|
160
175
|
// VITS/MMS emit 16 kHz; Kokoro 24 kHz. Take it from the model's own config
|
|
161
176
|
// where it says so, because guessing wrong plays the voice at the wrong pitch.
|
|
162
|
-
_rate = Number(net?.config?.sampling_rate) || (kind === '
|
|
177
|
+
_rate = Number(net?.config?.sampling_rate) || (kind === 'style-tts2' ? SAMPLE_RATE : 16000);
|
|
163
178
|
_state = 'ready'; _err = null; _progress = null;
|
|
164
179
|
log(`[tts] ready — model ${modelId} @ ${dtype} (${kind}, ${_rate} Hz, ${runtimeName()}, offline) — local speech active`);
|
|
165
180
|
return true;
|
|
@@ -260,7 +275,7 @@ export function splitSentences(text, maxChars = 300) {
|
|
|
260
275
|
* Synthesize ONE chunk. Returns 24 kHz mono Float32 PCM.
|
|
261
276
|
* @param {string} text @param {{voice?: string, speed?: number}} [opts]
|
|
262
277
|
*/
|
|
263
|
-
export async function synthChunk(text, { voice = DEFAULT_TTS_VOICE, speed = 1 } = {}) {
|
|
278
|
+
export async function synthChunk(text, { voice = DEFAULT_TTS_VOICE, speed = 1, speakerEmbedding = null } = {}) {
|
|
264
279
|
if (!isReady()) throw new Error('tts model not ready');
|
|
265
280
|
const tf = await import('@huggingface/transformers');
|
|
266
281
|
|
|
@@ -273,6 +288,20 @@ export async function synthChunk(text, { voice = DEFAULT_TTS_VOICE, speed = 1 }
|
|
|
273
288
|
return out.waveform.data;
|
|
274
289
|
}
|
|
275
290
|
|
|
291
|
+
// SpeechT5: conditioned by a 512-d speaker embedding, which is the whole point —
|
|
292
|
+
// it is the one architecture here that can be pointed at a person's voice.
|
|
293
|
+
// Without an embedding there is no voice to speak in, so this refuses rather
|
|
294
|
+
// than inventing one.
|
|
295
|
+
if (_arch === 'speecht5') {
|
|
296
|
+
if (!speakerEmbedding || speakerEmbedding.length !== 512) {
|
|
297
|
+
throw new Error('this model needs a saved voice — record one in Settings → Text-to-speech');
|
|
298
|
+
}
|
|
299
|
+
const { input_ids } = _tok(String(text));
|
|
300
|
+
const emb = new tf.Tensor('float32', Float32Array.from(speakerEmbedding), [1, 512]);
|
|
301
|
+
const { waveform } = await _net.generate_speech(input_ids, emb, { vocoder: _vocoder });
|
|
302
|
+
return waveform.data;
|
|
303
|
+
}
|
|
304
|
+
|
|
276
305
|
const { phonemize } = await import('phonemizer');
|
|
277
306
|
|
|
278
307
|
// G2P follows the VOICE, not the request — an American voice reading British
|
|
@@ -294,11 +323,11 @@ export async function synthChunk(text, { voice = DEFAULT_TTS_VOICE, speed = 1 }
|
|
|
294
323
|
}
|
|
295
324
|
|
|
296
325
|
/** Synthesize arbitrary-length text, chunk by chunk. `onChunk` sees each as it lands. */
|
|
297
|
-
export async function synth(text, { voice = DEFAULT_TTS_VOICE, speed = 1, onChunk = null } = {}) {
|
|
326
|
+
export async function synth(text, { voice = DEFAULT_TTS_VOICE, speed = 1, speakerEmbedding = null, onChunk = null } = {}) {
|
|
298
327
|
const chunks = splitSentences(text);
|
|
299
328
|
const out = [];
|
|
300
329
|
for (const c of chunks) {
|
|
301
|
-
const pcm = await synthChunk(c, { voice, speed });
|
|
330
|
+
const pcm = await synthChunk(c, { voice, speed, speakerEmbedding });
|
|
302
331
|
out.push(pcm);
|
|
303
332
|
onChunk?.(pcm);
|
|
304
333
|
}
|
|
@@ -337,6 +366,6 @@ export function toWav(pcm, sampleRate = SAMPLE_RATE) {
|
|
|
337
366
|
|
|
338
367
|
export function _reset() {
|
|
339
368
|
_state = 'off'; _model = null; _net = null; _tok = null; _dtype = null;
|
|
340
|
-
_err = null; _progress = null; _initPromise = null; _arch = null; _rate = SAMPLE_RATE;
|
|
369
|
+
_err = null; _progress = null; _initPromise = null; _arch = null; _rate = SAMPLE_RATE; _vocoder = null;
|
|
341
370
|
_voices.clear();
|
|
342
371
|
}
|
package/src/tts-models.js
CHANGED
|
@@ -55,6 +55,27 @@ export const TTS_MODEL_CATALOG = [
|
|
|
55
55
|
voices: true,
|
|
56
56
|
note: 'The Mandarin-tuned Kokoro. Same engine and voice mechanism as v1.0.',
|
|
57
57
|
},
|
|
58
|
+
{
|
|
59
|
+
id: 'Xenova/speecht5_tts',
|
|
60
|
+
label: 'SpeechT5 — your own voice',
|
|
61
|
+
lang: 'English',
|
|
62
|
+
tier: 'custom',
|
|
63
|
+
arch: 'speecht5',
|
|
64
|
+
// PINNED to fp32, and not as a preference. At the runtime default (q8 on
|
|
65
|
+
// native) this model renders about half of all speaker embeddings as
|
|
66
|
+
// near-silence — measured: 91-92% of samples under the noise floor, where the
|
|
67
|
+
// same embeddings at fp32 produce clean speech at ~47% silence, which is just
|
|
68
|
+
// the pauses between words. The quantization is destroying the speaker
|
|
69
|
+
// conditioning, and the failure looks exactly like "voice cloning does not
|
|
70
|
+
// work" rather than "the precision is wrong".
|
|
71
|
+
dtype: 'fp32',
|
|
72
|
+
approxMB: 190, // plus the ~50 MB HiFi-GAN vocoder it needs
|
|
73
|
+
ramMB: 500,
|
|
74
|
+
sampleRate: 16000,
|
|
75
|
+
voices: false, // no built-in voices…
|
|
76
|
+
customVoices: true, // …but it is the ONE model here that can use yours
|
|
77
|
+
note: 'The only model here that speaks in a voice you record. Rougher than Kokoro, and the match is approximate — see the note under Your voices.',
|
|
78
|
+
},
|
|
58
79
|
{
|
|
59
80
|
// One MMS entry so the second architecture is DISCOVERABLE from the list rather
|
|
60
81
|
// than only findable by search. The rest of the family (~1000 languages) is
|
|
@@ -123,6 +144,12 @@ export function ttsModelHasVoices(id) {
|
|
|
123
144
|
return m ? m.voices !== false : true;
|
|
124
145
|
}
|
|
125
146
|
|
|
147
|
+
// Can it speak in a voice the user recorded? Only SpeechT5 takes a speaker
|
|
148
|
+
// embedding — the other two are conditioned on something fixed.
|
|
149
|
+
export function ttsModelHasCustomVoices(id) {
|
|
150
|
+
return ttsModel(id)?.customVoices === true;
|
|
151
|
+
}
|
|
152
|
+
|
|
126
153
|
export function ttsModel(id) {
|
|
127
154
|
return TTS_MODEL_CATALOG.find((m) => m.id === id) || null;
|
|
128
155
|
}
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
// Custom voices — a speaker embedding derived from a sample the user recorded.
|
|
2
|
+
//
|
|
3
|
+
// A voice print is biometric-adjacent data about a specific person, so the rules
|
|
4
|
+
// here are deliberately tighter than for anything else this gateway stores:
|
|
5
|
+
// • It NEVER leaves the machine. The embedding is computed in-process by the
|
|
6
|
+
// speaker model already on disk (the one diarization uses), written under
|
|
7
|
+
// ~/.chatpanel, and read back only to condition local synthesis.
|
|
8
|
+
// • The SAMPLE is never persisted. Only the 512 floats derived from it — audio
|
|
9
|
+
// of someone speaking is far more revealing than the vector, and keeping it
|
|
10
|
+
// would serve no purpose once the embedding exists.
|
|
11
|
+
// • Deleting means the file is gone, not flagged. There is no soft-delete for
|
|
12
|
+
// someone's voice.
|
|
13
|
+
//
|
|
14
|
+
// What this can and cannot do is stated plainly in the UI, and the reason lives
|
|
15
|
+
// here: SpeechT5 was trained against speechbrain x-vectors and we embed with
|
|
16
|
+
// wavlm-base-plus-sv, which is a different 512-d space. The result is a distinct,
|
|
17
|
+
// STABLE voice derived from the sample — the same sample always gives the same
|
|
18
|
+
// voice, and different speakers give clearly different ones — but it is not a
|
|
19
|
+
// faithful reproduction of the speaker. Calling that "cloning" without saying so
|
|
20
|
+
// would be a lie told by a feature name.
|
|
21
|
+
|
|
22
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync, rmSync } from 'node:fs';
|
|
23
|
+
import { join } from 'node:path';
|
|
24
|
+
import { randomUUID } from 'node:crypto';
|
|
25
|
+
import os from 'node:os';
|
|
26
|
+
|
|
27
|
+
export const EMBED_DIM = 512;
|
|
28
|
+
const MAX_VOICES = 20;
|
|
29
|
+
const MAX_NAME = 60;
|
|
30
|
+
|
|
31
|
+
export function voicesDir() {
|
|
32
|
+
return process.env.CHATPANEL_TTS_VOICES || join(os.homedir(), '.chatpanel', 'tts-voices');
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
// An id becomes a FILENAME and arrives from the wire, so it is shape-checked, not
|
|
36
|
+
// trusted. UUIDs are what we mint; the pattern is what we accept.
|
|
37
|
+
export function isValidVoiceRef(id) {
|
|
38
|
+
return /^[A-Za-z0-9_-]{1,64}$/.test(String(id || ''));
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// The wire form is `custom:<id>` so one `voice` field can name either a built-in
|
|
42
|
+
// Kokoro voice or a saved one, with no ambiguity between the two namespaces.
|
|
43
|
+
export function parseCustomVoice(voice) {
|
|
44
|
+
const s = String(voice || '');
|
|
45
|
+
if (!s.startsWith('custom:')) return null;
|
|
46
|
+
const id = s.slice(7);
|
|
47
|
+
return isValidVoiceRef(id) ? id : null;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function fileFor(id) {
|
|
51
|
+
return join(voicesDir(), `${id}.json`);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function listVoices() {
|
|
55
|
+
const dir = voicesDir();
|
|
56
|
+
if (!existsSync(dir)) return [];
|
|
57
|
+
const out = [];
|
|
58
|
+
for (const f of readdirSync(dir)) {
|
|
59
|
+
if (!f.endsWith('.json')) continue;
|
|
60
|
+
try {
|
|
61
|
+
const v = JSON.parse(readFileSync(join(dir, f), 'utf8'));
|
|
62
|
+
// The vector is deliberately NOT returned by the listing: the UI needs a
|
|
63
|
+
// name and an id, and 512 floats of someone's voice print have no business
|
|
64
|
+
// in a settings page's JSON.
|
|
65
|
+
if (v?.id && v?.name) out.push({ id: v.id, name: v.name, createdAt: v.createdAt || 0, dim: v.vec?.length || 0 });
|
|
66
|
+
} catch { /* a corrupt file must not break the list */ }
|
|
67
|
+
}
|
|
68
|
+
return out.sort((a, b) => b.createdAt - a.createdAt);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function getVoice(id) {
|
|
72
|
+
if (!isValidVoiceRef(id)) return null;
|
|
73
|
+
try {
|
|
74
|
+
const v = JSON.parse(readFileSync(fileFor(id), 'utf8'));
|
|
75
|
+
return Array.isArray(v?.vec) && v.vec.length === EMBED_DIM ? v : null;
|
|
76
|
+
} catch { return null; }
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** Persist an embedding under a user-chosen name. Returns the stored record. */
|
|
80
|
+
export function saveVoice({ name, vec }) {
|
|
81
|
+
const clean = String(name || '').trim().slice(0, MAX_NAME);
|
|
82
|
+
if (!clean) throw new Error('a name is required');
|
|
83
|
+
if (!vec || vec.length !== EMBED_DIM) throw new Error(`expected a ${EMBED_DIM}-value embedding, got ${vec?.length || 0}`);
|
|
84
|
+
if (listVoices().length >= MAX_VOICES) throw new Error(`at most ${MAX_VOICES} saved voices — delete one first`);
|
|
85
|
+
const dir = voicesDir();
|
|
86
|
+
mkdirSync(dir, { recursive: true });
|
|
87
|
+
const rec = { id: randomUUID(), name: clean, createdAt: Date.now(), vec: Array.from(vec, (x) => Number(x) || 0) };
|
|
88
|
+
writeFileSync(fileFor(rec.id), JSON.stringify(rec));
|
|
89
|
+
return { id: rec.id, name: rec.name, createdAt: rec.createdAt, dim: rec.vec.length };
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Remove one permanently. Returns whether there was anything to remove. */
|
|
93
|
+
export function deleteVoice(id) {
|
|
94
|
+
if (!isValidVoiceRef(id)) return false;
|
|
95
|
+
const p = fileFor(id);
|
|
96
|
+
if (!existsSync(p)) return false;
|
|
97
|
+
rmSync(p, { force: true });
|
|
98
|
+
return true;
|
|
99
|
+
}
|