@chatpanel/gateway 0.6.52 → 0.6.53

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/gateway",
-  "version": "0.6.52",
+  "version": "0.6.53",
   "description": "Local privacy gateway — redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
   "type": "module",
   "bin": {

package/src/server.js

@@ -42,6 +42,7 @@ import { STT_MODEL_CATALOG, isKnownSttModel, isValidCustomSttId, DEFAULT_STT_MOD
 import * as ttsEngine from './tts-engine.js';
 import { TTS_MODEL_CATALOG, TTS_VOICES, isKnownTtsModel, isValidCustomTtsId, isKnownVoice, isValidVoiceId, DEFAULT_TTS_MODEL, DEFAULT_TTS_VOICE, TTS_DTYPES, isValidTtsDtype, MAX_TTS_CHARS } from './tts-models.js';
 import { ttsDestination, synthesizeRemote, isValidRemoteVoice } from './tts-remote.js';
+import * as ttsVoices from './tts-voices.js';
 import { resolvePro, checkQuota, consume, usage } from './freegate.js';
 import { publicConfig, applyConfigPatch, applyNerModelSelection, persistConfig, configPath } from './configstore.js';
 import { resolveDestination, aggregateModelsAsync, listDestinations } from './router.js';
@@ -52,7 +53,7 @@ import * as openai from './openai.js';
 import * as responses from './responses.js';
 import * as anthropic from './anthropic.js';
 
-export const VERSION = '0.6.52';
+export const VERSION = '0.6.53';
 
 // WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
 // store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
@@ -986,6 +987,7 @@ export function createGateway(cfg = loadConfig()) {
           // hide the picker rather than offer choices that cannot take effect.
           arch: ttsEngine.arch(),
           supportsVoices: ttsEngine.supportsVoices(),
+          supportsCustomVoices: ttsEngine.supportsCustomVoices(),
           sampleRate: ttsEngine.sampleRate(),
           voices: ttsEngine.arch() === 'vits' ? [] : TTS_VOICES.map((v) => ({ ...v, installed: ttsEngine.voiceOnDisk(v.id, active) })),
           dtype: cfg.tts?.dtype || 'auto',
@@ -1001,8 +1003,14 @@ export function createGateway(cfg = loadConfig()) {
         if (id && !(isKnownTtsModel(id) || isValidCustomTtsId(id))) return sendJson(res, 400, { error: { message: 'unknown or invalid model id', type: 'bad_model' } });
         // A voice id becomes a filename, so it is checked against the catalog AND
         // its shape before it is ever persisted.
+        // A default voice may be a built-in Kokoro one OR a saved custom one; both
+        // live in the same field, so both shapes are accepted and both validated.
         const voice = body && typeof body.voice === 'string' ? body.voice.trim() : null;
-        if (voice && !(isKnownVoice(voice) && isValidVoiceId(voice))) return sendJson(res, 400, { error: { message: 'unknown or invalid voice', type: 'bad_voice' } });
+        if (voice) {
+          const cid = ttsVoices.parseCustomVoice(voice);
+          const okVoice = cid ? !!ttsVoices.getVoice(cid) : (isKnownVoice(voice) && isValidVoiceId(voice));
+          if (!okVoice) return sendJson(res, 400, { error: { message: 'unknown or invalid voice', type: 'bad_voice' } });
+        }
         const dtype = body && typeof body.dtype === 'string' && isValidTtsDtype(body.dtype) ? body.dtype : undefined;
         if (!cfg.tts) cfg.tts = { enabled: true, model: DEFAULT_TTS_MODEL, voice: DEFAULT_TTS_VOICE, allowDownload: true };
         if (id) cfg.tts.model = id;
@@ -1014,6 +1022,59 @@ export function createGateway(cfg = loadConfig()) {
       }
     }
 
+    // --- Custom voices: a speaker embedding derived from a sample the user
+    // recorded. The AUDIO is embedded in-process and then discarded; only the 512
+    // floats are stored, under ~/.chatpanel, and they never leave this machine.
+    // See src/tts-voices.js for why the rules here are tighter than elsewhere.
+    if (pathname === '/tts/voices') {
+      if (req.method === 'GET') {
+        return sendJson(res, 200, {
+          voices: ttsVoices.listVoices(),
+          // Whether a saved voice can actually be USED right now depends on the
+          // active model — only SpeechT5 takes an embedding. Saying so here stops
+          // the UI offering voices that would be silently ignored.
+          usable: ttsEngine.supportsCustomVoices(),
+          embedder: diarizeEngine.DIARIZE_MODEL,
+          embedderReady: diarizeEngine.isReady(),
+        });
+      }
+      if (req.method === 'POST') {
+        let body = null;
+        try { body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')); } catch { body = null; }
+        const name = body && typeof body.name === 'string' ? body.name.trim() : '';
+        const pcm = body && Array.isArray(body.pcm) ? body.pcm : null;
+        if (!name) return sendJson(res, 400, { error: { message: 'a name is required', type: 'bad_request' } });
+        if (!pcm || pcm.length < 16000) {
+          // Under a second of audio produces an embedding dominated by whatever
+          // noise happened to be in it, and the resulting voice is arbitrary.
+          return sendJson(res, 400, { error: { message: 'need at least 1 second of 16 kHz mono audio', type: 'sample_too_short' } });
+        }
+        try {
+          // The embedder is the speaker model diarization already uses. If it is
+          // not resident yet, start it and say so — a ~100 MB download is not
+          // something to do silently while the user waits on a spinner.
+          if (!diarizeEngine.isReady()) {
+            diarizeEngine.download({ onLog: (m) => console.log(m) });
+            return sendJson(res, 503, {
+              error: { message: 'the speaker model is downloading (~100 MB) — try again in a moment', type: 'embedder_not_ready' },
+              progress: diarizeEngine.progress(),
+            });
+          }
+          const vec = await diarizeEngine.embed(Float32Array.from(pcm));
+          const saved = ttsVoices.saveVoice({ name, vec });
+          console.log(`[tts] saved custom voice "${saved.name}" (${saved.dim}-d, sample discarded)`);
+          return sendJson(res, 201, { ...saved, usable: ttsEngine.supportsCustomVoices() });
+        } catch (e) {
+          return sendJson(res, 400, { error: { message: e.message, type: 'save_failed' } });
+        }
+      }
+      if (req.method === 'DELETE') {
+        const id = url.searchParams.get('id') || '';
+        // Deleting someone's voice print is not a soft delete — the file is gone.
+        return sendJson(res, 200, { deleted: ttsVoices.deleteVoice(id) });
+      }
+    }
+
     // POST /tts — { text, voice?, speed? } → audio/wav — and its OpenAI-compatible
     // twin POST /v1/audio/speech ({ input, voice, speed, response_format }), so any
     // OpenAI client or tunnel can drive local speech with no ChatPanel-specific code.
@@ -1037,7 +1098,21 @@ export function createGateway(cfg = loadConfig()) {
       // not a Kokoro one), so the local catalog check would reject every valid id.
       const rawVoice = body && typeof body.voice === 'string' && body.voice.trim() ? body.voice.trim() : null;
       const voice = rawVoice || (dest ? dest.voice : (cfg.tts?.voice || DEFAULT_TTS_VOICE));
-      const voiceOk = dest ? isValidRemoteVoice(voice) : (isKnownVoice(voice) && isValidVoiceId(voice));
+      // `custom:<id>` names a saved voice. It is resolved to an embedding here so
+      // the engine never has to know where voices are stored.
+      const customId = dest ? null : ttsVoices.parseCustomVoice(voice);
+      let speakerEmbedding = null;
+      if (customId) {
+        const rec = ttsVoices.getVoice(customId);
+        if (!rec) return sendJson(res, 404, { error: { message: 'no such saved voice', type: 'bad_voice' } });
+        if (!ttsEngine.supportsCustomVoices() && ttsEngine.isReady()) {
+          return sendJson(res, 409, { error: { message: `the active model (${ttsEngine.arch()}) cannot use a recorded voice — switch to SpeechT5`, type: 'voice_unsupported' } });
+        }
+        speakerEmbedding = rec.vec;
+      }
+      const voiceOk = customId ? true
+        : dest ? isValidRemoteVoice(voice)
+          : (isKnownVoice(voice) && isValidVoiceId(voice));
       if (!voiceOk) return sendJson(res, 400, { error: { message: 'unknown or invalid voice', type: 'bad_voice' } });
       // We synthesize WAV only. Say so rather than returning WAV bytes under an mp3
       // content-type — a client that trusts the header would play noise.
@@ -1073,7 +1148,7 @@ export function createGateway(cfg = loadConfig()) {
           dtype: cfg.tts?.dtype || 'auto',
         });
         if (!ok) return sendJson(res, 503, { error: { message: ttsEngine.health().error || 'tts model not ready', type: 'tts_unavailable' } });
-        const pcm = await ttsEngine.synth(text, { voice, speed });
+        const pcm = await ttsEngine.synth(text, { voice, speed, speakerEmbedding });
         // The ACTIVE model's rate, not the constant: a VITS/MMS model emits 16 kHz
         // and writing it into a 24 kHz header plays it fast and chipmunked.
         const rate = ttsEngine.sampleRate();

package/src/tts-engine.js

@@ -41,14 +41,22 @@ let _dtype = null;
 let _err = null;
 let _progress = null;
 let _initPromise = null;
-let _arch = null;         // 'style-tts2' (Kokoro) | 'vits' (MMS et al)
+let _arch = null;         // 'style-tts2' (Kokoro) | 'vits' (MMS) | 'speecht5' (custom voices)
+let _vocoder = null;      // speecht5 only — mel → waveform
 let _rate = SAMPLE_RATE;  // the ACTIVE model's output rate
 const _voices = new Map(); // voice id → Float32Array style bank
 
 // The architectures this engine can actually drive. Anything else is refused at
 // load with a message naming what it is, rather than failing later inside a
 // forward pass with a shape error nobody can act on.
-export const SUPPORTED_ARCH = { style_text_to_speech_2: 'style-tts2', vits: 'vits' };
+export const SUPPORTED_ARCH = { style_text_to_speech_2: 'style-tts2', vits: 'vits', speecht5: 'speecht5' };
+
+// SpeechT5 is the only architecture here that takes a SPEAKER EMBEDDING, which is
+// what makes a custom voice possible at all: Kokoro's voices are fixed style banks
+// and VITS is single-speaker, so neither can be pointed at a person. It needs a
+// separate vocoder (mel → waveform), hence the extra model id.
+export const SPEECHT5_VOCODER = 'Xenova/speecht5_hifigan';
+export function supportsCustomVoices() { return _arch === 'speecht5'; }
 
 export function arch() { return _arch; }
 export function sampleRate() { return _rate; }
@@ -61,7 +69,7 @@ export function isReady() { return _state === 'ready' && !!_net; }
 export function progress() { return _progress; }
 
 export function health() {
-  return { configured: _state !== 'off', ok: isReady(), state: _state, model: _model, dtype: _dtype, runtime: runtimeName(), error: _err, arch: _arch, sampleRate: _rate, voices: supportsVoices() };
+  return { configured: _state !== 'off', ok: isReady(), state: _state, model: _model, dtype: _dtype, runtime: runtimeName(), error: _err, arch: _arch, sampleRate: _rate, voices: supportsVoices(), customVoices: supportsCustomVoices() };
 }
 
 export function modelDir(modelId) {
@@ -151,15 +159,22 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true, dtype:
     const onProgress = (p) => {
       if (p?.status === 'progress' && p.file) _progress = { model: modelId, file: p.file, pct: Math.round(p.progress || 0) };
     };
-    const Klass = kind === 'vits' ? tf.VitsModel : tf.StyleTextToSpeech2Model;
+    const Klass = kind === 'vits' ? tf.VitsModel
+      : kind === 'speecht5' ? tf.SpeechT5ForTextToSpeech
+        : tf.StyleTextToSpeech2Model;
     const [net, tok] = await Promise.all([
       Klass.from_pretrained(modelId, { dtype, progress_callback: onProgress }),
       tf.AutoTokenizer.from_pretrained(modelId),
     ]);
+    // The vocoder is a second download and a second failure point, so it is loaded
+    // only for the architecture that needs one.
+    _vocoder = kind === 'speecht5'
+      ? await tf.SpeechT5HifiGan.from_pretrained(SPEECHT5_VOCODER, { dtype, progress_callback: onProgress })
+      : null;
     _net = net; _tok = tok; _model = modelId; _dtype = dtype; _arch = kind;
     // VITS/MMS emit 16 kHz; Kokoro 24 kHz. Take it from the model's own config
     // where it says so, because guessing wrong plays the voice at the wrong pitch.
-    _rate = Number(net?.config?.sampling_rate) || (kind === 'vits' ? 16000 : SAMPLE_RATE);
+    _rate = Number(net?.config?.sampling_rate) || (kind === 'style-tts2' ? SAMPLE_RATE : 16000);
     _state = 'ready'; _err = null; _progress = null;
     log(`[tts] ready — model ${modelId} @ ${dtype} (${kind}, ${_rate} Hz, ${runtimeName()}, offline) — local speech active`);
     return true;
@@ -260,7 +275,7 @@ export function splitSentences(text, maxChars = 300) {
  * Synthesize ONE chunk. Returns 24 kHz mono Float32 PCM.
  * @param {string} text @param {{voice?: string, speed?: number}} [opts]
  */
-export async function synthChunk(text, { voice = DEFAULT_TTS_VOICE, speed = 1 } = {}) {
+export async function synthChunk(text, { voice = DEFAULT_TTS_VOICE, speed = 1, speakerEmbedding = null } = {}) {
   if (!isReady()) throw new Error('tts model not ready');
   const tf = await import('@huggingface/transformers');
 
@@ -273,6 +288,20 @@ export async function synthChunk(text, { voice = DEFAULT_TTS_VOICE, speed = 1 }
     return out.waveform.data;
   }
 
+  // SpeechT5: conditioned by a 512-d speaker embedding, which is the whole point —
+  // it is the one architecture here that can be pointed at a person's voice.
+  // Without an embedding there is no voice to speak in, so this refuses rather
+  // than inventing one.
+  if (_arch === 'speecht5') {
+    if (!speakerEmbedding || speakerEmbedding.length !== 512) {
+      throw new Error('this model needs a saved voice — record one in Settings → Text-to-speech');
+    }
+    const { input_ids } = _tok(String(text));
+    const emb = new tf.Tensor('float32', Float32Array.from(speakerEmbedding), [1, 512]);
+    const { waveform } = await _net.generate_speech(input_ids, emb, { vocoder: _vocoder });
+    return waveform.data;
+  }
+
   const { phonemize } = await import('phonemizer');
 
   // G2P follows the VOICE, not the request — an American voice reading British
@@ -294,11 +323,11 @@ export async function synthChunk(text, { voice = DEFAULT_TTS_VOICE, speed = 1 }
 }
 
 /** Synthesize arbitrary-length text, chunk by chunk. `onChunk` sees each as it lands. */
-export async function synth(text, { voice = DEFAULT_TTS_VOICE, speed = 1, onChunk = null } = {}) {
+export async function synth(text, { voice = DEFAULT_TTS_VOICE, speed = 1, speakerEmbedding = null, onChunk = null } = {}) {
   const chunks = splitSentences(text);
   const out = [];
   for (const c of chunks) {
-    const pcm = await synthChunk(c, { voice, speed });
+    const pcm = await synthChunk(c, { voice, speed, speakerEmbedding });
     out.push(pcm);
     onChunk?.(pcm);
   }
@@ -337,6 +366,6 @@ export function toWav(pcm, sampleRate = SAMPLE_RATE) {
 
 export function _reset() {
   _state = 'off'; _model = null; _net = null; _tok = null; _dtype = null;
-  _err = null; _progress = null; _initPromise = null; _arch = null; _rate = SAMPLE_RATE;
+  _err = null; _progress = null; _initPromise = null; _arch = null; _rate = SAMPLE_RATE; _vocoder = null;
   _voices.clear();
 }

package/src/tts-models.js

@@ -55,6 +55,27 @@ export const TTS_MODEL_CATALOG = [
     voices: true,
     note: 'The Mandarin-tuned Kokoro. Same engine and voice mechanism as v1.0.',
   },
+  {
+    id: 'Xenova/speecht5_tts',
+    label: 'SpeechT5 — your own voice',
+    lang: 'English',
+    tier: 'custom',
+    arch: 'speecht5',
+    // PINNED to fp32, and not as a preference. At the runtime default (q8 on
+    // native) this model renders about half of all speaker embeddings as
+    // near-silence — measured: 91-92% of samples under the noise floor, where the
+    // same embeddings at fp32 produce clean speech at ~47% silence, which is just
+    // the pauses between words. The quantization is destroying the speaker
+    // conditioning, and the failure looks exactly like "voice cloning does not
+    // work" rather than "the precision is wrong".
+    dtype: 'fp32',
+    approxMB: 190,   // plus the ~50 MB HiFi-GAN vocoder it needs
+    ramMB: 500,
+    sampleRate: 16000,
+    voices: false,          // no built-in voices…
+    customVoices: true,     // …but it is the ONE model here that can use yours
+    note: 'The only model here that speaks in a voice you record. Rougher than Kokoro, and the match is approximate — see the note under Your voices.',
+  },
   {
     // One MMS entry so the second architecture is DISCOVERABLE from the list rather
     // than only findable by search. The rest of the family (~1000 languages) is
@@ -123,6 +144,12 @@ export function ttsModelHasVoices(id) {
   return m ? m.voices !== false : true;
 }
 
+// Can it speak in a voice the user recorded? Only SpeechT5 takes a speaker
+// embedding — the other two are conditioned on something fixed.
+export function ttsModelHasCustomVoices(id) {
+  return ttsModel(id)?.customVoices === true;
+}
+
 export function ttsModel(id) {
   return TTS_MODEL_CATALOG.find((m) => m.id === id) || null;
 }

package/src/tts-voices.js

@@ -0,0 +1,99 @@
+// Custom voices — a speaker embedding derived from a sample the user recorded.
+//
+// A voice print is biometric-adjacent data about a specific person, so the rules
+// here are deliberately tighter than for anything else this gateway stores:
+//   • It NEVER leaves the machine. The embedding is computed in-process by the
+//     speaker model already on disk (the one diarization uses), written under
+//     ~/.chatpanel, and read back only to condition local synthesis.
+//   • The SAMPLE is never persisted. Only the 512 floats derived from it — audio
+//     of someone speaking is far more revealing than the vector, and keeping it
+//     would serve no purpose once the embedding exists.
+//   • Deleting means the file is gone, not flagged. There is no soft-delete for
+//     someone's voice.
+//
+// What this can and cannot do is stated plainly in the UI, and the reason lives
+// here: SpeechT5 was trained against speechbrain x-vectors and we embed with
+// wavlm-base-plus-sv, which is a different 512-d space. The result is a distinct,
+// STABLE voice derived from the sample — the same sample always gives the same
+// voice, and different speakers give clearly different ones — but it is not a
+// faithful reproduction of the speaker. Calling that "cloning" without saying so
+// would be a lie told by a feature name.
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync, readdirSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { randomUUID } from 'node:crypto';
+import os from 'node:os';
+
+export const EMBED_DIM = 512;
+const MAX_VOICES = 20;
+const MAX_NAME = 60;
+
+export function voicesDir() {
+  return process.env.CHATPANEL_TTS_VOICES || join(os.homedir(), '.chatpanel', 'tts-voices');
+}
+
+// An id becomes a FILENAME and arrives from the wire, so it is shape-checked, not
+// trusted. UUIDs are what we mint; the pattern is what we accept.
+export function isValidVoiceRef(id) {
+  return /^[A-Za-z0-9_-]{1,64}$/.test(String(id || ''));
+}
+
+// The wire form is `custom:<id>` so one `voice` field can name either a built-in
+// Kokoro voice or a saved one, with no ambiguity between the two namespaces.
+export function parseCustomVoice(voice) {
+  const s = String(voice || '');
+  if (!s.startsWith('custom:')) return null;
+  const id = s.slice(7);
+  return isValidVoiceRef(id) ? id : null;
+}
+
+function fileFor(id) {
+  return join(voicesDir(), `${id}.json`);
+}
+
+export function listVoices() {
+  const dir = voicesDir();
+  if (!existsSync(dir)) return [];
+  const out = [];
+  for (const f of readdirSync(dir)) {
+    if (!f.endsWith('.json')) continue;
+    try {
+      const v = JSON.parse(readFileSync(join(dir, f), 'utf8'));
+      // The vector is deliberately NOT returned by the listing: the UI needs a
+      // name and an id, and 512 floats of someone's voice print have no business
+      // in a settings page's JSON.
+      if (v?.id && v?.name) out.push({ id: v.id, name: v.name, createdAt: v.createdAt || 0, dim: v.vec?.length || 0 });
+    } catch { /* a corrupt file must not break the list */ }
+  }
+  return out.sort((a, b) => b.createdAt - a.createdAt);
+}
+
+export function getVoice(id) {
+  if (!isValidVoiceRef(id)) return null;
+  try {
+    const v = JSON.parse(readFileSync(fileFor(id), 'utf8'));
+    return Array.isArray(v?.vec) && v.vec.length === EMBED_DIM ? v : null;
+  } catch { return null; }
+}
+
+/** Persist an embedding under a user-chosen name. Returns the stored record. */
+export function saveVoice({ name, vec }) {
+  const clean = String(name || '').trim().slice(0, MAX_NAME);
+  if (!clean) throw new Error('a name is required');
+  if (!vec || vec.length !== EMBED_DIM) throw new Error(`expected a ${EMBED_DIM}-value embedding, got ${vec?.length || 0}`);
+  if (listVoices().length >= MAX_VOICES) throw new Error(`at most ${MAX_VOICES} saved voices — delete one first`);
+  const dir = voicesDir();
+  mkdirSync(dir, { recursive: true });
+  const rec = { id: randomUUID(), name: clean, createdAt: Date.now(), vec: Array.from(vec, (x) => Number(x) || 0) };
+  writeFileSync(fileFor(rec.id), JSON.stringify(rec));
+  return { id: rec.id, name: rec.name, createdAt: rec.createdAt, dim: rec.vec.length };
+}
+
+/** Remove one permanently. Returns whether there was anything to remove. */
+export function deleteVoice(id) {
+  if (!isValidVoiceRef(id)) return false;
+  const p = fileFor(id);
+  if (!existsSync(p)) return false;
+  rmSync(p, { force: true });
+  return true;
+}