@chatpanel/gateway 0.6.55 → 0.6.56

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/gateway",
-  "version": "0.6.55",
+  "version": "0.6.56",
   "description": "Local privacy gateway — redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
   "type": "module",
   "bin": {

package/src/pocket-tts-engine.js

@@ -0,0 +1,378 @@
+// Kyutai Pocket TTS — the engine that can actually speak in YOUR voice.
+//
+// Kokoro's voices are fixed style banks and SpeechT5 borrows a voice print from a
+// space it was not trained on, so neither reproduces a specific person. Pocket TTS
+// is built for it: its Mimi encoder turns a few seconds of audio into a voice
+// conditioning that seeds generation directly, which is why the same sample gives
+// back the same speaker rather than a distant relative of them.
+//
+// Ported from the reference browser implementation in the `KevinAHM/pocket-tts-web`
+// Hugging Face Space (Apache-2.0), adapted from a Web Worker to in-process Node:
+// fetch → fs, postMessage → return values, and the CDN onnxruntime → the one this
+// gateway already ships (parakeet-engine.js's getOrt() pattern, so the same code
+// runs on native ORT and on the binary's WASM build). Model weights are
+// CC-BY-4.0 from `KevinAHM/pocket-tts-onnx`.
+//
+// Five graphs, and the shape of a turn:
+//   text_conditioner  text tokens  → text embeddings
+//   mimi_encoder      voice sample → voice conditioning        (this is the cloning)
+//   flow_lm_main      autoregressive step → conditioning + an end-of-speech logit
+//   flow_lm_flow      flow-matching denoise of one latent frame
+//   mimi_decoder      latent frames → audio
+// flow_lm and mimi are STATEFUL: each call returns the state the next one needs,
+// described by a manifest in bundle.json rather than hardcoded here.
+
+import { join } from 'node:path';
+import { existsSync, mkdirSync, readFileSync, statSync, createWriteStream, renameSync } from 'node:fs';
+import { Readable } from 'node:stream';
+import { modelRoot } from './ner-engine.js';
+import { SentencePieceUnigram } from './sentencepiece.js';
+
+export const POCKET_REPO = 'KevinAHM/pocket-tts-onnx';
+export const DEFAULT_BUNDLE = 'english_2026-04';
+export const SAMPLE_RATE = 24000;
+
+// Generation constants, carried over from the reference implementation.
+const MAX_FRAMES = 500;          // hard ceiling per chunk (~40s at 12.5 fps)
+const LSD_STEPS = 1;             // flow-matching steps per frame
+const TEMPERATURE = 0.7;
+const EOS_LOGIT_THRESHOLD = -4.0;
+const FIRST_CHUNK_FRAMES = 3;    // decode early so audio starts sooner
+const NORMAL_CHUNK_FRAMES = 12;
+// The reference resets both states per text chunk; keeping them would let one
+// sentence's trailing state colour the next one's opening.
+const RESET_STATE_EACH_CHUNK = true;
+
+// voices.bin (the reference implementation's PREDEFINED speakers) is deliberately
+// absent: it lives only in the demo Space, not the weights repo, and this engine
+// exists to speak in a voice the user recorded. Kokoro already covers "pick a
+// stock voice", and far better.
+const FILES = (q = '_int8') => [
+  'bundle.json', 'tokenizer.model', 'bos_before_voice.npy',
+  `text_conditioner${q}.onnx`, `mimi_encoder${q}.onnx`, `mimi_decoder${q}.onnx`,
+  `flow_lm_main${q}.onnx`, `flow_lm_flow${q}.onnx`,
+];
+
+let _ortPromise = null;
+function getOrt() {
+  if (_ortPromise) return _ortPromise;
+  _ortPromise = (async () => {
+    const wasmPaths = globalThis.__CHATPANEL_WASM_PATHS__ || null;
+    const mod = await import(wasmPaths ? 'onnxruntime-web' : 'onnxruntime-node');
+    const ort = mod.InferenceSession ? mod : (mod.default || mod);
+    if (wasmPaths) {
+      try { ort.env.wasm.numThreads = 1; ort.env.wasm.proxy = false; ort.env.wasm.wasmPaths = wasmPaths; } catch { /* optional */ }
+    }
+    return ort;
+  })();
+  return _ortPromise;
+}
+
+export function bundleDir(bundle = DEFAULT_BUNDLE) {
+  return join(modelRoot(), 'pocket-tts', bundle);
+}
+
+export function bundleOnDisk(bundle = DEFAULT_BUNDLE, quant = '_int8') {
+  const dir = bundleDir(bundle);
+  try {
+    return FILES(quant).every((f) => { const p = join(dir, f); return existsSync(p) && statSync(p).size > 0; });
+  } catch { return false; }
+}
+
+async function downloadFile(bundle, file, dir, { onProgress, log } = {}) {
+  const url = `https://huggingface.co/${POCKET_REPO}/resolve/main/onnx/${bundle}/${file}`;
+  const res = await fetch(url, { redirect: 'follow' });
+  if (!res.ok || !res.body) throw new Error(`fetch ${file} → HTTP ${res.status}`);
+  const total = Number(res.headers.get('content-length')) || 0;
+  const tmp = join(dir, `${file}.part`);
+  const out = createWriteStream(tmp);
+  let got = 0, lastPct = -1;
+  const src = Readable.fromWeb(res.body);
+  src.on('data', (c) => {
+    got += c.length;
+    if (total) { const pct = Math.round((got / total) * 100); if (pct !== lastPct) { lastPct = pct; onProgress?.({ file, pct }); } }
+  });
+  await new Promise((resolve, reject) => { src.pipe(out); out.on('finish', resolve); out.on('error', reject); src.on('error', reject); });
+  // .part then rename, so an interrupted fetch never reads as installed.
+  renameSync(tmp, join(dir, file));
+  log?.(`[pocket-tts] fetched ${file}`);
+}
+
+export async function ensureBundle(bundle = DEFAULT_BUNDLE, quant = '_int8', { onProgress, log } = {}) {
+  const dir = bundleDir(bundle);
+  mkdirSync(dir, { recursive: true });
+  for (const file of FILES(quant)) {
+    const dest = join(dir, file);
+    if (existsSync(dest) && statSync(dest).size > 0) continue;
+    log?.(`[pocket-tts] downloading ${file}…`);
+    await downloadFile(bundle, file, dir, { onProgress, log });
+  }
+  return dir;
+}
+
+// ── .npy (float32) ──────────────────────────────────────────────────────────────
+export function parseNpyFloat32(buf) {
+  const magic = [0x93, 0x4e, 0x55, 0x4d, 0x50, 0x59];
+  for (let i = 0; i < magic.length; i++) if (buf[i] !== magic[i]) throw new Error('not an NPY file');
+  const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
+  const major = view.getUint8(6);
+  const headerLen = major === 1 ? view.getUint16(8, true) : view.getUint32(8, true);
+  const headerOffset = major === 1 ? 10 : 12;
+  const header = new TextDecoder().decode(buf.subarray(headerOffset, headerOffset + headerLen));
+  const m = /\(\s*([0-9,\s]*)\)/.exec(header);
+  if (!m) throw new Error('could not parse NPY shape');
+  const shape = m[1].split(',').map((x) => x.trim()).filter(Boolean).map((x) => parseInt(x, 10));
+  const start = headerOffset + headerLen;
+  const data = new Float32Array((buf.byteLength - start) / 4);
+  for (let i = 0; i < data.length; i++) data[i] = view.getFloat32(start + i * 4, true);
+  return { data, shape };
+}
+
+// ── state manifests ─────────────────────────────────────────────────────────────
+// Both stateful graphs describe their own state in bundle.json: which inputs to
+// seed, with what shape and fill, and which outputs feed them next turn. Reading
+// it beats hardcoding 18 + 56 tensor names that differ per language bundle.
+function filledArray(shape, dtype, fill) {
+  const size = shape.reduce((a, b) => a * b, 1);
+  if (dtype === 'int64') return new BigInt64Array(size);
+  if (dtype === 'bool') return new Uint8Array(size);
+  const d = new Float32Array(size);
+  if (fill === 'nan') d.fill(NaN);
+  else if (fill === 'ones') d.fill(1);
+  return d;
+}
+
+function initState(ort, manifest) {
+  const state = {};
+  for (const e of manifest) state[e.input_name] = new ort.Tensor(e.dtype, filledArray(e.shape, e.dtype, e.fill), e.shape);
+  return state;
+}
+
+function advanceState(state, result, manifest) {
+  for (const e of manifest) state[e.input_name] = result[e.output_name];
+}
+
+export class PocketTTS {
+  constructor() {
+    this.ready = false;
+    this.bundle = null;
+    this.meta = null;
+    this.tok = null;
+    this.bos = null;
+    this.sessions = null;
+    this.st = null;            // precomputed flow-matching s/t pairs
+    this.latentDim = 32;
+    this.condDim = 1024;
+    this.samplesPerFrame = 1920;
+  }
+
+  async load(bundle = DEFAULT_BUNDLE, { quant = '_int8', onProgress, log = () => {} } = {}) {
+    const dir = await ensureBundle(bundle, quant, { onProgress, log });
+    const ort = await getOrt();
+    this.meta = JSON.parse(readFileSync(join(dir, 'bundle.json'), 'utf8'));
+    this.tok = new SentencePieceUnigram(new Uint8Array(readFileSync(join(dir, 'tokenizer.model'))));
+    this.bos = this.meta.insert_bos_before_voice ? parseNpyFloat32(readFileSync(join(dir, 'bos_before_voice.npy'))) : null;
+    this.latentDim = Number(this.meta.latent_dim) || 32;
+    this.condDim = Number(this.meta.conditioning_dim) || 1024;
+    this.samplesPerFrame = Math.round(SAMPLE_RATE / (Number(this.meta.frame_rate) || 12.5));
+
+    const opts = { executionProviders: ['cpu'], graphOptimizationLevel: 'all', logSeverityLevel: 3 };
+    const [textConditioner, mimiEncoder, mimiDecoder, flowMain, flowFlow] = await Promise.all([
+      ort.InferenceSession.create(join(dir, `text_conditioner${quant}.onnx`), opts),
+      ort.InferenceSession.create(join(dir, `mimi_encoder${quant}.onnx`), opts),
+      ort.InferenceSession.create(join(dir, `mimi_decoder${quant}.onnx`), opts),
+      ort.InferenceSession.create(join(dir, `flow_lm_main${quant}.onnx`), opts),
+      ort.InferenceSession.create(join(dir, `flow_lm_flow${quant}.onnx`), opts),
+    ]);
+    this.sessions = { ort, textConditioner, mimiEncoder, mimiDecoder, flowMain, flowFlow };
+
+    // Flow matching walks s → t in fixed steps; the tensors never change, so they
+    // are built once rather than per frame (this runs MAX_FRAMES times a chunk).
+    this.st = [];
+    const dt = 1 / LSD_STEPS;
+    for (let i = 0; i < LSD_STEPS; i++) {
+      const s = i / LSD_STEPS;
+      this.st.push({
+        s: new ort.Tensor('float32', new Float32Array([s]), [1, 1]),
+        t: new ort.Tensor('float32', new Float32Array([s + dt]), [1, 1]),
+      });
+    }
+    this.bundle = bundle;
+    this.ready = true;
+    log(`[pocket-tts] ready — ${bundle} (${quant.replace('_', '') || 'fp32'}, ${SAMPLE_RATE} Hz)`);
+    return true;
+  }
+
+  /**
+   * THE CLONING STEP. A few seconds of 24 kHz mono audio → the voice conditioning
+   * that seeds generation. Returns { data, shape } to be stored as the voice print.
+   */
+  async encodeVoice(audio) {
+    if (!this.ready) throw new Error('pocket-tts not loaded');
+    const { ort, mimiEncoder } = this.sessions;
+    const pcm = audio instanceof Float32Array ? audio : Float32Array.from(audio);
+    const out = await mimiEncoder.run({ audio: new ort.Tensor('float32', pcm, [1, 1, pcm.length]) });
+    const emb = out[mimiEncoder.outputNames[0]];
+    let dims = emb.dims.slice();
+    while (dims.length > 3 && dims[0] === 1) dims = dims.slice(1);
+    if (dims.length < 3) dims = [1, dims[0], dims[1]];
+    return { data: Float32Array.from(emb.data), shape: dims };
+  }
+
+  // The voice conditioning is fed to flow_lm_main in the TEXT-embedding slot,
+  // preceded by the bundle's BOS frames — the model is told "this is how the
+  // speaker sounds" before it is told what to say.
+  #voiceTensor(voice) {
+    const { ort } = this.sessions;
+    let data = voice.data instanceof Float32Array ? voice.data : Float32Array.from(voice.data);
+    let dims = voice.shape.slice();
+    if (this.meta.insert_bos_before_voice && this.bos) {
+      const combined = new Float32Array(this.bos.data.length + data.length);
+      combined.set(this.bos.data, 0);
+      combined.set(data, this.bos.data.length);
+      data = combined;
+      dims = [1, dims[1] + this.bos.shape[1], dims[2]];
+    }
+    return new ort.Tensor('float32', data, dims);
+  }
+
+  async #voiceState(voice) {
+    const { ort, flowMain } = this.sessions;
+    const state = initState(ort, this.meta.flow_lm_state_manifest);
+    const result = await flowMain.run({
+      sequence: new ort.Tensor('float32', new Float32Array(0), [1, 0, this.latentDim]),
+      text_embeddings: this.#voiceTensor(voice),
+      ...state,
+    });
+    advanceState(state, result, this.meta.flow_lm_state_manifest);
+    return state;
+  }
+
+  // Normalization the model expects: one line, capitalized, terminated.
+  #prepare(text) {
+    let s = String(text).replace(/[\r\n]+/g, ' ').replace(/\s+/g, ' ').trim();
+    if (!s) return { text: '', framesAfterEos: 1 };
+    if (this.meta.remove_semicolons) s = s.replace(/;/g, ',');
+    const words = s.split(/\s+/).filter(Boolean).length;
+    let framesAfterEos = words <= 4 ? 3 : 1;
+    if (this.meta.model_recommended_frames_after_eos != null) framesAfterEos = Number(this.meta.model_recommended_frames_after_eos);
+    if (!/[A-ZÀ-Þ]/.test(s[0])) s = s[0].toUpperCase() + s.slice(1);
+    if (/[0-9A-Za-zÀ-ÿ]/.test(s[s.length - 1])) s += '.';
+    if (this.meta.pad_with_spaces_for_short_inputs && words < 5) s = `        ${s}`;
+    return { text: s, framesAfterEos };
+  }
+
+  // One forward pass per FRAME, so a long paragraph in a single chunk is a long
+  // time before any audio. Split on sentences, then on the model's token ceiling.
+  #chunks(text) {
+    const maxTokens = Number(this.meta.max_token_per_chunk) || 50;
+    const sentences = text.match(/[^.!?]+[.!?]+|[^.!?]+$/g) || [text];
+    const out = [];
+    for (const raw of sentences) {
+      const s = raw.trim();
+      if (!s) continue;
+      const ids = this.tok.encodeIds(s);
+      if (ids.length <= maxTokens) { out.push(s); continue; }
+      for (let i = 0; i < ids.length; i += maxTokens) {
+        const part = this.tok.decodeIds(ids.slice(i, i + maxTokens)).trim();
+        if (part) out.push(part);
+      }
+    }
+    return out.length ? out : [text];
+  }
+
+  /**
+   * Synthesize. `voice` is what encodeVoice() returned. `onAudio(pcm)` receives
+   * each decoded piece as it lands, so a caller can stream; the whole waveform is
+   * returned as well.
+   */
+  async synth(text, { voice, onAudio = null } = {}) {
+    if (!this.ready) throw new Error('pocket-tts not loaded');
+    if (!voice?.data?.length) throw new Error('pocket-tts needs a voice — record one');
+    const { ort, textConditioner, flowMain, flowFlow, mimiDecoder } = this.sessions;
+    const prepared = this.#prepare(text);
+    if (!prepared.text) return new Float32Array(0);
+    const chunks = this.#chunks(prepared.text);
+    const baseFlow = await this.#voiceState(voice);
+
+    const emptySeq = new ort.Tensor('float32', new Float32Array(0), [1, 0, this.latentDim]);
+    const emptyText = new ort.Tensor('float32', new Float32Array(0), [1, 0, this.condDim]);
+    let mimiState = initState(ort, this.meta.mimi_state_manifest);
+    let flowState = { ...baseFlow };
+    const pieces = [];
+    let first = true;
+
+    for (let c = 0; c < chunks.length; c++) {
+      if (RESET_STATE_EACH_CHUNK && c > 0) {
+        flowState = { ...baseFlow };
+        mimiState = initState(ort, this.meta.mimi_state_manifest);
+      }
+      const ids = this.tok.encodeIds(chunks[c]);
+      const tokens = new ort.Tensor('int64', BigInt64Array.from(ids, (t) => BigInt(t)), [1, ids.length]);
+      let textEmb = (await textConditioner.run({ token_ids: tokens }))[textConditioner.outputNames[0]];
+      if (textEmb.dims.length === 2) textEmb = new ort.Tensor('float32', Float32Array.from(textEmb.data), [1, textEmb.dims[0], textEmb.dims[1]]);
+
+      // Prime the LM with the text, then generate frames from silence.
+      advanceState(flowState, await flowMain.run({ sequence: emptySeq, text_embeddings: textEmb, ...flowState }), this.meta.flow_lm_state_manifest);
+
+      const latents = [];
+      let decoded = 0;
+      let cur = new ort.Tensor('float32', new Float32Array(this.latentDim).fill(NaN), [1, 1, this.latentDim]);
+      let eosAt = null;
+
+      for (let step = 0; step < MAX_FRAMES; step++) {
+        const ar = await flowMain.run({ sequence: cur, text_embeddings: emptyText, ...flowState });
+        const conditioning = ar.conditioning;
+        if (ar.eos_logit.data[0] > EOS_LOGIT_THRESHOLD && eosAt == null) eosAt = step;
+        const stop = eosAt != null && step >= eosAt + prepared.framesAfterEos;
+
+        // Start from Gaussian noise, then walk it along the learned flow field.
+        const std = Math.sqrt(TEMPERATURE);
+        const latent = new Float32Array(this.latentDim);
+        for (let i = 0; i < this.latentDim; i++) {
+          let u = 0, v = 0;
+          while (u === 0) u = Math.random();
+          while (v === 0) v = Math.random();
+          latent[i] = Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v) * std;
+        }
+        const dt = 1 / LSD_STEPS;
+        for (let k = 0; k < LSD_STEPS; k++) {
+          const f = await flowFlow.run({ c: conditioning, s: this.st[k].s, t: this.st[k].t, x: new ort.Tensor('float32', latent, [1, this.latentDim]) });
+          const dir = f.flow_dir.data;
+          for (let i = 0; i < this.latentDim; i++) latent[i] += dir[i] * dt;
+        }
+
+        latents.push(Float32Array.from(latent));
+        cur = new ort.Tensor('float32', latent, [1, 1, this.latentDim]);
+        advanceState(flowState, ar, this.meta.flow_lm_state_manifest);
+
+        // Decode in small batches — the first deliberately tiny so sound starts
+        // early, the rest larger because the decoder is cheaper in bulk.
+        const pending = latents.length - decoded;
+        let take = 0;
+        if (stop) take = pending;
+        else if (first && pending >= FIRST_CHUNK_FRAMES) take = FIRST_CHUNK_FRAMES;
+        else if (pending >= NORMAL_CHUNK_FRAMES) take = NORMAL_CHUNK_FRAMES;
+
+        if (take > 0) {
+          const buf = new Float32Array(take * this.latentDim);
+          for (let f = 0; f < take; f++) buf.set(latents[decoded + f], f * this.latentDim);
+          const dec = await mimiDecoder.run({ latent: new ort.Tensor('float32', buf, [1, take, this.latentDim]), ...mimiState });
+          advanceState(mimiState, dec, this.meta.mimi_state_manifest);
+          decoded += take;
+          first = false;
+          const pcm = Float32Array.from(dec[mimiDecoder.outputNames[0]].data);
+          pieces.push(pcm);
+          onAudio?.(pcm);
+        }
+        if (stop) break;
+      }
+    }
+
+    const total = pieces.reduce((n, p) => n + p.length, 0);
+    const out = new Float32Array(total);
+    let at = 0;
+    for (const p of pieces) { out.set(p, at); at += p.length; }
+    return out;
+  }
+}

package/src/sentencepiece.js

@@ -0,0 +1,184 @@
+// A minimal SentencePiece (Unigram) tokenizer — parse the .model protobuf and
+// encode with Viterbi.
+//
+// Why not a library: the reference implementation for pocket-tts ships a 4 MB
+// WASM-bundled sentencepiece build aimed at browsers. This gateway is
+// zero-runtime-dependency by design and also compiles into a Bun single-file
+// binary, so a 4 MB WASM blob for what is fundamentally a shortest-path search
+// over a 4,000-entry vocabulary is the wrong trade.
+//
+// Scope is deliberately narrow: UNIGRAM models with byte fallback, which is what
+// pocket-tts uses (4,000 pieces, each with a log-probability score, plus the 256
+// <0xNN> byte pieces). A BPE model would need a different algorithm and is
+// rejected at load rather than silently mis-tokenized.
+
+const UNK = 0, NORMAL_TYPE = 1, CONTROL = 3, BYTE = 6;
+// SentencePiece represents a space as U+2581 LOWER ONE EIGHTH BLOCK.
+const SPACE = '▁';
+
+function readVarint(buf, i) {
+  let result = 0, shift = 0;
+  for (;;) {
+    const b = buf[i++];
+    result += (b & 0x7f) * 2 ** shift;
+    if ((b & 0x80) === 0) return [result, i];
+    shift += 7;
+    if (shift > 49) throw new Error('varint too long');
+  }
+}
+
+// ModelProto { repeated SentencePiece pieces = 1; ... }
+// SentencePiece { string piece = 1; float score = 2; Type type = 3; }
+function parseModelProto(buf) {
+  const pieces = [];
+  let i = 0;
+  while (i < buf.length) {
+    let key;
+    [key, i] = readVarint(buf, i);
+    const field = key >> 3, wire = key & 7;
+    if (wire === 2) {
+      let len;
+      [len, i] = readVarint(buf, i);
+      const chunk = buf.subarray(i, i + len);
+      i += len;
+      if (field === 1) pieces.push(parsePiece(chunk));
+    } else if (wire === 0) {
+      [, i] = readVarint(buf, i);
+    } else if (wire === 5) {
+      i += 4;
+    } else if (wire === 1) {
+      i += 8;
+    } else {
+      throw new Error(`unsupported protobuf wire type ${wire}`);
+    }
+  }
+  return pieces;
+}
+
+function parsePiece(buf) {
+  let i = 0, piece = '', score = 0, type = NORMAL_TYPE;
+  while (i < buf.length) {
+    let key;
+    [key, i] = readVarint(buf, i);
+    const field = key >> 3, wire = key & 7;
+    if (wire === 2) {
+      let len;
+      [len, i] = readVarint(buf, i);
+      const val = buf.subarray(i, i + len);
+      i += len;
+      if (field === 1) piece = new TextDecoder().decode(val);
+    } else if (wire === 5) {
+      if (field === 2) score = new DataView(buf.buffer, buf.byteOffset + i, 4).getFloat32(0, true);
+      i += 4;
+    } else if (wire === 0) {
+      let v;
+      [v, i] = readVarint(buf, i);
+      if (field === 3) type = v;
+    } else {
+      break;
+    }
+  }
+  return { piece, score, type };
+}
+
+export class SentencePieceUnigram {
+  constructor(modelBytes) {
+    const pieces = parseModelProto(modelBytes);
+    if (!pieces.length) throw new Error('sentencepiece model contains no pieces');
+    // A unigram model scores every piece; a BPE model does not. Rejecting here
+    // beats producing plausible-looking but wrong token ids.
+    if (!pieces.some((p) => p.score !== 0)) {
+      throw new Error('this looks like a BPE sentencepiece model — only unigram is supported');
+    }
+    this.pieces = pieces;
+    this.vocab = new Map();
+    this.byteId = new Array(256).fill(-1);
+    this.unkId = 0;
+    for (let id = 0; id < pieces.length; id++) {
+      const { piece, type } = pieces[id];
+      if (type === UNK) this.unkId = id;
+      if (type === BYTE) {
+        const m = /^<0x([0-9A-Fa-f]{2})>$/.exec(piece);
+        if (m) this.byteId[parseInt(m[1], 16)] = id;
+        continue;
+      }
+      // Control pieces (<s>, </s>, <pad>) are addressable by id but must never be
+      // produced by encoding text — they are inserted by the caller if wanted.
+      if (type === CONTROL) continue;
+      if (!this.vocab.has(piece)) this.vocab.set(piece, id);
+    }
+    this.maxPieceLen = Math.max(...[...this.vocab.keys()].map((p) => p.length), 1);
+  }
+
+  get vocabSize() { return this.pieces.length; }
+
+  /** Text → token ids. Viterbi over piece scores, byte fallback for the rest. */
+  encodeIds(text) {
+    const norm = SPACE + String(text ?? '').normalize('NFKC').replace(/ /g, SPACE);
+    const n = norm.length;
+    // best[i] = { score, from, id } for the best segmentation of norm[0..i)
+    const best = new Array(n + 1).fill(null);
+    best[0] = { score: 0, from: -1, id: -1, bytes: null };
+
+    for (let i = 0; i < n; i++) {
+      if (!best[i]) continue;
+      let matched = false; // kept for readability of the fallback comment below
+      const limit = Math.min(n, i + this.maxPieceLen);
+      for (let j = i + 1; j <= limit; j++) {
+        const id = this.vocab.get(norm.slice(i, j));
+        if (id === undefined) continue;
+        matched = true;
+        const score = best[i].score + this.pieces[id].score;
+        if (!best[j] || score > best[j].score) best[j] = { score, from: i, id, bytes: null };
+      }
+      // Byte fallback is ALWAYS offered as an alternative, not only when nothing
+      // matched: a character can be in the vocabulary and still be the wrong split
+      // for the sentence around it. The per-byte penalty is far worse than any real
+      // piece's score, so Viterbi picks it only when it genuinely has to.
+      void matched;
+      {
+        const ch = String.fromCodePoint(norm.codePointAt(i));
+        const j = i + ch.length;
+        const bytes = new TextEncoder().encode(ch);
+        if (bytes.every((b) => this.byteId[b] >= 0)) {
+          const score = best[i].score + bytes.length * -10;
+          if (!best[j] || score > best[j].score) best[j] = { score, from: i, id: -1, bytes };
+        }
+      }
+    }
+
+    if (!best[n]) return [this.unkId];
+    const out = [];
+    for (let i = n; i > 0;) {
+      const node = best[i];
+      if (node.bytes) for (let k = node.bytes.length - 1; k >= 0; k--) out.push(this.byteId[node.bytes[k]]);
+      else out.push(node.id);
+      i = node.from;
+    }
+    return out.reverse();
+  }
+
+  /** Token ids → text. Byte pieces are reassembled before decoding as UTF-8. */
+  decodeIds(ids) {
+    const parts = [];
+    let pending = [];
+    const flush = () => {
+      if (!pending.length) return;
+      parts.push(new TextDecoder().decode(Uint8Array.from(pending)));
+      pending = [];
+    };
+    for (const id of ids) {
+      const p = this.pieces[id];
+      if (!p) continue;
+      if (p.type === BYTE) {
+        const m = /^<0x([0-9A-Fa-f]{2})>$/.exec(p.piece);
+        if (m) { pending.push(parseInt(m[1], 16)); continue; }
+      }
+      flush();
+      if (p.type === CONTROL || p.type === UNK) continue;
+      parts.push(p.piece);
+    }
+    flush();
+    return parts.join('').replace(new RegExp(SPACE, 'g'), ' ').replace(/^ /, '');
+  }
+}

package/src/server.js

@@ -53,7 +53,7 @@ import * as openai from './openai.js';
 import * as responses from './responses.js';
 import * as anthropic from './anthropic.js';
 
-export const VERSION = '0.6.55';
+export const VERSION = '0.6.56';
 
 // WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
 // store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
@@ -462,6 +462,23 @@ export function joinUpstream(base, pathname, search = '') {
 // short enough that a stuck download does not hold a request open forever.
 const EMBEDDER_WAIT_MS = 90_000;
 
+// Linear resample. Good enough for a voice-print sample — the encoder cares about
+// timbre, not the last decibel of fidelity — and it avoids a dependency for one
+// rate conversion.
+function resample(input, from, to) {
+  if (from === to) return input;
+  const ratio = from / to;
+  const out = new Float32Array(Math.floor(input.length / ratio));
+  for (let i = 0; i < out.length; i++) {
+    const pos = i * ratio;
+    const a = Math.floor(pos);
+    const b = Math.min(input.length - 1, a + 1);
+    const f = pos - a;
+    out[i] = input[a] * (1 - f) + input[b] * f;
+  }
+  return out;
+}
+
 const LOCAL_NAMESPACES = ['/tts', '/stt', '/ner', '/diarize', '/skills', '/config', '/logs', '/status', '/admin'];
 
 async function handleApi(req, res, { adapter, kind, pathname, search, base, destKey, destProtocol, harness, trace }, outBody, vault) {
@@ -1110,9 +1127,25 @@ export function createGateway(cfg = loadConfig()) {
               });
             }
           }
-          const vec = await diarizeEngine.embed(Float32Array.from(pcm));
-          const saved = ttsVoices.saveVoice({ name, vec });
-          console.log(`[tts] saved custom voice "${saved.name}" (${saved.dim}-d, sample discarded)`);
+          const audio = Float32Array.from(pcm);
+          const vec = await diarizeEngine.embed(audio);
+
+          // A voice print is engine-specific and the SAMPLE is about to be thrown
+          // away, so anything this voice might later need must be derived now.
+          // Pocket TTS is the engine that actually reproduces a speaker, so its
+          // conditioning is computed whenever its bundle is present — failing that
+          // is not fatal, it just means this voice works only with SpeechT5.
+          let pocket = null;
+          try {
+            const pt = await ttsEngine.pocketForEncoding({ allowDownload: cfg.tts?.allowDownload !== false, log: (m) => console.log(m) });
+            // The recorder sends 16 kHz; Mimi wants 24 kHz.
+            if (pt) pocket = await pt.encodeVoice(resample(audio, 16000, 24000));
+          } catch (e) {
+            console.log(`[tts] pocket conditioning unavailable for this voice (${e.message})`);
+          }
+
+          const saved = ttsVoices.saveVoice({ name, vec, pocket });
+          console.log(`[tts] saved custom voice "${saved.name}" (${saved.kinds.join(' + ')}, sample discarded)`);
           return sendJson(res, 201, { ...saved, usable: ttsEngine.supportsCustomVoices() });
         } catch (e) {
           return sendJson(res, 400, { error: { message: e.message, type: 'save_failed' } });
@@ -1225,7 +1258,23 @@ export function createGateway(cfg = loadConfig()) {
               },
             });
           }
-          speakerEmbedding = rec.vec;
+          // Which print to hand over depends on the engine: Pocket TTS takes its
+          // Mimi conditioning, SpeechT5 the 512-d x-vector. A voice saved before
+          // the pocket bundle existed has only the latter.
+          if (ttsEngine.isPocket()) {
+            const pk = ttsVoices.getPocketVoice(customId);
+            if (!pk) {
+              return sendJson(res, 409, {
+                error: {
+                  message: 'this voice was saved without a Pocket TTS conditioning — record it again with Pocket TTS selected',
+                  type: 'voice_kind_missing',
+                },
+              });
+            }
+            speakerEmbedding = pk;
+          } else {
+            speakerEmbedding = rec.vec;
+          }
           useVoice = `custom:${customId}`;
         } else if (customId) {
           // Explicitly asked for a recorded voice this model cannot use — say so.

package/src/tts-engine.js

@@ -24,8 +24,9 @@ import { ensureLib, modelRoot } from './ner-engine.js';
 import { runtimeDtype, runtimeName, DTYPE_SUFFIX } from './model-runtime.js';
 import {
   DEFAULT_TTS_MODEL, DEFAULT_TTS_VOICE, STYLE_DIM, MAX_PHONEME_TOKENS,
-  ttsModelDtype, isKnownTtsModel, isValidVoiceId, voiceLang,
+  ttsModelDtype, isKnownTtsModel, isValidVoiceId, voiceLang, ttsModelEngine, ttsModel,
 } from './tts-models.js';
+import { bundleOnDisk as pocketBundleOnDisk } from './pocket-tts-engine.js';
 
 // Kokoro's rate. Kept as a named export because it is the default and several
 // callers want a number before anything is loaded — but it is NOT universal: a
@@ -43,6 +44,8 @@ let _progress = null;
 let _initPromise = null;
 let _arch = null;         // 'style-tts2' (Kokoro) | 'vits' (MMS) | 'speecht5' (custom voices)
 let _vocoder = null;      // speecht5 only — mel → waveform
+let _pocket = null;       // pocket-tts only — its own PocketTTS instance
+let _encoder = null;      // a PocketTTS kept ONLY to encode voices, never to speak
 let _rate = SAMPLE_RATE;  // the ACTIVE model's output rate
 const _voices = new Map(); // voice id → Float32Array style bank
 
@@ -51,12 +54,38 @@ const _voices = new Map(); // voice id → Float32Array style bank
 // forward pass with a shape error nobody can act on.
 export const SUPPORTED_ARCH = { style_text_to_speech_2: 'style-tts2', vits: 'vits', speecht5: 'speecht5' };
 
+// Pocket TTS is not a transformers.js model — it is five raw ONNX graphs with a
+// hand-written generation loop, exactly like parakeet on the STT side — so it is
+// dispatched by catalog id rather than by a transformers config.model_type.
+export const POCKET_ARCH = 'pocket-tts';
+
 // SpeechT5 is the only architecture here that takes a SPEAKER EMBEDDING, which is
 // what makes a custom voice possible at all: Kokoro's voices are fixed style banks
 // and VITS is single-speaker, so neither can be pointed at a person. It needs a
 // separate vocoder (mel → waveform), hence the extra model id.
 export const SPEECHT5_VOCODER = 'Xenova/speecht5_hifigan';
-export function supportsCustomVoices() { return _arch === 'speecht5'; }
+// Both engines that can be pointed at a person. Pocket TTS is the one built for
+// it; SpeechT5 is kept because it is small and already downloaded for anyone who
+// tried it, but it borrows a voice rather than reproducing one.
+export function supportsCustomVoices() { return _arch === 'speecht5' || _arch === POCKET_ARCH; }
+export function isPocket() { return _arch === POCKET_ARCH; }
+
+/**
+ * A PocketTTS instance purely for ENCODING a voice, without disturbing whatever
+ * model is currently speaking. Saving a voice has to derive its conditioning while
+ * the sample still exists, and that must not silently switch the active engine out
+ * from under a conversation in progress.
+ */
+export async function pocketForEncoding({ allowDownload = true, log = () => {} } = {}) {
+  if (_pocket) return _pocket;
+  const { PocketTTS, bundleOnDisk, DEFAULT_BUNDLE } = await import('./pocket-tts-engine.js');
+  if (!bundleOnDisk(DEFAULT_BUNDLE) && !allowDownload) return null;
+  if (_encoder) return _encoder;
+  const pt = new PocketTTS();
+  await pt.load(DEFAULT_BUNDLE, { log });
+  _encoder = pt;
+  return pt;
+}
 
 export function arch() { return _arch; }
 export function sampleRate() { return _rate; }
@@ -79,6 +108,11 @@ export function modelDir(modelId) {
 // Present = the EXACT ONNX file this runtime will load, plus the tokenizer. Both
 // non-empty: a truncated download must not read as installed.
 export function modelOnDisk(modelId = _model || DEFAULT_TTS_MODEL, dtype = ttsModelDtype(modelId) || runtimeDtype()) {
+  // Pocket TTS keeps a bundle of five graphs under its own directory, not a single
+  // transformers-style onnx/ folder. bundleOnDisk is a pure fs predicate — the
+  // heavy onnxruntime import inside that module is dynamic — so importing it
+  // statically costs nothing.
+  if (ttsModelEngine(modelId) === POCKET_ARCH) return pocketBundleOnDisk(ttsModel(modelId)?.bundle);
   const dir = modelDir(modelId);
   const suffix = DTYPE_SUFFIX[dtype] ?? '';
   const need = [join(dir, 'onnx', `model${suffix}.onnx`), join(dir, 'tokenizer.json')];
@@ -104,6 +138,10 @@ export function init(cfg = {}) {
 }
 
 async function loadModel(modelId, { log = () => {}, allowDownload = true, dtype: dtypeOverride = null } = {}) {
+  // Pocket TTS has its own loader (raw onnxruntime, its own bundle layout), so it
+  // is routed before any transformers.js machinery is touched.
+  if (ttsModelEngine(modelId) === POCKET_ARCH) return loadPocket(modelId, { log, allowDownload });
+
   const prevNet = _net, prevModel = _model;
   let lib;
   try {
@@ -190,6 +228,36 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true, dtype:
   }
 }
 
+async function loadPocket(modelId, { log = () => {}, allowDownload = true } = {}) {
+  const prevArch = _arch, prevPocket = _pocket, prevModel = _model;
+  const { PocketTTS, bundleOnDisk, DEFAULT_BUNDLE, SAMPLE_RATE: PR } = await import('./pocket-tts-engine.js');
+  const bundle = ttsModel(modelId)?.bundle || DEFAULT_BUNDLE;
+  if (!bundleOnDisk(bundle) && !allowDownload) {
+    _state = 'error'; _err = 'model not on disk and downloads disabled';
+    return false;
+  }
+  _state = bundleOnDisk(bundle) ? 'loading' : 'downloading';
+  if (_state === 'downloading') _progress = { model: modelId, file: null, pct: 0 };
+  try {
+    const pt = new PocketTTS();
+    await pt.load(bundle, {
+      log,
+      onProgress: ({ file, pct }) => { _progress = { model: modelId, file, pct }; },
+    });
+    _pocket = pt; _net = pt; _tok = null; _vocoder = null;
+    _model = modelId; _arch = POCKET_ARCH; _dtype = 'int8'; _rate = PR;
+    _state = 'ready'; _err = null; _progress = null;
+    return true;
+  } catch (e) {
+    // A failed switch keeps whatever was working, same as every other engine here.
+    _pocket = prevPocket; _arch = prevArch; _model = prevModel;
+    _state = prevPocket || _net ? 'ready' : 'error';
+    _err = e.message; _progress = null;
+    log(`[pocket-tts] load failed (${e.message})`);
+    return false;
+  }
+}
+
 export async function setModel(modelId, { onLog = () => {}, allowDownload = true, dtype = 'auto' } = {}) {
   const want = dtype && dtype !== 'auto' ? dtype : (ttsModelDtype(modelId) || runtimeDtype());
   if (modelId === _model && isReady() && _dtype === want) return true;
@@ -288,6 +356,15 @@ export async function synthChunk(text, { voice = DEFAULT_TTS_VOICE, speed = 1, s
     return out.waveform.data;
   }
 
+  // Pocket TTS runs its own generation loop and chunking, so a whole utterance is
+  // handed over at once rather than being pre-split here.
+  if (_arch === POCKET_ARCH) {
+    if (!speakerEmbedding?.data?.length) {
+      throw new Error('this model needs a saved voice — record one in Settings → Text-to-speech');
+    }
+    return _pocket.synth(String(text), { voice: speakerEmbedding });
+  }
+
   // SpeechT5: conditioned by a 512-d speaker embedding, which is the whole point —
   // it is the one architecture here that can be pointed at a person's voice.
   // Without an embedding there is no voice to speak in, so this refuses rather
@@ -324,6 +401,9 @@ export async function synthChunk(text, { voice = DEFAULT_TTS_VOICE, speed = 1, s
 
 /** Synthesize arbitrary-length text, chunk by chunk. `onChunk` sees each as it lands. */
 export async function synth(text, { voice = DEFAULT_TTS_VOICE, speed = 1, speakerEmbedding = null, onChunk = null } = {}) {
+  // Pocket TTS splits internally against its own token ceiling, so splitting again
+  // here would cut sentences twice and reset its state mid-thought.
+  if (_arch === POCKET_ARCH) return synthChunk(text, { voice, speed, speakerEmbedding });
   const chunks = splitSentences(text);
   const out = [];
   for (const c of chunks) {
@@ -366,6 +446,6 @@ export function toWav(pcm, sampleRate = SAMPLE_RATE) {
 
 export function _reset() {
   _state = 'off'; _model = null; _net = null; _tok = null; _dtype = null;
-  _err = null; _progress = null; _initPromise = null; _arch = null; _rate = SAMPLE_RATE; _vocoder = null;
+  _err = null; _progress = null; _initPromise = null; _arch = null; _rate = SAMPLE_RATE; _vocoder = null; _pocket = null; _encoder = null;
   _voices.clear();
 }

package/src/tts-models.js

@@ -55,6 +55,26 @@ export const TTS_MODEL_CATALOG = [
     voices: true,
     note: 'The Mandarin-tuned Kokoro. Same engine and voice mechanism as v1.0.',
   },
+  {
+    // The one model here BUILT for cloning. Its Mimi encoder turns a sample into a
+    // conditioning the model generates from directly, rather than borrowing a
+    // voice print from a space it was not trained on — which is why it reproduces
+    // a speaker where SpeechT5 merely produces a consistent stranger. Also the
+    // fastest: measured 8-9x realtime at int8, against Kokoro's 2.1x.
+    id: 'kyutai/pocket-tts',
+    label: 'Pocket TTS — clone your voice',
+    lang: 'English',
+    tier: 'accurate',
+    arch: 'pocket-tts',
+    bundle: 'english_2026-04',
+    approxMB: 146,
+    ramMB: 700,
+    sampleRate: 24000,
+    voices: false,
+    customVoices: true,
+    recommended: true,
+    note: 'Kyutai Pocket TTS (MIT code, CC-BY-4.0 weights). Records a few seconds and speaks as you. Needs a saved voice; 146 MB one-time download.',
+  },
   {
     id: 'Xenova/speecht5_tts',
     label: 'SpeechT5 — your own voice',
@@ -74,7 +94,7 @@ export const TTS_MODEL_CATALOG = [
     sampleRate: 16000,
     voices: false,          // no built-in voices…
     customVoices: true,     // …but it is the ONE model here that can use yours
-    note: 'The only model here that speaks in a voice you record. Rougher than Kokoro, and the match is approximate — see the note under Your voices.',
+    note: 'Speaks using a voice you record — but it will NOT sound like you. It borrows pitch and timbre in a general way and produces a consistent voice of its own. Kokoro sounds better if you do not need a personal voice.',
   },
   {
     // One MMS entry so the second architecture is DISCOVERABLE from the list rather
@@ -137,6 +157,11 @@ export function ttsModelEngine(id) {
   return ttsModel(id)?.arch || 'style-tts2';
 }
 
+// The bundle a pocket-tts catalog entry uses (its language pack).
+export function ttsModelBundle(id) {
+  return ttsModel(id)?.bundle || null;
+}
+
 // Does this catalog entry have selectable voices? Kokoro picks one from a style
 // bank; VITS/MMS is single-speaker. Unknown (a searched model) resolves at load.
 export function ttsModelHasVoices(id) {

package/src/tts-voices.js

@@ -25,6 +25,17 @@ import { randomUUID } from 'node:crypto';
 import os from 'node:os';
 
 export const EMBED_DIM = 512;
+
+// A voice print is engine-specific: SpeechT5 wants a 512-d wavlm x-vector, Pocket
+// TTS wants its Mimi encoder's [1, N, 1024] conditioning. They are NOT
+// interchangeable, and the sample is discarded after saving, so whatever a voice
+// might later need has to be computed WHILE the audio is still in hand.
+//
+// The pocket conditioning is ~100k floats — small on disk, absurd inside a JSON
+// document that a settings page lists — so it lives beside the record as raw
+// Float32 and only its shape is stored in the JSON.
+export const KIND_SPEECHT5 = 'wavlm-512';
+export const KIND_POCKET = 'pocket-mimi';
 const MAX_VOICES = 20;
 const MAX_NAME = 60;
 
@@ -62,22 +73,50 @@ export function listVoices() {
       // The vector is deliberately NOT returned by the listing: the UI needs a
       // name and an id, and 512 floats of someone's voice print have no business
       // in a settings page's JSON.
-      if (v?.id && v?.name) out.push({ id: v.id, name: v.name, createdAt: v.createdAt || 0, dim: v.vec?.length || 0 });
+      if (v?.id && v?.name) {
+        out.push({
+          id: v.id, name: v.name, createdAt: v.createdAt || 0, dim: v.vec?.length || 0,
+          kinds: v.pocketShape ? [KIND_SPEECHT5, KIND_POCKET] : [KIND_SPEECHT5],
+        });
+      }
     } catch { /* a corrupt file must not break the list */ }
   }
   return out.sort((a, b) => b.createdAt - a.createdAt);
 }
 
+function pocketFileFor(id) {
+  return join(voicesDir(), `${id}.pocket.bin`);
+}
+
 export function getVoice(id) {
   if (!isValidVoiceRef(id)) return null;
   try {
     const v = JSON.parse(readFileSync(fileFor(id), 'utf8'));
-    return Array.isArray(v?.vec) && v.vec.length === EMBED_DIM ? v : null;
+    if (!Array.isArray(v?.vec) || v.vec.length !== EMBED_DIM) return null;
+    return v;
+  } catch { return null; }
+}
+
+/** The Pocket TTS conditioning for a voice, or null if it has none saved. */
+export function getPocketVoice(id) {
+  if (!isValidVoiceRef(id)) return null;
+  const meta = getVoice(id);
+  if (!meta?.pocketShape) return null;
+  try {
+    const buf = readFileSync(pocketFileFor(id));
+    return { data: new Float32Array(buf.buffer, buf.byteOffset, Math.floor(buf.byteLength / 4)), shape: meta.pocketShape };
   } catch { return null; }
 }
 
+/** Which engines can speak as this voice. Drives what the UI may offer. */
+export function voiceKinds(id) {
+  const v = getVoice(id);
+  if (!v) return [];
+  return v.pocketShape ? [KIND_SPEECHT5, KIND_POCKET] : [KIND_SPEECHT5];
+}
+
 /** Persist an embedding under a user-chosen name. Returns the stored record. */
-export function saveVoice({ name, vec }) {
+export function saveVoice({ name, vec, pocket = null }) {
   const clean = String(name || '').trim().slice(0, MAX_NAME);
   if (!clean) throw new Error('a name is required');
   if (!vec || vec.length !== EMBED_DIM) throw new Error(`expected a ${EMBED_DIM}-value embedding, got ${vec?.length || 0}`);
@@ -85,8 +124,13 @@ export function saveVoice({ name, vec }) {
   const dir = voicesDir();
   mkdirSync(dir, { recursive: true });
   const rec = { id: randomUUID(), name: clean, createdAt: Date.now(), vec: Array.from(vec, (x) => Number(x) || 0) };
+  if (pocket?.data?.length && Array.isArray(pocket.shape)) {
+    rec.pocketShape = pocket.shape;
+    const f32 = pocket.data instanceof Float32Array ? pocket.data : Float32Array.from(pocket.data);
+    writeFileSync(pocketFileFor(rec.id), Buffer.from(f32.buffer, f32.byteOffset, f32.byteLength));
+  }
   writeFileSync(fileFor(rec.id), JSON.stringify(rec));
-  return { id: rec.id, name: rec.name, createdAt: rec.createdAt, dim: rec.vec.length };
+  return { id: rec.id, name: rec.name, createdAt: rec.createdAt, dim: rec.vec.length, kinds: rec.pocketShape ? [KIND_SPEECHT5, KIND_POCKET] : [KIND_SPEECHT5] };
 }
 
 /** Remove one permanently. Returns whether there was anything to remove. */
@@ -95,5 +139,7 @@ export function deleteVoice(id) {
   const p = fileFor(id);
   if (!existsSync(p)) return false;
   rmSync(p, { force: true });
+  // Both halves, or the conditioning outlives the record that named it.
+  rmSync(pocketFileFor(id), { force: true });
   return true;
 }