@chatpanel/gateway 0.6.21 → 0.6.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/gateway",
-  "version": "0.6.21",
+  "version": "0.6.23",
   "description": "Local privacy gateway — redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
   "type": "module",
   "bin": {

package/src/config.js

@@ -104,6 +104,9 @@ const DEFAULTS = {
     enabled: true,
     model: 'onnx-community/whisper-base',
     allowDownload: true,
+    // Speaker diarization ("who said what") is an OPT-IN per-session stage (its
+    // model loads only when a session asks). Set false to disable it gateway-wide.
+    diarize: true,
   },
 
   // Log one line per request (method, tokens redacted) without any raw values.

package/src/diarize-engine.js

@@ -0,0 +1,140 @@
+// Speaker diarization ("who said what") — in-process, same engine + model dir as
+// STT/NER. Two layers:
+//   engine    — load Xenova/wavlm-base-plus-sv and embed a segment of audio into a
+//               512-d x-vector (speaker fingerprint). Download-on-demand, fail-open.
+//   Diarizer  — online clustering: keep a running centroid per speaker; a new
+//               segment joins the nearest centroid if cosine ≥ threshold, else it
+//               starts a new speaker. Stable "Speaker N" labels within a session.
+//
+// Honest limits: embeddings separate DIFFERENT speakers well but same-gender / very
+// similar voices can merge (synthetic TTS especially). In meetings the mic channel
+// is anchored as "You" and clustering only splits the remote channel — so a merge
+// there is far less costly. Opt-in per session (diarize:true); off for dictation.
+
+import { ensureLib, modelRoot } from './ner-engine.js';
+import { join } from 'node:path';
+import { existsSync, readdirSync } from 'node:fs';
+
+export const DIARIZE_MODEL = 'Xenova/wavlm-base-plus-sv';
+
+let _state = 'off';        // 'off' | 'loading' | 'downloading' | 'ready' | 'error'
+let _model = null;
+let _processor = null;
+let _err = null;
+let _initPromise = null;
+let _progress = null;
+
+// WASM can't load some quantized exports (see stt runtimeDtype); fp32 is the safe
+// cross-runtime choice on the binary, q8 on native.
+function runtimeDtype() {
+  return globalThis.__CHATPANEL_WASM_PATHS__ ? 'fp32' : 'q8';
+}
+
+export function state() { return _state; }
+export function isReady() { return _state === 'ready' && !!_model && !!_processor; }
+export function progress() { return _progress; }
+export function health() { return { configured: _state !== 'off', ok: isReady(), state: _state, model: _model, error: _err }; }
+
+export function modelOnDisk(modelId = DIARIZE_MODEL, dtype = runtimeDtype()) {
+  const dir = join(modelRoot(), ...modelId.split('/'), 'onnx');
+  if (!existsSync(dir)) return false;
+  const suffix = dtype === 'fp32' ? '' : (dtype === 'q8' ? '_quantized' : `_${dtype}`);
+  try { return readdirSync(dir).some((f) => f === `model${suffix}.onnx`); }
+  catch { return false; }
+}
+
+async function load({ log = () => {}, allowDownload = true } = {}) {
+  let lib;
+  try { lib = await ensureLib(); }
+  catch (e) { _state = 'error'; _err = e.message; log(`[diarize] engine unavailable (${e.message})`); return false; }
+
+  const haveLocal = modelOnDisk();
+  lib.env.allowRemoteModels = haveLocal ? false : !!allowDownload;
+  if (!haveLocal && !allowDownload) { _state = 'error'; _err = 'model not on disk and downloads disabled'; return false; }
+
+  const prevHost = lib.env.remoteHost;
+  if (!haveLocal) { try { lib.env.remoteHost = 'https://huggingface.co/'; } catch { /* wavlm isn't on the dl mirror */ } }
+  _state = haveLocal ? 'loading' : 'downloading';
+  if (!haveLocal) { _progress = { model: DIARIZE_MODEL, file: null, pct: 0 }; log(`[diarize] downloading ${DIARIZE_MODEL} (one-time)…`); }
+
+  try {
+    const { AutoModel, AutoProcessor } = lib;
+    const cb = (p) => {
+      if (!p) return;
+      const pct = typeof p.progress === 'number' ? Math.round(p.progress) : (_progress?.pct ?? 0);
+      if (p.status === 'progress' || p.status === 'download' || p.status === 'initiate') _progress = { model: DIARIZE_MODEL, file: p.file || _progress?.file || null, pct };
+    };
+    const processor = await AutoProcessor.from_pretrained(DIARIZE_MODEL, { progress_callback: cb });
+    const model = await AutoModel.from_pretrained(DIARIZE_MODEL, { dtype: runtimeDtype(), progress_callback: cb });
+    _processor = processor; _model = model; _state = 'ready'; _err = null; _progress = null;
+    log(`[diarize] ready — ${DIARIZE_MODEL} @ ${runtimeDtype()}`);
+    return true;
+  } catch (e) {
+    _err = e.message; _progress = null; _state = 'error';
+    log(`[diarize] load failed (${e.message}) — diarization off`);
+    return false;
+  } finally {
+    try { lib.env.remoteHost = prevHost; } catch { /* optional */ }
+  }
+}
+
+export function init(cfg = {}) {
+  if (_initPromise) return _initPromise;
+  _state = 'loading';
+  _initPromise = load({ log: cfg.onLog, allowDownload: cfg.allowDownload !== false });
+  return _initPromise;
+}
+
+// Embed a mono 16 kHz Float32 segment → Float32Array x-vector, or null if not ready.
+export async function embed(audio) {
+  if (!isReady() || !(audio instanceof Float32Array) || !audio.length) return null;
+  try {
+    const inputs = await _processor(audio);
+    const out = await _model(inputs);
+    const t = out.embeddings ?? out.logits ?? out.last_hidden_state;
+    return t?.data ? Float32Array.from(t.data) : null;
+  } catch { return null; }
+}
+
+export function _reset() { _state = 'off'; _model = null; _processor = null; _err = null; _initPromise = null; _progress = null; }
+export function _setForTest(model, processor) { _model = model; _processor = processor; _state = model ? 'ready' : 'off'; }
+
+// ── Online speaker clustering ────────────────────────────────────────────────────
+
+function cosine(a, b) {
+  let d = 0, na = 0, nb = 0;
+  for (let i = 0; i < a.length; i++) { d += a[i] * b[i]; na += a[i] * a[i]; nb += b[i] * b[i]; }
+  const denom = Math.sqrt(na) * Math.sqrt(nb);
+  return denom ? d / denom : 0;
+}
+
+// A per-session speaker tracker. `threshold` = min cosine to be the SAME speaker;
+// lower → fewer speakers (merges more), higher → more speakers (splits more).
+export class Diarizer {
+  constructor({ threshold = 0.75, maxSpeakers = 8 } = {}) {
+    this.threshold = threshold;
+    this.maxSpeakers = maxSpeakers;
+    this.centroids = []; // { id, label, vec: Float32Array, n }
+  }
+
+  // Assign an embedding to a speaker (nearest centroid ≥ threshold, else new).
+  // `pinnedLabel` forces a label (used for the mic channel = "You" in meetings).
+  assign(vec, { pinnedLabel = null } = {}) {
+    if (!vec) return null;
+    if (pinnedLabel) return this._merge(this._find(pinnedLabel) || this._create(pinnedLabel), vec);
+    let best = null, bestSim = -1;
+    for (const c of this.centroids) { const s = cosine(vec, c.vec); if (s > bestSim) { bestSim = s; best = c; } }
+    if (best && bestSim >= this.threshold) return this._merge(best, vec);
+    if (this.centroids.length >= this.maxSpeakers && best) return this._merge(best, vec); // cap: fold into nearest
+    return this._merge(this._create(`Speaker ${this.centroids.length + 1}`), vec);
+  }
+
+  _find(label) { return this.centroids.find((c) => c.label === label) || null; }
+  _create(label) { const c = { id: this.centroids.length + 1, label, vec: null, n: 0 }; this.centroids.push(c); return c; }
+  _merge(c, vec) {
+    if (!c.vec) { c.vec = Float32Array.from(vec); }
+    else { for (let i = 0; i < c.vec.length; i++) c.vec[i] = (c.vec[i] * c.n + vec[i]) / (c.n + 1); } // running mean centroid
+    c.n += 1;
+    return { id: c.id, label: c.label };
+  }
+}

package/src/ner-engine.js

@@ -162,7 +162,7 @@ export async function ensureLib() {
     try { Object.defineProperty(process, 'release', { value: { ...process.release, name: 'bun' }, configurable: true }); } catch { /* ignore */ }
   }
 
-  const { env, pipeline, Tensor } = await import('@huggingface/transformers');
+  const { env, pipeline, Tensor, AutoModel, AutoProcessor } = await import('@huggingface/transformers');
   env.cacheDir = root;          // where remote downloads are cached
   env.localModelPath = root;    // where local loads resolve — same dir, we control it
   try { env.remoteHost = MODEL_HOST; } catch { /* optional */ }
@@ -172,7 +172,7 @@ export async function ensureLib() {
     // failure when ORT-web runs outside a browser.
     try { env.backends.onnx.wasm.proxy = false; env.backends.onnx.wasm.wasmPaths = wasmPaths; } catch { /* optional */ }
   }
-  _lib = { env, pipeline, Tensor };
+  _lib = { env, pipeline, Tensor, AutoModel, AutoProcessor };
   return _lib;
 }
 

package/src/server.js

@@ -33,6 +33,7 @@ import { createHistoryStore } from './sqlite-store.js';
 import { ingestBackup } from './backup-ingest.js';
 import * as nerEngine from './ner-engine.js';
 import * as sttEngine from './stt-engine.js';
+import * as diarizeEngine from './diarize-engine.js';
 import { MODEL_CATALOG, isKnownModel, isValidCustomModelId } from './models.js';
 import { STT_MODEL_CATALOG, isKnownSttModel, isValidCustomSttId, DEFAULT_STT_MODEL } from './stt-models.js';
 import { resolvePro, checkQuota, consume, usage } from './freegate.js';
@@ -42,7 +43,7 @@ import * as openai from './openai.js';
 import * as responses from './responses.js';
 import * as anthropic from './anthropic.js';
 
-export const VERSION = '0.6.21';
+export const VERSION = '0.6.23';
 
 // WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
 // store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
@@ -706,10 +707,18 @@ export function createGateway(cfg = loadConfig()) {
       if (!sttEngine.isReady()) {
         sttEngine.init({ model: cfg.stt?.model || DEFAULT_STT_MODEL, allowDownload: cfg.stt?.allowDownload !== false, onLog: (m) => console.log(m) });
       }
+      // Diarization is another OPTIONAL stage: load its model only when a session
+      // asks for it (never on the dictation path).
+      const wantDiarize = body?.diarize === true && cfg.stt?.diarize !== false;
+      if (wantDiarize && !diarizeEngine.isReady()) {
+        diarizeEngine.init({ allowDownload: cfg.stt?.allowDownload !== false, onLog: (m) => console.log(m) });
+      }
       try {
         // `redact: true` chains the OPTIONAL redaction hop onto finals (STT → NER,
         // same composable model as everything else: any stage, with or without).
-        const { id } = sttEngine.createSession({ lang: body?.lang, redact: body?.redact === true });
+        // `diarize: true` (+ optional `speakerLabel` to pin the mic channel to a
+        // name) attaches a speaker to each final.
+        const { id } = sttEngine.createSession({ lang: body?.lang, redact: body?.redact === true, diarize: wantDiarize, speakerLabel: body?.speakerLabel });
         return sendJson(res, 201, { id, state: sttEngine.state() });
       } catch (e) {
         return sendJson(res, e.code === 'too_many_sessions' ? 429 : 500, { error: { message: e.message, type: e.code || 'stt_error' } });
@@ -751,9 +760,15 @@ export function createGateway(cfg = loadConfig()) {
           send({ type: 'state', state: sttEngine.state() });
           const progressTimer = setInterval(() => {
             const st = sttEngine.state();
-            if (st === 'downloading' || st === 'loading') send({ type: 'progress', state: st, ...(sttEngine.progress() || {}) });
-            else if (st === 'error') { send({ type: 'error', code: 'model_failed', message: sttEngine.health().error || 'model failed to load', fatal: true }); clearInterval(progressTimer); }
-            else { send({ type: 'state', state: st }); clearInterval(progressTimer); }
+            if (st === 'downloading' || st === 'loading') { send({ type: 'progress', state: st, ...(sttEngine.progress() || {}) }); return; }
+            if (st === 'error') { send({ type: 'error', code: 'model_failed', message: sttEngine.health().error || 'model failed to load', fatal: true }); clearInterval(progressTimer); return; }
+            // STT ready — if diarization is on, surface ITS one-time download too, so
+            // "who said what" doesn't silently lag while the speaker model fetches.
+            if (sess.diarize) {
+              const ds = diarizeEngine.state();
+              if (ds === 'downloading' || ds === 'loading') { send({ type: 'diarize-progress', state: ds, ...(diarizeEngine.progress() || {}) }); return; }
+            }
+            send({ type: 'state', state: st }); clearInterval(progressTimer);
           }, 500);
           progressTimer.unref?.();
           // Redaction is async — chain events so finals can't overtake interims.

package/src/stt-engine.js

@@ -21,6 +21,7 @@ import { existsSync, readdirSync } from 'node:fs';
 import { randomUUID } from 'node:crypto';
 import { ensureLib, modelRoot } from './ner-engine.js';
 import { DEFAULT_STT_MODEL, isEnglishOnly, sttModelDtype, isKnownSttModel } from './stt-models.js';
+import * as diarize from './diarize-engine.js';
 
 export const SAMPLE_RATE = 16000; // fixed wire contract: 16 kHz mono Float32 PCM
 
@@ -171,8 +172,8 @@ let _decodeChain = Promise.resolve(); // whisper is effectively single-threaded
 
 export function sessionCount() { return _sessions.size; }
 
-/** @param {{ lang?: string, redact?: boolean }} [opts] */
-export function createSession({ lang, redact = false } = {}) {
+/** @param {{ lang?: string, redact?: boolean, diarize?: boolean, speakerLabel?: string }} [opts] */
+export function createSession({ lang, redact = false, diarize: diarizeOpt = false, speakerLabel = null } = {}) {
   if (_sessions.size >= MAX_SESSIONS) {
     const e = /** @type {Error & { code?: string }} */ (new Error('too many concurrent dictation sessions'));
     e.code = 'too_many_sessions'; throw e;
@@ -184,6 +185,12 @@ export function createSession({ lang, redact = false } = {}) {
     // Opaque to this engine: the server applies the redaction hop to finals when
     // set. Pipeline stages stay independent — STT never imports NER.
     redact: !!redact,
+    // Diarization is opt-in. A per-session Diarizer clusters final segments into
+    // speakers; `speakerLabel` pins every final to one label (the mic channel =
+    // "You" in a meeting), so clustering only ever splits the other channels.
+    diarize: !!diarizeOpt,
+    speakerLabel: typeof speakerLabel === 'string' && speakerLabel ? speakerLabel.slice(0, 40) : null,
+    diarizer: diarizeOpt ? new diarize.Diarizer() : null,
     chunks: [],            // Float32Array pieces of the OPEN (unfinalized) segment
     samples: 0,
     listeners: new Set(),  // (event) => void — the SSE writers
@@ -263,6 +270,17 @@ function rms(audio, from = 0, to = audio.length) {
   return Math.sqrt(sum / n);
 }
 
+// Trim leading/trailing near-silence so a speaker embedding is computed on VOICE,
+// not room tone — otherwise the silence dominates and distinct speakers' vectors
+// converge (they'd all cluster as one). Window-scan at 20 ms granularity.
+function voicedRegion(audio) {
+  const w = Math.round(0.02 * SAMPLE_RATE);
+  let start = 0, end = audio.length;
+  for (let i = 0; i + w <= audio.length; i += w) { if (rms(audio, i, i + w) >= SILENCE_RMS) { start = i; break; } }
+  for (let i = audio.length - w; i >= 0; i -= w) { if (rms(audio, i, i + w) >= SILENCE_RMS) { end = i + w; break; } }
+  return end > start ? audio.subarray(start, end) : audio;
+}
+
 // Whisper's native language-ID, which transformers.js doesn't implement (its
 // pipeline just defaults to English): one decoder step from <|startoftranscript|>,
 // argmax over the 99 language tokens. Run ONCE per session on the first voiced
@@ -328,7 +346,16 @@ async function decodeSession(s, { flush = false } = {}) {
   if (flush || trailingQuiet || tooLong || overflow) {
     // Commit: the open segment becomes a final; the buffer restarts empty.
     s.chunks = []; s.samples = 0; s.lastInterim = '';
-    emit(s, { type: 'final', text });
+    // Diarize this committed segment (opt-in): embed its audio, cluster → speaker.
+    // A pinned label (mic = "You") skips clustering. Best-effort; never blocks text.
+    let speaker = null;
+    if (s.diarizer) {
+      try {
+        const vec = s.speakerLabel ? null : await diarize.embed(voicedRegion(audio));
+        speaker = s.diarizer.assign(vec, { pinnedLabel: s.speakerLabel });
+      } catch { /* diarization is additive — a failure never drops the transcript */ }
+    }
+    emit(s, speaker ? { type: 'final', text, speaker } : { type: 'final', text });
   } else if (text !== s.lastInterim) {
     s.lastInterim = text;
     emit(s, { type: 'interim', text });