@chatpanel/gateway 0.6.17 → 0.6.19

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/gateway",
-  "version": "0.6.17",
+  "version": "0.6.19",
   "description": "Local privacy gateway — redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
   "type": "module",
   "bin": {

package/src/config.js

@@ -96,6 +96,16 @@ const DEFAULTS = {
     enableFullTier: true,
   },
 
+  // Local speech-to-text (dictation) — whisper via the same in-process ONNX engine
+  // and model dir as NER (stt-engine.js). No autostart on purpose: the model
+  // downloads on FIRST dictation, never on gateway boot (first-run load time).
+  // Multilingual default so the spoken language is auto-detected per segment.
+  stt: {
+    enabled: true,
+    model: 'onnx-community/whisper-base',
+    allowDownload: true,
+  },
+
   // Log one line per request (method, tokens redacted) without any raw values.
   logRequests: true,
 

package/src/configstore.js

@@ -22,7 +22,7 @@ export function persistConfig(cfg, path = configPath()) {
     // restart drops them and every model falls back to the default OpenAI upstream.
     destinations: cfg.destinations,
     bridge: cfg.bridge, upstreams: cfg.upstreams, redaction: cfg.redaction,
-    ner: cfg.ner, allowedOrigins: cfg.allowedOrigins, maxBodyBytes: cfg.maxBodyBytes,
+    ner: cfg.ner, stt: cfg.stt, allowedOrigins: cfg.allowedOrigins, maxBodyBytes: cfg.maxBodyBytes,
     pro: cfg.pro, logRequests: cfg.logRequests, logDetail: cfg.logDetail, tools: cfg.tools,
   };
   writeFileSync(path, JSON.stringify(out, null, 2));
@@ -45,6 +45,7 @@ export function publicConfig(cfg, { proUnlocked = false } = {}) {
       dictionary: Array.isArray(cfg.redaction?.dictionary) ? cfg.redaction.dictionary : [],
     },
     ner: cfg.ner,
+    stt: cfg.stt,
     allowedOrigins: Array.isArray(cfg.allowedOrigins) ? cfg.allowedOrigins : [],
     // free = lifetime trial usage ({ used, cap, remaining }) — read-only; the cap
     // is fixed and the count is server-authoritative (never settable from the UI).
@@ -109,6 +110,13 @@ export function applyConfigPatch(cfg, patch = {}) {
     // its `used` count is server-authoritative — neither is editable here, so a
     // client can't raise the cap or reset its own trial.
   }
+  // Local dictation toggle. The MODEL is switched via POST /stt/models (like the
+  // NER manager), not patched here.
+  if (patch.stt && typeof patch.stt === 'object') {
+    cfg.stt = cfg.stt || { enabled: true, model: 'onnx-community/whisper-base', allowDownload: true };
+    if ('enabled' in patch.stt) cfg.stt.enabled = !!patch.stt.enabled;
+    if ('allowDownload' in patch.stt) cfg.stt.allowDownload = !!patch.stt.allowDownload;
+  }
   if (typeof patch.logRequests === 'boolean') cfg.logRequests = patch.logRequests;
   if (['off', 'types', 'values'].includes(patch.logDetail)) cfg.logDetail = patch.logDetail;
   if (patch.tools && typeof patch.tools === 'object') {

package/src/mcp.js

@@ -28,7 +28,7 @@ function baseUrl() {
 const TOOLS = [
   {
     name: 'search_history',
-    description: 'Full-text search the user\'s ChatPanel history (past chats and meeting transcripts) by keyword relevance. Use this to recall what was discussed when the current context does not already contain it.',
+    description: 'Full-text search the user\'s ChatPanel history — past chats, meeting transcripts, and notes — by keyword relevance. Use this to recall what was discussed or written when the current context does not already contain it.',
     inputSchema: {
       type: 'object',
       properties: {
@@ -40,10 +40,10 @@ const TOOLS = [
   },
   {
     name: 'get_record',
-    description: 'Fetch one full history record (its complete text) by id, e.g. chat:<id> or meeting:<id> returned by search_history.',
+    description: 'Fetch one full history record (its complete text) by id, e.g. chat:<id>, meeting:<id>, or note:<id> returned by search_history.',
     inputSchema: {
       type: 'object',
-      properties: { id: { type: 'string', description: 'Record id such as chat:abc or meeting:imp_123.' } },
+      properties: { id: { type: 'string', description: 'Record id such as chat:abc, meeting:imp_123, or note:xyz.' } },
       required: ['id'],
     },
   },

package/src/ner-engine.js

@@ -143,7 +143,9 @@ export async function fetchAdapter(_url, opts = {}) {
 export function progress() { return _progress; }
 
 // Import transformers ONCE and configure its env (model dir + WASM in a binary).
-async function ensureLib() {
+// Exported so sibling engines (stt-engine.js) share the same configured lib —
+// one env, one cache dir, one WASM setup; never a second copy of this logic.
+export async function ensureLib() {
   if (_lib) return _lib;
   const root = modelRoot();
   try { mkdirSync(root, { recursive: true }); } catch { /* best effort */ }
@@ -159,7 +161,7 @@ async function ensureLib() {
     try { Object.defineProperty(process, 'release', { value: { ...process.release, name: 'bun' }, configurable: true }); } catch { /* ignore */ }
   }
 
-  const { env, pipeline } = await import('@huggingface/transformers');
+  const { env, pipeline, Tensor } = await import('@huggingface/transformers');
   env.cacheDir = root;          // where remote downloads are cached
   env.localModelPath = root;    // where local loads resolve — same dir, we control it
   try { env.remoteHost = MODEL_HOST; } catch { /* optional */ }
@@ -169,7 +171,7 @@ async function ensureLib() {
     // failure when ORT-web runs outside a browser.
     try { env.backends.onnx.wasm.proxy = false; env.backends.onnx.wasm.wasmPaths = wasmPaths; } catch { /* optional */ }
   }
-  _lib = { env, pipeline };
+  _lib = { env, pipeline, Tensor };
   return _lib;
 }
 

package/src/server.js

@@ -20,7 +20,7 @@
 import { createServer } from 'node:http';
 import { loadConfig } from './config.js';
 import { startEntitlementRefresh, maybeRevalidate } from './entitlement-refresh.js';
-import { redactSegments } from './redact.js';
+import { redactSegments, segment } from './redact.js';
 import { pipeRestoredStream, pipeRestoredOpenAIStream, makeTokenRestorer } from './stream.js';
 import { restoreText, gatedDictionary, narrowSpecs, makeToolHarness, placeholderToolNote } from '@chatpanel/pii';
 import { streamBridgeChat, readBridgeToken, openBridgeChat } from './bridge.js';
@@ -32,7 +32,9 @@ import { saveBackupSecret, loadBackupSecret, hasBackupSecret } from './history-s
 import { createHistoryStore } from './sqlite-store.js';
 import { ingestBackup } from './backup-ingest.js';
 import * as nerEngine from './ner-engine.js';
+import * as sttEngine from './stt-engine.js';
 import { MODEL_CATALOG, isKnownModel } from './models.js';
+import { STT_MODEL_CATALOG, isKnownSttModel, isValidCustomSttId, DEFAULT_STT_MODEL } from './stt-models.js';
 import { resolvePro, checkQuota, consume, usage } from './freegate.js';
 import { publicConfig, applyConfigPatch, persistConfig, configPath } from './configstore.js';
 import { resolveDestination, aggregateModelsAsync } from './router.js';
@@ -40,7 +42,7 @@ import * as openai from './openai.js';
 import * as responses from './responses.js';
 import * as anthropic from './anthropic.js';
 
-export const VERSION = '0.6.17';
+export const VERSION = '0.6.19';
 
 // WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
 // store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
@@ -490,7 +492,14 @@ export function createGateway(cfg = loadConfig()) {
     if (req.method === 'OPTIONS') { res.writeHead(204); return res.end(); }
 
     if (req.method === 'GET' && pathname === '/health') {
-      return sendJson(res, 200, { ok: true, version: VERSION, backend: cfg.backend, tier: cfg.redaction.tier });
+      // `stt` is ADDITIVE (Tesla rule): old clients ignore it, new clients use it
+      // to auto-detect local dictation. `enabled` reflects config; the model only
+      // downloads on first use, so state may be 'off' while still available.
+      const stt = sttEngine.health();
+      return sendJson(res, 200, {
+        ok: true, version: VERSION, backend: cfg.backend, tier: cfg.redaction.tier,
+        stt: { enabled: cfg.stt?.enabled !== false, state: stt.state, ready: stt.ok, model: stt.model || cfg.stt?.model || DEFAULT_STT_MODEL },
+      });
     }
 
     // --- Config API (the extension's "Gateway" tab is a client of these) ---
@@ -646,6 +655,118 @@ export function createGateway(cfg = loadConfig()) {
         return sendJson(res, 202, { accepted: true, active: id, state: nerEngine.state(), progress: nerEngine.progress() });
       }
     }
+    // --- Local speech-to-text (dictation). Whisper runs IN-PROCESS (stt-engine.js,
+    // same ONNX engine + model dir as NER); audio arrives as 16 kHz mono Float32 PCM
+    // chunks over loopback and ONLY TEXT ever leaves this process. Wire contract
+    // (additive; see docs in the hub repo):
+    //   POST   /stt/sessions               {lang?} → 201 { id }   (ensures the model)
+    //   POST   /stt/sessions/:id/audio     binary Float32 PCM chunk → { ok }
+    //   GET    /stt/sessions/:id/events    SSE: progress | interim | final | error | end
+    //   DELETE /stt/sessions/:id           flush tail → { ok }
+    //   GET/POST /stt/models               catalog + progress / switch (mirrors /ner/models)
+    if (pathname === '/stt/models') {
+      if (req.method === 'GET') {
+        const active = sttEngine.health().model || cfg.stt?.model || DEFAULT_STT_MODEL;
+        const available = /** @type {any[]} */ (STT_MODEL_CATALOG.map((m) => ({ ...m, installed: sttEngine.modelOnDisk(m.id) })));
+        // Surface an active CUSTOM (non-catalog) model so the UI can show it too.
+        if (active && !available.some((m) => m.id === active)) {
+          available.push({ id: active, label: active, lang: '—', tier: 'custom', custom: true, installed: sttEngine.modelOnDisk(active), note: 'Custom model (from Hugging Face).' });
+        }
+        return sendJson(res, 200, {
+          active,
+          state: sttEngine.state(),
+          progress: sttEngine.progress(),
+          available,
+        });
+      }
+      if (req.method === 'POST') {
+        let body = null;
+        try { body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')); } catch { body = null; }
+        const id = body && typeof body.id === 'string' ? body.id.trim() : null;
+        // Curated catalog OR a strictly-validated custom whisper id (Advanced).
+        if (!id || !(isKnownSttModel(id) || isValidCustomSttId(id))) return sendJson(res, 400, { error: { message: 'unknown or invalid model id', type: 'bad_model' } });
+        if (cfg.stt) cfg.stt.model = id; else cfg.stt = { enabled: true, model: id, allowDownload: true };
+        try { persistConfig(cfg, configPath()); } catch { /* best effort */ }
+        sttEngine.setModel(id, { onLog: (m) => console.log(m) });
+        return sendJson(res, 202, { accepted: true, active: id, state: sttEngine.state(), progress: sttEngine.progress() });
+      }
+    }
+    if (pathname === '/stt/sessions' && req.method === 'POST') {
+      if (cfg.stt?.enabled === false) return sendJson(res, 403, { error: { message: 'STT disabled in gateway config', type: 'stt_disabled' } });
+      let body = null;
+      try { body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8') || '{}'); } catch { body = null; }
+      // Kick the model load on first use (single-flight; downloads once). The
+      // client follows progress on the session's SSE stream.
+      if (!sttEngine.isReady()) {
+        sttEngine.init({ model: cfg.stt?.model || DEFAULT_STT_MODEL, allowDownload: cfg.stt?.allowDownload !== false, onLog: (m) => console.log(m) });
+      }
+      try {
+        // `redact: true` chains the OPTIONAL redaction hop onto finals (STT → NER,
+        // same composable model as everything else: any stage, with or without).
+        const { id } = sttEngine.createSession({ lang: body?.lang, redact: body?.redact === true });
+        return sendJson(res, 201, { id, state: sttEngine.state() });
+      } catch (e) {
+        return sendJson(res, e.code === 'too_many_sessions' ? 429 : 500, { error: { message: e.message, type: e.code || 'stt_error' } });
+      }
+    }
+    {
+      const m = pathname.match(/^\/stt\/sessions\/([0-9a-f-]{36})(\/audio|\/events)?$/);
+      if (m) {
+        const sid = m[1];
+        if (m[2] === '/audio' && req.method === 'POST') {
+          try {
+            const raw = await readBody(req, cfg.maxBodyBytes);
+            sttEngine.pushAudio(sid, sttEngine.toFloat32(raw));
+            return sendJson(res, 200, { ok: true, state: sttEngine.state() });
+          } catch (e) {
+            return sendJson(res, e.code === 'no_session' ? 404 : 400, { error: { message: e.message, type: e.code || 'stt_error' } });
+          }
+        }
+        if (m[2] === '/events' && req.method === 'GET') {
+          const sess = sttEngine.getSession(sid);
+          if (!sess) return sendJson(res, 404, { error: { message: 'no such session', type: 'no_session' } });
+          res.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache', connection: 'keep-alive' });
+          const send = (ev) => res.write(`data: ${JSON.stringify(ev)}\n\n`);
+          // Optional STT → NER hop: redact FINALS through the same shared guard as
+          // chat traffic (one implementation, composed — never a second redactor).
+          // Interims stay raw (transient, loopback-only). NOTE: the vault is
+          // discarded, so these placeholders are permanent — that's the point.
+          const sttIsPro = sess.redact ? await resolvePro(cfg.pro?.entitlementToken) : true;
+          const maybeRedact = async (ev) => {
+            if (ev.type !== 'final' || !sess.redact) return ev;
+            try {
+              let t = ev.text;
+              await redactSegments([segment(() => t, (v) => { t = v; })], cfg.redaction, { isPro: sttIsPro });
+              return { ...ev, text: t };
+            } catch { return ev; } // fail-open: raw text is still local-only
+          };
+          // While the model is loading/downloading, stream progress so the UI can
+          // show "downloading 43%" instead of dead air on first-ever dictation.
+          send({ type: 'state', state: sttEngine.state() });
+          const progressTimer = setInterval(() => {
+            const st = sttEngine.state();
+            if (st === 'downloading' || st === 'loading') send({ type: 'progress', state: st, ...(sttEngine.progress() || {}) });
+            else if (st === 'error') { send({ type: 'error', code: 'model_failed', message: sttEngine.health().error || 'model failed to load', fatal: true }); clearInterval(progressTimer); }
+            else { send({ type: 'state', state: st }); clearInterval(progressTimer); }
+          }, 500);
+          progressTimer.unref?.();
+          // Redaction is async — chain events so finals can't overtake interims.
+          let evChain = Promise.resolve();
+          const unsub = sttEngine.subscribe(sid, (ev) => {
+            evChain = evChain.then(async () => {
+              send(await maybeRedact(ev));
+              if (ev.type === 'end') { clearInterval(progressTimer); res.end(); }
+            }).catch(() => {});
+          });
+          req.on('close', () => { clearInterval(progressTimer); unsub?.(); });
+          return;
+        }
+        if (!m[2] && req.method === 'DELETE') {
+          await sttEngine.endSession(sid);
+          return sendJson(res, 200, { ok: true });
+        }
+      }
+    }
     if (pathname === '/logs' && req.method === 'GET') {
       return sendJson(res, 200, { entries: [...recentRequests].reverse() }); // newest first; counts only, unless logDetail enriches each entry
     }

package/src/stt-engine.js

@@ -0,0 +1,368 @@
+// In-process speech-to-text — whisper via transformers.js, zero native deps.
+//
+// Same design as ner-engine.js (and deliberately the same file shape): an ONNX
+// model run IN-PROCESS, downloaded once on first use into ~/.chatpanel/models,
+// fully offline afterwards — audio NEVER leaves the machine. This is what makes
+// dictation private: the extension streams mic PCM over loopback, we transcribe
+// locally, and only text goes back.
+//
+// Two layers:
+//   engine   — load/switch the whisper pipeline (mirrors ner-engine verbatim)
+//   sessions — rolling-buffer streaming decode: interim results every ~1.2s,
+//              finals on trailing silence or when a segment grows too long.
+//              Whisper isn't natively streaming, so we re-decode the open
+//              segment and commit it when the speaker pauses.
+//
+// Fail-open by design: if the model can't load, sessions emit one error event
+// and dictation falls back to the browser engine client-side.
+
+import { join } from 'node:path';
+import { existsSync, readdirSync } from 'node:fs';
+import { randomUUID } from 'node:crypto';
+import { ensureLib, modelRoot } from './ner-engine.js';
+import { DEFAULT_STT_MODEL, isEnglishOnly, sttModelDtype, isKnownSttModel } from './stt-models.js';
+
+export const SAMPLE_RATE = 16000; // fixed wire contract: 16 kHz mono Float32 PCM
+
+// The right whisper quantization depends on the ONNX RUNTIME, not the OS:
+//   • native onnxruntime-node (the npm gateway) loads the small, fast `q8`
+//     (_quantized) exports — best size + speed.
+//   • onnxruntime-web WASM (the standalone binary — SAME wasm on macOS/Windows/
+//     Linux, so this is inherently cross-platform) CANNOT load the block-quantized
+//     exports (q8/int8/uint8 → MatMulNBits "missing scale"; fp16 → graph error);
+//     of the loadable ones only `fp32` is fast enough for real-time (q4/bnb4 are
+//     ~8× slower). Verified empirically against the bundled ORT-web build.
+// The binary entry sets __CHATPANEL_WASM_PATHS__, so that global tells us which
+// runtime we're on. A model may override via `dtype` in the STT catalog.
+export function runtimeDtype() {
+  return globalThis.__CHATPANEL_WASM_PATHS__ ? 'fp32' : 'q8';
+}
+
+let _state = 'off';        // 'off' | 'loading' | 'downloading' | 'ready' | 'error'
+let _model = null;         // active model id
+let _pipe = null;          // the loaded automatic-speech-recognition pipeline
+let _err = null;           // last error message (for /health)
+let _initPromise = null;   // single-flight init
+let _progress = null;      // { model, file, pct } while downloading, else null
+
+// transformers.js dtype → the ONNX filename suffix it loads. Presence must check
+// the EXACT file the current runtime will fetch — otherwise a q8 install (native)
+// looks "present" to the WASM runtime, which actually needs the fp32 file, and the
+// offline load fails. Checking the real target makes a runtime switch re-download.
+const DTYPE_SUFFIX = {
+  fp32: '', q8: '_quantized', int8: '_int8', uint8: '_uint8',
+  fp16: '_fp16', q4: '_q4', bnb4: '_bnb4', q4f16: '_q4f16',
+};
+
+export function modelOnDisk(modelId = _model || DEFAULT_STT_MODEL, dtype = sttModelDtype(modelId) || runtimeDtype()) {
+  const dir = join(modelRoot(), ...modelId.split('/'), 'onnx');
+  if (!existsSync(dir)) return false;
+  const suffix = DTYPE_SUFFIX[dtype] ?? '';
+  try {
+    const files = readdirSync(dir);
+    return files.includes(`encoder_model${suffix}.onnx`)
+      && files.some((f) => f === `decoder_model_merged${suffix}.onnx` || f === `decoder_model${suffix}.onnx`);
+  } catch { return false; }
+}
+
+export function state() { return _state; }
+export function isReady() { return _state === 'ready' && !!_pipe; }
+export function progress() { return _progress; }
+
+export function health() {
+  return { configured: _state !== 'off', ok: isReady(), state: _state, model: _model, error: _err };
+}
+
+// (Re)load a model into _pipe. Same contract as ner-engine.loadModel: fail-open,
+// and a failed SWITCH keeps the previous working pipeline.
+/** @param {string} modelId @param {{ log?: (m: string) => void, allowDownload?: boolean }} [opts] */
+async function loadModel(modelId, { log = () => {}, allowDownload = true } = {}) {
+  const prevPipe = _pipe;
+  const prevModel = _model;
+  let lib;
+  try {
+    lib = await ensureLib();
+  } catch (e) {
+    _state = 'error'; _err = `engine load failed: ${e.message}`;
+    log(`[stt] transformers.js not available (${e.message}) — dictation falls back to the browser engine`);
+    return false;
+  }
+
+  const haveLocal = modelOnDisk(modelId);
+  lib.env.allowRemoteModels = haveLocal ? false : !!allowDownload;
+  if (!haveLocal && !allowDownload) {
+    _state = 'error'; _err = 'model not on disk and downloads disabled';
+    log(`[stt] model ${modelId} not installed and downloads disabled`);
+    return false;
+  }
+
+  // Curated catalog models come from the private dl.chatpanel.net mirror (ensureLib
+  // already set that as remoteHost). A CUSTOM ("Advanced") id isn't mirrored, so
+  // fetch it from Hugging Face directly — only for this load, then restore.
+  const prevHost = lib.env.remoteHost;
+  const isCustom = !isKnownSttModel(modelId);
+  if (!haveLocal && isCustom) { try { lib.env.remoteHost = 'https://huggingface.co/'; } catch { /* optional */ } }
+
+  _state = haveLocal ? 'loading' : 'downloading';
+  if (!haveLocal) { _progress = { model: modelId, file: null, pct: 0 }; log(`[stt] downloading model ${modelId} (one-time${isCustom ? ', from Hugging Face' : ''})…`); }
+
+  try {
+    // Per-model override wins (catalog `dtype`), else the runtime-appropriate default.
+    const dtype = sttModelDtype(modelId) || runtimeDtype();
+    const pipe = await lib.pipeline('automatic-speech-recognition', modelId, {
+      dtype,
+      progress_callback: (p) => {
+        if (!p) return;
+        const pct = typeof p.progress === 'number' ? Math.round(p.progress) : (_progress?.pct ?? 0);
+        if (p.status === 'progress' || p.status === 'download' || p.status === 'initiate') {
+          _progress = { model: modelId, file: p.file || _progress?.file || null, pct };
+        } else if (p.status === 'done' && p.file) {
+          log(`[stt] fetched ${p.file}`);
+        }
+      },
+    });
+    _pipe = pipe; _model = modelId; _state = 'ready'; _err = null; _progress = null;
+    if (prevPipe && prevPipe !== pipe) { try { await prevPipe.dispose?.(); } catch { /* ignore */ } }
+    log(`[stt] ready — model ${modelId} (in-process, offline) — local dictation active`);
+    return true;
+  } catch (e) {
+    _err = e.message; _progress = null;
+    if (prevPipe) { _pipe = prevPipe; _model = prevModel; _state = 'ready'; }
+    else { _state = 'error'; }
+    log(`[stt] model load failed (${e.message})${prevPipe ? ' — keeping previous model' : ''}`);
+    return false;
+  } finally {
+    try { lib.env.remoteHost = prevHost; } catch { /* optional */ } // restore the mirror for NER + catalog loads
+  }
+}
+
+// Load the configured model once, on FIRST USE (never at gateway startup — the
+// download is deferred until someone actually dictates). Single-flight.
+export function init(cfg = {}) {
+  if (_initPromise) return _initPromise;
+  const log = typeof cfg.onLog === 'function' ? cfg.onLog : () => {};
+  _model = cfg.model || DEFAULT_STT_MODEL;
+  _state = 'loading';
+  _initPromise = loadModel(_model, { log, allowDownload: cfg.allowDownload !== false });
+  return _initPromise;
+}
+
+export async function setModel(modelId, opts = {}) {
+  const log = typeof opts.onLog === 'function' ? opts.onLog : () => {};
+  if (!modelId) return false;
+  if (modelId === _model && isReady()) return true;
+  return loadModel(modelId, { log, allowDownload: opts.allowDownload !== false });
+}
+
+// ── Streaming sessions ──────────────────────────────────────────────────────────
+
+const MAX_SESSIONS = 4;
+const IDLE_MS = 60_000;             // no audio for a minute → session expires
+const DECODE_GAP_MS = 1200;         // min spacing between decodes of one session
+const MAX_SEGMENT_S = 12;           // force-finalize an open segment past this
+const SILENCE_FINAL_MS = 700;       // trailing quiet that commits a segment
+const SILENCE_RMS = 0.008;          // "quiet" threshold on Float32 PCM
+const MAX_BUFFER_S = 30;            // hard cap on the open segment (whisper ctx)
+
+const _sessions = new Map();
+let _decodeChain = Promise.resolve(); // whisper is effectively single-threaded —
+                                      // serialize decodes across ALL sessions
+
+export function sessionCount() { return _sessions.size; }
+
+/** @param {{ lang?: string, redact?: boolean }} [opts] */
+export function createSession({ lang, redact = false } = {}) {
+  if (_sessions.size >= MAX_SESSIONS) {
+    const e = /** @type {Error & { code?: string }} */ (new Error('too many concurrent dictation sessions'));
+    e.code = 'too_many_sessions'; throw e;
+  }
+  const s = {
+    id: randomUUID(),
+    lang: typeof lang === 'string' && lang ? lang.slice(0, 12) : null,
+    langTried: false, // language auto-detect runs once per session (multilingual models)
+    // Opaque to this engine: the server applies the redaction hop to finals when
+    // set. Pipeline stages stay independent — STT never imports NER.
+    redact: !!redact,
+    chunks: [],            // Float32Array pieces of the OPEN (unfinalized) segment
+    samples: 0,
+    listeners: new Set(),  // (event) => void — the SSE writers
+    lastDecodeAt: 0,
+    decodeTimer: null,
+    lastInterim: '',
+    closed: false,
+    idleTimer: null,
+  };
+  _sessions.set(s.id, s);
+  touch(s);
+  return { id: s.id };
+}
+
+export function getSession(id) { return _sessions.get(String(id || '')) || null; }
+
+export function subscribe(id, fn) {
+  const s = getSession(id);
+  if (!s) return null;
+  s.listeners.add(fn);
+  return () => s.listeners.delete(fn);
+}
+
+function emit(s, ev) {
+  for (const fn of s.listeners) { try { fn(ev); } catch { /* listener's problem */ } }
+}
+
+function touch(s) {
+  if (s.idleTimer) clearTimeout(s.idleTimer);
+  s.idleTimer = setTimeout(() => endSession(s.id).catch(() => {}), IDLE_MS);
+  s.idleTimer.unref?.();
+}
+
+// Accept a chunk of 16 kHz mono Float32 PCM and schedule a decode.
+export function pushAudio(id, float32) {
+  const s = getSession(id);
+  if (!s || s.closed) {
+    const e = /** @type {Error & { code?: string }} */ (new Error('no such session'));
+    e.code = 'no_session'; throw e;
+  }
+  if (!(float32 instanceof Float32Array) || !float32.length) return;
+  s.chunks.push(float32);
+  s.samples += float32.length;
+  touch(s);
+  scheduleDecode(s);
+}
+
+// HTTP bodies arrive as Buffers whose byteOffset may not be 4-aligned — copy.
+export function toFloat32(buf) {
+  const bytes = buf.length - (buf.length % 4);
+  const out = new Float32Array(bytes / 4);
+  for (let i = 0; i < out.length; i++) out[i] = buf.readFloatLE(i * 4);
+  return out;
+}
+
+function scheduleDecode(s) {
+  if (s.closed || s.decodeTimer) return;
+  const wait = Math.max(0, s.lastDecodeAt + DECODE_GAP_MS - Date.now());
+  s.decodeTimer = setTimeout(() => {
+    s.decodeTimer = null;
+    _decodeChain = _decodeChain.then(() => decodeSession(s).catch(() => {}));
+  }, wait);
+  s.decodeTimer.unref?.();
+}
+
+function concatBuffer(s) {
+  const out = new Float32Array(s.samples);
+  let o = 0;
+  for (const c of s.chunks) { out.set(c, o); o += c.length; }
+  return out;
+}
+
+function rms(audio, from = 0, to = audio.length) {
+  let sum = 0;
+  const n = Math.max(1, to - from);
+  for (let i = from; i < to; i++) sum += audio[i] * audio[i];
+  return Math.sqrt(sum / n);
+}
+
+// Whisper's native language-ID, which transformers.js doesn't implement (its
+// pipeline just defaults to English): one decoder step from <|startoftranscript|>,
+// argmax over the 99 language tokens. Run ONCE per session on the first voiced
+// segment, then pinned — so "speak any language, it just works" without a setting.
+async function detectLanguage(audio) {
+  try {
+    const lib = await ensureLib();
+    const gc = _pipe?.model?.generation_config;
+    if (!gc?.lang_to_id || !gc.decoder_start_token_id || !lib.Tensor) return null; // english-only model (or exotic export)
+    const clip = audio.length > SAMPLE_RATE * 8 ? audio.subarray(0, SAMPLE_RATE * 8) : audio;
+    const feats = await _pipe.processor(clip);
+    const decoder_input_ids = new lib.Tensor('int64', new BigInt64Array([BigInt(gc.decoder_start_token_id)]), [1, 1]);
+    const out = await _pipe.model({ ...feats, decoder_input_ids });
+    let best = null, bestVal = -Infinity;
+    for (const [tok, id] of Object.entries(gc.lang_to_id)) {
+      const v = Number(out.logits.data[id]);
+      if (v > bestVal) { bestVal = v; best = tok; }
+    }
+    return best ? best.slice(2, -2) : null; // '<|fr|>' → 'fr'
+  } catch { return null; } // fail-open: whisper's English default still transcribes
+}
+
+async function decodeSession(s, { flush = false } = {}) {
+  if (s.closed && !flush) return;
+  if (!isReady() || !s.samples) return;
+  s.lastDecodeAt = Date.now();
+
+  const audio = concatBuffer(s);
+  const tail = Math.round((SILENCE_FINAL_MS / 1000) * SAMPLE_RATE);
+  const trailingQuiet = audio.length > tail && rms(audio, audio.length - tail) < SILENCE_RMS;
+
+  // Nothing but room tone? Don't decode (whisper hallucinates on silence) and
+  // don't let the buffer grow unbounded — keep only the last second.
+  if (rms(audio) < SILENCE_RMS) {
+    if (audio.length > SAMPLE_RATE * 5) {
+      s.chunks = [audio.subarray(audio.length - SAMPLE_RATE)];
+      s.samples = SAMPLE_RATE;
+    }
+    return;
+  }
+
+  // Auto-detect the spoken language on the session's first voiced audio (≥1s),
+  // then pin it. An explicit client `lang` wins; `.en` models skip all of this.
+  if (!s.lang && !s.langTried && !isEnglishOnly(_model) && audio.length >= SAMPLE_RATE) {
+    s.langTried = true;
+    const detected = await detectLanguage(audio);
+    if (detected) { s.lang = detected; emit(s, { type: 'language', lang: detected }); }
+  }
+
+  let text = '';
+  try {
+    const opts = isEnglishOnly(_model) ? {} : { language: s.lang || undefined, task: 'transcribe' };
+    const out = await _pipe(audio, opts);
+    text = String(out?.text || '').trim();
+  } catch (e) {
+    emit(s, { type: 'error', code: 'decode_failed', message: e.message, fatal: false });
+    return;
+  }
+  if (!text) return;
+
+  const tooLong = audio.length >= MAX_SEGMENT_S * SAMPLE_RATE;
+  const overflow = audio.length >= MAX_BUFFER_S * SAMPLE_RATE;
+  if (flush || trailingQuiet || tooLong || overflow) {
+    // Commit: the open segment becomes a final; the buffer restarts empty.
+    s.chunks = []; s.samples = 0; s.lastInterim = '';
+    emit(s, { type: 'final', text });
+  } else if (text !== s.lastInterim) {
+    s.lastInterim = text;
+    emit(s, { type: 'interim', text });
+  }
+  // More audio may have arrived while decoding — the next push reschedules.
+}
+
+// Stop a session: flush any remaining speech as a last final, notify, clean up.
+export async function endSession(id) {
+  const s = getSession(id);
+  if (!s || s.closed) return false;
+  s.closed = true;
+  if (s.idleTimer) clearTimeout(s.idleTimer);
+  if (s.decodeTimer) { clearTimeout(s.decodeTimer); s.decodeTimer = null; }
+  try {
+    await (_decodeChain = _decodeChain.then(() => decodeSession(s, { flush: true }).catch(() => {})));
+  } finally {
+    emit(s, { type: 'end' });
+    s.listeners.clear();
+    _sessions.delete(s.id);
+  }
+  return true;
+}
+
+// Test hook: reset module state.
+export function _reset() {
+  for (const s of _sessions.values()) {
+    if (s.idleTimer) clearTimeout(s.idleTimer);
+    if (s.decodeTimer) clearTimeout(s.decodeTimer);
+  }
+  _sessions.clear();
+  _state = 'off'; _model = null; _pipe = null; _err = null; _initPromise = null; _progress = null;
+}
+
+// Test hook: inject a fake pipeline (so session logic is testable without a model).
+export function _setPipeForTest(pipe, model = 'test/fake') {
+  _pipe = pipe; _model = model; _state = pipe ? 'ready' : 'off';
+}

package/src/stt-models.js

@@ -0,0 +1,85 @@
+// Catalog of speech-to-text (whisper) models the gateway can run, surfaced in the
+// extension's Gateway settings and dictation UI so users pick by their machine's
+// resources. All are ONNX (transformers.js) automatic-speech-recognition models —
+// same engine, download plumbing, and model root as the NER catalog (models.js).
+//
+// `tier` groups them for the picker (light / balanced / accurate / max). `ramMB`
+// is a rough working-set hint. `approxMB` is the DOWNLOAD size for the dtype the
+// current runtime will actually fetch — it differs by runtime (the WASM binary
+// downloads fp32; the native npm build downloads q8), so it's a range, and the
+// gateway reports the exact bytes as they stream.
+//
+// Adding a model: any onnx-community/* (or Xenova/*) whisper export works. Verify
+// it loads on BOTH runtimes (native q8 + WASM fp32) before listing it — the WASM
+// runtime can't load block-quantized (q8/int8) or fp16 exports (see stt-engine
+// runtimeDtype). `.en` models are English-only and reject language/task options.
+
+export const DEFAULT_STT_MODEL = 'onnx-community/whisper-base';
+
+export const STT_MODEL_CATALOG = [
+  {
+    id: 'onnx-community/whisper-tiny.en',
+    label: 'Tiny (English)',
+    lang: 'English',
+    tier: 'light',
+    approxMB: 150,   // fp32 on WASM; ~40 on native q8
+    ramMB: 400,
+    note: 'Fastest, lightest. English only — great for quick dictation on any machine.',
+  },
+  {
+    id: 'onnx-community/whisper-base',
+    label: 'Base (multilingual)',
+    lang: '99 languages (auto-detected)',
+    tier: 'balanced',
+    approxMB: 300,   // fp32 on WASM; ~80 on native q8
+    ramMB: 700,
+    note: 'Default. Detects the spoken language automatically; good accuracy at real-time speed.',
+  },
+  {
+    id: 'onnx-community/whisper-small',
+    label: 'Small (multilingual)',
+    lang: '99 languages (auto-detected)',
+    tier: 'accurate',
+    approxMB: 950,   // fp32 on WASM; ~250 on native q8
+    ramMB: 1600,
+    note: 'Noticeably more accurate; needs a faster CPU to keep up in real time.',
+  },
+  {
+    id: 'onnx-community/whisper-large-v3-turbo',
+    label: 'Large v3 Turbo (multilingual)',
+    lang: '99 languages (auto-detected)',
+    tier: 'max',
+    approxMB: 1600,  // native q8 recommended; heavy on WASM
+    ramMB: 3200,
+    note: 'Best accuracy. For powerful machines; use the native (npm) gateway for speed.',
+  },
+];
+
+export function isKnownSttModel(id) {
+  return STT_MODEL_CATALOG.some((m) => m.id === id);
+}
+
+export function sttModel(id) {
+  return STT_MODEL_CATALOG.find((m) => m.id === id) || null;
+}
+
+// English-only exports reject `language`/`task` generation options.
+export function isEnglishOnly(id) {
+  return /\.en$/.test(String(id || ''));
+}
+
+// A model may pin an explicit dtype (overriding the runtime default). None do by
+// default — the engine's runtimeDtype() picks q8 (native) / fp32 (WASM) — but
+// this is the seam for a model that needs a specific quantization.
+export function sttModelDtype(id) {
+  return sttModel(id)?.dtype || null;
+}
+
+// Accept a user-supplied ("Advanced") whisper model id. STRICT: `org/name` shape,
+// must be a whisper export, no path traversal. Curated catalog ids come from the
+// private dl.chatpanel.net mirror; custom ids are fetched from Hugging Face
+// directly (the engine points remoteHost there only for the custom load).
+export function isValidCustomSttId(id) {
+  const s = String(id || '');
+  return /^[A-Za-z0-9][\w.-]*\/[\w.-]+$/.test(s) && /whisper/i.test(s) && !s.includes('..');
+}