@chatpanel/gateway 0.6.25 → 0.6.27

package/README.md

@@ -54,6 +54,15 @@ chatpanel-gateway
 #     backend  : bridge (agent: codex, via http://127.0.0.1:4319)
 ```
 
+> **Binary vs. npm — same features, very different local-AI speed.** Both run
+> identical redaction/routing. But the standalone binary runs the local models
+> (speech-to-text, diarization, NER) on the **WASM** runtime — **fp32-only,
+> single-threaded, slow**. The **npm** install uses the **native** runtime with
+> **quantized (q8)** models — in our tests **~10× faster** speech-to-text
+> (real-time even on larger, more accurate models). If you'll use voice/meeting
+> features, install via **npm**. Don't run both — they can shadow each other on
+> `PATH`; check `GET /health` → `stt.runtime` (`native` vs `wasm`).
+
 Then point your front-end agent at it. **opencode** (`opencode.json`):
 
 ```jsonc

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/gateway",
-  "version": "0.6.25",
+  "version": "0.6.27",
   "description": "Local privacy gateway — redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
   "type": "module",
   "bin": {

package/src/ner-engine.js

@@ -179,6 +179,7 @@ export async function ensureLib() {
 // (Re)load a specific model into _pipe. Downloads it first if missing (and allowed).
 // Reusable by init() and setModel(). Fail-open: on error, state='error', _pipe stays
 // whatever it was (so a failed SWITCH doesn't kill a working detector).
+/** @param {string} modelId @param {{ log?: (m: string) => void, allowDownload?: boolean }} [opts] */
 async function loadModel(modelId, { log = () => {}, allowDownload = true } = {}) {
   const prevPipe = _pipe;
   const prevModel = _model;
@@ -211,18 +212,26 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true } = {})
   if (!haveLocal) { _progress = { model: modelId, file: null, pct: 0 }; log(`[ner] downloading model ${modelId} (one-time${isCustom ? ', from Hugging Face' : ''})…`); }
 
   try {
-    const pipe = await lib.pipeline('token-classification', modelId, {
-      dtype: 'q8',
-      progress_callback: (p) => {
-        if (!p) return;
-        const pct = typeof p.progress === 'number' ? Math.round(p.progress) : (_progress?.pct ?? 0);
-        if (p.status === 'progress' || p.status === 'download' || p.status === 'initiate') {
-          _progress = { model: modelId, file: p.file || _progress?.file || null, pct };
-        } else if (p.status === 'done' && p.file) {
-          log(`[ner] fetched ${p.file}`);
-        }
-      },
-    });
+    const progress_callback = (p) => {
+      if (!p) return;
+      const pct = typeof p.progress === 'number' ? Math.round(p.progress) : (_progress?.pct ?? 0);
+      if (p.status === 'progress' || p.status === 'download' || p.status === 'initiate') {
+        _progress = { model: modelId, file: p.file || _progress?.file || null, pct };
+      } else if (p.status === 'done' && p.file) {
+        log(`[ner] fetched ${p.file}`);
+      }
+    };
+    let pipe;
+    try {
+      pipe = await lib.pipeline('token-classification', modelId, { dtype: 'q8', progress_callback });
+    } catch (e) {
+      // Mirror gap → fall back to Hugging Face (same as stt-engine).
+      if (haveLocal || isCustom || !allowDownload) throw e;
+      log(`[ner] mirror fetch failed (${String(e.message).slice(0, 80)}) — retrying from Hugging Face…`);
+      lib.env.remoteHost = 'https://huggingface.co/';
+      lib.env.allowRemoteModels = true;
+      pipe = await lib.pipeline('token-classification', modelId, { dtype: 'q8', progress_callback });
+    }
     // Swap in the new pipeline, dispose the old one (free its WASM/native session).
     _pipe = pipe; _model = modelId; _state = 'ready'; _err = null; _progress = null;
     if (prevPipe && prevPipe !== pipe) { try { await prevPipe.dispose?.(); } catch { /* ignore */ } }

package/src/server.js

@@ -35,7 +35,7 @@ import * as nerEngine from './ner-engine.js';
 import * as sttEngine from './stt-engine.js';
 import * as diarizeEngine from './diarize-engine.js';
 import { MODEL_CATALOG, isKnownModel, isValidCustomModelId } from './models.js';
-import { STT_MODEL_CATALOG, isKnownSttModel, isValidCustomSttId, DEFAULT_STT_MODEL } from './stt-models.js';
+import { STT_MODEL_CATALOG, isKnownSttModel, isValidCustomSttId, DEFAULT_STT_MODEL, STT_DTYPES, isValidDtype } from './stt-models.js';
 import { resolvePro, checkQuota, consume, usage } from './freegate.js';
 import { publicConfig, applyConfigPatch, persistConfig, configPath } from './configstore.js';
 import { resolveDestination, aggregateModelsAsync } from './router.js';
@@ -43,7 +43,7 @@ import * as openai from './openai.js';
 import * as responses from './responses.js';
 import * as anthropic from './anthropic.js';
 
-export const VERSION = '0.6.25';
+export const VERSION = '0.6.27';
 
 // WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
 // store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
@@ -686,6 +686,12 @@ export function createGateway(cfg = loadConfig()) {
           state: sttEngine.state(),
           progress: sttEngine.progress(),
           available,
+          // Precision (quantization) picker: current choice + selectable options +
+          // the dtype actually loaded. On WASM only fp32 loads (see runtimeDtype).
+          dtype: cfg.stt?.dtype || 'auto',
+          loadedDtype: sttEngine.health().dtype,
+          runtime: sttEngine.health().runtime,
+          dtypes: STT_DTYPES,
         });
       }
       if (req.method === 'POST') {
@@ -694,10 +700,13 @@ export function createGateway(cfg = loadConfig()) {
         const id = body && typeof body.id === 'string' ? body.id.trim() : null;
         // Curated catalog OR a strictly-validated custom whisper id (Advanced).
         if (!id || !(isKnownSttModel(id) || isValidCustomSttId(id))) return sendJson(res, 400, { error: { message: 'unknown or invalid model id', type: 'bad_model' } });
+        // Optional precision override (q8/q4/fp16/…); 'auto' clears it.
+        const dtype = typeof body.dtype === 'string' && isValidDtype(body.dtype) ? body.dtype : undefined;
         if (cfg.stt) cfg.stt.model = id; else cfg.stt = { enabled: true, model: id, allowDownload: true };
+        if (dtype) cfg.stt.dtype = dtype === 'auto' ? null : dtype;
         try { persistConfig(cfg, configPath()); } catch { /* best effort */ }
-        sttEngine.setModel(id, { onLog: (m) => console.log(m) });
-        return sendJson(res, 202, { accepted: true, active: id, state: sttEngine.state(), progress: sttEngine.progress() });
+        sttEngine.setModel(id, { onLog: (m) => console.log(m), dtype: dtype || cfg.stt.dtype || 'auto' });
+        return sendJson(res, 202, { accepted: true, active: id, dtype: cfg.stt.dtype || 'auto', state: sttEngine.state(), progress: sttEngine.progress() });
       }
     }
     // Speaker (diarization) model manager — the "who said what" x-vector model.
@@ -725,7 +734,7 @@ export function createGateway(cfg = loadConfig()) {
       // Kick the model load on first use (single-flight; downloads once). The
       // client follows progress on the session's SSE stream.
       if (!sttEngine.isReady()) {
-        sttEngine.init({ model: cfg.stt?.model || DEFAULT_STT_MODEL, allowDownload: cfg.stt?.allowDownload !== false, onLog: (m) => console.log(m) });
+        sttEngine.init({ model: cfg.stt?.model || DEFAULT_STT_MODEL, allowDownload: cfg.stt?.allowDownload !== false, dtype: cfg.stt?.dtype || undefined, onLog: (m) => console.log(m) });
       }
       // Diarization is another OPTIONAL stage: load its model only when a session
       // asks for it (never on the dictation path).

package/src/stt-engine.js

@@ -85,8 +85,8 @@ export function health() {
 
 // (Re)load a model into _pipe. Same contract as ner-engine.loadModel: fail-open,
 // and a failed SWITCH keeps the previous working pipeline.
-/** @param {string} modelId @param {{ log?: (m: string) => void, allowDownload?: boolean }} [opts] */
-async function loadModel(modelId, { log = () => {}, allowDownload = true } = {}) {
+/** @param {string} modelId @param {{ log?: (m: string) => void, allowDownload?: boolean, dtype?: string }} [opts] */
+async function loadModel(modelId, { log = () => {}, allowDownload = true, dtype: dtypeOverride = null } = {}) {
   const prevPipe = _pipe;
   const prevModel = _model;
   let lib;
@@ -98,7 +98,13 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true } = {})
     return false;
   }
 
-  const haveLocal = modelOnDisk(modelId);
+  // Resolve the precision up front so the on-disk check targets the RIGHT files —
+  // else switching precision (e.g. q8→fp16) would see the q8 files as "present" and
+  // try to load fp16 offline, which fails.
+  const chosen = dtypeOverride && dtypeOverride !== 'auto' ? dtypeOverride : null;
+  const dtype = chosen || sttModelDtype(modelId) || runtimeDtype();
+
+  const haveLocal = modelOnDisk(modelId, dtype);
   lib.env.allowRemoteModels = haveLocal ? false : !!allowDownload;
   if (!haveLocal && !allowDownload) {
     _state = 'error'; _err = 'model not on disk and downloads disabled';
@@ -117,20 +123,28 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true } = {})
   if (!haveLocal) { _progress = { model: modelId, file: null, pct: 0 }; log(`[stt] downloading model ${modelId} (one-time${isCustom ? ', from Hugging Face' : ''})…`); }
 
   try {
-    // Per-model override wins (catalog `dtype`), else the runtime-appropriate default.
-    const dtype = sttModelDtype(modelId) || runtimeDtype();
-    const pipe = await lib.pipeline('automatic-speech-recognition', modelId, {
-      dtype,
-      progress_callback: (p) => {
-        if (!p) return;
-        const pct = typeof p.progress === 'number' ? Math.round(p.progress) : (_progress?.pct ?? 0);
-        if (p.status === 'progress' || p.status === 'download' || p.status === 'initiate') {
-          _progress = { model: modelId, file: p.file || _progress?.file || null, pct };
-        } else if (p.status === 'done' && p.file) {
-          log(`[stt] fetched ${p.file}`);
-        }
-      },
-    });
+    const progress_callback = (p) => {
+      if (!p) return;
+      const pct = typeof p.progress === 'number' ? Math.round(p.progress) : (_progress?.pct ?? 0);
+      if (p.status === 'progress' || p.status === 'download' || p.status === 'initiate') {
+        _progress = { model: modelId, file: p.file || _progress?.file || null, pct };
+      } else if (p.status === 'done' && p.file) {
+        log(`[stt] fetched ${p.file}`);
+      }
+    };
+    let pipe;
+    try {
+      pipe = await lib.pipeline('automatic-speech-recognition', modelId, { dtype, progress_callback });
+    } catch (e) {
+      // The dl.chatpanel.net mirror only proxies an allowlist; a catalog model
+      // missing from it (or any mirror hiccup) 403s. Fall back to Hugging Face so
+      // a download is never blocked by a mirror gap. (Custom ids already use HF.)
+      if (haveLocal || isCustom || !allowDownload) throw e;
+      log(`[stt] mirror fetch failed (${String(e.message).slice(0, 80)}) — retrying from Hugging Face…`);
+      lib.env.remoteHost = 'https://huggingface.co/';
+      lib.env.allowRemoteModels = true;
+      pipe = await lib.pipeline('automatic-speech-recognition', modelId, { dtype, progress_callback });
+    }
     _pipe = pipe; _model = modelId; _state = 'ready'; _err = null; _progress = null; _dtype = dtype;
     if (prevPipe && prevPipe !== pipe) { try { await prevPipe.dispose?.(); } catch { /* ignore */ } }
     log(`[stt] ready — model ${modelId} @ ${dtype} (in-process, offline) — local dictation active`);
@@ -153,15 +167,17 @@ export function init(cfg = {}) {
   const log = typeof cfg.onLog === 'function' ? cfg.onLog : () => {};
   _model = cfg.model || DEFAULT_STT_MODEL;
   _state = 'loading';
-  _initPromise = loadModel(_model, { log, allowDownload: cfg.allowDownload !== false });
+  _initPromise = loadModel(_model, { log, allowDownload: cfg.allowDownload !== false, dtype: cfg.dtype });
   return _initPromise;
 }
 
 export async function setModel(modelId, opts = {}) {
   const log = typeof opts.onLog === 'function' ? opts.onLog : () => {};
   if (!modelId) return false;
-  if (modelId === _model && isReady()) return true;
-  return loadModel(modelId, { log, allowDownload: opts.allowDownload !== false });
+  // Re-load if the model OR the requested precision changed.
+  const wantDtype = opts.dtype && opts.dtype !== 'auto' ? opts.dtype : (sttModelDtype(modelId) || runtimeDtype());
+  if (modelId === _model && isReady() && _dtype === wantDtype) return true;
+  return loadModel(modelId, { log, allowDownload: opts.allowDownload !== false, dtype: opts.dtype });
 }
 
 // ── Streaming sessions ──────────────────────────────────────────────────────────

package/src/stt-models.js

@@ -85,3 +85,19 @@ export function isValidCustomSttId(id) {
   const s = String(id || '');
   return /^[A-Za-z0-9][\w.-]*\/[\w.-]+$/.test(s) && !s.includes('..');
 }
+
+// Selectable quantizations (precision). 'auto' = let the runtime decide (q8 on
+// native, fp32 on WASM). Each is a transformers.js dtype; a model must actually
+// ship that ONNX variant (the load fails open otherwise). Ordered fastest→best.
+export const STT_DTYPES = [
+  { id: 'auto', label: 'Auto (recommended)', note: 'q8 on the native gateway, fp32 on the WASM binary.' },
+  { id: 'q8', label: 'q8 — fast, balanced', note: 'Int8. Best speed/quality trade-off (native default).' },
+  { id: 'q4', label: 'q4 — smallest & fastest', note: '4-bit. Least memory; slightly lower accuracy.' },
+  { id: 'int8', label: 'int8', note: 'Alternative int8 export.' },
+  { id: 'fp16', label: 'fp16 — more accurate', note: 'Half precision. Larger, a bit slower.' },
+  { id: 'fp32', label: 'fp32 — most accurate (slow)', note: 'Full precision. Largest + slowest; the only one that loads on the WASM binary.' },
+];
+
+export function isValidDtype(d) {
+  return STT_DTYPES.some((x) => x.id === d);
+}