@chatpanel/gateway 0.6.18 → 0.6.20

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/gateway",
-  "version": "0.6.18",
+  "version": "0.6.20",
   "description": "Local privacy gateway — redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
   "type": "module",
   "bin": {

package/src/models.js

@@ -36,3 +36,12 @@ export const MODEL_CATALOG = [
 export function isKnownModel(id) {
   return MODEL_CATALOG.some((m) => m.id === id);
 }
+
+// Accept a user-supplied ("bring your own") NER model id. STRICT `org/name` shape,
+// no path traversal. We can't verify it's token-classification from the id alone —
+// the engine fails open if the download/labels don't fit. Custom ids fetch from
+// Hugging Face directly (the engine points remoteHost there for the custom load).
+export function isValidCustomModelId(id) {
+  const s = String(id || '');
+  return /^[A-Za-z0-9][\w.-]*\/[\w.-]+$/.test(s) && !s.includes('..');
+}

package/src/ner-engine.js

@@ -17,6 +17,7 @@
 import os from 'node:os';
 import { join } from 'node:path';
 import { existsSync, mkdirSync } from 'node:fs';
+import { isKnownModel } from './models.js';
 
 const DEFAULT_MODEL = 'Xenova/bert-base-NER';
 
@@ -199,8 +200,15 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true } = {})
     return false;
   }
 
+  // Curated catalog models come from the private dl.chatpanel.net mirror
+  // (ensureLib set that as remoteHost). A user's BYO id isn't mirrored, so fetch
+  // it from Hugging Face directly — only for this load, then restore.
+  const prevHost = lib.env.remoteHost;
+  const isCustom = !isKnownModel(modelId);
+  if (!haveLocal && isCustom) { try { lib.env.remoteHost = 'https://huggingface.co/'; } catch { /* optional */ } }
+
   _state = haveLocal ? 'loading' : 'downloading';
-  if (!haveLocal) { _progress = { model: modelId, file: null, pct: 0 }; log(`[ner] downloading model ${modelId} (one-time)…`); }
+  if (!haveLocal) { _progress = { model: modelId, file: null, pct: 0 }; log(`[ner] downloading model ${modelId} (one-time${isCustom ? ', from Hugging Face' : ''})…`); }
 
   try {
     const pipe = await lib.pipeline('token-classification', modelId, {
@@ -227,6 +235,8 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true } = {})
     else { _state = 'error'; }
     log(`[ner] model load failed (${e.message})${prevPipe ? ' — keeping previous model' : ' — deterministic-only'}`);
     return false;
+  } finally {
+    try { lib.env.remoteHost = prevHost; } catch { /* optional */ } // restore the mirror
   }
 }
 

package/src/server.js

@@ -33,8 +33,8 @@ import { createHistoryStore } from './sqlite-store.js';
 import { ingestBackup } from './backup-ingest.js';
 import * as nerEngine from './ner-engine.js';
 import * as sttEngine from './stt-engine.js';
-import { MODEL_CATALOG, isKnownModel } from './models.js';
-import { STT_MODEL_CATALOG, isKnownSttModel, DEFAULT_STT_MODEL } from './stt-models.js';
+import { MODEL_CATALOG, isKnownModel, isValidCustomModelId } from './models.js';
+import { STT_MODEL_CATALOG, isKnownSttModel, isValidCustomSttId, DEFAULT_STT_MODEL } from './stt-models.js';
 import { resolvePro, checkQuota, consume, usage } from './freegate.js';
 import { publicConfig, applyConfigPatch, persistConfig, configPath } from './configstore.js';
 import { resolveDestination, aggregateModelsAsync } from './router.js';
@@ -42,7 +42,7 @@ import * as openai from './openai.js';
 import * as responses from './responses.js';
 import * as anthropic from './anthropic.js';
 
-export const VERSION = '0.6.18';
+export const VERSION = '0.6.20';
 
 // WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
 // store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
@@ -632,9 +632,14 @@ export function createGateway(cfg = loadConfig()) {
     // model (downloading it first if needed) and persists the choice.
     if (pathname === '/ner/models') {
       if (req.method === 'GET') {
-        const available = MODEL_CATALOG.map((m) => ({ ...m, installed: nerEngine.modelOnDisk(m.id) }));
+        const active = nerEngine.health().model || cfg.ner?.model || null;
+        const available = /** @type {any[]} */ (MODEL_CATALOG.map((m) => ({ ...m, installed: nerEngine.modelOnDisk(m.id) })));
+        // Surface an active BYO (non-catalog) model so the UI shows it too.
+        if (active && !available.some((m) => m.id === active)) {
+          available.push({ id: active, label: active, lang: '—', custom: true, installed: nerEngine.modelOnDisk(active), note: 'Custom model (from Hugging Face).' });
+        }
         return sendJson(res, 200, {
-          active: nerEngine.health().model || cfg.ner?.model || null,
+          active,
           state: nerEngine.state(),
           progress: nerEngine.progress(),
           available,
@@ -643,8 +648,9 @@ export function createGateway(cfg = loadConfig()) {
       if (req.method === 'POST') {
         let body = null;
         try { body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')); } catch { body = null; }
-        const id = body && typeof body.id === 'string' ? body.id : null;
-        if (!id || !isKnownModel(id)) return sendJson(res, 400, { error: { message: 'unknown model id', type: 'bad_model' } });
+        const id = body && typeof body.id === 'string' ? body.id.trim() : null;
+        // Curated catalog OR a strictly-validated BYO id (org/name, from HF).
+        if (!id || !(isKnownModel(id) || isValidCustomModelId(id))) return sendJson(res, 400, { error: { message: 'unknown or invalid model id', type: 'bad_model' } });
         // Persist first so a restart keeps the choice, then (re)load. Don't block the
         // response on a possibly-long download — the client polls GET for progress.
         if (cfg.ner) cfg.ner.model = id; else cfg.ner = { autostart: true, model: id, allowDownload: true, enableFullTier: true };
@@ -666,9 +672,14 @@ export function createGateway(cfg = loadConfig()) {
     //   GET/POST /stt/models               catalog + progress / switch (mirrors /ner/models)
     if (pathname === '/stt/models') {
       if (req.method === 'GET') {
-        const available = STT_MODEL_CATALOG.map((m) => ({ ...m, installed: sttEngine.modelOnDisk(m.id) }));
+        const active = sttEngine.health().model || cfg.stt?.model || DEFAULT_STT_MODEL;
+        const available = /** @type {any[]} */ (STT_MODEL_CATALOG.map((m) => ({ ...m, installed: sttEngine.modelOnDisk(m.id) })));
+        // Surface an active CUSTOM (non-catalog) model so the UI can show it too.
+        if (active && !available.some((m) => m.id === active)) {
+          available.push({ id: active, label: active, lang: '—', tier: 'custom', custom: true, installed: sttEngine.modelOnDisk(active), note: 'Custom model (from Hugging Face).' });
+        }
         return sendJson(res, 200, {
-          active: sttEngine.health().model || cfg.stt?.model || DEFAULT_STT_MODEL,
+          active,
           state: sttEngine.state(),
           progress: sttEngine.progress(),
           available,
@@ -677,8 +688,9 @@ export function createGateway(cfg = loadConfig()) {
       if (req.method === 'POST') {
         let body = null;
         try { body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')); } catch { body = null; }
-        const id = body && typeof body.id === 'string' ? body.id : null;
-        if (!id || !isKnownSttModel(id)) return sendJson(res, 400, { error: { message: 'unknown model id', type: 'bad_model' } });
+        const id = body && typeof body.id === 'string' ? body.id.trim() : null;
+        // Curated catalog OR a strictly-validated custom whisper id (Advanced).
+        if (!id || !(isKnownSttModel(id) || isValidCustomSttId(id))) return sendJson(res, 400, { error: { message: 'unknown or invalid model id', type: 'bad_model' } });
         if (cfg.stt) cfg.stt.model = id; else cfg.stt = { enabled: true, model: id, allowDownload: true };
         try { persistConfig(cfg, configPath()); } catch { /* best effort */ }
         sttEngine.setModel(id, { onLog: (m) => console.log(m) });

package/src/stt-engine.js

@@ -20,24 +20,50 @@ import { join } from 'node:path';
 import { existsSync, readdirSync } from 'node:fs';
 import { randomUUID } from 'node:crypto';
 import { ensureLib, modelRoot } from './ner-engine.js';
-import { DEFAULT_STT_MODEL, isEnglishOnly } from './stt-models.js';
+import { DEFAULT_STT_MODEL, isEnglishOnly, sttModelDtype, isKnownSttModel } from './stt-models.js';
 
 export const SAMPLE_RATE = 16000; // fixed wire contract: 16 kHz mono Float32 PCM
 
+// The right whisper quantization depends on the ONNX RUNTIME, not the OS:
+//   • native onnxruntime-node (the npm gateway) loads the small, fast `q8`
+//     (_quantized) exports — best size + speed.
+//   • onnxruntime-web WASM (the standalone binary — SAME wasm on macOS/Windows/
+//     Linux, so this is inherently cross-platform) CANNOT load the block-quantized
+//     exports (q8/int8/uint8 → MatMulNBits "missing scale"; fp16 → graph error);
+//     of the loadable ones only `fp32` is fast enough for real-time (q4/bnb4 are
+//     ~8× slower). Verified empirically against the bundled ORT-web build.
+// The binary entry sets __CHATPANEL_WASM_PATHS__, so that global tells us which
+// runtime we're on. A model may override via `dtype` in the STT catalog.
+export function runtimeDtype() {
+  return globalThis.__CHATPANEL_WASM_PATHS__ ? 'fp32' : 'q8';
+}
+
 let _state = 'off';        // 'off' | 'loading' | 'downloading' | 'ready' | 'error'
 let _model = null;         // active model id
 let _pipe = null;          // the loaded automatic-speech-recognition pipeline
 let _err = null;           // last error message (for /health)
 let _initPromise = null;   // single-flight init
 let _progress = null;      // { model, file, pct } while downloading, else null
-
-// Whisper exports ship encoder+decoder ONNX files (not the single model.onnx the
-// NER check looks for), so presence = "any encoder_model*.onnx on disk".
-export function modelOnDisk(modelId = _model || DEFAULT_STT_MODEL) {
+let _dtype = null;         // the quantization actually loaded (fp32 on WASM, q8 native)
+
+// transformers.js dtype → the ONNX filename suffix it loads. Presence must check
+// the EXACT file the current runtime will fetch — otherwise a q8 install (native)
+// looks "present" to the WASM runtime, which actually needs the fp32 file, and the
+// offline load fails. Checking the real target makes a runtime switch re-download.
+const DTYPE_SUFFIX = {
+  fp32: '', q8: '_quantized', int8: '_int8', uint8: '_uint8',
+  fp16: '_fp16', q4: '_q4', bnb4: '_bnb4', q4f16: '_q4f16',
+};
+
+export function modelOnDisk(modelId = _model || DEFAULT_STT_MODEL, dtype = sttModelDtype(modelId) || runtimeDtype()) {
   const dir = join(modelRoot(), ...modelId.split('/'), 'onnx');
   if (!existsSync(dir)) return false;
-  try { return readdirSync(dir).some((f) => /^encoder_model.*\.onnx$/.test(f)); }
-  catch { return false; }
+  const suffix = DTYPE_SUFFIX[dtype] ?? '';
+  try {
+    const files = readdirSync(dir);
+    return files.includes(`encoder_model${suffix}.onnx`)
+      && files.some((f) => f === `decoder_model_merged${suffix}.onnx` || f === `decoder_model${suffix}.onnx`);
+  } catch { return false; }
 }
 
 export function state() { return _state; }
@@ -45,7 +71,7 @@ export function isReady() { return _state === 'ready' && !!_pipe; }
 export function progress() { return _progress; }
 
 export function health() {
-  return { configured: _state !== 'off', ok: isReady(), state: _state, model: _model, error: _err };
+  return { configured: _state !== 'off', ok: isReady(), state: _state, model: _model, dtype: _dtype, error: _err };
 }
 
 // (Re)load a model into _pipe. Same contract as ner-engine.loadModel: fail-open,
@@ -71,12 +97,21 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true } = {})
     return false;
   }
 
+  // Curated catalog models come from the private dl.chatpanel.net mirror (ensureLib
+  // already set that as remoteHost). A CUSTOM ("Advanced") id isn't mirrored, so
+  // fetch it from Hugging Face directly — only for this load, then restore.
+  const prevHost = lib.env.remoteHost;
+  const isCustom = !isKnownSttModel(modelId);
+  if (!haveLocal && isCustom) { try { lib.env.remoteHost = 'https://huggingface.co/'; } catch { /* optional */ } }
+
   _state = haveLocal ? 'loading' : 'downloading';
-  if (!haveLocal) { _progress = { model: modelId, file: null, pct: 0 }; log(`[stt] downloading model ${modelId} (one-time)…`); }
+  if (!haveLocal) { _progress = { model: modelId, file: null, pct: 0 }; log(`[stt] downloading model ${modelId} (one-time${isCustom ? ', from Hugging Face' : ''})…`); }
 
   try {
+    // Per-model override wins (catalog `dtype`), else the runtime-appropriate default.
+    const dtype = sttModelDtype(modelId) || runtimeDtype();
     const pipe = await lib.pipeline('automatic-speech-recognition', modelId, {
-      dtype: 'q8',
+      dtype,
       progress_callback: (p) => {
         if (!p) return;
         const pct = typeof p.progress === 'number' ? Math.round(p.progress) : (_progress?.pct ?? 0);
@@ -87,9 +122,9 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true } = {})
         }
       },
     });
-    _pipe = pipe; _model = modelId; _state = 'ready'; _err = null; _progress = null;
+    _pipe = pipe; _model = modelId; _state = 'ready'; _err = null; _progress = null; _dtype = dtype;
     if (prevPipe && prevPipe !== pipe) { try { await prevPipe.dispose?.(); } catch { /* ignore */ } }
-    log(`[stt] ready — model ${modelId} (in-process, offline) — local dictation active`);
+    log(`[stt] ready — model ${modelId} @ ${dtype} (in-process, offline) — local dictation active`);
     return true;
   } catch (e) {
     _err = e.message; _progress = null;
@@ -97,6 +132,8 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true } = {})
     else { _state = 'error'; }
     log(`[stt] model load failed (${e.message})${prevPipe ? ' — keeping previous model' : ''}`);
     return false;
+  } finally {
+    try { lib.env.remoteHost = prevHost; } catch { /* optional */ } // restore the mirror for NER + catalog loads
   }
 }
 

package/src/stt-models.js

@@ -1,38 +1,57 @@
 // Catalog of speech-to-text (whisper) models the gateway can run, surfaced in the
-// extension's Gateway settings and dictation UI. All are ONNX (transformers.js)
-// automatic-speech-recognition models — same engine, download plumbing, and model
-// root as the NER catalog (models.js). Sizes are the on-disk q8 footprint, approx.
+// extension's Gateway settings and dictation UI so users pick by their machine's
+// resources. All are ONNX (transformers.js) automatic-speech-recognition models —
+// same engine, download plumbing, and model root as the NER catalog (models.js).
+//
+// `tier` groups them for the picker (light / balanced / accurate / max). `ramMB`
+// is a rough working-set hint. `approxMB` is the DOWNLOAD size for the dtype the
+// current runtime will actually fetch — it differs by runtime (the WASM binary
+// downloads fp32; the native npm build downloads q8), so it's a range, and the
+// gateway reports the exact bytes as they stream.
 //
 // Adding a model: any onnx-community/* (or Xenova/*) whisper export works. Verify
-// it loads + transcribes before listing it here. `.en` models are English-only and
-// reject language/task options — sttDecodeOptions() below handles that split.
+// it loads on BOTH runtimes (native q8 + WASM fp32) before listing it — the WASM
+// runtime can't load block-quantized (q8/int8) or fp16 exports (see stt-engine
+// runtimeDtype). `.en` models are English-only and reject language/task options.
 
-// Default is MULTILINGUAL on purpose: whisper auto-detects the spoken language
-// per segment when no `language` is pinned — dictation must "just work" in any
-// language without a setting. `.en` models are the speed opt-in, not the default.
 export const DEFAULT_STT_MODEL = 'onnx-community/whisper-base';
 
 export const STT_MODEL_CATALOG = [
+  {
+    id: 'onnx-community/whisper-tiny.en',
+    label: 'Tiny (English)',
+    lang: 'English',
+    tier: 'light',
+    approxMB: 150,   // fp32 on WASM; ~40 on native q8
+    ramMB: 400,
+    note: 'Fastest, lightest. English only — great for quick dictation on any machine.',
+  },
   {
     id: 'onnx-community/whisper-base',
-    label: 'Multilingual — balanced',
+    label: 'Base (multilingual)',
     lang: '99 languages (auto-detected)',
-    approxMB: 105,
-    note: 'Default. Detects the spoken language automatically; good accuracy.',
+    tier: 'balanced',
+    approxMB: 300,   // fp32 on WASM; ~80 on native q8
+    ramMB: 700,
+    note: 'Default. Detects the spoken language automatically; good accuracy at real-time speed.',
   },
   {
-    id: 'onnx-community/whisper-tiny.en',
-    label: 'English — fastest',
-    lang: 'English',
-    approxMB: 60,
-    note: 'Real-time on modest hardware; English only.',
+    id: 'onnx-community/whisper-small',
+    label: 'Small (multilingual)',
+    lang: '99 languages (auto-detected)',
+    tier: 'accurate',
+    approxMB: 950,   // fp32 on WASM; ~250 on native q8
+    ramMB: 1600,
+    note: 'Noticeably more accurate; needs a faster CPU to keep up in real time.',
   },
   {
-    id: 'onnx-community/whisper-small',
-    label: 'Multilingual — accurate',
+    id: 'onnx-community/whisper-large-v3-turbo',
+    label: 'Large v3 Turbo (multilingual)',
     lang: '99 languages (auto-detected)',
-    approxMB: 330,
-    note: 'Best accuracy; needs a faster machine for real-time use.',
+    tier: 'max',
+    approxMB: 1600,  // native q8 recommended; heavy on WASM
+    ramMB: 3200,
+    note: 'Best accuracy. For powerful machines; use the native (npm) gateway for speed.',
   },
 ];
 
@@ -40,7 +59,27 @@ export function isKnownSttModel(id) {
   return STT_MODEL_CATALOG.some((m) => m.id === id);
 }
 
+export function sttModel(id) {
+  return STT_MODEL_CATALOG.find((m) => m.id === id) || null;
+}
+
 // English-only exports reject `language`/`task` generation options.
 export function isEnglishOnly(id) {
   return /\.en$/.test(String(id || ''));
 }
+
+// A model may pin an explicit dtype (overriding the runtime default). None do by
+// default — the engine's runtimeDtype() picks q8 (native) / fp32 (WASM) — but
+// this is the seam for a model that needs a specific quantization.
+export function sttModelDtype(id) {
+  return sttModel(id)?.dtype || null;
+}
+
+// Accept a user-supplied ("Advanced") whisper model id. STRICT: `org/name` shape,
+// must be a whisper export, no path traversal. Curated catalog ids come from the
+// private dl.chatpanel.net mirror; custom ids are fetched from Hugging Face
+// directly (the engine points remoteHost there only for the custom load).
+export function isValidCustomSttId(id) {
+  const s = String(id || '');
+  return /^[A-Za-z0-9][\w.-]*\/[\w.-]+$/.test(s) && /whisper/i.test(s) && !s.includes('..');
+}