@chatpanel/gateway 0.6.26 → 0.6.27

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/gateway",
-  "version": "0.6.26",
+  "version": "0.6.27",
   "description": "Local privacy gateway — redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
   "type": "module",
   "bin": {

package/src/server.js

@@ -35,7 +35,7 @@ import * as nerEngine from './ner-engine.js';
 import * as sttEngine from './stt-engine.js';
 import * as diarizeEngine from './diarize-engine.js';
 import { MODEL_CATALOG, isKnownModel, isValidCustomModelId } from './models.js';
-import { STT_MODEL_CATALOG, isKnownSttModel, isValidCustomSttId, DEFAULT_STT_MODEL } from './stt-models.js';
+import { STT_MODEL_CATALOG, isKnownSttModel, isValidCustomSttId, DEFAULT_STT_MODEL, STT_DTYPES, isValidDtype } from './stt-models.js';
 import { resolvePro, checkQuota, consume, usage } from './freegate.js';
 import { publicConfig, applyConfigPatch, persistConfig, configPath } from './configstore.js';
 import { resolveDestination, aggregateModelsAsync } from './router.js';
@@ -43,7 +43,7 @@ import * as openai from './openai.js';
 import * as responses from './responses.js';
 import * as anthropic from './anthropic.js';
 
-export const VERSION = '0.6.26';
+export const VERSION = '0.6.27';
 
 // WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
 // store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
@@ -686,6 +686,12 @@ export function createGateway(cfg = loadConfig()) {
           state: sttEngine.state(),
           progress: sttEngine.progress(),
           available,
+          // Precision (quantization) picker: current choice + selectable options +
+          // the dtype actually loaded. On WASM only fp32 loads (see runtimeDtype).
+          dtype: cfg.stt?.dtype || 'auto',
+          loadedDtype: sttEngine.health().dtype,
+          runtime: sttEngine.health().runtime,
+          dtypes: STT_DTYPES,
         });
       }
       if (req.method === 'POST') {
@@ -694,10 +700,13 @@ export function createGateway(cfg = loadConfig()) {
         const id = body && typeof body.id === 'string' ? body.id.trim() : null;
         // Curated catalog OR a strictly-validated custom whisper id (Advanced).
         if (!id || !(isKnownSttModel(id) || isValidCustomSttId(id))) return sendJson(res, 400, { error: { message: 'unknown or invalid model id', type: 'bad_model' } });
+        // Optional precision override (q8/q4/fp16/…); 'auto' clears it.
+        const dtype = typeof body.dtype === 'string' && isValidDtype(body.dtype) ? body.dtype : undefined;
         if (cfg.stt) cfg.stt.model = id; else cfg.stt = { enabled: true, model: id, allowDownload: true };
+        if (dtype) cfg.stt.dtype = dtype === 'auto' ? null : dtype;
         try { persistConfig(cfg, configPath()); } catch { /* best effort */ }
-        sttEngine.setModel(id, { onLog: (m) => console.log(m) });
-        return sendJson(res, 202, { accepted: true, active: id, state: sttEngine.state(), progress: sttEngine.progress() });
+        sttEngine.setModel(id, { onLog: (m) => console.log(m), dtype: dtype || cfg.stt.dtype || 'auto' });
+        return sendJson(res, 202, { accepted: true, active: id, dtype: cfg.stt.dtype || 'auto', state: sttEngine.state(), progress: sttEngine.progress() });
       }
     }
     // Speaker (diarization) model manager — the "who said what" x-vector model.
@@ -725,7 +734,7 @@ export function createGateway(cfg = loadConfig()) {
       // Kick the model load on first use (single-flight; downloads once). The
       // client follows progress on the session's SSE stream.
       if (!sttEngine.isReady()) {
-        sttEngine.init({ model: cfg.stt?.model || DEFAULT_STT_MODEL, allowDownload: cfg.stt?.allowDownload !== false, onLog: (m) => console.log(m) });
+        sttEngine.init({ model: cfg.stt?.model || DEFAULT_STT_MODEL, allowDownload: cfg.stt?.allowDownload !== false, dtype: cfg.stt?.dtype || undefined, onLog: (m) => console.log(m) });
       }
       // Diarization is another OPTIONAL stage: load its model only when a session
       // asks for it (never on the dictation path).

package/src/stt-engine.js

@@ -85,8 +85,8 @@ export function health() {
 
 // (Re)load a model into _pipe. Same contract as ner-engine.loadModel: fail-open,
 // and a failed SWITCH keeps the previous working pipeline.
-/** @param {string} modelId @param {{ log?: (m: string) => void, allowDownload?: boolean }} [opts] */
-async function loadModel(modelId, { log = () => {}, allowDownload = true } = {}) {
+/** @param {string} modelId @param {{ log?: (m: string) => void, allowDownload?: boolean, dtype?: string }} [opts] */
+async function loadModel(modelId, { log = () => {}, allowDownload = true, dtype: dtypeOverride = null } = {}) {
   const prevPipe = _pipe;
   const prevModel = _model;
   let lib;
@@ -98,7 +98,13 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true } = {})
     return false;
   }
 
-  const haveLocal = modelOnDisk(modelId);
+  // Resolve the precision up front so the on-disk check targets the RIGHT files —
+  // else switching precision (e.g. q8→fp16) would see the q8 files as "present" and
+  // try to load fp16 offline, which fails.
+  const chosen = dtypeOverride && dtypeOverride !== 'auto' ? dtypeOverride : null;
+  const dtype = chosen || sttModelDtype(modelId) || runtimeDtype();
+
+  const haveLocal = modelOnDisk(modelId, dtype);
   lib.env.allowRemoteModels = haveLocal ? false : !!allowDownload;
   if (!haveLocal && !allowDownload) {
     _state = 'error'; _err = 'model not on disk and downloads disabled';
@@ -117,8 +123,6 @@ async function loadModel(modelId, { log = () => {}, allowDownload = true } = {})
   if (!haveLocal) { _progress = { model: modelId, file: null, pct: 0 }; log(`[stt] downloading model ${modelId} (one-time${isCustom ? ', from Hugging Face' : ''})…`); }
 
   try {
-    // Per-model override wins (catalog `dtype`), else the runtime-appropriate default.
-    const dtype = sttModelDtype(modelId) || runtimeDtype();
     const progress_callback = (p) => {
       if (!p) return;
       const pct = typeof p.progress === 'number' ? Math.round(p.progress) : (_progress?.pct ?? 0);
@@ -163,15 +167,17 @@ export function init(cfg = {}) {
   const log = typeof cfg.onLog === 'function' ? cfg.onLog : () => {};
   _model = cfg.model || DEFAULT_STT_MODEL;
   _state = 'loading';
-  _initPromise = loadModel(_model, { log, allowDownload: cfg.allowDownload !== false });
+  _initPromise = loadModel(_model, { log, allowDownload: cfg.allowDownload !== false, dtype: cfg.dtype });
   return _initPromise;
 }
 
 export async function setModel(modelId, opts = {}) {
   const log = typeof opts.onLog === 'function' ? opts.onLog : () => {};
   if (!modelId) return false;
-  if (modelId === _model && isReady()) return true;
-  return loadModel(modelId, { log, allowDownload: opts.allowDownload !== false });
+  // Re-load if the model OR the requested precision changed.
+  const wantDtype = opts.dtype && opts.dtype !== 'auto' ? opts.dtype : (sttModelDtype(modelId) || runtimeDtype());
+  if (modelId === _model && isReady() && _dtype === wantDtype) return true;
+  return loadModel(modelId, { log, allowDownload: opts.allowDownload !== false, dtype: opts.dtype });
 }
 
 // ── Streaming sessions ──────────────────────────────────────────────────────────

package/src/stt-models.js

@@ -85,3 +85,19 @@ export function isValidCustomSttId(id) {
   const s = String(id || '');
   return /^[A-Za-z0-9][\w.-]*\/[\w.-]+$/.test(s) && !s.includes('..');
 }
+
+// Selectable quantizations (precision). 'auto' = let the runtime decide (q8 on
+// native, fp32 on WASM). Each is a transformers.js dtype; a model must actually
+// ship that ONNX variant (the load fails open otherwise). Ordered fastest→best.
+export const STT_DTYPES = [
+  { id: 'auto', label: 'Auto (recommended)', note: 'q8 on the native gateway, fp32 on the WASM binary.' },
+  { id: 'q8', label: 'q8 — fast, balanced', note: 'Int8. Best speed/quality trade-off (native default).' },
+  { id: 'q4', label: 'q4 — smallest & fastest', note: '4-bit. Least memory; slightly lower accuracy.' },
+  { id: 'int8', label: 'int8', note: 'Alternative int8 export.' },
+  { id: 'fp16', label: 'fp16 — more accurate', note: 'Half precision. Larger, a bit slower.' },
+  { id: 'fp32', label: 'fp32 — most accurate (slow)', note: 'Full precision. Largest + slowest; the only one that loads on the WASM binary.' },
+];
+
+export function isValidDtype(d) {
+  return STT_DTYPES.some((x) => x.id === d);
+}