@chatpanel/gateway 0.6.10 → 0.6.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/gateway",
-  "version": "0.6.10",
+  "version": "0.6.12",
   "description": "Local privacy gateway — redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
   "type": "module",
   "bin": {
@@ -27,7 +27,7 @@
     "node": ">=18"
   },
   "dependencies": {
-    "@chatpanel/pii": "^0.2.10",
+    "@chatpanel/pii": "^0.2.11",
     "@huggingface/transformers": "^4.2.0",
     "onnxruntime-web": "1.26.0-dev.20260416-b7804b056c"
   },

package/src/history-store.js

@@ -0,0 +1,155 @@
+// WARM-tier record store + encrypted-at-rest persistence (gateway side).
+//
+// The SearchIndex keeps only term-frequencies + light meta, so it can rank but
+// can't hand back a full record. This store keeps the raw records so the gateway
+// can (a) survive a restart WITHOUT the extension re-ingesting everything — the
+// user's "no cold start" requirement — and (b) serve read endpoints (list/get)
+// that an external UI renders. The BM25 index is derived from the store and
+// rebuilt on load (cheap, in-memory), so only the store is persisted.
+//
+// ENCRYPTED AT REST: records are AES-256-GCM'd with a key generated once and kept
+// at ~/.chatpanel/history-key (0600). This is the LOCAL/on-device tier — a local
+// key is correct here; zero-knowledge (keys never on the box) is the future CLOUD
+// tier, not this one. The file on disk is useless without the local key.
+
+import { randomBytes, createCipheriv, createDecipheriv } from 'node:crypto';
+import { readFileSync, writeFileSync, existsSync, mkdirSync, chmodSync } from 'node:fs';
+import { join, dirname } from 'node:path';
+import os from 'node:os';
+import { SearchIndex } from './search-index.js';
+
+const DIR = join(os.homedir(), '.chatpanel');
+const STORE_PATH = process.env.CHATPANEL_HISTORY_STORE || join(DIR, 'history-store.enc');
+const KEY_PATH = process.env.CHATPANEL_HISTORY_KEY || join(DIR, 'history-key');
+
+function loadOrCreateKey() {
+  try {
+    if (existsSync(KEY_PATH)) return Buffer.from(readFileSync(KEY_PATH, 'utf8').trim(), 'base64');
+  } catch {
+    /* regenerate below */
+  }
+  const key = randomBytes(32);
+  mkdirSync(dirname(KEY_PATH), { recursive: true });
+  writeFileSync(KEY_PATH, key.toString('base64'), { mode: 0o600 });
+  try {
+    chmodSync(KEY_PATH, 0o600);
+  } catch {
+    /* best effort on platforms without POSIX perms */
+  }
+  return key;
+}
+
+function encrypt(key, plaintextBuf) {
+  const iv = randomBytes(12);
+  const cipher = createCipheriv('aes-256-gcm', key, iv);
+  const ct = Buffer.concat([cipher.update(plaintextBuf), cipher.final()]);
+  return { v: 1, iv: iv.toString('base64'), tag: cipher.getAuthTag().toString('base64'), ct: ct.toString('base64') };
+}
+
+function decrypt(key, env) {
+  const decipher = createDecipheriv('aes-256-gcm', key, Buffer.from(env.iv, 'base64'));
+  decipher.setAuthTag(Buffer.from(env.tag, 'base64'));
+  return Buffer.concat([decipher.update(Buffer.from(env.ct, 'base64')), decipher.final()]);
+}
+
+// Records-plus-index with lazy, debounced encrypted persistence.
+export class HistoryStore {
+  constructor({ storePath = STORE_PATH, persistMs = 2000 } = {}) {
+    this.records = new Map(); // id -> { id, text, title, type, date }
+    this.index = new SearchIndex();
+    this.storePath = storePath;
+    this.persistMs = persistMs;
+    this._key = null;
+    this._timer = null;
+    this._dirty = false;
+  }
+
+  get size() {
+    return this.records.size;
+  }
+
+  key() {
+    if (!this._key) this._key = loadOrCreateKey();
+    return this._key;
+  }
+
+  // Load the encrypted store from disk and rebuild the index. Safe on a missing/
+  // corrupt file — starts empty rather than throwing (fail-open for a cache).
+  load() {
+    try {
+      if (!existsSync(this.storePath)) return this;
+      const env = JSON.parse(readFileSync(this.storePath, 'utf8'));
+      const records = JSON.parse(decrypt(this.key(), env).toString('utf8'));
+      this.records = new Map(records.map((r) => [r.id, r]));
+      this.index = new SearchIndex();
+      for (const r of this.records.values()) this.index.upsert(r);
+    } catch {
+      this.records = new Map();
+      this.index = new SearchIndex();
+    }
+    return this;
+  }
+
+  // Write the encrypted store now (synchronous). Callers normally use the
+  // debounced schedulePersist(); this is the flush.
+  persistNow() {
+    this._dirty = false;
+    if (this._timer) {
+      clearTimeout(this._timer);
+      this._timer = null;
+    }
+    const buf = Buffer.from(JSON.stringify([...this.records.values()]), 'utf8');
+    const env = encrypt(this.key(), buf);
+    mkdirSync(dirname(this.storePath), { recursive: true });
+    writeFileSync(this.storePath, JSON.stringify(env), { mode: 0o600 });
+  }
+
+  schedulePersist() {
+    this._dirty = true;
+    if (this._timer) return;
+    this._timer = setTimeout(() => {
+      this._timer = null;
+      if (this._dirty) {
+        try {
+          this.persistNow();
+        } catch {
+          /* keep serving from memory even if disk write fails */
+        }
+      }
+    }, this.persistMs);
+    if (this._timer.unref) this._timer.unref(); // don't hold the process open
+  }
+
+  // Apply upserts/removes to BOTH the record store and the index, then schedule a
+  // persist. Returns the new size. Mirrors SearchIndex.bulk's shape.
+  bulk({ upserts = [], removes = [] } = {}) {
+    for (const id of removes) {
+      this.records.delete(id);
+      this.index.remove(id);
+    }
+    for (const d of upserts) {
+      if (!d || !d.id) continue;
+      const rec = { id: d.id, text: String(d.text || ''), title: d.title || '', type: d.type || '', date: d.date || 0 };
+      this.records.set(rec.id, rec);
+      this.index.upsert(rec);
+    }
+    this.schedulePersist();
+    return this.records.size;
+  }
+
+  search(query, opts) {
+    return this.index.search(query, opts);
+  }
+
+  // Metadata list for an external UI, newest first, paginated. No bodies.
+  list({ limit = 50, offset = 0 } = {}) {
+    const all = [...this.records.values()]
+      .map((r) => ({ id: r.id, title: r.title, type: r.type, date: r.date, chars: r.text.length }))
+      .sort((a, b) => (b.date || 0) - (a.date || 0));
+    return { total: all.length, items: all.slice(offset, offset + limit) };
+  }
+
+  get(id) {
+    return this.records.get(id) || null;
+  }
+}

package/src/search-index.js

@@ -0,0 +1,134 @@
+// WARM tier — full-corpus search on the LOCAL gateway.
+//
+// The gateway is the user's own process, so it may hold decrypted records + the key and
+// build a real index outside the browser's memory/CPU (see docs/architecture-data-tiers).
+// This is the compute core: a BM25 doc store you can incrementally upsert/remove and
+// query. Persistence (encrypted at rest) and the HTTP endpoints layer on top of this.
+//
+// The tokenizer + BM25 are a VENDORED copy of the extension's meeting-index.js so the
+// gateway ranks identically to the browser hot tier — keep them in sync. Pure, no deps.
+
+const STOP = new Set((
+  'the a an and or but of to in on at for with is are was were be been being it this that these those as i you he '
+  + 'if am because need needs needed its lets let '
+  + 'she they we us my your our their me him her them so no yes not do does did have has had will would can could '
+  + 'should from by about into over under again further then once here there all any both each few more most other '
+  + 'some such only own same than too very just dont cant couldnt didnt doesnt hadnt hasnt havent im ive id isnt '
+  + 'youre were werent theyre thats theres whats wheres whos wont wouldnt shouldnt okay yeah uh um like really '
+  + 'going get got know think mean right well say said also one two how what when where which who whom why'
+).split(/\s+/));
+
+function normalizeToken(raw) {
+  let word = String(raw || '')
+    .toLowerCase()
+    .replace(/[’‘`]/g, "'")
+    .replace(/[‐‑‒–—]/g, '-')
+    .replace(/^['+_-]+|['+_-]+$/g, '');
+  if (word.endsWith("'s")) word = word.slice(0, -2);
+  const compact = word.replace(/['_-]+/g, '');
+  if (/^[a-z]\+\+$/.test(word)) return word;
+  if (compact.length < 2 || /^\d+$/.test(compact)) return '';
+  if (STOP.has(word) || STOP.has(compact)) return '';
+  return word.replace(/'/g, '');
+}
+
+export function tokenize(text) {
+  const m = String(text || '').toLowerCase().match(/[a-z0-9][a-z0-9'_+-]{1,}/g);
+  if (!m) return [];
+  return m.map(normalizeToken).filter(Boolean);
+}
+
+// A mutable BM25 index over documents. Each doc: { id, text, title?, type?, date? }.
+// upsert/remove are incremental (no full rebuild); search() ranks lazily off the current
+// term stats. Deletes are honest removals (the whole point of tombstoned sync upstream).
+export class SearchIndex {
+  constructor() {
+    this.docs = new Map();  // id -> { tf: Map<term,count>, len, meta }
+    this.df = new Map();    // term -> #docs containing it
+    this.totalLen = 0;
+  }
+
+  get size() { return this.docs.size; }
+
+  _removeStats(prev) {
+    this.totalLen -= prev.len;
+    for (const term of prev.tf.keys()) {
+      const n = (this.df.get(term) || 0) - 1;
+      if (n <= 0) this.df.delete(term); else this.df.set(term, n);
+    }
+  }
+
+  upsert(doc) {
+    if (!doc || !doc.id) return;
+    const prev = this.docs.get(doc.id);
+    if (prev) this._removeStats(prev);
+    const terms = tokenize(`${doc.title || ''}\n${doc.text || ''}`);
+    const tf = new Map();
+    for (const t of terms) tf.set(t, (tf.get(t) || 0) + 1);
+    for (const t of tf.keys()) this.df.set(t, (this.df.get(t) || 0) + 1);
+    this.totalLen += terms.length;
+    this.docs.set(doc.id, {
+      tf,
+      len: terms.length,
+      meta: { id: doc.id, title: doc.title || '', type: doc.type || '', date: doc.date || 0 },
+    });
+  }
+
+  remove(id) {
+    const prev = this.docs.get(id);
+    if (!prev) return false;
+    this._removeStats(prev);
+    this.docs.delete(id);
+    return true;
+  }
+
+  bulk({ upserts = [], removes = [] } = {}) {
+    for (const id of removes) this.remove(id);
+    for (const d of upserts) this.upsert(d);
+    return this.size;
+  }
+
+  // Okapi BM25. Returns [{ id, score, title, type, date }] sorted desc, top `limit`.
+  search(query, { limit = 10, k1 = 1.5, b = 0.75 } = {}) {
+    const qterms = [...new Set(tokenize(query))];
+    if (!qterms.length || !this.docs.size) return [];
+    const N = this.docs.size;
+    const avgdl = this.totalLen / N || 1;
+    const idf = new Map();
+    for (const t of qterms) {
+      const n = this.df.get(t) || 0;
+      idf.set(t, n ? Math.log(1 + (N - n + 0.5) / (n + 0.5)) : 0);
+    }
+    const out = [];
+    for (const [id, d] of this.docs) {
+      let s = 0;
+      for (const t of qterms) {
+        const f = d.tf.get(t);
+        if (!f) continue;
+        s += (idf.get(t) || 0) * (f * (k1 + 1)) / (f + k1 * (1 - b + b * (d.len / avgdl)));
+      }
+      if (s > 0) out.push({ id, score: s, ...d.meta });
+    }
+    out.sort((a, c) => c.score - a.score);
+    return out.slice(0, Math.max(1, limit));
+  }
+
+  // Serialize/restore for encrypted-at-rest persistence (next slice). Maps → arrays.
+  toJSON() {
+    return {
+      v: 1,
+      docs: [...this.docs].map(([id, d]) => ({ id, len: d.len, meta: d.meta, tf: [...d.tf] })),
+    };
+  }
+
+  static fromJSON(blob) {
+    const idx = new SearchIndex();
+    for (const d of (blob?.docs || [])) {
+      const tf = new Map(d.tf || []);
+      idx.docs.set(d.id, { tf, len: d.len || 0, meta: d.meta || { id: d.id } });
+      idx.totalLen += d.len || 0;
+      for (const t of tf.keys()) idx.df.set(t, (idx.df.get(t) || 0) + 1);
+    }
+    return idx;
+  }
+}

package/src/server.js

@@ -28,6 +28,7 @@ import { createRelaySession, getRelaySession, endRelaySession, pumpBridgeStream,
 import { shaperFor } from './shape.js';
 import { startNer } from './ner.js';
 import { installTimestampedConsole } from './log.js';
+import { HistoryStore } from './history-store.js';
 import * as nerEngine from './ner-engine.js';
 import { MODEL_CATALOG, isKnownModel } from './models.js';
 import { resolvePro, checkQuota, consume, usage } from './freegate.js';
@@ -37,7 +38,12 @@ import * as openai from './openai.js';
 import * as responses from './responses.js';
 import * as anthropic from './anthropic.js';
 
-export const VERSION = '0.6.10';
+export const VERSION = '0.6.12';
+
+// WARM search tier — one record store + BM25 index per gateway process, fed by the
+// extension's ingest sync. Encrypted at rest under ~/.chatpanel, loaded on start so a
+// restart doesn't need a full re-ingest (no cold start). See docs/architecture-data-tiers.
+const historyStore = new HistoryStore().load();
 
 const KNOWN_AGENTS = new Set(['codex', 'claude', 'opencode', 'pi', 'kiro', 'antigravity']);
 
@@ -502,6 +508,50 @@ export function createGateway(cfg = loadConfig()) {
         uptimeSeconds: Math.floor((Date.now() - STARTED_AT) / 1000),
       });
     }
+    // --- WARM search tier. The extension pushes its DECRYPTED records to this LOCAL
+    // process (on-device, loopback- + origin-gated above) which holds a BM25 index off
+    // the browser thread and answers full-corpus search — for the panel AND other local
+    // tools. Ranks identically to the browser hot tier (shared tokenizer/BM25).
+    //   POST /v1/history/ingest  { upserts:[{id,text,title,type,date}], removes:[id] } → { size }
+    //   POST /v1/history/search  { query, limit } → { results }
+    //   GET  /v1/history/status  → { size }
+    //   GET  /v1/history/list?limit&offset → { total, items:[{id,title,type,date,chars}] }
+    //   GET  /v1/history/get?id=… → { record:{id,title,type,date,text} }  (for external UIs)
+    if (pathname === '/v1/history/ingest' && req.method === 'POST') {
+      try {
+        const body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')) || {};
+        const size = historyStore.bulk({
+          upserts: Array.isArray(body.upserts) ? body.upserts : [],
+          removes: Array.isArray(body.removes) ? body.removes : [],
+        });
+        return sendJson(res, 200, { ok: true, size });
+      } catch (e) {
+        return sendJson(res, 400, { error: { message: `ingest failed: ${e.message}`, type: 'ingest_error' } });
+      }
+    }
+    if (pathname === '/v1/history/search' && req.method === 'POST') {
+      try {
+        const body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')) || {};
+        const results = historyStore.search(String(body.query || ''), { limit: Number(body.limit) || 10 });
+        return sendJson(res, 200, { ok: true, size: historyStore.size, results });
+      } catch (e) {
+        return sendJson(res, 400, { error: { message: `search failed: ${e.message}`, type: 'search_error' } });
+      }
+    }
+    if (pathname === '/v1/history/status' && req.method === 'GET') {
+      return sendJson(res, 200, { ok: true, size: historyStore.size });
+    }
+    if (pathname === '/v1/history/list' && req.method === 'GET') {
+      const limit = Math.min(500, Math.max(1, Number(url.searchParams.get('limit')) || 50));
+      const offset = Math.max(0, Number(url.searchParams.get('offset')) || 0);
+      return sendJson(res, 200, { ok: true, ...historyStore.list({ limit, offset }) });
+    }
+    if (pathname === '/v1/history/get' && req.method === 'GET') {
+      const record = historyStore.get(String(url.searchParams.get('id') || ''));
+      if (!record) return sendJson(res, 404, { error: { message: 'no such record', type: 'not_found' } });
+      return sendJson(res, 200, { ok: true, record });
+    }
+
     // The detector, on the gateway's own port (no second port). GET → health;
     // POST {text} → {entities}. The bundled engine runs IN-PROCESS; a user's own
     // external detector (if configured) is proxied for back-compat.