@chatpanel/gateway 0.6.12 → 0.6.14

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/gateway",
-  "version": "0.6.12",
+  "version": "0.6.14",
   "description": "Local privacy gateway — redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
   "type": "module",
   "bin": {

package/src/backup-decrypt.js

@@ -0,0 +1,44 @@
+// Decrypt a ChatPanel backup envelope — the gateway counterpart to the extension's
+// crypto-backup.js. Same wire format so the gateway can read the user's own daily
+// encrypted backups (with the passphrase they hand off) even when the extension
+// isn't running. WebCrypto (PBKDF2 + AES-GCM) + zlib gunzip; no dependencies.
+//
+// Envelope (v2): { type, version, kdf:{iterations,salt}, cipher:'AES-GCM',
+//   compression:'gzip'|'none'|absent, iv, ct }. v1 (no `compression`) → plaintext.
+
+import { gunzipSync } from 'node:zlib';
+
+const ENCRYPTED_TYPE = 'chatpanel-backup-encrypted';
+const b64 = (s) => Buffer.from(String(s || ''), 'base64');
+
+async function deriveKey(passphrase, salt, iterations) {
+  const base = await crypto.subtle.importKey('raw', new TextEncoder().encode(passphrase), 'PBKDF2', false, ['deriveKey']);
+  return crypto.subtle.deriveKey(
+    { name: 'PBKDF2', salt, iterations, hash: 'SHA-256' },
+    base,
+    { name: 'AES-GCM', length: 256 },
+    false,
+    ['decrypt'],
+  );
+}
+
+export function isEncryptedBackup(obj) {
+  return !!obj && typeof obj === 'object' && obj.type === ENCRYPTED_TYPE;
+}
+
+// Envelope + passphrase → the original backup data object. Throws a friendly error
+// on a wrong passphrase or a tampered file (AES-GCM auth-tag mismatch catches both).
+export async function decryptBackupEnvelope(envelope, passphrase) {
+  if (!isEncryptedBackup(envelope)) throw new Error('not an encrypted ChatPanel backup');
+  if (!passphrase) throw new Error('a passphrase is required to decrypt this backup');
+  const key = await deriveKey(passphrase, b64(envelope.kdf?.salt), envelope.kdf?.iterations || 250000);
+  let payload;
+  try {
+    payload = new Uint8Array(await crypto.subtle.decrypt({ name: 'AES-GCM', iv: b64(envelope.iv) }, key, b64(envelope.ct)));
+  } catch {
+    throw new Error('wrong passphrase, or the backup file is corrupted');
+  }
+  // v2 gzips before encrypting; v1 (no `compression` key) is plaintext JSON.
+  if (envelope.compression === 'gzip') payload = new Uint8Array(gunzipSync(Buffer.from(payload)));
+  return JSON.parse(Buffer.from(payload).toString('utf8'));
+}

package/src/backup-ingest.js

@@ -0,0 +1,76 @@
+// Backup-ingest: seed the warm store from the user's own daily encrypted backup so
+// the gateway holds the full corpus even when the extension isn't running. Finds the
+// newest backup file, decrypts it with the handed-off passphrase, and turns it into
+// warm records.
+//
+// The record extraction here is a SIMPLIFIED mirror of the extension's rich
+// conversationSource/meetingSource — same ids (chat:<id> / meeting:<id>), so when the
+// extension IS running its live-sync upserts its exact records right over these. This
+// path only needs "searchable text + title + date + id"; it is the fallback, not the
+// source of truth.
+
+import { readFileSync, existsSync, readdirSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+import os from 'node:os';
+import { decryptBackupEnvelope } from './backup-decrypt.js';
+
+const BACKUP_DIR = process.env.CHATPANEL_BACKUP_DIR || join(os.homedir(), 'Downloads', 'ChatPanel Backups');
+const BACKUP_RE = /^chatpanel-backup-[A-Za-z]+\.encrypted\.json$/;
+
+// Newest chatpanel-backup-*.encrypted.json in the backups dir, or null.
+export function findLatestBackup(dir = BACKUP_DIR) {
+  try {
+    let best = null;
+    for (const f of readdirSync(dir)) {
+      if (!BACKUP_RE.test(f)) continue;
+      const path = join(dir, f);
+      const mtime = statSync(path).mtimeMs;
+      if (!best || mtime > best.mtime) best = { path, mtime };
+    }
+    return best?.path || null;
+  } catch {
+    return null;
+  }
+}
+
+// Decrypted backup data → warm records [{ id, text, title, type, date }].
+export function backupToRecords(data) {
+  const out = [];
+  for (const c of data?.conversations || []) {
+    if (!c?.id) continue;
+    const title = c.title || 'Chat';
+    const date = c.updatedAt || c.createdAt || 0;
+    const body = (c.messages || [])
+      .filter((m) => m && m.content)
+      .map((m) => `${m.role || 'user'}: ${m.content}`)
+      .join('\n');
+    out.push({ id: `chat:${c.id}`, type: 'chat', title, date, text: `CHAT: ${title}\n${body}`.trim() });
+  }
+  for (const m of data?.meetings || []) {
+    const id = m?.id;
+    if (!id) continue;
+    const title = m.title || 'Meeting';
+    const date = m.startedAt || 0;
+    const segs = (m.segments || []).map((s) => `${s.speaker || ''}: ${s.text || ''}`).join('\n');
+    out.push({ id: `meeting:${id}`, type: 'meeting', title, date, text: `MEETING: ${title}\n${segs}`.trim() });
+  }
+  return out;
+}
+
+// Decrypt the latest (or given) backup and upsert its records into the store.
+// Returns { ok, ingested, size } or { ok:false, reason }. Never throws on a missing
+// file / passphrase; surfaces a decrypt failure as reason:'decrypt'.
+export async function ingestBackup(store, passphrase, { path } = {}) {
+  const file = path || findLatestBackup();
+  if (!file || !existsSync(file)) return { ok: false, reason: 'no-backup' };
+  if (!passphrase) return { ok: false, reason: 'no-passphrase' };
+  let data;
+  try {
+    data = await decryptBackupEnvelope(JSON.parse(readFileSync(file, 'utf8')), passphrase);
+  } catch (e) {
+    return { ok: false, reason: 'decrypt', error: String(e?.message || e) };
+  }
+  const records = backupToRecords(data);
+  store.bulk({ upserts: records });
+  return { ok: true, file, ingested: records.length, size: store.size };
+}

package/src/history-store.js

@@ -21,6 +21,7 @@ import { SearchIndex } from './search-index.js';
 const DIR = join(os.homedir(), '.chatpanel');
 const STORE_PATH = process.env.CHATPANEL_HISTORY_STORE || join(DIR, 'history-store.enc');
 const KEY_PATH = process.env.CHATPANEL_HISTORY_KEY || join(DIR, 'history-key');
+const SECRET_PATH = process.env.CHATPANEL_HISTORY_SECRET || join(DIR, 'history-secret.enc');
 
 function loadOrCreateKey() {
   try {
@@ -52,6 +53,28 @@ function decrypt(key, env) {
   return Buffer.concat([decipher.update(Buffer.from(env.ct, 'base64')), decipher.final()]);
 }
 
+// The handed-off backup passphrase, encrypted at rest with the same local key.
+// Kept separate from the records file: it's a credential, not corpus data, and the
+// gateway needs it before the store is even loaded (startup backup-ingest).
+export function saveBackupSecret(passphrase) {
+  const env = encrypt(loadOrCreateKey(), Buffer.from(String(passphrase || ''), 'utf8'));
+  mkdirSync(dirname(SECRET_PATH), { recursive: true });
+  writeFileSync(SECRET_PATH, JSON.stringify(env), { mode: 0o600 });
+}
+
+export function loadBackupSecret() {
+  try {
+    if (!existsSync(SECRET_PATH)) return '';
+    return decrypt(loadOrCreateKey(), JSON.parse(readFileSync(SECRET_PATH, 'utf8'))).toString('utf8');
+  } catch {
+    return '';
+  }
+}
+
+export function hasBackupSecret() {
+  return existsSync(SECRET_PATH);
+}
+
 // Records-plus-index with lazy, debounced encrypted persistence.
 export class HistoryStore {
   constructor({ storePath = STORE_PATH, persistMs = 2000 } = {}) {

package/src/server.js

@@ -28,7 +28,9 @@ import { createRelaySession, getRelaySession, endRelaySession, pumpBridgeStream,
 import { shaperFor } from './shape.js';
 import { startNer } from './ner.js';
 import { installTimestampedConsole } from './log.js';
-import { HistoryStore } from './history-store.js';
+import { saveBackupSecret, loadBackupSecret, hasBackupSecret } from './history-store.js';
+import { createHistoryStore } from './sqlite-store.js';
+import { ingestBackup } from './backup-ingest.js';
 import * as nerEngine from './ner-engine.js';
 import { MODEL_CATALOG, isKnownModel } from './models.js';
 import { resolvePro, checkQuota, consume, usage } from './freegate.js';
@@ -38,12 +40,13 @@ import * as openai from './openai.js';
 import * as responses from './responses.js';
 import * as anthropic from './anthropic.js';
 
-export const VERSION = '0.6.12';
+export const VERSION = '0.6.14';
 
-// WARM search tier — one record store + BM25 index per gateway process, fed by the
-// extension's ingest sync. Encrypted at rest under ~/.chatpanel, loaded on start so a
-// restart doesn't need a full re-ingest (no cold start). See docs/architecture-data-tiers.
-const historyStore = new HistoryStore().load();
+// WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
+// store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
+// Persistent + memory-mapped, so a restart needs no re-ingest (no cold start).
+// See docs/architecture-data-tiers.
+const historyStore = await createHistoryStore();
 
 const KNOWN_AGENTS = new Set(['codex', 'claude', 'opencode', 'pi', 'kiro', 'antigravity']);
 
@@ -551,6 +554,34 @@ export function createGateway(cfg = loadConfig()) {
       if (!record) return sendJson(res, 404, { error: { message: 'no such record', type: 'not_found' } });
       return sendJson(res, 200, { ok: true, record });
     }
+    // Key-handoff (loopback only): store the user's backup passphrase so the gateway
+    // can decrypt their daily backups unattended. Encrypted at rest with the local
+    // key. { passphrase } → stores it and does one immediate ingest. { passphrase:'' }
+    // forgets it. GET → whether a key is held (never returns the key itself).
+    if (pathname === '/v1/history/key' && req.method === 'GET') {
+      return sendJson(res, 200, { ok: true, hasKey: hasBackupSecret() });
+    }
+    if (pathname === '/v1/history/key' && req.method === 'POST') {
+      try {
+        const body = JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8')) || {};
+        saveBackupSecret(String(body.passphrase || ''));
+        const result = body.passphrase ? await ingestBackup(historyStore, String(body.passphrase)) : { ok: true, ingested: 0 };
+        return sendJson(res, 200, { ok: true, hasKey: !!body.passphrase, ...result });
+      } catch (e) {
+        return sendJson(res, 400, { error: { message: `key handoff failed: ${e.message}`, type: 'key_error' } });
+      }
+    }
+    // Trigger a backup-ingest now (uses the stored key). Optional { path } overrides
+    // which backup file to read. Returns how many records were seeded.
+    if (pathname === '/v1/history/ingest-backup' && req.method === 'POST') {
+      try {
+        const body = JSON.parse((await readBody(req, cfg.maxBodyBytes) || '{}').toString('utf8') || '{}') || {};
+        const result = await ingestBackup(historyStore, loadBackupSecret(), { path: body.path });
+        return sendJson(res, result.ok ? 200 : 409, result);
+      } catch (e) {
+        return sendJson(res, 400, { error: { message: `backup ingest failed: ${e.message}`, type: 'ingest_error' } });
+      }
+    }
 
     // The detector, on the gateway's own port (no second port). GET → health;
     // POST {text} → {entities}. The bundled engine runs IN-PROCESS; a user's own
@@ -782,6 +813,14 @@ export function start(cfg = loadConfig()) {
     console.log(`  redaction: ${cfg.redaction.tier}` + (cfg.redaction.detection?.backend && cfg.redaction.detection.backend !== 'off'
       ? ` + ${cfg.redaction.detection.backend} detector` : (cfg.ner?.autostart ? ' (+ NER starting…)' : '')));
   });
+  // If the user handed off a backup key, refresh the warm store from the latest
+  // daily backup in the background — so the gateway stays current even when the
+  // extension never runs. Best-effort; never blocks startup or crashes it.
+  if (hasBackupSecret()) {
+    ingestBackup(historyStore, loadBackupSecret())
+      .then((r) => { if (r?.ok) console.log(`  warm     : seeded ${r.ingested} records from ${r.file}`); })
+      .catch(() => {});
+  }
   const shutdown = () => { ner?.stop(); entitlement.stop(); server.close(() => process.exit(0)); };
   process.on('SIGINT', shutdown);
   process.on('SIGTERM', shutdown);

package/src/sqlite-store.js

@@ -0,0 +1,143 @@
+// WARM store — SQLite + FTS5 backend (the scale engine behind the same interface
+// as history-store.js's HistoryStore). SQLite is memory-mapped, so a year of
+// chats/meetings is searchable without loading the whole corpus into RAM, with
+// battle-tested BM25 full-text search and O(1) record lookups.
+//
+// Dual runtime: the npm gateway runs on Node (node:sqlite, built in since 22); the
+// standalone binary is compiled with Bun (bun:sqlite). Both ship FTS5. The
+// specifier is computed so neither bundler tries to resolve the other runtime's
+// module. createHistoryStore() falls back to the encrypted-JSON HistoryStore if
+// SQLite can't load at all, so this can never break a gateway.
+//
+// AT REST: a local .db file (0600) under ~/.chatpanel, protected by OS disk
+// encryption — the on-device warm tier. Zero-knowledge encryption is the COLD/cloud
+// tier's job, not this one. The backup passphrase (a credential) stays encrypted
+// via history-store.js's saveBackupSecret.
+
+import { join } from 'node:path';
+import { mkdirSync } from 'node:fs';
+import os from 'node:os';
+import { HistoryStore } from './history-store.js';
+
+const DIR = join(os.homedir(), '.chatpanel');
+const DB_PATH = process.env.CHATPANEL_HISTORY_DB || join(DIR, 'history.db');
+
+// Silence node:sqlite's one-time "experimental" warning for a clean CLI; pass
+// every other warning through untouched.
+if (typeof Bun === 'undefined') {
+  const emit = process.emitWarning.bind(process);
+  process.emitWarning = (w, ...a) => (typeof w === 'string' && w.includes('SQLite is an experimental') ? undefined : emit(w, ...a));
+}
+
+// Normalize node:sqlite (DatabaseSync) and bun:sqlite (Database) to one tiny API.
+async function openDb(path) {
+  if (typeof Bun !== 'undefined') {
+    const { Database } = await import('bun:sqlite');
+    const db = new Database(path, { create: true });
+    return {
+      exec: (sql) => db.run(sql),
+      run: (sql, p = []) => db.prepare(sql).run(...p),
+      all: (sql, p = []) => db.query(sql).all(...p),
+      get: (sql, p = []) => db.query(sql).get(...p),
+    };
+  }
+  const spec = 'node' + ':sqlite'; // computed so the Bun bundler won't touch it
+  const { DatabaseSync } = await import(spec);
+  const db = new DatabaseSync(path);
+  return {
+    exec: (sql) => db.exec(sql),
+    run: (sql, p = []) => db.prepare(sql).run(...p),
+    all: (sql, p = []) => db.prepare(sql).all(...p),
+    get: (sql, p = []) => db.prepare(sql).get(...p),
+  };
+}
+
+// query text → a safe FTS5 MATCH string. Each term is quoted (so FTS5 operators in
+// user text can't inject), joined with OR for recall — bm25 handles the ranking.
+function ftsMatch(query) {
+  const terms = String(query || '').toLowerCase().match(/[a-z0-9][a-z0-9'_+-]*/g);
+  if (!terms || !terms.length) return null;
+  return terms.map((t) => `"${t.replace(/"/g, '""')}"`).join(' OR ');
+}
+
+export class SqliteHistoryStore {
+  constructor({ path = DB_PATH } = {}) {
+    this.path = path;
+    this.db = null;
+  }
+
+  async init() {
+    if (this.path !== ':memory:') mkdirSync(DIR, { recursive: true });
+    this.db = await openDb(this.path);
+    this.db.exec('PRAGMA journal_mode=WAL');
+    this.db.exec('CREATE TABLE IF NOT EXISTS records(id TEXT PRIMARY KEY, title TEXT, type TEXT, date INTEGER, chars INTEGER)');
+    this.db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(id UNINDEXED, title, text, tokenize='unicode61')");
+    return this;
+  }
+
+  // Kept for interface-compatibility with HistoryStore (SQLite is already loaded).
+  load() {
+    return this;
+  }
+
+  get size() {
+    return this.db.get('SELECT COUNT(*) c FROM records')?.c || 0;
+  }
+
+  bulk({ upserts = [], removes = [] } = {}) {
+    this.db.exec('BEGIN');
+    try {
+      for (const id of removes) {
+        this.db.run('DELETE FROM records WHERE id = ?', [id]);
+        this.db.run('DELETE FROM fts WHERE id = ?', [id]);
+      }
+      for (const d of upserts) {
+        if (!d || !d.id) continue;
+        const text = String(d.text || '');
+        this.db.run('DELETE FROM fts WHERE id = ?', [d.id]); // FTS5 has no UPSERT on UNINDEXED id
+        this.db.run('INSERT INTO fts(id, title, text) VALUES(?, ?, ?)', [d.id, d.title || '', text]);
+        this.db.run('INSERT OR REPLACE INTO records(id, title, type, date, chars) VALUES(?, ?, ?, ?, ?)', [d.id, d.title || '', d.type || '', d.date || 0, text.length]);
+      }
+      this.db.exec('COMMIT');
+    } catch (e) {
+      this.db.exec('ROLLBACK');
+      throw e;
+    }
+    return this.size;
+  }
+
+  // [{ id, score, title, type, date }] — score higher = better (bm25 is negated).
+  search(query, { limit = 10 } = {}) {
+    const match = ftsMatch(query);
+    if (!match) return [];
+    const rows = this.db.all(
+      'SELECT r.id id, r.title title, r.type type, r.date date, bm25(fts) b FROM fts JOIN records r ON r.id = fts.id WHERE fts MATCH ? ORDER BY b LIMIT ?',
+      [match, limit],
+    );
+    return rows.map((r) => ({ id: r.id, score: -r.b, title: r.title, type: r.type, date: r.date }));
+  }
+
+  list({ limit = 50, offset = 0 } = {}) {
+    const total = this.db.get('SELECT COUNT(*) c FROM records')?.c || 0;
+    const items = this.db.all('SELECT id, title, type, date, chars FROM records ORDER BY date DESC LIMIT ? OFFSET ?', [limit, offset]);
+    return { total, items };
+  }
+
+  get(id) {
+    const meta = this.db.get('SELECT id, title, type, date FROM records WHERE id = ?', [id]);
+    if (!meta) return null;
+    const body = this.db.get('SELECT text FROM fts WHERE id = ?', [id]);
+    return { ...meta, text: body?.text || '' };
+  }
+}
+
+// Pick the best available warm engine. SQLite when it loads; otherwise the
+// encrypted-JSON HistoryStore — so a gateway is never left without a warm store.
+export async function createHistoryStore(opts = {}) {
+  try {
+    return await new SqliteHistoryStore(opts).init();
+  } catch (e) {
+    console.log(`[warm] SQLite unavailable (${e.message}); using the encrypted file store`);
+    return new HistoryStore(opts).load();
+  }
+}