@chatpanel/gateway 0.6.71 → 0.6.73

package/bin/chatpanel-gateway.js

@@ -3,6 +3,9 @@
 //
 //   chatpanel-gateway              start the gateway (foreground)
 //   chatpanel-gateway mcp          stdio MCP server exposing warm history as tools
+//   chatpanel-gateway tools list   the same tools, from a shell — names and one-liners
+//   chatpanel-gateway tools schema <tool>   one tool's full schema
+//   chatpanel-gateway call <tool> '<json>'  run one tool (exit 0 ok · 1 tool error · 2 usage)
 //   chatpanel-gateway local        show the local runtime — bridge + gateway, one view
 //   chatpanel-gateway connect      point your CLI agents (Codex, Claude Code, …) at this server
 //   chatpanel-gateway --install    register login auto-start + start now
@@ -21,6 +24,11 @@ try {
     // server.js (which would open a second handle on the warm SQLite store).
     const { runMcpServer } = await import('../src/mcp.js');
     await runMcpServer();
+  } else if (arg === 'tools' || arg === 'call') {
+    // MCP2CLI: the MCP tools as shell verbs, for agents that only have a shell. Same
+    // in-process dispatcher as `mcp`, same no-server.js rule.
+    const { runMcpCli } = await import('../src/mcp-cli.js');
+    process.exit(await runMcpCli(process.argv.slice(2)));
   } else if (arg === 'local') {
     // Read-only unified view of both services. No server.js import — just HTTP probes.
     const { localStatus, formatLocalStatus } = await import('../src/local-status.js');
@@ -52,7 +60,7 @@ try {
         start();
         break;
       default:
-        console.error(`unknown option: ${arg}\nUsage: chatpanel-gateway [mcp|--install|--uninstall|--status|--version]`);
+        console.error(`unknown option: ${arg}\nUsage: chatpanel-gateway [mcp|tools list|tools schema <tool>|call <tool> '<json>'|local|connect|--install|--uninstall|--status|--version]`);
         process.exit(2);
     }
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/gateway",
-  "version": "0.6.71",
+  "version": "0.6.73",
   "description": "Local privacy gateway \u2014 redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
   "type": "module",
   "bin": {

package/src/mcp-cli.js

@@ -0,0 +1,113 @@
+// `chatpanel-gateway tools list | tools schema <tool> | call <tool> '<json>'` — the same
+// tools the MCP server offers, reached from a shell.
+//
+// An MCP connection is a standing cost: every tool's schema sits in the agent's context on
+// every turn, called or not. A shell verb costs nothing until it is used — `tools list`
+// prints names and one-liners, `tools schema` pulls ONE schema at the moment of calling,
+// `call` runs the tool. So an agent that has only a shell (a skill's `scripts/`, a CI job,
+// a CLI with no MCP support) reaches history, memory and briefs at a cost that scales with
+// use rather than with how many tools exist. The daemon's secrets never move: the CLI
+// talks to the gateway exactly as the MCP server does, over loopback with the gateway
+// token, and prints results.
+//
+// Exit codes, so a script can tell them apart: 0 the tool answered; 1 the tool itself
+// reported an error (a real tool-level failure — the record was not found); 2 a usage or
+// transport failure (bad JSON, unknown verb, gateway unreachable).
+
+import { readFileSync } from 'node:fs';
+import { handleRpc } from './mcp.js';
+
+const USAGE = [
+  'Usage:',
+  '  chatpanel-gateway tools list                 names and one-liners (no schemas)',
+  '  chatpanel-gateway tools schema <tool>        one tool\'s full input schema',
+  '  chatpanel-gateway call <tool> [\'<json>\']     run a tool; arguments as a JSON object',
+  '  chatpanel-gateway call <tool> --file <path>  arguments from a JSON file (no shell quoting)',
+  '',
+  'Exit codes: 0 ok · 1 the tool reported an error · 2 usage or transport failure',
+].join('\n');
+
+/** First sentence, whitespace collapsed, capped — enough to choose a tool, not to call it. */
+export function oneLiner(description, max = 100) {
+  const s = String(description || '').replace(/\s+/g, ' ').trim();
+  const cut = s.search(/[.!?]\s/);
+  const first = cut > 20 ? s.slice(0, cut + 1) : s;
+  return first.length > max ? `${first.slice(0, max - 1).trimEnd()}…` : first;
+}
+
+export function formatToolsList(tools) {
+  const width = Math.min(24, Math.max(...tools.map((t) => t.name.length), 4));
+  return tools.map((t) => {
+    const req = Array.isArray(t.inputSchema?.required) && t.inputSchema.required.length ? `(${t.inputSchema.required.join(', ')})` : '()';
+    return `${t.name.padEnd(width)}  ${req.padEnd(22)} ${oneLiner(t.description)}`;
+  }).join('\n');
+}
+
+function parseArgs(argv) {
+  const rest = [...argv];
+  let file = null;
+  const i = rest.indexOf('--file');
+  if (i >= 0) { file = rest[i + 1]; rest.splice(i, 2); }
+  return { rest, file };
+}
+
+/**
+ * Run one CLI invocation. Returns the exit code; writes through `out`/`err` so a test can
+ * capture without a process. `rpc` is the MCP dispatcher — `handleRpc` in production.
+ */
+export async function runMcpCli(argv, { out = (s) => process.stdout.write(s), err = (s) => process.stderr.write(s), rpc = handleRpc } = {}) {
+  const [verb, ...more] = argv;
+  const list = async () => (await rpc({ jsonrpc: '2.0', id: 1, method: 'tools/list' }))?.result?.tools || [];
+
+  if (verb === 'tools') {
+    const [sub, name] = more;
+    if (sub === 'list') {
+      out(`${formatToolsList(await list())}\n`);
+      return 0;
+    }
+    if (sub === 'schema') {
+      if (!name) { err(`tools schema: which tool?\n${USAGE}\n`); return 2; }
+      const tool = (await list()).find((t) => t.name === name);
+      if (!tool) { err(`no tool named "${name}". Run: chatpanel-gateway tools list\n`); return 2; }
+      out(`${JSON.stringify({ name: tool.name, description: tool.description, inputSchema: tool.inputSchema }, null, 2)}\n`);
+      return 0;
+    }
+    err(`${USAGE}\n`);
+    return 2;
+  }
+
+  if (verb === 'call') {
+    const { rest, file } = parseArgs(more);
+    const [name, inline] = rest;
+    if (!name) { err(`call: which tool?\n${USAGE}\n`); return 2; }
+    let args = {};
+    try {
+      const text = file ? readFileSync(file, 'utf8') : (inline ?? '{}');
+      args = JSON.parse(text);
+    } catch (e) {
+      err(`call: arguments must be a JSON object (${e.message})\n`);
+      return 2;
+    }
+    if (!args || typeof args !== 'object' || Array.isArray(args)) { err('call: arguments must be a JSON object\n'); return 2; }
+    let reply;
+    try {
+      reply = await rpc({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name, arguments: args } });
+    } catch (e) {
+      err(`call: ${e.message}\n`);
+      return 2;
+    }
+    if (reply?.error) { err(`call: ${reply.error.message}\n`); return 2; }
+    const text = (reply?.result?.content || []).filter((c) => c.type === 'text').map((c) => c.text).join('\n');
+    if (reply?.result?.isError) {
+      // The gateway being down looks like a tool error from the MCP layer; it is transport.
+      const transport = /fetch failed|ECONNREFUSED|not running|unreachable/i.test(text);
+      err(`${text}\n`);
+      return transport ? 2 : 1;
+    }
+    out(`${text}\n`);
+    return 0;
+  }
+
+  err(`${USAGE}\n`);
+  return 2;
+}

package/src/ner.js

@@ -8,6 +8,10 @@
 // deterministic-only redaction. redact.js consults the engine directly, so we do
 // NOT mutate cfg.redaction.detection here (that field is reserved for a user's own
 // external detector, which takes precedence — see below).
+//
+// `ensureNer` (bottom) is the same thing on demand, for the case autostart cannot cover:
+// a config that says autostart:false, weights already on disk, and a client asking for
+// detection right now.
 
 import * as engine from './ner-engine.js';
 import { persistConfig, configPath } from './configstore.js';
@@ -54,3 +58,56 @@ export function startNer(cfg) {
   // Nothing to kill (no child process); just stop logging after shutdown.
   return { stop() { stopped = true; } };
 }
+
+/**
+ * START THE BUNDLED DETECTOR BECAUSE SOMEONE ASKED FOR DETECTION.
+ *
+ * `startNer` only runs at boot, and only when `ner.autostart` is on. Everything after that
+ * assumed the engine was either running or deliberately not wanted — so a config carrying
+ * `autostart:false` (older gateways wrote it, and it survives every upgrade) left POST /ner
+ * answering 503 forever, with the weights sitting on disk the whole time. The extension's
+ * composer showed "name detection is not answering", and every real turn quietly fell back
+ * to deterministic-only redaction: names, organisations and places went to the model in full.
+ *
+ * A request for entity detection IS the intent to detect, so this starts the engine on that
+ * request instead of waiting for a restart the user has no reason to perform.
+ *
+ * Two limits keep it from being a surprise. It never DOWNLOADS: weights arrive through the
+ * model manager, which shows progress, so a preview keystroke can never kick off a hundred
+ * megabytes. And it defers to a user's own external detector exactly as `startNer` does —
+ * that field means "use mine", not "use mine if it happens to be up".
+ *
+ * Returns the engine state after the attempt; never throws.
+ */
+export async function ensureNer(cfg, { log = (m) => console.log(m) } = {}) {
+  const det = cfg?.redaction?.detection;
+  if (det && det.backend && det.backend !== 'off') return 'external';
+
+  const st = engine.state();
+  if (st === 'ready') return st;
+  // A load already in flight (or one that already failed) — join it rather than starting a
+  // second one. engine.init() is single-flight, so this is just the wait.
+  if (st === 'loading' || st === 'downloading' || st === 'error') {
+    try { await engine.init(); } catch { /* fail-open: deterministic-only */ }
+    return engine.state();
+  }
+
+  const n = cfg?.ner || {};
+  const model = n.model || undefined;
+  if (!engine.modelOnDisk(model)) return 'not-downloaded';
+
+  try {
+    await engine.init({ model, allowDownload: false, onLog: log });
+  } catch (e) {
+    log(`[ner] on-demand load failed (${e.message}) — deterministic-only`);
+    return engine.state();
+  }
+  if (engine.isReady()) {
+    log(`[ner] started on demand — model ${engine.health().model} — entity detection active`);
+    if (n.enableFullTier !== false && cfg.redaction && cfg.redaction.tier !== 'full') {
+      cfg.redaction.tier = 'full';
+      log('[ner] full tier on — name/org redaction active');
+    }
+  }
+  return engine.state();
+}

package/src/server.js

@@ -29,7 +29,7 @@ import { secureFetch } from './secure-fetch.js';
 import { streamBridgeChat, readBridgeToken, openBridgeChat } from './bridge.js';
 import { createRelaySession, getRelaySession, endRelaySession, pumpBridgeStream, deliverToolResult, toolsToSpecs, parseToolCallId } from './toolrelay.js';
 import { shaperFor } from './shape.js';
-import { startNer } from './ner.js';
+import { startNer, ensureNer } from './ner.js';
 import { installTimestampedConsole } from './log.js';
 import { saveBackupSecret, clearBackupSecret, loadBackupSecret, hasBackupSecret } from './history-store.js';
 import { createMemoryStore } from './memory-store.js';
@@ -56,7 +56,7 @@ import * as openai from './openai.js';
 import * as responses from './responses.js';
 import * as anthropic from './anthropic.js';
 
-export const VERSION = '0.6.71';
+export const VERSION = '0.6.73';
 
 // WARM search tier — SQLite + FTS5 record store (falls back to an encrypted-JSON
 // store if SQLite can't load), fed by the extension's ingest sync + backup-ingest.
@@ -281,6 +281,43 @@ function nerBaseUrl(cfg) {
   return url;
 }
 
+/**
+ * CAN THIS GATEWAY REDACT NAMES RIGHT NOW — the question every privacy UI has to answer
+ * before it draws a shield.
+ *
+ * Deterministic patterns (emails, phones, cards, keys) always run, so a redaction with no
+ * detector behind it produces text that LOOKS redacted. `coverage` names the difference:
+ * 'names' means people, organisations and places are covered too; 'patterns' means they are
+ * not, and the client is expected to say so rather than let the shield imply it.
+ *
+ * Synchronous on purpose — it reports the engine's own state and never probes the network,
+ * so it can sit on a per-keystroke route.
+ */
+function detectorStatus(cfg) {
+  const det = cfg?.redaction?.detection;
+  const tier = cfg?.redaction?.tier === 'full' ? 'full' : 'basic';
+  if (det && det.backend && det.backend !== 'off') {
+    // A user's own detector: we cannot know it is up without sending it text, and a probe
+    // per keystroke is not a trade worth making. Report it as configured and let the turn
+    // itself be the test.
+    return { source: 'external', backend: det.backend, state: 'external', model: det.model || null, ready: true, tier, coverage: tier === 'full' ? 'names' : 'patterns' };
+  }
+  const h = nerEngine.health();
+  const ready = h.ok && tier === 'full';
+  return {
+    source: 'bundled',
+    state: h.state,                 // 'off' | 'loading' | 'downloading' | 'ready' | 'error'
+    model: h.model || cfg?.ner?.model || null,
+    // Weights on disk with the engine off is the recoverable case (it starts on the next
+    // request); no weights is the case that needs the user to install a model.
+    installed: nerEngine.modelOnDisk(cfg?.ner?.model || undefined),
+    error: h.error || null,
+    ready,
+    tier,
+    coverage: ready ? 'names' : 'patterns',
+  };
+}
+
 // Health of the detector for /status. The bundled IN-PROCESS engine takes
 // precedence; its public contract URL is the gateway's own /ner (no second port).
 // A user-configured external detector is probed over HTTP as before.
@@ -690,6 +727,9 @@ export function createGateway(cfg = loadConfig()) {
           model: health.model,         // e.g. "en_core_web_sm"
           url: health.url,
         },
+        // Additive: one block that answers "are names covered right now", for clients that
+        // draw a privacy indicator and must not overstate it. See detectorStatus.
+        detector: detectorStatus(cfg),
         pro: { unlocked: proUnlocked }, usage: usage(cfg),
         uptimeSeconds: Math.floor((Date.now() - STARTED_AT) / 1000),
       });
@@ -928,8 +968,30 @@ export function createGateway(cfg = loadConfig()) {
         return sendJson(res, health.ok ? 200 : 503, health);
       }
       if (req.method === 'POST') {
+        // A POST here IS the request to detect, so start the bundled engine if it is not
+        // running and its weights are already on disk (see ensureNer — it never downloads).
+        // Without this a config with `ner.autostart:false` answered 503 until a restart,
+        // and the client read that as "redaction is broken" while the model sat unused.
+        await ensureNer(cfg);
         // In-process engine path.
         if (nerEngine.state() !== 'off') {
+          // NOT READY IS NOT AN ANSWER. `detect()` returns [] when the pipeline is still
+          // loading or failed to load, and 200 {entities:[]} is indistinguishable from
+          // "this text contains no names" — the one lie a privacy layer must never tell.
+          // Say which state it is in instead, and let the caller show "starting…".
+          if (!nerEngine.isReady()) {
+            const h = nerEngine.health();
+            return sendJson(res, 503, {
+              error: {
+                message: h.state === 'error'
+                  ? `NER model failed to load: ${h.error || 'unknown error'}`
+                  : `NER model is ${h.state} — not ready yet`,
+                type: h.state === 'error' ? 'ner_error' : 'ner_loading',
+              },
+              state: h.state,
+              model: h.model,
+            });
+          }
           try {
             const body = await readBody(req, cfg.maxBodyBytes);
             let text = '';
@@ -942,7 +1004,22 @@ export function createGateway(cfg = loadConfig()) {
         }
         // External detector proxy (user-configured endpoint).
         const url = nerBaseUrl(cfg);
-        if (!url) return sendJson(res, 503, { error: { message: 'NER not configured — deterministic-only redaction', type: 'ner_off' } });
+        if (!url) {
+          // The bundled engine is the intended detector here and it is not running. Say
+          // WHY — "not configured" sent a user hunting through settings that were already
+          // correct, when the real answer was that the model was never downloaded.
+          const onDisk = nerEngine.modelOnDisk(cfg.ner?.model || undefined);
+          return sendJson(res, 503, {
+            error: {
+              message: onDisk
+                ? 'the bundled detector could not start — deterministic-only redaction'
+                : 'no entity detector is installed — deterministic-only redaction (install one from Gateway settings)',
+              type: onDisk ? 'ner_error' : 'ner_not_installed',
+            },
+            state: nerEngine.state(),
+            model: cfg.ner?.model || null,
+          });
+        }
         try {
           const body = await readBody(req, cfg.maxBodyBytes);
           // secureFetch: scheme/host policy + resolved-IP check before POSTing raw text to the detector.
@@ -973,10 +1050,13 @@ export function createGateway(cfg = loadConfig()) {
       let text = '';
       try { text = String(JSON.parse((await readBody(req, cfg.maxBodyBytes)).toString('utf8'))?.text || ''); }
       catch { text = ''; }
-      if (!text) return sendJson(res, 200, { text: '', count: 0, sanitized: 0, entities: [] });
+      if (!text) return sendJson(res, 200, { text: '', count: 0, sanitized: 0, entities: [], detector: detectorStatus(cfg) });
       try {
         let out = text;
         const isPro = await resolvePro(cfg.pro?.entitlementToken);
+        // Same start-on-demand as a real turn, for the same reason — and so the preview
+        // keeps describing the request rather than a weaker version of it.
+        await ensureNer(cfg);
         const r = await redactSegments(
           [segment(() => out, (v) => { out = v; })],
           cfg.redaction,
@@ -988,6 +1068,11 @@ export function createGateway(cfg = loadConfig()) {
           sanitized: r.sanitized || 0,
           tier: cfg.redaction?.tier === 'full' ? 'full' : 'basic',
           entities: redactionDetail(r.vault, 'types') || [],
+          // WHAT THIS PREVIEW COULD NOT SEE — additive, and the whole reason a client can be
+          // honest. Patterns catch emails, phones and card numbers with no detector at all,
+          // so a preview with the detector down looks identical to a clean one: same text,
+          // same shield, names intact. `coverage` is the difference, said out loud.
+          detector: detectorStatus(cfg),
         });
       } catch (e) {
         // Fail LOUD. A preview that silently returns the original text would tell the user
@@ -1672,6 +1757,11 @@ export function createGateway(cfg = loadConfig()) {
         // was off for this turn" rather than "0 redactions", which would read as "nothing to
         // redact". Anonymous callers cannot switch it off: that is what the token is for.
         redactionOff = String(req.headers['x-chatpanel-redaction'] || '').trim().toLowerCase() === 'off' && isAdminAuthorized(req);
+        // The detector must be up BEFORE the text is redacted, not after someone notices it
+        // wasn't. redactSegments consults the engine and falls open when it is not ready, so
+        // a gateway whose engine never autostarted sent names through at full tier without a
+        // word. Cheap after the first call: ready → a state check, absent weights → nothing.
+        if (!redactionOff) await ensureNer(cfg);
         const segs = redactionOff ? [] : r.adapter.collectSegments(body, cfg.redaction);
         const ac = new AbortController();
         req.on('close', () => ac.abort());