@chatpanel/gateway 0.4.2 → 0.5.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/gateway",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "description": "Local privacy gateway — redacts PII out of OpenAI/Anthropic API traffic before it reaches a model, then restores it in the reply. Point opencode, codex, aider, Claude Code, etc. at it.",
   "type": "module",
   "bin": {
@@ -30,7 +30,7 @@
     "node": ">=18"
   },
   "dependencies": {
-    "@chatpanel/pii": "^0.2.1"
+    "@chatpanel/pii": "^0.2.3"
   },
   "homepage": "https://chatpanel.net",
   "repository": {

package/src/configstore.js

@@ -48,6 +48,7 @@ export function publicConfig(cfg, { proUnlocked = false } = {}) {
       autoNarrow: cfg.tools?.autoNarrow !== false,
       maxPerTurn: Number(cfg.tools?.maxPerTurn) > 0 ? Number(cfg.tools.maxPerTurn) : 8,
       narrowAll: !!cfg.tools?.narrowAll,
+      toolData: cfg.tools?.toolData === 'redactRemote' ? 'redactRemote' : 'real',
     },
   };
 }
@@ -105,6 +106,7 @@ export function applyConfigPatch(cfg, patch = {}) {
     cfg.tools = cfg.tools || {};
     if ('autoNarrow' in patch.tools) cfg.tools.autoNarrow = !!patch.tools.autoNarrow;
     if ('narrowAll' in patch.tools) cfg.tools.narrowAll = !!patch.tools.narrowAll;
+    if (patch.tools.toolData === 'real' || patch.tools.toolData === 'redactRemote') cfg.tools.toolData = patch.tools.toolData;
     const cap = Number(patch.tools.maxPerTurn);
     if (Number.isFinite(cap) && cap >= 1) cfg.tools.maxPerTurn = Math.floor(cap);
   }

package/src/openai.js

@@ -55,16 +55,19 @@ export function extractLatestToolResult(body) {
 }
 
 // Restore a buffered (non-streaming) response: assistant text + tool-call args.
-export function restoreResponse(json, vault) {
+export function restoreResponse(json, vault, harness = null) {
   for (const choice of json?.choices || []) {
     const msg = choice?.message;
     if (!msg) continue;
-    // Visible text keeps the pseudonym (restoreText); tool-call args get the REAL
-    // value (restoreDeepAliases) so the client runs the tool on real data.
+    // Visible text keeps the pseudonym (restoreText); tool-call args go through the
+    // shared harness — real values so the client runs the tool on real data, or the
+    // redacted token kept for remote MCP tools under "redact remote".
     if (typeof msg.content === 'string') msg.content = restoreText(msg.content, vault);
     for (const tc of msg.tool_calls || []) {
       if (tc?.function && typeof tc.function.arguments === 'string') {
-        tc.function.arguments = restoreDeepAliases(tc.function.arguments, vault);
+        tc.function.arguments = harness
+          ? harness.toTool(tc.function.name, tc.function.arguments)
+          : restoreDeepAliases(tc.function.arguments, vault);
       }
     }
   }

package/src/server.js

@@ -21,7 +21,7 @@ import { createServer } from 'node:http';
 import { loadConfig } from './config.js';
 import { redactSegments } from './redact.js';
 import { pipeRestoredStream, pipeRestoredOpenAIStream, makeTokenRestorer } from './stream.js';
-import { restoreText, effectiveTier, gatedDictionary, narrowSpecs } from '@chatpanel/pii';
+import { restoreText, effectiveTier, gatedDictionary, narrowSpecs, makeToolHarness } from '@chatpanel/pii';
 import { streamBridgeChat, readBridgeToken, openBridgeChat } from './bridge.js';
 import { createRelaySession, getRelaySession, endRelaySession, pumpBridgeStream, deliverToolResult, toolsToSpecs, parseToolCallId } from './toolrelay.js';
 import { shaperFor } from './shape.js';
@@ -33,7 +33,7 @@ import * as openai from './openai.js';
 import * as responses from './responses.js';
 import * as anthropic from './anthropic.js';
 
-export const VERSION = '0.4.2';
+export const VERSION = '0.5.0';
 
 const KNOWN_AGENTS = new Set(['codex', 'claude', 'opencode', 'pi', 'kiro', 'antigravity']);
 
@@ -150,6 +150,28 @@ function sendJson(res, status, obj) {
   res.end(JSON.stringify(obj));
 }
 
+// The detector URL (…:9009/ner) and its sibling /health. Returns null when no
+// detector is wired (deterministic-only).
+function nerBaseUrl(cfg) {
+  const url = cfg.redaction?.detection?.url;
+  if (!url || cfg.redaction?.detection?.backend === 'off') return null;
+  return url;
+}
+
+// Live health probe of the bundled/configured NER (GET /health → { ok, model }).
+async function probeNerHealth(cfg) {
+  const url = nerBaseUrl(cfg);
+  if (!url) return { configured: false, ok: false, url: null, model: null };
+  try {
+    const r = await fetch(url.replace(/\/ner\/?$/, '') + '/health', { signal: AbortSignal.timeout(2000) });
+    if (!r.ok) return { configured: true, ok: false, url, model: null };
+    const j = await r.json().catch(() => ({}));
+    return { configured: true, ok: true, url, model: j.model || null };
+  } catch {
+    return { configured: true, ok: false, url, model: null };
+  }
+}
+
 function forwardHeaders(headers, base) {
   const out = {};
   for (const [k, v] of Object.entries(headers)) {
@@ -179,12 +201,12 @@ async function pumpRelay(res, s, shaper) {
 }
 
 // New tool-enabled turn: open the bridge with the client's tools as MCP specs.
-async function startRelay(req, res, { kind, adapter, agent }, body, vault, cfg, isPro, tools) {
+async function startRelay(req, res, { kind, adapter, agent }, body, vault, cfg, isPro, tools, harness = null) {
   const { messages, system } = adapter.toTurn(body);
   const token = readBridgeToken(cfg.bridge.token);
   const shaper = shaperFor(kind, body?.model || agent);
   const redactOpts = { tier: effectiveTier({ tier: cfg.redaction.tier }, isPro), dictionary: gatedDictionary(cfg.redaction, isPro), entities: [] };
-  const s = createRelaySession({ vault, redactOpts, bridgeUrl: cfg.bridge.url, token });
+  const s = createRelaySession({ vault, redactOpts, bridgeUrl: cfg.bridge.url, token, harness });
   const ttl = setTimeout(() => endRelaySession(s.id), 135_000); // bridge tool-call timeout is 120s
   let resp;
   try {
@@ -206,7 +228,7 @@ async function resumeRelay(res, s, toolContent, model) {
   return pumpRelay(res, s, shaper);
 }
 
-async function handleBridge(req, res, { kind, adapter, redactable, pathname, agentOverride }, body, vault, cfg, isPro) {
+async function handleBridge(req, res, { kind, adapter, redactable, pathname, agentOverride, harness }, body, vault, cfg, isPro) {
   if (!redactable) {
     return sendJson(res, 404, { error: `endpoint ${pathname} not supported by the bridge backend` });
   }
@@ -222,7 +244,7 @@ async function handleBridge(req, res, { kind, adapter, redactable, pathname, age
     }
     const tools = adapter.extractTools(body);
     if (tools.length && body?.stream === true) {
-      return startRelay(req, res, { kind, adapter, agent: agentOverride || pickAgent(body?.model, cfg) }, body, vault, cfg, isPro, tools);
+      return startRelay(req, res, { kind, adapter, agent: agentOverride || pickAgent(body?.model, cfg) }, body, vault, cfg, isPro, tools, harness);
     }
   }
 
@@ -266,7 +288,7 @@ async function handleBridge(req, res, { kind, adapter, redactable, pathname, age
 
 // ---- backend: api ----------------------------------------------------------
 
-async function handleApi(req, res, { adapter, kind, pathname, search, base, destKey, destProtocol }, outBody, vault) {
+async function handleApi(req, res, { adapter, kind, pathname, search, base, destKey, destProtocol, harness }, outBody, vault) {
   let upstream;
   try {
     const headers = forwardHeaders(req.headers, base);
@@ -291,16 +313,17 @@ async function handleApi(req, res, { adapter, kind, pathname, search, base, dest
 
   if (ct.includes('text/event-stream') && upstream.body) {
     res.writeHead(upstream.status, resHeaders);
-    // OpenAI streaming: restore tool-call args with real values (aliases undone)
-    // while keeping visible text pseudonymized. Other protocols: generic restore.
-    const pipe = kind === 'openai' ? pipeRestoredOpenAIStream : pipeRestoredStream;
-    return pipe(upstream.body, res, vault);
+    // OpenAI streaming: restore tool-call args via the harness (real, or kept
+    // redacted for remote MCP under redactRemote) while keeping visible text
+    // pseudonymized. Other protocols: generic restore.
+    if (kind === 'openai') return pipeRestoredOpenAIStream(upstream.body, res, vault, harness);
+    return pipeRestoredStream(upstream.body, res, vault);
   }
 
   const buf = Buffer.from(await upstream.arrayBuffer());
   if (vault && ct.includes('application/json')) {
     try {
-      const json = adapter.restoreResponse(JSON.parse(buf.toString('utf8')), vault);
+      const json = adapter.restoreResponse(JSON.parse(buf.toString('utf8')), vault, harness);
       res.writeHead(upstream.status, { ...resHeaders, 'content-type': 'application/json' });
       return res.end(Buffer.from(JSON.stringify(json), 'utf8'));
     } catch { /* fall through */ }
@@ -330,14 +353,41 @@ export function createGateway(cfg = loadConfig()) {
     // --- Config API (the extension's "Gateway" tab is a client of these) ---
     if (pathname === '/status' && req.method === 'GET') {
       const proUnlocked = await resolvePro(cfg.pro?.entitlementToken);
-      const nerOn = cfg.redaction?.detection?.backend && cfg.redaction.detection.backend !== 'off';
+      const health = await probeNerHealth(cfg); // live GET /health on the detector
       return sendJson(res, 200, {
         ok: true, version: VERSION, backend: cfg.backend, tier: cfg.redaction.tier,
-        ner: { autostart: !!cfg.ner?.autostart, ready: !!nerOn },
+        ner: {
+          autostart: !!cfg.ner?.autostart,
+          configured: health.configured,
+          ready: health.ok,            // the detector actually answered /health
+          model: health.model,         // e.g. "en_core_web_sm"
+          url: health.url,
+        },
         pro: { unlocked: proUnlocked }, usage: usage(cfg),
         uptimeSeconds: Math.floor((Date.now() - STARTED_AT) / 1000),
       });
     }
+    // Proxy the detector so it's checkable on the gateway's own port (the NER runs
+    // on its own port, e.g. 9009 — hitting :4320/ner used to 404). GET → health.
+    if (pathname === '/ner') {
+      const url = nerBaseUrl(cfg);
+      if (!url) return sendJson(res, 503, { error: { message: 'NER not configured — deterministic-only redaction', type: 'ner_off' } });
+      if (req.method === 'GET') {
+        const health = await probeNerHealth(cfg);
+        return sendJson(res, health.ok ? 200 : 503, health);
+      }
+      if (req.method === 'POST') {
+        try {
+          const body = await readBody(req, cfg.maxBodyBytes);
+          const r = await fetch(url, { method: 'POST', headers: { 'content-type': 'application/json' }, body, signal: AbortSignal.timeout(8000) });
+          const text = await r.text();
+          res.writeHead(r.status, { 'content-type': 'application/json' });
+          return res.end(text);
+        } catch (e) {
+          return sendJson(res, 502, { error: { message: `NER unreachable: ${e.message}`, type: 'ner_unreachable' } });
+        }
+      }
+    }
     if (pathname === '/logs' && req.method === 'GET') {
       return sendJson(res, 200, { entries: [...recentRequests].reverse() }); // newest first; counts only
     }
@@ -412,6 +462,12 @@ export function createGateway(cfg = loadConfig()) {
       }
     }
 
+    // THE shared tool harness — same one the extension uses. The gateway only needs
+    // ② (tool args): restore to real for the client to run, or keep the redacted
+    // token for remote MCP tools when tools.toolData is "redactRemote". Results are
+    // re-redacted by the NEXT request's normal redaction, so ③ isn't needed here.
+    const harness = makeToolHarness({ vault, toolData: cfg.tools?.toolData });
+
     // Route by the requested model → a destination (agent via the bridge, or an
     // API we forward to). Falls back to the legacy backend when none configured.
     const dest = resolveDestination(body?.model, cfg, r.kind);
@@ -424,9 +480,9 @@ export function createGateway(cfg = loadConfig()) {
       if (isSelfUrl(dest.baseUrl, cfg)) {
         return sendJson(res, 508, { error: { message: `destination "${dest.id}" points back at the gateway (${dest.baseUrl}) — refusing to forward (would loop).`, type: 'loop_detected' } });
       }
-      return handleApi(req, res, { ...r, pathname, search: url.search, base: dest.baseUrl, destKey: dest.apiKey, destProtocol: dest.protocol }, outBody, vault);
+      return handleApi(req, res, { ...r, pathname, search: url.search, base: dest.baseUrl, destKey: dest.apiKey, destProtocol: dest.protocol, harness }, outBody, vault);
     }
-    return handleBridge(req, res, { ...r, pathname, agentOverride: dest?.agent }, body, vault, cfg, isPro);
+    return handleBridge(req, res, { ...r, pathname, agentOverride: dest?.agent, harness }, body, vault, cfg, isPro);
   });
 }
 

package/src/stream.js

@@ -97,12 +97,16 @@ function makeFieldRestorer(vault, restoreFn) {
 // model + user keep the pseudonym) but TOOL-CALL argument deltas with
 // restoreWithAliases (the client runs the tool on the REAL value). Passes through
 // any non-JSON event untouched.
-export async function pipeRestoredOpenAIStream(upstreamBody, nodeRes, vault) {
+export async function pipeRestoredOpenAIStream(upstreamBody, nodeRes, vault, harness = null) {
   const reader = upstreamBody.getReader();
   const decoder = new TextDecoder();
   let buf = '';
   const contentR = makeFieldRestorer(vault, restoreText);
-  const argRs = new Map(); // tool_call index -> field restorer (aliases)
+  const argRs = new Map();      // tool_call index -> field restorer
+  const toolNames = new Map();  // tool_call index -> name (for redact-remote decisions)
+  // Tool args go through the shared harness (real, or kept-redacted for remote MCP
+  // under redactRemote); without a harness, default to real values (aliases undone).
+  const argFn = (idx) => (text, v) => (harness ? harness.toTool(toolNames.get(idx) || '', text) : restoreWithAliases(text, v));
 
   const handleBlock = (block) => {
     const out = [];
@@ -123,8 +127,9 @@ export async function pipeRestoredOpenAIStream(upstreamBody, nodeRes, vault) {
         if (typeof d.content === 'string') d.content = contentR.push(d.content);
         for (const tc of d.tool_calls || []) {
           const idx = typeof tc.index === 'number' ? tc.index : 0;
+          if (tc.function && typeof tc.function.name === 'string' && tc.function.name) toolNames.set(idx, tc.function.name);
           if (tc.function && typeof tc.function.arguments === 'string') {
-            if (!argRs.has(idx)) argRs.set(idx, makeFieldRestorer(vault, restoreWithAliases));
+            if (!argRs.has(idx)) argRs.set(idx, makeFieldRestorer(vault, argFn(idx)));
             tc.function.arguments = argRs.get(idx).push(tc.function.arguments);
           }
         }

package/src/toolrelay.js

@@ -35,9 +35,9 @@ export function toolsToSpecs(tools) {
     .map((t) => ({ name: t.function.name, description: t.function.description || '', parameters: t.function.parameters || { type: 'object', properties: {} } }));
 }
 
-export function createRelaySession({ vault, redactOpts, bridgeUrl, token }) {
+export function createRelaySession({ vault, redactOpts, bridgeUrl, token, harness = null }) {
   const id = randomUUID().slice(0, 8);
-  const s = { id, reader: null, decoder: new TextDecoder(), buf: '', bridgeSessionId: null, toolId: null, vault: vault || createVault(), redactOpts: redactOpts || { tier: 'basic' }, bridgeUrl, token, done: false };
+  const s = { id, reader: null, decoder: new TextDecoder(), buf: '', bridgeSessionId: null, toolId: null, vault: vault || createVault(), redactOpts: redactOpts || { tier: 'basic' }, bridgeUrl, token, harness, done: false };
   sessions.set(id, s);
   return s;
 }
@@ -68,8 +68,11 @@ export async function pumpBridgeStream(s, handlers) {
           handlers.onText(evt.text);
         } else if (evt.type === 'tool_request') {
           s.bridgeSessionId = evt.session; s.toolId = evt.id;
-          // restore placeholders so the CLIENT runs the tool on REAL values
-          const restoredArgs = restoreDeep(evt.input ?? {}, s.vault);
+          // ② via the shared harness: real values so the CLIENT runs the tool on
+          // them, or the redacted token kept for remote MCP under "redact remote".
+          const restoredArgs = s.harness
+            ? s.harness.toTool(evt.name, evt.input ?? {})
+            : restoreDeep(evt.input ?? {}, s.vault);
           handlers.onToolRequest({ name: evt.name, restoredArgs, toolId: encodeToolCallId(s.id, evt.id) });
           return 'parked';
         } else if (evt.type === 'done') {