@chatpanel/pii 0.2.15 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/pii",
-  "version": "0.2.15",
+  "version": "0.4.0",
   "description": "The canonical ChatPanel privacy engine \u2014 reversible PII redaction + pseudonymization with local entity detection. Pure, dependency-free ESM shared by the ChatPanel extension, gateway, and bridge.",
   "type": "module",
   "main": "index.js",

package/pii-detect.js

@@ -101,10 +101,19 @@ export function parseJsonLoose(s) {
   try { return JSON.parse(String(s).slice(a, b + 1)); } catch { return null; }
 }
 
+// The instruction WITHOUT the shape. The shape now comes from ENTITIES_SCHEMA in
+// `@chatpanel/events` — the one object that renders the prompt block, builds the
+// `response_format` a server enforces, and reads the reply. INJECTED, not imported: this
+// package ships zero dependencies so the bridge can vendor it. A host without it still works.
 export const EXTRACT_SYS = 'You extract sensitive entities from text for redaction. '
-  + 'Return ONLY JSON: {"entities":[{"value":"<verbatim text>","type":"PERSON|ORG|LOCATION|ID|EMAIL|PHONE|OTHER"}]}. '
   + 'Copy each value exactly as it appears. Include people, organizations, locations, and account/ID numbers. No commentary, no code fences.';
 
+const FALLBACK_SHAPE = 'Return ONLY JSON: {"entities":[{"value":"<verbatim text>",'
+  + '"type":"PERSON|ORG|LOCATION|ID|EMAIL|PHONE|OTHER"}]}. No commentary, no code fences.';
+
+// The seam: { block, format(mode), parse(text) }. Absent, everything below behaves as before.
+const NO_STRUCTURE = Object.freeze({ block: '', format: null, parse: null });
+
 async function detectViaEndpoint(text, det, signal, fetchImpl) {
   const res = await fetchImpl(det.url, {
     method: 'POST',
@@ -116,7 +125,7 @@ async function detectViaEndpoint(text, det, signal, fetchImpl) {
   return normalizeEntities(await res.json(), det.types);
 }
 
-async function detectViaOpenAI(text, det, signal, fetchImpl) {
+async function detectViaOpenAI(text, det, signal, fetchImpl, structured = NO_STRUCTURE) {
   const base = String(det.url || '').replace(/\/$/, '');
   // Build the chat URL the SAME way the chat path does. An OpenAI-compatible baseUrl
   // already ends in /v1 (Ollama, OpenRouter, NVIDIA, OpenAI…) → only add
@@ -125,25 +134,70 @@ async function detectViaOpenAI(text, det, signal, fetchImpl) {
   const url = /\/chat\/completions$/.test(base) ? base
     : /\/v\d+$/.test(base) ? `${base}/chat/completions`
       : `${base}/v1/chat/completions`;
-  const res = await fetchImpl(url, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json', ...(det.apiKey ? { Authorization: `Bearer ${det.apiKey}` } : {}) },
-    body: JSON.stringify({
-      model: det.model || 'local',
-      temperature: 0,
-      max_tokens: det.maxTokens || 256,
-      messages: [{ role: 'system', content: EXTRACT_SYS }, { role: 'user', content: text }],
-    }),
-    signal,
-  });
-  if (!res.ok) throw new Error(`detect HTTP ${res.status}`);
-  const json = await res.json();
+  const sys = `${EXTRACT_SYS}\n\n${structured.block || FALLBACK_SHAPE}`;
+  const ask = async (mode) => {
+    const fmt = structured.format ? structured.format(mode) : null;
+    const res = await fetchImpl(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', ...(det.apiKey ? { Authorization: `Bearer ${det.apiKey}` } : {}) },
+      body: JSON.stringify({
+        model: det.model || 'local',
+        temperature: 0,
+        max_tokens: det.maxTokens || 256,
+        messages: [{ role: 'system', content: sys }, { role: 'user', content: text }],
+        ...(fmt || {}),
+      }),
+      signal,
+    });
+    // 400/422 is the server saying it does not understand the body — a different thing from
+    // being down, and the only one worth retrying with a weaker one.
+    if (!res.ok) { const e = new Error(`detect HTTP ${res.status}`); e.status = res.status; throw e; }
+    return res.json();
+  };
+
+  // Grammar first, then plain JSON mode, then nothing. `json_schema` constrains the decoder to
+  // this exact shape — the difference between a 3B local model that answers and one that
+  // writes a paragraph — but many servers reject the field, so each rung is tried once.
+  let json = null;
+  if (structured.format) {
+    for (const mode of ['schema', 'object', 'none']) {
+      try { json = await ask(mode); break; }
+      catch (e) { if (mode === 'none' || (e.status !== 400 && e.status !== 422)) throw e; }
+    }
+  } else {
+    json = await ask('none');
+  }
   const content = json?.choices?.[0]?.message?.content ?? json?.content ?? '';
-  return normalizeEntities(parseJsonLoose(content), det.types);
+  // The schema-aligned reader when the host has one; the loose slice otherwise.
+  const parsed = structured.parse ? structured.parse(content) : parseJsonLoose(content);
+  return normalizeEntities(parsed, det.types);
+}
+
+// Never throws: an egress record that could break redaction is worse than no record.
+function report(onEgress, det, sent, t0, count, err) {
+  if (typeof onEgress !== 'function') return;
+  try {
+    onEgress({
+      backend: det.backend || '',
+      host: hostOf(det.url),
+      chars: sent.length,
+      entities: count,
+      ms: Date.now() - t0,
+      ok: !err,
+      error: err ? String(err.message || err).slice(0, 200) : '',
+    });
+  } catch { /* observability must never be the reason detection fails */ }
 }
 
 // Returns [{value, type}] spans for `text`, or [] (fail-open) on any error/timeout.
-export async function detectEntities(text, cfg, { signal, fetchImpl = globalThis.fetch, strict = false } = {}) {
+// `onEgress` reports that RAW text left for a detector — the FACT, never the text. This is
+// the one call that sends un-redacted content off the device (you cannot redact before you
+// have detected); it is SSRF-guarded but was logged nowhere, and det.url accepts any public
+// host. Injected, like `structured`: this package has no logger. The record carries the HOST
+// (never the full URL, which can hold a token) and counts — never values.
+const hostOf = (u) => { try { return new URL(String(u)).host; } catch { return ''; } };
+
+export async function detectEntities(text, cfg, { signal, fetchImpl = globalThis.fetch, strict = false, structured = NO_STRUCTURE, onEgress = null } = {}) {
   const det = cfg?.detection;
   if (!det || !det.backend || det.backend === 'off' || !det.url || typeof fetchImpl !== 'function') return [];
   const capped = String(text || '').slice(0, det.maxChars || 8000);
@@ -158,7 +212,11 @@ export async function detectEntities(text, cfg, { signal, fetchImpl = globalThis
     // is the normal case. A blocked URL fails open (deterministic-only), or surfaces
     // to the Test button in strict mode.
     assertEndpointUrl(det.url);
-    ents = await withTimeout(run(capped, det, signal, fetchImpl), det.timeoutMs || 1500, signal);
+    const t0 = Date.now();
+    try {
+      ents = await withTimeout(run(capped, det, signal, fetchImpl, structured), det.timeoutMs || 1500, signal);
+      report(onEgress, det, capped, t0, ents.length, null);
+    } catch (e) { report(onEgress, det, capped, t0, 0, e); throw e; }
   } catch (e) {
     if (strict) throw e; // surface errors to the Test button
     ents = []; // otherwise fail open — deterministic redaction still applies

package/pii-redact.js

@@ -33,7 +33,7 @@ const TOLERANT_TOKEN_RE = /\[{0,2}([A-Z][A-Z0-9]*_\d+)\]{0,2}/g;
 // A vault is the per-conversation mapping between placeholders and originals. Keep
 // one per conversation so PERSON_1 means the same entity across turns.
 export function createVault() {
-  // `aliases` maps a pseudonym (e.g. "Alex") back to the real value (e.g. "Suresh")
+  // `aliases` maps a pseudonym (e.g. "Robin") back to the real value (e.g. "Alex Rivera")
   // so LOCAL tool calls (history/meeting search) can run on real data. The reply
   // restorer ignores it — pseudonyms stay permanent in the user's view.
   return { byToken: new Map(), byValue: new Map(), counts: new Map(), aliases: new Map() };

package/tool-harness.js

@@ -77,6 +77,16 @@ export function placeholderToolNote({ toolData = 'real' } = {}) {
   return intro + remote + rules;
 }
 
+// Tools whose results come from the PUBLIC web rather than from the user's own machine or
+// accounts. Deliberately a short, explicit allowlist rather than a heuristic: being wrong in
+// the "public" direction would send real PII to a model, so a tool earns its place here only
+// when its output is public by construction. Page tools are NOT here — the user's open tab
+// may be an internal app.
+const PUBLIC_SOURCE_TOOLS = new Set(['web_search', 'web_fetch', 'fetch_url']);
+export function isPublicSourceTool(name) {
+  return PUBLIC_SOURCE_TOOLS.has(String(name || '').toLowerCase());
+}
+
 export function makeToolHarness({ vault = null, toolData = 'real', redactOpts = null, redactResults = true, remoteTools = null } = {}) {
   const on = !!vault;                       // privacy enabled for this turn?
   const redactRemote = toolData === 'redactRemote';
@@ -109,6 +119,20 @@ export function makeToolHarness({ vault = null, toolData = 'real', redactOpts =
     // via a nested field the old string/{text}-only path skipped.
     toModelResult(name, raw) {
       if (!on || !redactResults || !redactOpts) return raw;
+      // PUBLIC RESULTS ARE NOT THE USER'S DATA.
+      //
+      // Redaction exists to stop the user's information LEAVING the device. Text coming back
+      // from a public web search never left it — the model's provider could fetch the same
+      // page itself — so rewriting it buys no privacy and actively corrupts facts: a
+      // dictionary pseudonym (a user's own name → a stand-in) renamed a same-named public
+      // figure inside search results, and the answer came back about a person who does not
+      // exist. The detectors
+      // (emails, phones, keys) also fire on unrelated strangers' details in fetched pages.
+      //
+      // So public-source results pass through intact. Everything local or private — history,
+      // meetings, notes, the user's own page, any MCP server — is redacted exactly as before,
+      // which is where a leak could actually happen.
+      if (isPublicSourceTool(name)) return raw;
       return redactResultShape(raw, vault, redactOpts);
     },