@chatpanel/pii 0.3.0 → 0.7.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chatpanel/pii",
-  "version": "0.3.0",
+  "version": "0.7.0",
   "description": "The canonical ChatPanel privacy engine \u2014 reversible PII redaction + pseudonymization with local entity detection. Pure, dependency-free ESM shared by the ChatPanel extension, gateway, and bridge.",
   "type": "module",
   "main": "index.js",

package/pii-detect.js

@@ -39,15 +39,57 @@ export function withTimeout(promise, ms, signal) {
   });
 }
 
-// Map common NER labels (spaCy, HF, Presidio) onto our placeholder types.
+// Map common NER labels onto our placeholder types.
+//
+// FOUR VOCABULARIES, not one, and an unmapped label is SILENTLY DROPPED — `keepEntity` sends
+// anything it does not recognise to the digit-count fallback, where a name has no digits and
+// fails. So a missing row here does not degrade redaction, it turns it off for that type,
+// with nothing on screen to say so.
+//
+// That is not hypothetical. The `multilang-pii-ner` model emits the ai4privacy vocabulary —
+// GIVENNAME, SURNAME, TELEPHONENUM, CITY — and none of those were mapped, so selecting it
+// (it is the default in some builds) meant person names sailed through to the model in
+// plaintext while the shield in the composer still read as on. The deterministic detectors
+// kept catching emails and card numbers, which is exactly what made it hard to notice.
+//
+//   • spaCy / OntoNotes      PER, ORG, GPE, LOC, NORP
+//   • HF bert-base-NER       PER, ORG, LOC, MISC
+//   • Presidio               PERSON, PHONE_NUMBER, EMAIL_ADDRESS, US_SSN…
+//   • ai4privacy / multilang GIVENNAME, SURNAME, STREET, ZIPCODE, TELEPHONENUM…
+//
+// When adding a model, run one sentence through it and map every label it returns. An
+// unrecognised label is a hole, and it is an invisible one.
 function normType(t) {
   const s = String(t || 'ENTITY').toUpperCase().replace(/[^A-Z0-9]/g, '') || 'ENTITY';
   const map = {
+    // People
     PER: 'PERSON', PERSON: 'PERSON', PERSONNAME: 'PERSON',
-    ORG: 'ORG', ORGANIZATION: 'ORG',
+    GIVENNAME: 'PERSON', FIRSTNAME: 'PERSON', MIDDLENAME: 'PERSON',
+    SURNAME: 'PERSON', LASTNAME: 'PERSON', FULLNAME: 'PERSON',
+    // Organisations
+    ORG: 'ORG', ORGANIZATION: 'ORG', COMPANYNAME: 'ORG', COMPANY: 'ORG',
+    // Places. An address PART is still an address — a building number and a postcode
+    // identify a household as surely as the street does.
     GPE: 'LOCATION', LOC: 'LOCATION', LOCATION: 'LOCATION',
-    NORP: 'GROUP', EMAIL: 'EMAIL', EMAILADDRESS: 'EMAIL',
-    PHONE: 'PHONE', PHONENUMBER: 'PHONE',
+    CITY: 'LOCATION', STATE: 'LOCATION', COUNTY: 'LOCATION', COUNTRY: 'LOCATION',
+    STREET: 'ADDRESS', BUILDINGNUM: 'ADDRESS', BUILDINGNUMBER: 'ADDRESS',
+    ZIPCODE: 'ADDRESS', POSTCODE: 'ADDRESS', SECADDRESS: 'ADDRESS', ADDRESS: 'ADDRESS',
+    NORP: 'GROUP',
+    // Contact
+    EMAIL: 'EMAIL', EMAILADDRESS: 'EMAIL',
+    PHONE: 'PHONE', PHONENUMBER: 'PHONE', TELEPHONENUM: 'PHONE', PHONEIMEI: 'ID',
+    // Numbers that identify a person. These are ALWAYS redacted (see ALWAYS_KEEP), which is
+    // the point of naming them rather than leaving them to the digit-count fallback.
+    SOCIALNUM: 'SSN', USSSN: 'SSN', SSN: 'SSN',
+    CREDITCARDNUMBER: 'CREDITCARD', CREDITCARD: 'CREDITCARD',
+    IBAN: 'IBAN', IBANCODE: 'IBAN',
+    ACCOUNTNUM: 'ID', ACCOUNTNUMBER: 'ID', TAXNUM: 'ID', IDCARDNUM: 'ID',
+    DRIVERLICENSENUM: 'ID', PASSPORTNUM: 'ID', VEHICLEVRM: 'ID',
+    // A date of birth identifies; a plain date does not, and small models tag "today".
+    DATEOFBIRTH: 'ID', DOB: 'ID',
+    // Handles and secrets
+    USERNAME: 'ID', USERID: 'ID', IP: 'ID', IPADDRESS: 'ID', MAC: 'ID',
+    PASSWORD: 'SECRET', APIKEY: 'SECRET', SECRET: 'SECRET',
   };
   return map[s] || s;
 }
@@ -57,7 +99,9 @@ function normType(t) {
 // questions still work if "location" is turned off, etc. Numeric/temporal labels
 // (DATE, CARDINAL, ORDINAL…) are noisy — small NER models tag "today" / "4" — so
 // they only count when the value is a long digit run (phone/account/ID).
-const ALWAYS_KEEP = new Set(['EMAIL', 'PHONE', 'SSN', 'CREDITCARD', 'IBAN', 'ID']);
+// SECRET joined these: a password or a key must never reach a model, and leaving it to the
+// per-category toggles would let "turn off numbers" switch it off.
+const ALWAYS_KEEP = new Set(['EMAIL', 'PHONE', 'SSN', 'CREDITCARD', 'IBAN', 'ID', 'SECRET']);
 const LOCATION_TYPES = new Set(['LOCATION', 'FAC', 'ADDRESS', 'GROUP', 'NRP']);
 
 function keepEntity(value, type, types) {
@@ -101,10 +145,19 @@ export function parseJsonLoose(s) {
   try { return JSON.parse(String(s).slice(a, b + 1)); } catch { return null; }
 }
 
+// The instruction WITHOUT the shape. The shape now comes from ENTITIES_SCHEMA in
+// `@chatpanel/events` — the one object that renders the prompt block, builds the
+// `response_format` a server enforces, and reads the reply. INJECTED, not imported: this
+// package ships zero dependencies so the bridge can vendor it. A host without it still works.
 export const EXTRACT_SYS = 'You extract sensitive entities from text for redaction. '
-  + 'Return ONLY JSON: {"entities":[{"value":"<verbatim text>","type":"PERSON|ORG|LOCATION|ID|EMAIL|PHONE|OTHER"}]}. '
   + 'Copy each value exactly as it appears. Include people, organizations, locations, and account/ID numbers. No commentary, no code fences.';
 
+const FALLBACK_SHAPE = 'Return ONLY JSON: {"entities":[{"value":"<verbatim text>",'
+  + '"type":"PERSON|ORG|LOCATION|ID|EMAIL|PHONE|OTHER"}]}. No commentary, no code fences.';
+
+// The seam: { block, format(mode), parse(text) }. Absent, everything below behaves as before.
+const NO_STRUCTURE = Object.freeze({ block: '', format: null, parse: null });
+
 async function detectViaEndpoint(text, det, signal, fetchImpl) {
   const res = await fetchImpl(det.url, {
     method: 'POST',
@@ -116,7 +169,7 @@ async function detectViaEndpoint(text, det, signal, fetchImpl) {
   return normalizeEntities(await res.json(), det.types);
 }
 
-async function detectViaOpenAI(text, det, signal, fetchImpl) {
+async function detectViaOpenAI(text, det, signal, fetchImpl, structured = NO_STRUCTURE) {
   const base = String(det.url || '').replace(/\/$/, '');
   // Build the chat URL the SAME way the chat path does. An OpenAI-compatible baseUrl
   // already ends in /v1 (Ollama, OpenRouter, NVIDIA, OpenAI…) → only add
@@ -125,25 +178,70 @@ async function detectViaOpenAI(text, det, signal, fetchImpl) {
   const url = /\/chat\/completions$/.test(base) ? base
     : /\/v\d+$/.test(base) ? `${base}/chat/completions`
       : `${base}/v1/chat/completions`;
-  const res = await fetchImpl(url, {
-    method: 'POST',
-    headers: { 'Content-Type': 'application/json', ...(det.apiKey ? { Authorization: `Bearer ${det.apiKey}` } : {}) },
-    body: JSON.stringify({
-      model: det.model || 'local',
-      temperature: 0,
-      max_tokens: det.maxTokens || 256,
-      messages: [{ role: 'system', content: EXTRACT_SYS }, { role: 'user', content: text }],
-    }),
-    signal,
-  });
-  if (!res.ok) throw new Error(`detect HTTP ${res.status}`);
-  const json = await res.json();
+  const sys = `${EXTRACT_SYS}\n\n${structured.block || FALLBACK_SHAPE}`;
+  const ask = async (mode) => {
+    const fmt = structured.format ? structured.format(mode) : null;
+    const res = await fetchImpl(url, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json', ...(det.apiKey ? { Authorization: `Bearer ${det.apiKey}` } : {}) },
+      body: JSON.stringify({
+        model: det.model || 'local',
+        temperature: 0,
+        max_tokens: det.maxTokens || 256,
+        messages: [{ role: 'system', content: sys }, { role: 'user', content: text }],
+        ...(fmt || {}),
+      }),
+      signal,
+    });
+    // 400/422 is the server saying it does not understand the body — a different thing from
+    // being down, and the only one worth retrying with a weaker one.
+    if (!res.ok) { const e = new Error(`detect HTTP ${res.status}`); e.status = res.status; throw e; }
+    return res.json();
+  };
+
+  // Grammar first, then plain JSON mode, then nothing. `json_schema` constrains the decoder to
+  // this exact shape — the difference between a 3B local model that answers and one that
+  // writes a paragraph — but many servers reject the field, so each rung is tried once.
+  let json = null;
+  if (structured.format) {
+    for (const mode of ['schema', 'object', 'none']) {
+      try { json = await ask(mode); break; }
+      catch (e) { if (mode === 'none' || (e.status !== 400 && e.status !== 422)) throw e; }
+    }
+  } else {
+    json = await ask('none');
+  }
   const content = json?.choices?.[0]?.message?.content ?? json?.content ?? '';
-  return normalizeEntities(parseJsonLoose(content), det.types);
+  // The schema-aligned reader when the host has one; the loose slice otherwise.
+  const parsed = structured.parse ? structured.parse(content) : parseJsonLoose(content);
+  return normalizeEntities(parsed, det.types);
+}
+
+// Never throws: an egress record that could break redaction is worse than no record.
+function report(onEgress, det, sent, t0, count, err) {
+  if (typeof onEgress !== 'function') return;
+  try {
+    onEgress({
+      backend: det.backend || '',
+      host: hostOf(det.url),
+      chars: sent.length,
+      entities: count,
+      ms: Date.now() - t0,
+      ok: !err,
+      error: err ? String(err.message || err).slice(0, 200) : '',
+    });
+  } catch { /* observability must never be the reason detection fails */ }
 }
 
 // Returns [{value, type}] spans for `text`, or [] (fail-open) on any error/timeout.
-export async function detectEntities(text, cfg, { signal, fetchImpl = globalThis.fetch, strict = false } = {}) {
+// `onEgress` reports that RAW text left for a detector — the FACT, never the text. This is
+// the one call that sends un-redacted content off the device (you cannot redact before you
+// have detected); it is SSRF-guarded but was logged nowhere, and det.url accepts any public
+// host. Injected, like `structured`: this package has no logger. The record carries the HOST
+// (never the full URL, which can hold a token) and counts — never values.
+const hostOf = (u) => { try { return new URL(String(u)).host; } catch { return ''; } };
+
+export async function detectEntities(text, cfg, { signal, fetchImpl = globalThis.fetch, strict = false, structured = NO_STRUCTURE, onEgress = null } = {}) {
   const det = cfg?.detection;
   if (!det || !det.backend || det.backend === 'off' || !det.url || typeof fetchImpl !== 'function') return [];
   const capped = String(text || '').slice(0, det.maxChars || 8000);
@@ -158,7 +256,11 @@ export async function detectEntities(text, cfg, { signal, fetchImpl = globalThis
     // is the normal case. A blocked URL fails open (deterministic-only), or surfaces
     // to the Test button in strict mode.
     assertEndpointUrl(det.url);
-    ents = await withTimeout(run(capped, det, signal, fetchImpl), det.timeoutMs || 1500, signal);
+    const t0 = Date.now();
+    try {
+      ents = await withTimeout(run(capped, det, signal, fetchImpl, structured), det.timeoutMs || 1500, signal);
+      report(onEgress, det, capped, t0, ents.length, null);
+    } catch (e) { report(onEgress, det, capped, t0, 0, e); throw e; }
   } catch (e) {
     if (strict) throw e; // surface errors to the Test button
     ents = []; // otherwise fail open — deterministic redaction still applies

package/pii-redact.js

@@ -22,6 +22,46 @@ import { stripHidden, confusablesSkeleton } from './sanitize.js';
 
 const TOKEN_RE = /\[\[([A-Z][A-Z0-9]*)_(\d+)\]\]/g;
 
+/**
+ * THE TOKEN FORMAT, PUBLISHED — because `[[TYPE_n]]` collides with `[[wikilink]]`.
+ *
+ * The placeholder grammar was chosen to be visually obvious in a prompt, and it is
+ * character-for-character the wikilink syntax notes and briefs use. So `[[PERSON_1]]` in a
+ * stored message reads as a link to a page called "PERSON_1", and downstream that became a
+ * backlink, a graph node, and a subject with its own page. The name of a person we
+ * deliberately did not learn was being filed as a thing we know about.
+ *
+ * A placeholder is the ABSENCE of an identity. It must never become a link, a subject, a tag
+ * or a topic — and it must never be restored into anything derived and persisted, because
+ * that would put the PII back on disk in a second place.
+ *
+ * Exported rather than left private so consumers ASK instead of re-deriving the pattern:
+ * this package owns the format, and `CLAUDE.md` lists it as a wire contract that only
+ * changes additively.
+ */
+export const REDACTION_TOKEN_TYPES = Object.freeze([
+  'PERSON', 'ORG', 'LOCATION', 'ADDRESS', 'EMAIL', 'PHONE', 'ID', 'SSN', 'IBAN',
+  'CREDITCARD', 'CARD', 'POST', 'FAC', 'GROUP', 'NRP', 'ENTITY', 'KEY', 'SECRET',
+  'TERM', 'PII', 'OTHER',
+]);
+
+/**
+ * Is this bare string one of OUR placeholders?
+ *
+ * Matched against the known type vocabulary rather than the bare `[A-Z]+_\d+` shape, and
+ * that holds even inside brackets: `[[Q3_2026]]` and `[[PHASE_2]]` are links people
+ * genuinely write, so a shape test would trade one invisible bug for another. A custom
+ * dictionary type is the accepted gap — it is user-chosen, so a downstream consumer filing
+ * it is a name the user picked, not a stranger's identity.
+ *
+ * Bracket-tolerant: callers ask both before and after a wikilink parser has stripped them.
+ */
+export function isRedactionToken(value) {
+  const bare = String(value ?? '').trim().replace(/^\[{1,2}|\]{1,2}$/g, '');
+  const m = /^([A-Z][A-Z0-9]*)_\d+$/.exec(bare);
+  return !!m && REDACTION_TOKEN_TYPES.includes(m[1]);
+}
+
 // Bracket-TOLERANT match of the same token. Smaller models routinely drop or mangle
 // the [[ ]] when echoing a placeholder into tool-call JSON — e.g. they emit "ORG_1"
 // or "[ORG_1]" instead of "[[ORG_1]]" — which the strict TOKEN_RE misses, leaving
@@ -33,7 +73,7 @@ const TOLERANT_TOKEN_RE = /\[{0,2}([A-Z][A-Z0-9]*_\d+)\]{0,2}/g;
 // A vault is the per-conversation mapping between placeholders and originals. Keep
 // one per conversation so PERSON_1 means the same entity across turns.
 export function createVault() {
-  // `aliases` maps a pseudonym (e.g. "Alex") back to the real value (e.g. "Suresh")
+  // `aliases` maps a pseudonym (e.g. "Robin") back to the real value (e.g. "Alex Rivera")
   // so LOCAL tool calls (history/meeting search) can run on real data. The reply
   // restorer ignores it — pseudonyms stay permanent in the user's view.
   return { byToken: new Map(), byValue: new Map(), counts: new Map(), aliases: new Map() };
@@ -339,6 +379,37 @@ export function restoreWithAliases(text, vault) {
   return out;
 }
 
+/**
+ * THE LAST LINE BEFORE A HUMAN READS IT.
+ *
+ * `restoreText` undoes what a given vault minted. This asks the harder question a UI has to
+ * answer: is there ANY placeholder left in what I am about to show? A reply is redacted for
+ * the model's benefit, never the reader's — so a token reaching the screen is always a bug,
+ * and one that is invisible to the code that caused it, because by then the turn is over.
+ *
+ * It exists because a turn can mint tokens in one vault and be restored against another (or
+ * against none): a local agent under "redact for remote only" gets no vault at all, while
+ * tool results reaching it may already carry placeholders from somewhere else. Every one of
+ * those paths ends at the same render call, so the check belongs there.
+ *
+ * Returns `{ text, unresolved }` — restored where the vault knows the token, and the list of
+ * the ones it could not, so the caller can decide (mask, warn, log) rather than silently
+ * shipping `[[PERSON_5]]` to a person reading about their own colleagues.
+ */
+export function scrubPlaceholders(text, vault) {
+  const src = String(text ?? '');
+  if (!src) return { text: src, unresolved: [] };
+  const unresolved = [];
+  const out = src.replace(TOKEN_RE, (match) => {
+    const value = vault?.byToken?.get(match);
+    if (value != null) return value;
+    unresolved.push(match);
+    return match;
+  });
+  TOKEN_RE.lastIndex = 0;
+  return { text: out, unresolved };
+}
+
 // True if the text still contains any redaction placeholder (useful for streaming
 // restore — buffer a tail when a token may be split across chunks).
 export function hasToken(text) {

package/tool-harness.js

@@ -124,8 +124,9 @@ export function makeToolHarness({ vault = null, toolData = 'real', redactOpts =
       // Redaction exists to stop the user's information LEAVING the device. Text coming back
       // from a public web search never left it — the model's provider could fetch the same
       // page itself — so rewriting it buys no privacy and actively corrupts facts: a
-      // dictionary pseudonym (Suresh → John) renamed a public actor inside search results,
-      // and the answer came back about "Mysore Seshaiah John Babu Naidu". The detectors
+      // dictionary pseudonym (a user's own name → a stand-in) renamed a same-named public
+      // figure inside search results, and the answer came back about a person who does not
+      // exist. The detectors
       // (emails, phones, keys) also fire on unrelated strangers' details in fetched pages.
       //
       // So public-source results pass through intact. Everything local or private — history,