@chatpanel/pii 0.4.0 → 0.7.1

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@chatpanel/pii",
-  "version": "0.4.0",
-  "description": "The canonical ChatPanel privacy engine \u2014 reversible PII redaction + pseudonymization with local entity detection. Pure, dependency-free ESM shared by the ChatPanel extension, gateway, and bridge.",
+  "version": "0.7.1",
+  "description": "The canonical ChatPanel privacy engine — reversible PII redaction + pseudonymization with local entity detection. Pure, dependency-free ESM shared by the ChatPanel extension, gateway, and bridge.",
   "type": "module",
   "main": "index.js",
   "exports": {

package/pii-detect.js

@@ -39,15 +39,57 @@ export function withTimeout(promise, ms, signal) {
   });
 }
 
-// Map common NER labels (spaCy, HF, Presidio) onto our placeholder types.
+// Map common NER labels onto our placeholder types.
+//
+// FOUR VOCABULARIES, not one, and an unmapped label is SILENTLY DROPPED — `keepEntity` sends
+// anything it does not recognise to the digit-count fallback, where a name has no digits and
+// fails. So a missing row here does not degrade redaction, it turns it off for that type,
+// with nothing on screen to say so.
+//
+// That is not hypothetical. The `multilang-pii-ner` model emits the ai4privacy vocabulary —
+// GIVENNAME, SURNAME, TELEPHONENUM, CITY — and none of those were mapped, so selecting it
+// (it is the default in some builds) meant person names sailed through to the model in
+// plaintext while the shield in the composer still read as on. The deterministic detectors
+// kept catching emails and card numbers, which is exactly what made it hard to notice.
+//
+//   • spaCy / OntoNotes      PER, ORG, GPE, LOC, NORP
+//   • HF bert-base-NER       PER, ORG, LOC, MISC
+//   • Presidio               PERSON, PHONE_NUMBER, EMAIL_ADDRESS, US_SSN…
+//   • ai4privacy / multilang GIVENNAME, SURNAME, STREET, ZIPCODE, TELEPHONENUM…
+//
+// When adding a model, run one sentence through it and map every label it returns. An
+// unrecognised label is a hole, and it is an invisible one.
 function normType(t) {
   const s = String(t || 'ENTITY').toUpperCase().replace(/[^A-Z0-9]/g, '') || 'ENTITY';
   const map = {
+    // People
     PER: 'PERSON', PERSON: 'PERSON', PERSONNAME: 'PERSON',
-    ORG: 'ORG', ORGANIZATION: 'ORG',
+    GIVENNAME: 'PERSON', FIRSTNAME: 'PERSON', MIDDLENAME: 'PERSON',
+    SURNAME: 'PERSON', LASTNAME: 'PERSON', FULLNAME: 'PERSON',
+    // Organisations
+    ORG: 'ORG', ORGANIZATION: 'ORG', COMPANYNAME: 'ORG', COMPANY: 'ORG',
+    // Places. An address PART is still an address — a building number and a postcode
+    // identify a household as surely as the street does.
     GPE: 'LOCATION', LOC: 'LOCATION', LOCATION: 'LOCATION',
-    NORP: 'GROUP', EMAIL: 'EMAIL', EMAILADDRESS: 'EMAIL',
-    PHONE: 'PHONE', PHONENUMBER: 'PHONE',
+    CITY: 'LOCATION', STATE: 'LOCATION', COUNTY: 'LOCATION', COUNTRY: 'LOCATION',
+    STREET: 'ADDRESS', BUILDINGNUM: 'ADDRESS', BUILDINGNUMBER: 'ADDRESS',
+    ZIPCODE: 'ADDRESS', POSTCODE: 'ADDRESS', SECADDRESS: 'ADDRESS', ADDRESS: 'ADDRESS',
+    NORP: 'GROUP',
+    // Contact
+    EMAIL: 'EMAIL', EMAILADDRESS: 'EMAIL',
+    PHONE: 'PHONE', PHONENUMBER: 'PHONE', TELEPHONENUM: 'PHONE', PHONEIMEI: 'ID',
+    // Numbers that identify a person. These are ALWAYS redacted (see ALWAYS_KEEP), which is
+    // the point of naming them rather than leaving them to the digit-count fallback.
+    SOCIALNUM: 'SSN', USSSN: 'SSN', SSN: 'SSN',
+    CREDITCARDNUMBER: 'CREDITCARD', CREDITCARD: 'CREDITCARD',
+    IBAN: 'IBAN', IBANCODE: 'IBAN',
+    ACCOUNTNUM: 'ID', ACCOUNTNUMBER: 'ID', TAXNUM: 'ID', IDCARDNUM: 'ID',
+    DRIVERLICENSENUM: 'ID', PASSPORTNUM: 'ID', VEHICLEVRM: 'ID',
+    // A date of birth identifies; a plain date does not, and small models tag "today".
+    DATEOFBIRTH: 'ID', DOB: 'ID',
+    // Handles and secrets
+    USERNAME: 'ID', USERID: 'ID', IP: 'ID', IPADDRESS: 'ID', MAC: 'ID',
+    PASSWORD: 'SECRET', APIKEY: 'SECRET', SECRET: 'SECRET',
   };
   return map[s] || s;
 }
@@ -57,7 +99,9 @@ function normType(t) {
 // questions still work if "location" is turned off, etc. Numeric/temporal labels
 // (DATE, CARDINAL, ORDINAL…) are noisy — small NER models tag "today" / "4" — so
 // they only count when the value is a long digit run (phone/account/ID).
-const ALWAYS_KEEP = new Set(['EMAIL', 'PHONE', 'SSN', 'CREDITCARD', 'IBAN', 'ID']);
+// SECRET joined these: a password or a key must never reach a model, and leaving it to the
+// per-category toggles would let "turn off numbers" switch it off.
+const ALWAYS_KEEP = new Set(['EMAIL', 'PHONE', 'SSN', 'CREDITCARD', 'IBAN', 'ID', 'SECRET']);
 const LOCATION_TYPES = new Set(['LOCATION', 'FAC', 'ADDRESS', 'GROUP', 'NRP']);
 
 function keepEntity(value, type, types) {
@@ -200,6 +244,24 @@ const hostOf = (u) => { try { return new URL(String(u)).host; } catch { return '
 export async function detectEntities(text, cfg, { signal, fetchImpl = globalThis.fetch, strict = false, structured = NO_STRUCTURE, onEgress = null } = {}) {
   const det = cfg?.detection;
   if (!det || !det.backend || det.backend === 'off' || !det.url || typeof fetchImpl !== 'function') return [];
+  // AN IN-PROCESS DETECTOR SENDS NOTHING ANYWHERE, so the network guard below must not
+  // judge it by a URL it never dials.
+  //
+  // This is not a hypothetical. A host that runs the model in its own process passes a
+  // sentinel URL and a fetchImpl that ignores it entirely — and the sentinel failed the
+  // http(s) scheme check, threw, and was swallowed by the fail-open path. The result was a
+  // detector that reported itself ready, answered its own health route correctly, and
+  // contributed NOTHING to a single redaction: names, organisations and places went to the
+  // model in full while the UI said full tier.
+  //
+  // The opt-out is deliberately narrow. It requires the caller to have supplied its OWN
+  // fetch, so a `transport: 'in-process'` line in a config file cannot turn the SSRF guard
+  // off for a real network address — without an injected transport there is no in-process
+  // anything, and the flag is refused rather than honoured.
+  const inProcess = det.transport === 'in-process';
+  if (inProcess && fetchImpl === globalThis.fetch) {
+    throw new Error("detection.transport 'in-process' needs an injected fetch; refusing to treat a network call as in-process");
+  }
   const capped = String(text || '').slice(0, det.maxChars || 8000);
   if (capped.trim().length < 8) return [];
   const key = cacheKey(capped, det);
@@ -211,12 +273,12 @@ export async function detectEntities(text, cfg, { signal, fetchImpl = globalThis
     // only, never cloud metadata. Loopback/LAN allowed — a local NER server / Ollama
     // is the normal case. A blocked URL fails open (deterministic-only), or surfaces
     // to the Test button in strict mode.
-    assertEndpointUrl(det.url);
+    if (!inProcess) assertEndpointUrl(det.url);
     const t0 = Date.now();
     try {
       ents = await withTimeout(run(capped, det, signal, fetchImpl, structured), det.timeoutMs || 1500, signal);
-      report(onEgress, det, capped, t0, ents.length, null);
-    } catch (e) { report(onEgress, det, capped, t0, 0, e); throw e; }
+      if (!inProcess) report(onEgress, det, capped, t0, ents.length, null);
+    } catch (e) { if (!inProcess) report(onEgress, det, capped, t0, 0, e); throw e; }
   } catch (e) {
     if (strict) throw e; // surface errors to the Test button
     ents = []; // otherwise fail open — deterministic redaction still applies

package/pii-redact.js

@@ -22,6 +22,46 @@ import { stripHidden, confusablesSkeleton } from './sanitize.js';
 
 const TOKEN_RE = /\[\[([A-Z][A-Z0-9]*)_(\d+)\]\]/g;
 
+/**
+ * THE TOKEN FORMAT, PUBLISHED — because `[[TYPE_n]]` collides with `[[wikilink]]`.
+ *
+ * The placeholder grammar was chosen to be visually obvious in a prompt, and it is
+ * character-for-character the wikilink syntax notes and briefs use. So `[[PERSON_1]]` in a
+ * stored message reads as a link to a page called "PERSON_1", and downstream that became a
+ * backlink, a graph node, and a subject with its own page. The name of a person we
+ * deliberately did not learn was being filed as a thing we know about.
+ *
+ * A placeholder is the ABSENCE of an identity. It must never become a link, a subject, a tag
+ * or a topic — and it must never be restored into anything derived and persisted, because
+ * that would put the PII back on disk in a second place.
+ *
+ * Exported rather than left private so consumers ASK instead of re-deriving the pattern:
+ * this package owns the format, and `CLAUDE.md` lists it as a wire contract that only
+ * changes additively.
+ */
+export const REDACTION_TOKEN_TYPES = Object.freeze([
+  'PERSON', 'ORG', 'LOCATION', 'ADDRESS', 'EMAIL', 'PHONE', 'ID', 'SSN', 'IBAN',
+  'CREDITCARD', 'CARD', 'POST', 'FAC', 'GROUP', 'NRP', 'ENTITY', 'KEY', 'SECRET',
+  'TERM', 'PII', 'OTHER',
+]);
+
+/**
+ * Is this bare string one of OUR placeholders?
+ *
+ * Matched against the known type vocabulary rather than the bare `[A-Z]+_\d+` shape, and
+ * that holds even inside brackets: `[[Q3_2026]]` and `[[PHASE_2]]` are links people
+ * genuinely write, so a shape test would trade one invisible bug for another. A custom
+ * dictionary type is the accepted gap — it is user-chosen, so a downstream consumer filing
+ * it is a name the user picked, not a stranger's identity.
+ *
+ * Bracket-tolerant: callers ask both before and after a wikilink parser has stripped them.
+ */
+export function isRedactionToken(value) {
+  const bare = String(value ?? '').trim().replace(/^\[{1,2}|\]{1,2}$/g, '');
+  const m = /^([A-Z][A-Z0-9]*)_\d+$/.exec(bare);
+  return !!m && REDACTION_TOKEN_TYPES.includes(m[1]);
+}
+
 // Bracket-TOLERANT match of the same token. Smaller models routinely drop or mangle
 // the [[ ]] when echoing a placeholder into tool-call JSON — e.g. they emit "ORG_1"
 // or "[ORG_1]" instead of "[[ORG_1]]" — which the strict TOKEN_RE misses, leaving
@@ -339,6 +379,37 @@ export function restoreWithAliases(text, vault) {
   return out;
 }
 
+/**
+ * THE LAST LINE BEFORE A HUMAN READS IT.
+ *
+ * `restoreText` undoes what a given vault minted. This asks the harder question a UI has to
+ * answer: is there ANY placeholder left in what I am about to show? A reply is redacted for
+ * the model's benefit, never the reader's — so a token reaching the screen is always a bug,
+ * and one that is invisible to the code that caused it, because by then the turn is over.
+ *
+ * It exists because a turn can mint tokens in one vault and be restored against another (or
+ * against none): a local agent under "redact for remote only" gets no vault at all, while
+ * tool results reaching it may already carry placeholders from somewhere else. Every one of
+ * those paths ends at the same render call, so the check belongs there.
+ *
+ * Returns `{ text, unresolved }` — restored where the vault knows the token, and the list of
+ * the ones it could not, so the caller can decide (mask, warn, log) rather than silently
+ * shipping `[[PERSON_5]]` to a person reading about their own colleagues.
+ */
+export function scrubPlaceholders(text, vault) {
+  const src = String(text ?? '');
+  if (!src) return { text: src, unresolved: [] };
+  const unresolved = [];
+  const out = src.replace(TOKEN_RE, (match) => {
+    const value = vault?.byToken?.get(match);
+    if (value != null) return value;
+    unresolved.push(match);
+    return match;
+  });
+  TOKEN_RE.lastIndex = 0;
+  return { text: out, unresolved };
+}
+
 // True if the text still contains any redaction placeholder (useful for streaming
 // restore — buffer a tail when a token may be split across chunks).
 export function hasToken(text) {