@chatpanel/bridge 0.10.42 → 0.11.0

package/src/pii/sanitize.js

@@ -0,0 +1,179 @@
+// GENERATED — do not edit.
+// Source of truth: chatpanel-pii/sanitize.js (npm @chatpanel/pii).
+// Edit there, then run: npm run sync:pii
+//
+// Vendored rather than depended on: the bridge ships zero runtime dependencies so a
+// curl one-liner install cannot fail on someone's registry, and so the compiled
+// single-file binary has nothing to resolve.
+
+// Unicode de-steganography for text that flows through the privacy boundary.
+//
+// Invisible and format-control characters are a single vector with three abuses,
+// all relevant to a redaction product:
+//
+//   1. Redaction bypass - splitting a value with zero-width chars (j<ZWSP>o<ZWSP>hn@x.com)
+//      hides it from the regex/NER detector, then the model reassembles the real
+//      value. The deterministic engine works on text, so the smuggled PII leaks.
+//   2. Hidden prompt injection - Unicode Tag characters (U+E0000-E007F) render as
+//      nothing but encode a full ASCII instruction the model reads ("ASCII smuggling").
+//   3. Fingerprinting / watermarking - steganographic markers injected into a prompt
+//      (e.g. a client classifying a custom gateway and encoding a bit into invisible
+//      punctuation). A privacy proxy should scrub these - and never emit its own.
+//
+// We strip the channels that have no legitimate place in plain prompt text, while
+// PRESERVING the few legitimate uses (emoji ZWJ/variation sequences, normal accents).
+//
+// The patterns are BUILT FROM NUMERIC CODE POINTS below - there are deliberately no
+// literal invisible characters anywhere in this source (auditable, and fitting for a
+// de-steg module). Pure + dependency-free ESM. Call it BEFORE detection so obfuscated
+// PII becomes matchable, and on model output before restoration so a token can't be
+// split/spoofed with invisibles.
+
+// Code-point ranges (inclusive) per category, by their abuse.
+const RANGES = {
+  // Unicode Tag block - the ASCII-smuggling channel.
+  tags: [[0xE0000, 0xE007F]],
+  // Bidi controls - reorder/override visible text to hide reversed instructions.
+  bidi: [[0x061C, 0x061C], [0x200E, 0x200F], [0x202A, 0x202E], [0x2066, 0x2069]],
+  // Zero-width & assorted invisible format chars: soft hyphen, Hangul/Mongolian
+  // fillers, ZWSP, word/invisible joiners, deprecated format controls, BOM/ZWNBSP,
+  // interlinear annotation, object replacement.
+  zeroWidth: [
+    [0x00AD, 0x00AD], [0x115F, 0x1160], [0x180E, 0x180E], [0x200B, 0x200B],
+    [0x2060, 0x2064], [0x206A, 0x206F], [0x3164, 0x3164], [0xFEFF, 0xFEFF],
+    [0xFFA0, 0xFFA0], [0xFFF9, 0xFFFB], [0xFFFC, 0xFFFC],
+  ],
+  // Supplementary variation selectors - the byte-smuggling range. Never legit in text.
+  supVS: [[0xE0100, 0xE01EF]],
+  // Line/paragraph separators - converted to '\n' (kill parser tricks, keep the break).
+  lineSep: [[0x2028, 0x2029]],
+  // ZWJ/ZWNJ + BMP variation selectors - legit ONLY next to an emoji base, so these
+  // are stripped contextually (see ANOMALOUS_JOIN_VS), not unconditionally.
+  joinVS: [[0x200C, 0x200D], [0xFE00, 0xFE0F]],
+};
+
+const u = (cp) => `\\u{${cp.toString(16).toUpperCase()}}`;
+const cls = (ranges) => ranges.map(([a, b]) => (a === b ? u(a) : `${u(a)}-${u(b)}`)).join('');
+
+const TAGS = new RegExp(`[${cls(RANGES.tags)}]`, 'gu');
+const BIDI = new RegExp(`[${cls(RANGES.bidi)}]`, 'gu');
+const ZERO_WIDTH = new RegExp(`[${cls(RANGES.zeroWidth)}]`, 'gu');
+const SUP_VS = new RegExp(`[${cls(RANGES.supVS)}]`, 'gu');
+const LINE_SEP = new RegExp(`[${cls(RANGES.lineSep)}]`, 'gu');
+// Strip ZWJ/ZWNJ/VS only when NOT preceded by an emoji base (so emoji sequences and
+// regional-indicator flags survive); supplementary VS are always stripped.
+const ANOMALOUS_JOIN_VS = new RegExp(
+  `(?<![\\p{Extended_Pictographic}${u(0x1F1E6)}-${u(0x1F1FF)}])[${cls(RANGES.joinVS)}]|[${cls(RANGES.supVS)}]`,
+  'gu',
+);
+// Runs of combining marks (Zalgo / bit-stuffing). A real stacked diacritic is 1-3
+// marks; anything past the cap is signalling, not language.
+const COMBINING_RUN = /\p{M}+/gu;
+
+// Cheap boolean for hot paths / UI ("does this contain anything hidden?"). Excludes
+// the context-dependent joinVS so legitimate emoji aren't flagged - sanitizeUnicode()
+// stays the source of truth for those.
+const ANY_HIDDEN = new RegExp(
+  `[${cls(RANGES.bidi)}${cls(RANGES.zeroWidth)}]|[${cls(RANGES.tags)}]|[${cls(RANGES.supVS)}]`,
+  'u',
+);
+
+export function hasHiddenChars(text) {
+  return typeof text === 'string' && ANY_HIDDEN.test(text);
+}
+
+// sanitizeUnicode(text, opts) -> { clean, removed, findings }
+//   clean    - text with the smuggling channels stripped/normalized
+//   removed  - total count of stripped/collapsed characters (0 = nothing hidden)
+//   findings - per-category counts (only non-zero keys), for transparent reporting
+//
+// opts.normalize: 'NFC' (default, appearance-preserving) | 'NFKC' (also folds
+//   fullwidth/homoglyph compatibility forms - stronger for detection, but rewrites
+//   some visible glyphs) | 'none'.
+// opts.collapseCombiningOver: max combining marks kept per run (default 4).
+export function sanitizeUnicode(text, { normalize = 'NFC', collapseCombiningOver = 4 } = {}) {
+  if (typeof text !== 'string' || text === '') return { clean: text ?? '', removed: 0, findings: {} };
+  let s = text;
+  const findings = {};
+
+  // Strip one category, counting by code point (spread iterates code points, so a
+  // supplementary char like a Tag counts as 1, not 2 UTF-16 units).
+  const strip = (re, key) => {
+    let n = 0;
+    s = s.replace(re, (m) => { n += [...m].length; return ''; });
+    if (n) findings[key] = n;
+  };
+
+  let lineSep = 0;
+  s = s.replace(LINE_SEP, () => { lineSep++; return '\n'; });
+  if (lineSep) findings.lineSep = lineSep;
+
+  strip(TAGS, 'tags');
+  strip(BIDI, 'bidi');
+  strip(ZERO_WIDTH, 'zeroWidth');
+  strip(ANOMALOUS_JOIN_VS, 'joinersVS');
+
+  // Compose canonically so split/decomposed forms can't dodge the detector.
+  if (normalize && normalize !== 'none') {
+    try { s = s.normalize(normalize); } catch { /* invalid form name -> skip */ }
+  }
+
+  let combining = 0;
+  s = s.replace(COMBINING_RUN, (run) => {
+    const marks = [...run];
+    if (marks.length <= collapseCombiningOver) return run;
+    combining += marks.length - collapseCombiningOver;
+    return marks.slice(0, collapseCombiningOver).join('');
+  });
+  if (combining) findings.combining = combining;
+
+  const removed = (findings.tags || 0) + (findings.bidi || 0) + (findings.zeroWidth || 0)
+    + (findings.joinersVS || 0) + (findings.combining || 0);
+  return { clean: s, removed, findings };
+}
+
+// Convenience for the common "just give me clean text" caller.
+export function stripHidden(text, opts) {
+  return sanitizeUnicode(text, opts).clean;
+}
+
+// ── Confusables skeleton ─────────────────────────────────────────────────────
+// Fold single-code-point Latin LOOK-ALIKES (Cyrillic / Greek / fullwidth) to their
+// ASCII skeleton so a homoglyph-obfuscated value (jоhn@x.com with a Cyrillic 'о')
+// becomes matchable by the ASCII regexes. STRICTLY 1:1 per code point — every mapping
+// is one char → one char — so a match's indices in the skeleton line up exactly with
+// the original text. Use it for DETECTION only and redact the ORIGINAL span, so
+// legitimate Cyrillic/Greek/CJK text is never rewritten (only deceptively-Latin
+// values that actually match a detector get touched). Built from numeric code points
+// (no literal confusables in source, like the rest of this module).
+const CONFUSABLE = new Map([
+  // Cyrillic lowercase → Latin
+  [0x0430, 'a'], [0x0435, 'e'], [0x043E, 'o'], [0x0440, 'p'], [0x0441, 'c'],
+  [0x0443, 'y'], [0x0445, 'x'], [0x0455, 's'], [0x0456, 'i'], [0x0458, 'j'],
+  [0x04BB, 'h'], [0x043C, 'm'], [0x043D, 'h'], [0x0442, 't'], [0x043A, 'k'],
+  // Cyrillic uppercase → Latin
+  [0x0410, 'A'], [0x0412, 'B'], [0x0415, 'E'], [0x041A, 'K'], [0x041C, 'M'],
+  [0x041D, 'H'], [0x041E, 'O'], [0x0420, 'P'], [0x0421, 'C'], [0x0422, 'T'],
+  [0x0425, 'X'], [0x0406, 'I'], [0x0408, 'J'], [0x0405, 'S'],
+  // Greek → Latin
+  [0x03BF, 'o'], [0x03C1, 'p'], [0x03B1, 'a'], [0x03BD, 'v'], [0x03B9, 'i'],
+  [0x0391, 'A'], [0x0392, 'B'], [0x0395, 'E'], [0x0396, 'Z'], [0x0397, 'H'],
+  [0x0399, 'I'], [0x039A, 'K'], [0x039C, 'M'], [0x039D, 'N'], [0x039F, 'O'],
+  [0x03A1, 'P'], [0x03A4, 'T'], [0x03A5, 'Y'], [0x03A7, 'X'],
+]);
+
+export function confusablesSkeleton(text) {
+  if (typeof text !== 'string' || text === '') return text ?? '';
+  let out = '';
+  for (const ch of text) {
+    const cp = ch.codePointAt(0);
+    if (cp >= 0xFF01 && cp <= 0xFF5E) { out += String.fromCharCode(cp - 0xFEE0); continue; } // fullwidth ASCII
+    const mapped = CONFUSABLE.get(cp);
+    out += mapped != null ? mapped : ch;
+  }
+  return out;
+}
+
+// Exposed for tests / external auditing.
+export const SANITIZE_RANGES = RANGES;
+export const CONFUSABLE_MAP = CONFUSABLE;

package/src/pii/tool-harness.js

@@ -0,0 +1,152 @@
+// GENERATED — do not edit.
+// Source of truth: chatpanel-pii/tool-harness.js (npm @chatpanel/pii).
+// Edit there, then run: npm run sync:pii
+//
+// Vendored rather than depended on: the bridge ships zero runtime dependencies so a
+// curl one-liner install cannot fail on someone's registry, and so the compiled
+// single-file binary has nothing to resolve.
+
+// THE tool harness — one interception layer shared by every orchestrator
+// (ChatPanel API + agent, gateway API + relay) so tool handling can't drift. It
+// owns the boundaries around a model turn:
+//
+//   ⓪ selectTools  — inject + narrow the tools the model is offered (MCP-auto).
+//   ② toTool       — what a tool RECEIVES: real values (so on-device / remote
+//                    lookups work), or the redacted token for remote MCP tools
+//                    when the user chose "redact remote".
+//   ③ toModelResult— what the MODEL sees back: the tool result re-redacted so it
+//                    stays blinded.
+//   ④ toUser       — the final reply: reversible tokens restored (pseudonyms stay).
+//
+// PRIVACY IS OPTIONAL. With no `vault` (redaction off), ②③④ are pass-throughs —
+// no latency, no placeholder confusion — but ⓪ selectTools STILL narrows, because
+// not every turn is privacy-sensitive yet every turn benefits from fewer tools.
+//
+// Self-contained on the SYNCED engine files (pii-redact.js, tool-rank.js), so the
+// extension (browser ESM) and the gateway (npm) run the exact same code. The caller
+// passes the already-gated `redactOpts` ({tier, entities, dictionary}) it computed
+// from cfg+isPro — tier/dictionary selection stays out of the harness.
+
+import { restoreText, restoreWithAliases, redactResultShape } from './pii-redact.js';
+import { narrowSpecs } from './tool-rank.js';
+
+// MCP / remote tools are server-prefixed (mcp_server__tool). Local tools
+// (history/meeting/page, or a client's core bash/read) are not — they always get
+// real values and are never narrowed away.
+export const isRemoteToolName = (name) => /^mcp[_-]/i.test(String(name || ''));
+
+// Deep restore of a tool-call argument value, undoing reversible tokens AND
+// pseudonyms (tools run locally / on real data; only the model stays blinded).
+export function restoreToolArgs(value, vault) {
+  if (!vault) return value;
+  if (typeof value === 'string') return restoreWithAliases(value, vault);
+  if (Array.isArray(value)) return value.map((v) => restoreToolArgs(v, vault));
+  if (value && typeof value === 'object') {
+    const out = {};
+    for (const k of Object.keys(value)) out[k] = restoreToolArgs(value[k], vault);
+    return out;
+  }
+  return value;
+}
+
+// System-prompt note that tells the model how to behave around placeholders when
+// tools are armed. WITHOUT this, privacy-aware models (Codex, Claude) recognize a
+// [[LOCATION_1]] token as redacted and REFUSE to use it for a lookup ("I can't see
+// your real city") — the opposite of what we want. Weak models call the tool blindly
+// and it works (the harness restores the real value), so the note levels them up.
+export function placeholderToolNote({ toolData = 'real' } = {}) {
+  const intro =
+    'PRIVACY PLACEHOLDERS: some values in this conversation are tokens like [[PERSON_1]], '
+    + '[[LOCATION_1]], [[ORG_1]] that stand in for the user\'s real private data. ';
+  const remote = toolData === 'redactRemote'
+    ? 'When you call a LOCAL tool the placeholder is automatically replaced with the real '
+      + 'value before the tool runs; REMOTE (MCP) tools deliberately receive the placeholder '
+      + 'to keep private data off third-party servers. '
+    : 'When you call ANY tool, these placeholders are AUTOMATICALLY replaced with the real '
+      + 'values before the tool executes — the tool receives the TRUE value and returns correct '
+      + 'results. ';
+  const rules =
+    'So: treat each placeholder as a CONCRETE, specific value you already have — it is enough to '
+    + 'act on, NOT missing or unknown information. CALL THE TOOL using the placeholder exactly as '
+    + 'written, as if it were the real value. '
+    // The common failure isn\'t a privacy refusal — it\'s the model deciding it "lacks data"
+    // because the value is a token, and answering from general knowledge instead of looking up.
+    + 'If answering needs the real data behind a placeholder (e.g. which city [[LOCATION_1]] is, '
+    + 'who [[PERSON_1]] is, what [[ORG_1]] does), do NOT reply that you lack information or cannot '
+    + 'answer. Instead pick the most relevant available tool for that placeholder\'s TYPE — LOCATION '
+    + '→ geography / place lookups, PERSON → people lookups, ORG → company/org lookups, dates/IDs → '
+    + 'the matching lookup — and pass the placeholder straight through as the argument. The harness '
+    + 'restores the true value before the tool runs, so the lookup returns correct results. Make your '
+    + 'best-guess tool call FIRST; only conclude you lack data AFTER a tool has actually returned '
+    + 'nothing useful. '
+    + 'Do NOT ask the user to re-type the value and do NOT refuse on privacy grounds — the lookup '
+    + 'will work. The real values are restored in your final answer automatically, so write your '
+    + 'answer using the placeholders too.';
+  return intro + remote + rules;
+}
+
+// Tools whose results come from the PUBLIC web rather than from the user's own machine or
+// accounts. Deliberately a short, explicit allowlist rather than a heuristic: being wrong in
+// the "public" direction would send real PII to a model, so a tool earns its place here only
+// when its output is public by construction. Page tools are NOT here — the user's open tab
+// may be an internal app.
+const PUBLIC_SOURCE_TOOLS = new Set(['web_search', 'web_fetch', 'fetch_url']);
+export function isPublicSourceTool(name) {
+  return PUBLIC_SOURCE_TOOLS.has(String(name || '').toLowerCase());
+}
+
+export function makeToolHarness({ vault = null, toolData = 'real', redactOpts = null, redactResults = true, remoteTools = null } = {}) {
+  const on = !!vault;                       // privacy enabled for this turn?
+  const redactRemote = toolData === 'redactRemote';
+  // How we decide a tool is REMOTE (must not receive real PII under redactRemote).
+  // Prefer an EXPLICIT set/predicate the caller derived from the toolset (a remote
+  // tool not named mcp_* would otherwise be misclassified as local and get real
+  // values); fall back to the mcp_* name heuristic when the caller passes nothing.
+  const isRemoteTool = typeof remoteTools === 'function' ? remoteTools
+    : (remoteTools instanceof Set ? (name) => remoteTools.has(name)
+      : isRemoteToolName);
+  return {
+    enabled: on,
+    isRemoteTool,
+
+    // ⓪ Always-on tool selection (privacy-independent). `available` is any spec
+    // list; `opts` forwards { cap, keep, name, description } to the shared ranker.
+    selectTools(available, query, opts = {}) {
+      return narrowSpecs(available, query, opts);
+    },
+
+    // ② What the tool receives.
+    toTool(name, args) {
+      if (!on) return args;                                    // privacy off → already real
+      if (redactRemote && isRemoteTool(name)) return args;      // keep PII off remote MCP
+      return restoreToolArgs(args, vault);                      // real values for the tool
+    },
+
+    // ③ What the model sees back (re-redacted). Walks string / { text } / array /
+    // MCP { content:[{text}] } shapes so a tool result can't leak PII to the model
+    // via a nested field the old string/{text}-only path skipped.
+    toModelResult(name, raw) {
+      if (!on || !redactResults || !redactOpts) return raw;
+      // PUBLIC RESULTS ARE NOT THE USER'S DATA.
+      //
+      // Redaction exists to stop the user's information LEAVING the device. Text coming back
+      // from a public web search never left it — the model's provider could fetch the same
+      // page itself — so rewriting it buys no privacy and actively corrupts facts: a
+      // dictionary pseudonym (a user's own name → a stand-in) renamed a same-named public
+      // figure inside search results, and the answer came back about a person who does not
+      // exist. The detectors
+      // (emails, phones, keys) also fire on unrelated strangers' details in fetched pages.
+      //
+      // So public-source results pass through intact. Everything local or private — history,
+      // meetings, notes, the user's own page, any MCP server — is redacted exactly as before,
+      // which is where a leak could actually happen.
+      if (isPublicSourceTool(name)) return raw;
+      return redactResultShape(raw, vault, redactOpts);
+    },
+
+    // ④ The final reply the user sees.
+    toUser(text) {
+      return on ? restoreText(text, vault) : text;
+    },
+  };
+}

package/src/pii/tool-rank.js

@@ -0,0 +1,110 @@
+// GENERATED — do not edit.
+// Source of truth: chatpanel-pii/tool-rank.js (npm @chatpanel/pii).
+// Edit there, then run: npm run sync:pii
+//
+// Vendored rather than depended on: the bridge ships zero runtime dependencies so a
+// curl one-liner install cannot fail on someone's registry, and so the compiled
+// single-file binary has nothing to resolve.
+
+// Deterministic, model-free tool ranking — shared by the extension's side panel
+// and the gateway, so "auto mode" narrows the same way everywhere (single source
+// of truth, per the no-duplication rule).
+//
+// Ranks tool specs by lexical relevance to a query, weighting each query word by
+// INVERSE DOCUMENT FREQUENCY across the toolset: a distinctive word like "wiki"
+// (in 1–2 tools) counts far more than a common one like "search" (in many) — so
+// "use wiki search" ranks the Wikipedia tool above generic search tools instead
+// of tying them. Latency-sensitive: pure string ops, runs on every turn, no model
+// call. Generic over the spec shape via name/description accessors (the extension
+// uses { name, description }; the gateway uses OpenAI's { function: { name, … } }).
+
+const STOP = new Set([
+  'the', 'and', 'for', 'with', 'that', 'this', 'use', 'can', 'you', 'your', 'please',
+  'about', 'from', 'what', 'who', 'how', 'are', 'was', 'will', 'just', 'tell', 'find',
+  'get', 'into', 'them', 'they', 'their', 'name', 'one', 'but', 'not', 'all',
+]);
+
+const defName = (s) => (s && s.name) || '';
+const defDesc = (s) => (s && s.description) || '';
+
+// Names of GENERAL entry-point tools — preferred when a query doesn't pin a specific
+// tool. Matches the tool segment (after the server prefix): e.g. ...__wikipedia_search,
+// ...__ask_pipeworx, ...__get_summary, ...__search_wikipedia.
+const GENERAL_TOOL_RE = /(?:^|_)(search|ask|lookup|find|answer|summary|wiki)(?:_|$)/i;
+
+// Returns specs scored + sorted most-relevant first, as [{ s, i, n }] (i = original
+// index, n = score). Stable for ties (preserves original order).
+export function scoreToolSpecs(specs, query, { name = defName, description = defDesc } = {}) {
+  const q = String(query || '').toLowerCase();
+  const words = [...new Set(q.split(/[^a-z0-9]+/).filter((w) => w.length > 2 && !STOP.has(w)))];
+  const list = [...(specs || [])];
+  const names = list.map((s) => String(name(s) || '').toLowerCase());
+  const hays = list.map((s, i) => `${names[i]} ${String(description(s) || '').toLowerCase()}`);
+  const N = list.length || 1;
+  const df = {}; // how many tools mention each query word
+  for (const w of words) df[w] = hays.reduce((n, h) => n + (h.includes(w) ? 1 : 0), 0);
+  const idf = (w) => Math.log(1 + N / (1 + (df[w] || 0))); // rarer → higher weight
+  const score = (i) => {
+    let n = 0;
+    for (const w of words) if (hays[i].includes(w)) n += idf(w);
+    for (const part of names[i].split(/[^a-z0-9]+/)) {
+      if (part.length > 2 && q.includes(part)) n += 2 + idf(part); // tool explicitly named
+    }
+    // General-purpose ENTRY-POINT tools (search / ask / lookup / get-summary / answer)
+    // are the right default when the query doesn't keyword-match a specific tool —
+    // e.g. "which state is Seattle in" → a wikipedia SEARCH/ASK tool, not one of 20
+    // dataset-query tools. A small tie-breaker boost (below a real keyword match) so
+    // those generic tools win when nothing else distinguishes them.
+    if (GENERAL_TOOL_RE.test(names[i])) n += 1.5;
+    return n;
+  };
+  return list.map((s, i) => ({ s, i, n: score(i) })).sort((a, b) => (b.n - a.n) || (a.i - b.i));
+}
+
+// Rank tool specs most-relevant first (stable for ties).
+export function rankToolSpecs(specs, query, accessors) {
+  return scoreToolSpecs(specs, query, accessors).map((x) => x.s);
+}
+
+// Narrow a flat spec list to at most `cap` entries that DON'T match `keep`, always
+// retaining everything that does (e.g. local page/history tools). `cap` therefore
+// bounds the NARROWABLE (MCP) tools; kept tools ride along free. Returns the list
+// unchanged when there's no cap or the narrowable set already fits.
+// The MCP server a tool belongs to: mcp_<server>__<tool> → "mcp_<server>". Tools
+// without that shape are their own "server" (never grouped together).
+function serverKey(n) {
+  const s = String(n || '');
+  const i = s.indexOf('__');
+  return i > 0 ? s.slice(0, i) : s;
+}
+
+export function narrowSpecs(specs, query, { cap = 0, keep, name = defName, description = defDesc } = {}) {
+  const list = specs || [];
+  if (!cap || cap < 1) return list;
+  const kept = keep ? list.filter(keep) : [];
+  const rest = keep ? list.filter((s) => !kept.includes(s)) : list;
+  if (rest.length <= cap) return list;
+  // SERVER-DIVERSE selection: rank all narrowable tools, then pick ROUND-ROBIN across
+  // servers — each server's best tool first, then seconds, … up to `cap`. This keeps
+  // a relevant server (e.g. wikipedia) from being crowded out of the top-K by another
+  // server that happens to have many tools. Servers are visited best-first (the order
+  // their top-ranked tool appears in the global ranking).
+  const ranked = rankToolSpecs(rest, query, { name, description });
+  const queues = new Map(); // serverKey -> [tools] in rank order (insertion = best-first)
+  for (const s of ranked) {
+    const k = serverKey(name(s));
+    if (!queues.has(k)) queues.set(k, []);
+    queues.get(k).push(s);
+  }
+  const lanes = [...queues.values()];
+  const chosen = new Set();
+  for (let round = 0; chosen.size < cap; round++) {
+    let advanced = false;
+    for (const lane of lanes) {
+      if (chosen.size >= cap) break;
+      if (lane.length > round) { chosen.add(lane[round]); advanced = true; }
+    }
+    if (!advanced) break;
+  }
+  return list.filter((s) => kept.includes(s) || chosen.has(s)); // preserve original order
+}

package/src/sanitize.js

@@ -1,137 +1,7 @@
-// VENDORED COPY of chatpanel-pii/sanitize.js - keep in sync with the canonical
-// engine (the bridge stays dependency-free, so this is a copy, not an import).
-// Regenerate by copying ../chatpanel-pii/sanitize.js over this file.
-
-// Unicode de-steganography for text that flows through the privacy boundary.
+// The Unicode sanitizer, from the one engine that owns it.
 //
-// Invisible and format-control characters are a single vector with three abuses,
-// all relevant to a redaction product:
-//
-//   1. Redaction bypass - splitting a value with zero-width chars (j<ZWSP>o<ZWSP>hn@x.com)
-//      hides it from the regex/NER detector, then the model reassembles the real
-//      value. The deterministic engine works on text, so the smuggled PII leaks.
-//   2. Hidden prompt injection - Unicode Tag characters (U+E0000-E007F) render as
-//      nothing but encode a full ASCII instruction the model reads ("ASCII smuggling").
-//   3. Fingerprinting / watermarking - steganographic markers injected into a prompt
-//      (e.g. a client classifying a custom gateway and encoding a bit into invisible
-//      punctuation). A privacy proxy should scrub these - and never emit its own.
-//
-// We strip the channels that have no legitimate place in plain prompt text, while
-// PRESERVING the few legitimate uses (emoji ZWJ/variation sequences, normal accents).
-//
-// The patterns are BUILT FROM NUMERIC CODE POINTS below - there are deliberately no
-// literal invisible characters anywhere in this source (auditable, and fitting for a
-// de-steg module). Pure + dependency-free ESM. Call it BEFORE detection so obfuscated
-// PII becomes matchable, and on model output before restoration so a token can't be
-// split/spoofed with invisibles.
-
-// Code-point ranges (inclusive) per category, by their abuse.
-const RANGES = {
-  // Unicode Tag block - the ASCII-smuggling channel.
-  tags: [[0xE0000, 0xE007F]],
-  // Bidi controls - reorder/override visible text to hide reversed instructions.
-  bidi: [[0x061C, 0x061C], [0x200E, 0x200F], [0x202A, 0x202E], [0x2066, 0x2069]],
-  // Zero-width & assorted invisible format chars: soft hyphen, Hangul/Mongolian
-  // fillers, ZWSP, word/invisible joiners, deprecated format controls, BOM/ZWNBSP,
-  // interlinear annotation, object replacement.
-  zeroWidth: [
-    [0x00AD, 0x00AD], [0x115F, 0x1160], [0x180E, 0x180E], [0x200B, 0x200B],
-    [0x2060, 0x2064], [0x206A, 0x206F], [0x3164, 0x3164], [0xFEFF, 0xFEFF],
-    [0xFFA0, 0xFFA0], [0xFFF9, 0xFFFB], [0xFFFC, 0xFFFC],
-  ],
-  // Supplementary variation selectors - the byte-smuggling range. Never legit in text.
-  supVS: [[0xE0100, 0xE01EF]],
-  // Line/paragraph separators - converted to '\n' (kill parser tricks, keep the break).
-  lineSep: [[0x2028, 0x2029]],
-  // ZWJ/ZWNJ + BMP variation selectors - legit ONLY next to an emoji base, so these
-  // are stripped contextually (see ANOMALOUS_JOIN_VS), not unconditionally.
-  joinVS: [[0x200C, 0x200D], [0xFE00, 0xFE0F]],
-};
-
-const u = (cp) => `\\u{${cp.toString(16).toUpperCase()}}`;
-const cls = (ranges) => ranges.map(([a, b]) => (a === b ? u(a) : `${u(a)}-${u(b)}`)).join('');
-
-const TAGS = new RegExp(`[${cls(RANGES.tags)}]`, 'gu');
-const BIDI = new RegExp(`[${cls(RANGES.bidi)}]`, 'gu');
-const ZERO_WIDTH = new RegExp(`[${cls(RANGES.zeroWidth)}]`, 'gu');
-const SUP_VS = new RegExp(`[${cls(RANGES.supVS)}]`, 'gu');
-const LINE_SEP = new RegExp(`[${cls(RANGES.lineSep)}]`, 'gu');
-// Strip ZWJ/ZWNJ/VS only when NOT preceded by an emoji base (so emoji sequences and
-// regional-indicator flags survive); supplementary VS are always stripped.
-const ANOMALOUS_JOIN_VS = new RegExp(
-  `(?<![\\p{Extended_Pictographic}${u(0x1F1E6)}-${u(0x1F1FF)}])[${cls(RANGES.joinVS)}]|[${cls(RANGES.supVS)}]`,
-  'gu',
-);
-// Runs of combining marks (Zalgo / bit-stuffing). A real stacked diacritic is 1-3
-// marks; anything past the cap is signalling, not language.
-const COMBINING_RUN = /\p{M}+/gu;
-
-// Cheap boolean for hot paths / UI ("does this contain anything hidden?"). Excludes
-// the context-dependent joinVS so legitimate emoji aren't flagged - sanitizeUnicode()
-// stays the source of truth for those.
-const ANY_HIDDEN = new RegExp(
-  `[${cls(RANGES.bidi)}${cls(RANGES.zeroWidth)}]|[${cls(RANGES.tags)}]|[${cls(RANGES.supVS)}]`,
-  'u',
-);
-
-export function hasHiddenChars(text) {
-  return typeof text === 'string' && ANY_HIDDEN.test(text);
-}
-
-// sanitizeUnicode(text, opts) -> { clean, removed, findings }
-//   clean    - text with the smuggling channels stripped/normalized
-//   removed  - total count of stripped/collapsed characters (0 = nothing hidden)
-//   findings - per-category counts (only non-zero keys), for transparent reporting
-//
-// opts.normalize: 'NFC' (default, appearance-preserving) | 'NFKC' (also folds
-//   fullwidth/homoglyph compatibility forms - stronger for detection, but rewrites
-//   some visible glyphs) | 'none'.
-// opts.collapseCombiningOver: max combining marks kept per run (default 4).
-export function sanitizeUnicode(text, { normalize = 'NFC', collapseCombiningOver = 4 } = {}) {
-  if (typeof text !== 'string' || text === '') return { clean: text ?? '', removed: 0, findings: {} };
-  let s = text;
-  const findings = {};
-
-  // Strip one category, counting by code point (spread iterates code points, so a
-  // supplementary char like a Tag counts as 1, not 2 UTF-16 units).
-  const strip = (re, key) => {
-    let n = 0;
-    s = s.replace(re, (m) => { n += [...m].length; return ''; });
-    if (n) findings[key] = n;
-  };
-
-  let lineSep = 0;
-  s = s.replace(LINE_SEP, () => { lineSep++; return '\n'; });
-  if (lineSep) findings.lineSep = lineSep;
-
-  strip(TAGS, 'tags');
-  strip(BIDI, 'bidi');
-  strip(ZERO_WIDTH, 'zeroWidth');
-  strip(ANOMALOUS_JOIN_VS, 'joinersVS');
-
-  // Compose canonically so split/decomposed forms can't dodge the detector.
-  if (normalize && normalize !== 'none') {
-    try { s = s.normalize(normalize); } catch { /* invalid form name -> skip */ }
-  }
-
-  let combining = 0;
-  s = s.replace(COMBINING_RUN, (run) => {
-    const marks = [...run];
-    if (marks.length <= collapseCombiningOver) return run;
-    combining += marks.length - collapseCombiningOver;
-    return marks.slice(0, collapseCombiningOver).join('');
-  });
-  if (combining) findings.combining = combining;
-
-  const removed = (findings.tags || 0) + (findings.bidi || 0) + (findings.zeroWidth || 0)
-    + (findings.joinersVS || 0) + (findings.combining || 0);
-  return { clean: s, removed, findings };
-}
-
-// Convenience for the common "just give me clean text" caller.
-export function stripHidden(text, opts) {
-  return sanitizeUnicode(text, opts).clean;
-}
-
-// Exposed for tests / external auditing.
-export const SANITIZE_RANGES = RANGES;
+// This file used to be a HAND copy of chatpanel-pii/sanitize.js, kept in step by memory — the
+// exact drift the vendoring scripts exist to prevent. The engine is now vendored properly under
+// src/pii/ (npm run sync:pii, with a --check drift guard), so this is a re-export and there is
+// only one copy in the repo to diverge from. Existing importers are unchanged.
+export { hasHiddenChars, sanitizeUnicode, stripHidden, SANITIZE_RANGES } from './pii/sanitize.js';