agent-sanitizer 2.63.0 → 2.63.1

package/src/confusables.mjs

@@ -11,9 +11,11 @@
  * non-ASCII glyphs — see isTokenBoundary). A token folds only when both hold:
  *
  *   1. Folding makes it pure ASCII — every non-ASCII code point in it is
- *      flagged. The bypass requires the folded token to come out byte-equal to
- *      an ASCII deny-rule target, so a token left holding an unmapped glyph
- *      could never match one anyway; skipping it costs no enforcement.
+ *      flagged AND an ASCII pretender (see isAsciiPretender: a letter of a real
+ *      Latin orthography, `İ` or `í`, is a word's own letter, not a disguise).
+ *      The bypass requires the folded token to come out byte-equal to an ASCII
+ *      deny-rule target, so a token left holding an unmapped glyph could never
+ *      match one anyway; skipping it costs no enforcement.
  *   2. It is more than a lone non-ASCII glyph standing between two boundaries.
  *      A one-code-point token is a one-letter foreign word — Russian "с", "о",
  *      "у", "а" are among the most frequent words in the language — as readily
@@ -58,6 +60,8 @@
  */
 import { createRequire } from "node:module";
 
+import { GENERAL_CATEGORY_MARK, inSpans } from "./unicode-properties.mjs";
+
 /**
  * Default path/command fields to fold per tool. Agent-agnostic: the keys are
  * the conventional Claude/Anthropic tool names, but a caller with a different
@@ -163,6 +167,34 @@ export function hasNonAscii(value) {
   return false;
 }
 
+// The letters of living Latin-script orthographies: Latin-1 Supplement (minus
+// × and ÷, which are symbols), Latin Extended-A/B and Latin Extended Additional
+// (Vietnamese). IPA, phonetic small capitals and fullwidth forms are Latin-script
+// too, but no orthography spells a word with them, so they stay look-alikes.
+const ORTHOGRAPHIC_LATIN =
+  /^[\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u024F\u1E00-\u1EFF]$/u;
+
+/**
+ * True when `glyph` holds a non-ASCII code point that is neither a combining
+ * mark nor a letter of a real Latin-script orthography. THE predicate both the
+ * host rule's WARNING tier and the input fold gate on, so they cannot disagree.
+ * Turkish `ı`, Spanish `í`, Polish `ł` are a word's own letters (`altın.com`,
+ * `día.txt`), and flagging them is the worse failure; a cross-script glyph
+ * (Cyrillic `а`), an IPA/small-capital one (`ɡ`, `ᴀ`) or anything NFKC folds
+ * to ASCII (`ａ`, `fi`) stays a pretender.
+ * @param {string} glyph one or more code points
+ * @returns {boolean}
+ */
+export function isAsciiPretender(glyph) {
+  for (const ch of glyph) {
+    const cp = /** @type {number} */ (ch.codePointAt(0));
+    if (!hasNonAscii(ch) || inSpans(GENERAL_CATEGORY_MARK, cp)) continue;
+    if (!ORTHOGRAPHIC_LATIN.test(ch) || !hasNonAscii(ch.normalize("NFKC")))
+      return true;
+  }
+  return false;
+}
+
 /**
  * Model-facing note naming the fields whose confusables were folded.
  * @param {string[]} normalized
@@ -308,12 +340,18 @@ function isTokenBoundary(ch) {
 export function selectFoldableFindings(text, findings) {
   for (const finding of findings)
     assertFinding(finding, bytesAt(text, finding));
+  // A same-script orthographic letter (`İ` in `İstanbul`, `í` in `día`) is not
+  // a disguise, so it counts as an unmapped glyph below: its token is a word,
+  // and none of it folds.
+  const pretenders = findings.filter((finding) =>
+    isAsciiPretender(finding.char),
+  );
 
   // Every UTF-16 offset a finding covers, not just its start: a scanner is free
   // to report a multi-code-point match, and treating only the first offset as
   // flagged would read the rest as unmapped and reject an otherwise ASCII token.
   const flagged = new Set();
-  for (const finding of findings)
+  for (const finding of pretenders)
     for (let i = 0; i < finding.char.length; i++)
       flagged.add(finding.index + i);
 
@@ -357,7 +395,7 @@ export function selectFoldableFindings(text, findings) {
   // units, which is what `foldableAt` is indexed by: an astral glyph occupies
   // two of them, so a code-point walk would leave the tail of a `char` that
   // carries one unchecked.
-  return findings.filter((finding) => {
+  return pretenders.filter((finding) => {
     for (let i = 0; i < finding.char.length; i++)
       if (foldableAt[finding.index + i] !== 1) return false;
     return true;

package/src/gates.mjs

@@ -7,28 +7,31 @@
  * `export … from "./html.mjs"` would eagerly evaluate that ~200ms module on
  * every root import, defeating the lazy-load design. This module imports
  * nothing, so re-exporting it is free.
+ *
+ * INVARIANT: every gate is a SUPERSET of its parser (a narrower one silently
+ * turns its layer off; a needless fire only costs a parse), checked by
+ * test/gates-superset.test.mjs.
  */
 
 // ─── Cheap pre-gates ─────────────────────────────────────────────────────────
 
 /**
- * Matches any HTML tag-like construct: opening tags, closing tags (`</`),
- * comments and bogus declarations (`<!`), and processing instructions / bogus
- * comments (`<?…?>`, which the HTML tokenizer hides exactly like a comment).
- * The `<!`/`<?` arms carry a comment-only document into the pipeline at all:
- * without them it would skip both Layer 2's splice of the comment and Layer 3's
- * exfil scan over the comment interior.
- * Gate for Layer 2 (HTML sanitization) and the HTML img/a exfil path in
- * Layer 3.
+ * Matches every point where the HTML tokenizer leaves the data state: `<` then
+ * an ASCII letter, `/`, `!` or `?`. No closing `>` on purpose: a comment holding
+ * `1 < 2`, or left unterminated, still hides everything after it. Gate for
+ * Layer 2 and the HTML img/a exfil path in Layer 3.
  */
-export const HTML_TAG_PRESENT = /<[a-zA-Z/!?][^<>]*>/;
+export const HTML_TAG_PRESENT = /<[a-zA-Z/!?]/;
 
 /**
  * Matches markdown link/image syntax (`](`, `![`) and reference link
- * definitions (`[label]: url` at line start). Gate for Layer 3 (markdown
- * exfiltration detection).
+ * definitions (`[label]: url` at a line start, also nested in a blockquote or
+ * list item). The label admits backslash escapes and line endings, as
+ * CommonMark labels do, but no unescaped `[`, which also keeps the scan linear.
+ * Gate for Layer 3 (markdown exfiltration detection).
  */
-export const MD_LINK_HINT = /\]\(|!\[|^[ \t]*\[[^[\]\n]+\]:\s/m;
+export const MD_LINK_HINT =
+  /\]\(|!\[|^[ \t]*(?:(?:>|[-+*]|\d{1,9}[.)])[ \t]*)*\[(?:[^\\[\]]|\\[\s\S])+\]:/m;
 
 /**
  * True when `text` is worth handing to the heavy remark/rehype graph at all:
@@ -45,12 +48,12 @@ export function needsMarkdownPipeline(text) {
 }
 
 /**
- * Matches an absolute http(s) URL anywhere in the text. Gate for the Layer-3
- * URL detectors, which read the URLs a GFM autolink literal yields and so find
- * a look-alike host or an exfil-shaped query in bare prose — no tag and no
- * link syntax required.
+ * Matches the start of every URL a GFM autolink literal yields (`http://`,
+ * `https://`, scheme-less `www.`). Gate for the Layer-3 URL detectors, which
+ * find a look-alike host or exfil-shaped query in bare prose. (A bare e-mail
+ * becomes a host-less `mailto:` URL that neither detector can flag.)
  */
-export const URL_PRESENT = /https?:\/\//i;
+export const URL_PRESENT = /https?:\/\/|www\./i;
 
 /**
  * True when Layer 3 (exfil + confusable-host detection) can find something.
@@ -65,6 +68,24 @@ export function needsUrlScan(text) {
   return URL_PRESENT.test(text) || needsMarkdownPipeline(text);
 }
 
+// A comment / bogus-comment open: hides everything after it, closed or not.
+const MARKUP_DECLARATION_OPEN = /<[!?]/;
+
+/**
+ * POLICY, not a gate: true when `text` reads as HTML rather than data that
+ * merely holds a `<` (a comment open, or a tag open with a `>` after it). The
+ * output hook uses it to decide whether an MCP result earns the Layer-2 splice,
+ * so `a<b` stays verbatim; kept apart from {@link HTML_TAG_PRESENT} so widening
+ * the gate never widens the policy. Linear, unlike a `<…[^>]*>` regex.
+ * @param {string} text
+ * @returns {boolean}
+ */
+export function isHtmlShaped(text) {
+  if (MARKUP_DECLARATION_OPEN.test(text)) return true;
+  const open = text.search(HTML_TAG_PRESENT);
+  return open !== -1 && text.includes(">", open + 2);
+}
+
 // ─── Secret-shape pre-gate (Layer 3 URL-param reuse) ─────────────────────────
 // Cheap shape match that decides whether a URL parameter value carries a
 // credential (Layer 3). This hand-duplicates credential-shape knowledge that
@@ -97,12 +118,23 @@ export const SECRET_HINT_EXT =
   /(?:AC|SK)[a-z0-9]{32}|SG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}|sq0csp-[0-9A-Za-z_-]{43}|(?<![0-9])[0-9]{8,10}:[0-9A-Za-z_-]{35}|(?<![0-9a-z])[0-9a-z]{32}-us[0-9]{1,2}|(?<![A-Za-z0-9_-])[MNO][A-Za-z0-9_-]{23,25}\.[A-Za-z0-9_-]{6}\.[A-Za-z0-9_-]{27}|T3BlbkFJ|pypi-AgE|(?<![A-Za-z0-9])AKC[A-Za-z0-9]{10}|(?<![A-Za-z0-9])AP[0-9A-Fa-f][A-Za-z0-9]{8}|:\/\/[^\s:/@]{1,64}:[^\s:/@]{1,64}@|(?:key|pw|pass)["']?[\s:=>]+["']?[A-Za-z0-9_/+-]{20}/i;
 
 /**
- * True when either pre-gate alternation shape-matches `text`. Split into two
- * literals (see SECRET_HINT) and OR'd so neither grows into a
- * polynomial-backtracking shape.
+ * Either alternation against `text` exactly as written.
  * @param {string} text
  * @returns {boolean}
  */
-export function matchesSecretHint(text) {
+function hintMatchesView(text) {
   return SECRET_HINT.test(text) || SECRET_HINT_EXT.test(text);
 }
+
+/**
+ * True when either pre-gate alternation shape-matches `text`, or the same text
+ * with every `\n` removed (the view the engine's `_redact_cross_line` scans, so
+ * this stays a superset of the redactor). Two literals OR'd so neither grows
+ * into a polynomial-backtracking shape (see SECRET_HINT).
+ * @param {string} text
+ * @returns {boolean}
+ */
+export function matchesSecretHint(text) {
+  if (hintMatchesView(text)) return true;
+  return text.includes("\n") && hintMatchesView(text.replace(/\n/g, ""));
+}

package/src/html.mjs

@@ -2451,7 +2451,9 @@ const VALUE_HAS_DIGIT_RE = /\d/;
 // signed-CDN token (imgix `?s=`, KeyCDN `?secure=`) are all a bare digest under
 // a generic name, and every one of them was reported before: hex is a subset of
 // base64's alphabet, so the 40-char base64 arm caught them on charset alone.
-const BLOB_VALUE_B64_RE = /^[A-Za-z0-9+/]{40,}={0,2}$/;
+// Spelled `X{40}X*`, not the equivalent `X{40,}`: irregexp runs `{40,}` as a
+// counted loop pushing a backtrack entry per character (4x slower on 64 KB).
+const BLOB_VALUE_B64_RE = /^[A-Za-z0-9+/]{40}[A-Za-z0-9+/]*={0,2}$/;
 const HEX_ONLY_RE = /^[A-Fa-f0-9]+$/;
 // 16 bytes, the floor the hex arm has always carried.
 const HEX_BLOB_MIN_LEN = 32;
@@ -2473,7 +2475,8 @@ const DIGEST_HEX_LENGTHS = new Set([32, 40, 56, 64, 96, 128]);
 // (`?d=<200-char base64url of cookies>`) routinely dodged it. The mix test keeps
 // the slug benign (no uppercase) while catching the scattered-separator blob the
 // run gate missed. Anchored to the whole value for the same RAW-query reason.
-const BLOB_VALUE_B64URL_RE = /^[A-Za-z0-9_-]{40,}={0,2}$/;
+// `X{40}X*` for the reason given at BLOB_VALUE_B64_RE.
+const BLOB_VALUE_B64URL_RE = /^[A-Za-z0-9_-]{40}[A-Za-z0-9_-]*={0,2}$/;
 
 /**
  * True when `value` carries at least one uppercase letter AND one digit — the
@@ -3149,41 +3152,54 @@ function collectUrls(text) {
  * how loudly (see the exfil tier in ./output.mjs).
  * @param {string} text
  * @param {{ flagDigestValues?: boolean }} [options] see {@link checkExfilUrl}
- * @returns {Array<{ isImage: boolean, autoFetched: boolean, reason: string, target: string }> | null}
+ * @returns {Array<{ isImage: boolean, autoFetched: boolean, reason: string, target: string, unscannable?: true }> | null}
  */
 export function detectExfil(text, options = {}) {
   if (!needsUrlScan(text)) return null;
+  return scanUrlsOrSentinel(
+    () => {
+      /** @type {Array<{ isImage: boolean, autoFetched: boolean, reason: string, target: string, unscannable?: true }>} */
+      const threats = [];
+      for (const { url, isImage, autoFetched, context } of collectUrls(text)) {
+        const reason =
+          checkExfilUrl(url, options) ||
+          (context !== "resource" && isOffOrigin(url)
+            ? OFF_ORIGIN_REASON[context]
+            : null);
+        if (!reason) continue;
+        threats.push({ isImage, autoFetched, reason, target: urlHost(url) });
+      }
+      return threats;
+    },
+    {
+      isImage: false,
+      // Fail closed on the severity tier too: an input the scanner could not
+      // read is not evidence that what it hides is harmless.
+      autoFetched: true,
+      reason: "input too deeply nested to scan for exfil URLs",
+      target: "(unparseable HTML)",
+      unscannable: true,
+    },
+  );
+}
 
-  /** @type {Array<{ isImage: boolean, autoFetched: boolean, reason: string, target: string }>} */
-  const threats = [];
-
+/**
+ * Run one URL scan; null when it finds nothing. A stack overflow (pathological
+ * nesting) fails CLOSED as the lone `unscannable` sentinel; any other throw is
+ * a scan bug and propagates. Shared so both detectors have one rethrow arm.
+ * @template T
+ * @param {() => T[]} scan
+ * @param {T} sentinel
+ * @returns {T[] | null}
+ */
+function scanUrlsOrSentinel(scan, sentinel) {
+  let threats;
   try {
-    for (const { url, isImage, autoFetched, context } of collectUrls(text)) {
-      const reason =
-        checkExfilUrl(url, options) ||
-        (context !== "resource" && isOffOrigin(url)
-          ? OFF_ORIGIN_REASON[context]
-          : null);
-      if (!reason) continue;
-      threats.push({ isImage, autoFetched, reason, target: urlHost(url) });
-    }
-  } catch {
-    // The parse/visit blew up (stack overflow on pathological nesting). Fail
-    // CLOSED so the never-throw contract holds: report one sentinel threat so
-    // the caller still warns rather than crashing, since an input too nested to
-    // scan could itself be hiding an exfil URL.
-    return [
-      {
-        isImage: false,
-        // Fail closed on the severity tier too: an input the scanner could not
-        // read is not evidence that what it hides is harmless.
-        autoFetched: true,
-        reason: "input too deeply nested to scan for exfil URLs",
-        target: "(unparseable HTML)",
-      },
-    ];
+    threats = scan();
+  } catch (err) {
+    if (!(err instanceof RangeError)) throw err;
+    return [sentinel];
   }
-
   return threats.length > 0 ? threats : null;
 }
 
@@ -3196,35 +3212,32 @@ export function detectExfil(text, options = {}) {
  *
  * Fails CLOSED on a parse blow-up for the same reason detectExfil does.
  * @param {string} text
- * @returns {Array<{ severity: string, description: string }> | null}
+ * @returns {Array<{ severity: string, description: string, unscannable?: true }> | null}
  */
 export function detectConfusableHosts(text) {
   if (!needsUrlScan(text)) return null;
-
-  /** @type {Array<{ severity: string, description: string }>} */
-  const threats = [];
-  /** @type {Set<string>} */
-  const seen = new Set();
-
-  try {
-    for (const { url } of collectUrls(text)) {
-      const found = confusableHost(url);
-      // One host repeated across a document is one deception, not N.
-      if (!found || seen.has(found.ascii)) continue;
-      seen.add(found.ascii);
-      threats.push({
-        severity: found.severity,
-        description: describeConfusableHost(found),
-      });
-    }
-  } catch {
-    return [
-      {
-        severity: SEVERITY.WARNING,
-        description: "input too deeply nested to scan for confusable hosts",
-      },
-    ];
-  }
-
-  return threats.length > 0 ? threats : null;
+  return scanUrlsOrSentinel(
+    () => {
+      /** @type {Array<{ severity: string, description: string, unscannable?: true }>} */
+      const threats = [];
+      /** @type {Set<string>} */
+      const seen = new Set();
+      for (const { url } of collectUrls(text)) {
+        const found = confusableHost(url);
+        // One host repeated across a document is one deception, not N.
+        if (!found || seen.has(found.ascii)) continue;
+        seen.add(found.ascii);
+        threats.push({
+          severity: found.severity,
+          description: describeConfusableHost(found),
+        });
+      }
+      return threats;
+    },
+    {
+      severity: SEVERITY.WARNING,
+      description: "input too deeply nested to scan for confusable hosts",
+      unscannable: true,
+    },
+  );
 }

package/src/index.mjs

@@ -63,6 +63,7 @@ export {
 // would eagerly load on every root import.
 export {
   HTML_TAG_PRESENT,
+  isHtmlShaped,
   MD_LINK_HINT,
   SECRET_HINT,
   SECRET_HINT_EXT,

package/src/invisible.mjs

@@ -103,6 +103,7 @@ export const CATEGORY = Object.freeze({
   HIDDEN_HTML: "hidden-html",
   EXFIL_URLS: "exfil-urls",
   CONFUSABLE_HOST: "confusable-host",
+  UNSCANNABLE: "unscannable",
 });
 
 // code -> human label, used only to build `warnings` text. Decoupled from
@@ -118,6 +119,7 @@ export const CATEGORY_LABELS = Object.freeze({
   [CATEGORY.HIDDEN_HTML]: "hidden HTML",
   [CATEGORY.EXFIL_URLS]: "exfil URLs",
   [CATEGORY.CONFUSABLE_HOST]: "confusable hosts",
+  [CATEGORY.UNSCANNABLE]: "input too deeply nested to scan for URLs",
 });
 
 /** @type {Array<[string, RegExp]>} Each entry pairs a CATEGORY code with its detector. */