agent-sanitizer 2.19.6 → 2.19.8

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-sanitizer",
-  "version": "2.19.6",
+  "version": "2.19.8",
   "description": "Defend an agent against hidden-content injection: strip payload-capable invisible Unicode and ANSI, splice out human-invisible HTML, and flag data-exfil URLs in untrusted text before any model sees it.",
   "type": "module",
   "repository": {

package/src/gates.mjs

@@ -28,6 +28,20 @@ export const HTML_TAG_PRESENT = /<[a-zA-Z/!?][^<>]*>/;
  */
 export const MD_LINK_HINT = /\]\(|!\[|^[ \t]*\[[^[\]\n]+\]:\s/m;
 
+/**
+ * True when `text` is worth handing to the heavy remark/rehype graph at all:
+ * Layers 2 and 3 can only find something in text that carries an HTML tag or a
+ * markdown link. THE pre-gate for both entry points that run those layers
+ * (`sanitize()` in ./index.mjs, `sanitizeText()` in ./output.mjs) — it lives
+ * here, next to the two regexes it composes, so the two cannot gate on
+ * different conditions and pay (or skip) the ~200ms import for different inputs.
+ * @param {string} text
+ * @returns {boolean}
+ */
+export function needsMarkdownPipeline(text) {
+  return HTML_TAG_PRESENT.test(text) || MD_LINK_HINT.test(text);
+}
+
 // ─── Secret-shape pre-gate (Layer 3 URL-param reuse) ─────────────────────────
 // Cheap shape match that decides whether a URL parameter value carries a
 // credential (Layer 3). This hand-duplicates credential-shape knowledge that

package/src/index.mjs

@@ -11,7 +11,14 @@
  * the convenience wrapper.
  */
 import { CATEGORY, describeStripped } from "./invisible.mjs";
+import { needsMarkdownPipeline } from "./gates.mjs";
 import { applyLayer1, LONE_SURROGATE_RE } from "./layer1.mjs";
+import {
+  describeExfil,
+  describeHtmlSanitized,
+  describeWarned,
+  LONE_SURROGATE_WARNING,
+} from "./warnings.mjs";
 
 // Layer 1 lives in the zero-dependency `./layer1.mjs`, shared verbatim with the
 // tool-output pipeline (`./output`) and the Edit-repair rehydrator
@@ -48,25 +55,6 @@ export {
   matchesSecretHint,
 } from "./gates.mjs";
 
-/** @param {{ comments: number, hidden: number }} removed */
-function describeRemoved(removed) {
-  const parts = [];
-  if (removed.comments > 0) parts.push(`${removed.comments} HTML comment(s)`);
-  if (removed.hidden > 0) parts.push(`${removed.hidden} hidden element(s)`);
-  return parts.join(", ");
-}
-
-/** @param {{ tags: Record<string, number>, dataSrc: number }} warned */
-function describeWarned(warned) {
-  const parts = Object.entries(warned.tags).map(
-    ([tag, count]) => `${tag}×${count}`,
-  );
-  if (warned.dataSrc > 0) parts.push(`data: URI×${warned.dataSrc}`);
-  return parts.length > 0
-    ? `Preserved but reported (page source kept inspectable): ${parts.join(", ")}`
-    : "";
-}
-
 /**
  * Sanitize untrusted text before any LLM sees it.
  *
@@ -108,10 +96,16 @@ export async function sanitize(text, options) {
   if (wellFormed !== cleaned) {
     cleaned = wellFormed;
     found.push(CATEGORY.LONE_SURROGATES);
-    warnings.push("Normalized lone UTF-16 surrogates");
+    warnings.push(LONE_SURROGATE_WARNING);
   }
 
-  if (!html) return { cleaned, found, warnings };
+  // Layers 2 and 3 can only find something in text carrying an HTML tag or a
+  // markdown link, so the shared pre-gate decides whether the heavy
+  // remark/rehype graph is imported at all — the same gate `sanitizeText()`
+  // applies, so both entry points pay for (and skip) the import on exactly the
+  // same inputs.
+  if (!html || !needsMarkdownPipeline(cleaned))
+    return { cleaned, found, warnings };
 
   let sanitizeHtml, detectExfil;
   /* c8 ignore start -- a rejected dynamic import of a module that ships in
@@ -140,9 +134,7 @@ export async function sanitize(text, options) {
       cleaned = layer2.text;
       if (layer2.removed.comments > 0) found.push(CATEGORY.HTML_COMMENTS);
       if (layer2.removed.hidden > 0) found.push(CATEGORY.HIDDEN_HTML);
-      warnings.push(
-        `HTML sanitized: ${describeRemoved(layer2.removed)} replaced with placeholders`,
-      );
+      warnings.push(describeHtmlSanitized(layer2.removed));
     }
     const preserved = describeWarned(layer2.warned);
     if (preserved) warnings.push(preserved);
@@ -151,15 +143,7 @@ export async function sanitize(text, options) {
   const threats = detectExfil(preSplice);
   if (threats) {
     found.push(CATEGORY.EXFIL_URLS);
-    const reasons = [
-      ...new Set(
-        threats.map(
-          (threat) =>
-            `${threat.isImage ? "image" : "link"} to ${threat.target}: ${threat.reason}`,
-        ),
-      ),
-    ];
-    warnings.push(`Exfil-shaped URLs detected: ${reasons.join("; ")}`);
+    warnings.push(describeExfil(threats));
   }
 
   return { cleaned, found, warnings };

package/src/invisible.mjs

@@ -9,7 +9,7 @@
  * sibling data module, not a package), so it decides preservation from the
  * actual cursive-join semantics rather than a hand-rolled script guess.
  */
-import { joiningType, isVirama } from "./joining-type.mjs";
+import { joiningType, isVirama, isBrahmicConsonant } from "./joining-type.mjs";
 import { isStandardizedVariant } from "./standardized-variants.mjs";
 import { CF_CODEPOINTS } from "./cf-charset.mjs";
 import { scanAnsi, TOKEN_KIND } from "./ansi.mjs";
@@ -364,52 +364,32 @@ function isCjkIdeograph(ch) {
   return CJK_IDEOGRAPH_RE.test(ch);
 }
 
-// Consonant (KA..HA and script-specific additional-consonant) ranges of the
-// Brahmic scripts the joiner carve-out serves. A virama does half-form/conjunct
-// work ONLY on a consonant base; a bare or base-less halant + ZWJ carries no
-// rendering and is a smuggling channel, so the Indic joiner is preserved only
-// when its virama sits on one of these. Broad per-block spans — precision here
-// only needs "a real Brahmic letter of this script", not an exact consonant set.
+// Brahmic consonants: the only base a virama does half-form/conjunct work on.
+// A bare or base-less halant + ZWJ carries no rendering and is a smuggling
+// channel, so the Indic joiner is preserved only over one of these.
 //
-// UNLIKE the CJK and Hangul gates these CANNOT be derived from a property
-// escape: the UCD property that names them is Indic_Syllabic_Category=Consonant,
-// and JS RegExp exposes only General_Category, Script, Script_Extensions and the
-// binary properties. \p{Script=Devanagari} is the wrong shape — it also holds the
-// independent vowels (U+0904–U+0914), which a virama never attaches to, so
-// switching to it would preserve joiners after a bare vowel + halant. The table
-// therefore stays literal, and
-// test/invisible-unicode-tables.test.mjs pins the strongest contract that IS
-// derivable: every assigned code point in each span is a letter of the script
-// the span names, and each span holds at least one such letter. Drift (a future Unicode
-// filling a hole with a non-letter, or a typo'd span crossing into a neighbouring
-// script's block) then fails CI instead of shipping.
-// Keyed by script name so the contract test can check each span against the
-// script it claims; exported for exactly that test.
-/** @type {ReadonlyArray<readonly [string, number, number]>} */
-export const BRAHMIC_CONSONANT_RANGES = [
-  ["Devanagari", 0x0915, 0x0939], // KA–HA
-  ["Devanagari", 0x0958, 0x095f], // additional consonants
-  ["Bengali", 0x0995, 0x09b9],
-  ["Bengali", 0x09dc, 0x09df], // additional consonants
-  ["Gurmukhi", 0x0a15, 0x0a39],
-  ["Gurmukhi", 0x0a59, 0x0a5e], // additional consonants
-  ["Gujarati", 0x0a95, 0x0ab9],
-  ["Oriya", 0x0b15, 0x0b39],
-  ["Oriya", 0x0b5c, 0x0b5f], // additional consonants
-  ["Tamil", 0x0b95, 0x0bb9],
-  ["Telugu", 0x0c15, 0x0c39],
-  ["Telugu", 0x0c58, 0x0c5a], // additional consonants
-  ["Kannada", 0x0c95, 0x0cb9],
-  ["Malayalam", 0x0d15, 0x0d3a],
-  ["Sinhala", 0x0d9a, 0x0dc6],
-];
-
-/** True when `cp` is a Brahmic consonant — the only base a virama attaches to.
- * @param {number} cp @returns {boolean} */
-function isBrahmicConsonant(cp) {
-  for (const [, start, end] of BRAHMIC_CONSONANT_RANGES)
-    if (cp >= start && cp <= end) return true;
-  return false;
+// The spans are GENERATED from the UCD (Indic_Syllabic_Category=Consonant,
+// restricted by Script) and live in ./joining-type.mjs alongside the virama
+// table they are read against — see scripts/gen-joining-type.mjs. They used to
+// be hand-typed per-block KA–HA approximations here, which swept up the holes
+// between the real consonants; ECMAScript exposes no
+// \p{Indic_Syllabic_Category=…} escape, and \p{Script=Devanagari} is the wrong
+// shape on its own (it also holds the independent vowels U+0904–U+0914, which a
+// virama never attaches to), so a generated table is the only drift-proof
+// answer. Re-exported here because it was part of this module's surface before
+// it moved, and because test/invisible-unicode-tables.test.mjs checks each span
+// against the script it claims.
+export { BRAHMIC_CONSONANT_RANGES } from "./joining-type.mjs";
+
+/** True when `ch` is a Brahmic consonant. Takes a CHARACTER, like its sibling
+ * predicates here (`isCjkIdeograph`, `isJoinControl`), over the code-point
+ * `isBrahmicConsonant` it wraps in ./joining-type.mjs, where every predicate
+ * takes a code point.
+ * @param {string} ch @returns {boolean} */
+function isBrahmicConsonantChar(ch) {
+  return (
+    ch !== "" && isBrahmicConsonant(/** @type {number} */ (ch.codePointAt(0)))
+  );
 }
 
 // ─── Blank-filler carve-out (Braille / archaic Hangul) ───────────────────────
@@ -532,11 +512,7 @@ function followsBrahmicConjunct(cps, i) {
   while (j >= 0 && !isJoinControl(cps[j]) && classify(cps[j]) !== null) j--;
   if (j < 0 || !isVirama(/** @type {number} */ (cps[j].codePointAt(0))))
     return false;
-  const base = effectiveNeighbor(cps, j, -1);
-  return (
-    base !== "" &&
-    isBrahmicConsonant(/** @type {number} */ (base.codePointAt(0)))
-  );
+  return isBrahmicConsonantChar(effectiveNeighbor(cps, j, -1));
 }
 
 /**

package/src/joining-type.mjs

@@ -1,8 +1,8 @@
 /**
  * GENERATED by scripts/gen-joining-type.mjs from ucd-full@17.0.0 — DO NOT EDIT.
  *
- * Unicode Joining_Type and Indic virama range tables backing the ZWNJ/ZWJ
- * carve-out in invisible.mjs. Regenerate with `pnpm gen:joining-type`;
+ * Unicode Joining_Type, Indic virama and Brahmic consonant range tables backing
+ * the ZWNJ/ZWJ carve-out in invisible.mjs. Regenerate with `pnpm gen:joining-type`;
  * test/joining-type.test.mjs fails if this drifts from the pinned UCD.
  */
 
@@ -573,6 +573,62 @@ const VIRAMA_RANGES = [
   [0x119e0, 0x119e0],
   [0x11c3f, 0x11c3f],
 ];
+
+// [start, end, Script] Indic_Syllabic_Category = Consonant ranges restricted to
+// the Brahmic scripts the joiner carve-out covers — the only bases a virama
+// attaches to. Keyed by script name so the contract test can check each span
+// against the script it claims.
+/** @type {ReadonlyArray<readonly [string, number, number]>} */
+export const BRAHMIC_CONSONANT_RANGES = [
+  ["Devanagari", 0x0915, 0x0939],
+  ["Devanagari", 0x0958, 0x095f],
+  ["Devanagari", 0x0978, 0x097f],
+  ["Bengali", 0x0995, 0x09a8],
+  ["Bengali", 0x09aa, 0x09b0],
+  ["Bengali", 0x09b2, 0x09b2],
+  ["Bengali", 0x09b6, 0x09b9],
+  ["Bengali", 0x09dc, 0x09dd],
+  ["Bengali", 0x09df, 0x09df],
+  ["Bengali", 0x09f0, 0x09f1],
+  ["Gurmukhi", 0x0a15, 0x0a28],
+  ["Gurmukhi", 0x0a2a, 0x0a30],
+  ["Gurmukhi", 0x0a32, 0x0a33],
+  ["Gurmukhi", 0x0a35, 0x0a36],
+  ["Gurmukhi", 0x0a38, 0x0a39],
+  ["Gurmukhi", 0x0a59, 0x0a5c],
+  ["Gurmukhi", 0x0a5e, 0x0a5e],
+  ["Gujarati", 0x0a95, 0x0aa8],
+  ["Gujarati", 0x0aaa, 0x0ab0],
+  ["Gujarati", 0x0ab2, 0x0ab3],
+  ["Gujarati", 0x0ab5, 0x0ab9],
+  ["Gujarati", 0x0af9, 0x0af9],
+  ["Oriya", 0x0b15, 0x0b28],
+  ["Oriya", 0x0b2a, 0x0b30],
+  ["Oriya", 0x0b32, 0x0b33],
+  ["Oriya", 0x0b35, 0x0b39],
+  ["Oriya", 0x0b5c, 0x0b5d],
+  ["Oriya", 0x0b5f, 0x0b5f],
+  ["Oriya", 0x0b71, 0x0b71],
+  ["Tamil", 0x0b95, 0x0b95],
+  ["Tamil", 0x0b99, 0x0b9a],
+  ["Tamil", 0x0b9c, 0x0b9c],
+  ["Tamil", 0x0b9e, 0x0b9f],
+  ["Tamil", 0x0ba3, 0x0ba4],
+  ["Tamil", 0x0ba8, 0x0baa],
+  ["Tamil", 0x0bae, 0x0bb9],
+  ["Telugu", 0x0c15, 0x0c28],
+  ["Telugu", 0x0c2a, 0x0c39],
+  ["Telugu", 0x0c58, 0x0c5a],
+  ["Kannada", 0x0c95, 0x0ca8],
+  ["Kannada", 0x0caa, 0x0cb3],
+  ["Kannada", 0x0cb5, 0x0cb9],
+  ["Kannada", 0x0cde, 0x0cde],
+  ["Malayalam", 0x0d15, 0x0d3a],
+  ["Sinhala", 0x0d9a, 0x0db1],
+  ["Sinhala", 0x0db3, 0x0dbb],
+  ["Sinhala", 0x0dbd, 0x0dbd],
+  ["Sinhala", 0x0dc0, 0x0dc6],
+];
 // Stryker restore all
 
 /**
@@ -614,3 +670,16 @@ export function joiningType(cp) {
 export function isVirama(cp) {
   return /** @type {boolean} */ (lookup(VIRAMA_RANGES, cp, false));
 }
+
+/**
+ * True when `cp` is a Brahmic consonant — the only base a virama attaches to,
+ * and therefore the only base after which a ZWJ/ZWNJ is a real conjunct
+ * request rather than a zero-width payload.
+ * @param {number} cp
+ * @returns {boolean}
+ */
+export function isBrahmicConsonant(cp) {
+  for (const [, start, end] of BRAHMIC_CONSONANT_RANGES)
+    if (cp >= start && cp <= end) return true;
+  return false;
+}