agent-input-sanitizer 1.0.0

package/src/index.mjs

@@ -0,0 +1,221 @@
+/**
+ * Top-level convenience entry for agent-input-sanitizer.
+ *
+ * `sanitize` always runs the zero-dependency Layer 1 (invisible-char + ANSI
+ * stripping, lone-surrogate normalization) and, when `html` is requested,
+ * lazy-loads the heavier HTML layer (Layers 2 & 3) so the remark/rehype graph
+ * is only paid for by callers that ask for it.
+ *
+ * The low-level building blocks stay public via the `./invisible` and `./html`
+ * subpath entries; import those directly when you want a single layer without
+ * the convenience wrapper.
+ */
+import { stripInvisibleWithReport, LONG_RUN_RE } from "./invisible.mjs";
+
+export {
+  stripInvisible,
+  stripInvisibleWithReport,
+  isSgrOnly,
+  STRIP,
+  SGR_RE,
+  CHECKS,
+  VS,
+  BLANK_NON_CF,
+  LONG_RUN_RE,
+  LONG_RUN_THRESHOLD,
+  SCATTERED_THRESHOLD,
+} from "./invisible.mjs";
+
+// Layer 2/3 cheap pre-gates. Re-exported from the dependency-free `./gates.mjs`
+// (not `./html.mjs`) so consumers can share the exact HTML-tag/markdown-link
+// hints and secret-shape pre-gate without duplicating the regexes — and without
+// pulling in the heavy remark/rehype graph that a re-export from `./html.mjs`
+// would eagerly load on every root import.
+export {
+  HTML_TAG_PRESENT,
+  MD_LINK_HINT,
+  SECRET_HINT,
+  SECRET_HINT_EXT,
+  matchesSecretHint,
+} from "./gates.mjs";
+
+// The two raw control introducers an ANSI sequence can start with: 7-bit
+// ESC (U+001B) and the 8-bit C1 CSI (U+009B). Both are category Cc, so the
+// invisible-char pass (which targets Cf / variation / blank fillers) never
+// removes them; the residual sweep below is what guarantees neither survives.
+// eslint-disable-next-line no-control-regex -- matching the raw introducers is the point
+const CONTROL_INTRODUCER_RE = /[\u001b\u009b]/g;
+
+// Full ANSI escape grammar (CSI/SGR/OSC-with-BEL), not just SGR: the Layer-1
+// guarantee is that no control introducer survives, and a cursor-move or
+// erase sequence is as much a display-spoofing hazard as a color one. The
+// pattern is linear (every quantified run is bounded or non-overlapping), so it
+// carries no catastrophic-backtracking risk on adversarial input.
+// prettier-ignore
+// eslint-disable-next-line no-control-regex -- matching ESC-led sequences is the point
+const ANSI_RE = /[\u001b\u009b][[\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\d/#&.:=?%@~_]+)*|[a-zA-Z\d]+(?:;[-a-zA-Z\d/#&.:=?%@~_]*)*)?\u0007)|(?:(?:\d{1,4}(?:;\d{0,4})*)?[\dA-PR-TZcf-ntqry=><~]))/g;
+
+// Unpaired UTF-16 surrogates (high not followed by low, or low not preceded by
+// high). Normalized before the HTML parser, which throws on a stray byte —
+// which would otherwise let a single malformed code unit suppress all output.
+const LONE_SURROGATE_RE =
+  /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/g;
+
+/**
+ * Strip ANSI escape sequences to a fixed point. Removing one sequence can
+ * reconstitute another around it (a lone ESC left of `ESC[32m[0m` gains the
+ * trailing `[0m` once the inner sequence is removed, forming a brand-new valid
+ * sequence the single pass would miss), so iterate until stable: every changed
+ * pass consumes at least one ESC introducer, so the pass count is bounded by
+ * the input's ESC count, and ANSI-free text exits after one pass.
+ * @param {string} input
+ * @returns {string}
+ */
+function stripAnsiFully(input) {
+  let prev = input;
+  let out = prev.replace(ANSI_RE, "");
+  while (out !== prev) {
+    prev = out;
+    out = prev.replace(ANSI_RE, "");
+  }
+  return out;
+}
+
+/**
+ * Layer 1: ANSI + invisible-char strip with a result guaranteed free of every
+ * raw ANSI control introducer (7-bit ESC U+001B and 8-bit C1 CSI U+009B).
+ *
+ * Removing an invisible character can reconstitute an escape its split hid from
+ * the ANSI pass (`ESC`<ZWSP>`[32m` → `ESC[32m`), so strip ANSI again after the
+ * invisible pass — but only when stripInvisible changed something, since
+ * reconstitution is impossible otherwise and the re-strip is a wasted pass on
+ * the hot clean path. The ANSI strip still cannot match an *incomplete*
+ * reconstituted sequence (a lone `ESC[` left when an inner complete sequence is
+ * removed from a nested split), so a final sweep removes every residual raw
+ * introducer outright — that sweep, not the regex matching, is the guarantee
+ * that no control introducer survives. `deAnsi` is the ANSI strip of the
+ * original (invisible runs intact), the scope a LONG_RUN payload check needs.
+ * @param {string} text
+ * @returns {{ cleaned: string, deAnsi: string, found: string[] }}
+ */
+function applyLayer1(text) {
+  const deAnsi = stripAnsiFully(text);
+  // stripInvisibleWithReport returns `found` for exactly the categories it
+  // removed — so a ZWNJ/ZWJ the carve-out PRESERVES never registers as a strip,
+  // and the leading-BOM exception is already handled inside it.
+  const { cleaned: afterInvis, found } = stripInvisibleWithReport(deAnsi);
+  let ansiFound = deAnsi.length !== text.length;
+
+  let cleaned = afterInvis;
+  if (afterInvis !== deAnsi) {
+    const reStripped = stripAnsiFully(afterInvis);
+    if (reStripped.length !== afterInvis.length) ansiFound = true;
+    cleaned = reStripped;
+  }
+  const swept = cleaned.replace(CONTROL_INTRODUCER_RE, "");
+  if (swept !== cleaned) {
+    cleaned = swept;
+    ansiFound = true;
+  }
+
+  if (ansiFound) found.push("ANSI escapes");
+  return { cleaned, deAnsi, found };
+}
+
+/** @param {{ comments: number, hidden: number }} removed */
+function describeRemoved(removed) {
+  const parts = [];
+  if (removed.comments > 0) parts.push(`${removed.comments} HTML comment(s)`);
+  if (removed.hidden > 0) parts.push(`${removed.hidden} hidden element(s)`);
+  return parts.join(", ");
+}
+
+/** @param {{ tags: Record<string, number>, dataSrc: number }} warned */
+function describeWarned(warned) {
+  const parts = Object.entries(warned.tags).map(
+    ([tag, count]) => `${tag}×${count}`,
+  );
+  if (warned.dataSrc > 0) parts.push(`data: URI×${warned.dataSrc}`);
+  return parts.length > 0
+    ? `Preserved but reported (page source kept inspectable): ${parts.join(", ")}`
+    : "";
+}
+
+/**
+ * Sanitize untrusted text before any LLM sees it.
+ *
+ * Always runs Layer 1 (invisible-char + ANSI stripping, lone-surrogate
+ * normalization). When `html` is true, also lazy-loads the HTML layer to splice
+ * out human-invisible HTML (comments, hidden elements — Layer 2) and detect
+ * data-exfil-shaped URLs (Layer 3); the heavy remark/rehype dependency is only
+ * imported on that path. The exfil scan runs on the pre-splice text so a beacon
+ * URL hidden inside a `display:none` element is still reported, not buried by
+ * its own removal.
+ *
+ * `found` names the categories neutralized; `warnings` carries the
+ * operator-facing notices. `cleaned` is always a string, never throws, and
+ * changes only carry a warning (no silent suppression).
+ * @param {string} text
+ * @param {{ html?: boolean }} [options]
+ * @returns {Promise<{ cleaned: string, found: string[], warnings: string[] }>}
+ */
+export async function sanitize(text, { html = false } = {}) {
+  /** @type {string[]} */ const found = [];
+  /** @type {string[]} */ const warnings = [];
+
+  const { cleaned: layer1, deAnsi, found: invisFound } = applyLayer1(text);
+  let cleaned = layer1;
+  if (invisFound.length > 0) {
+    found.push(...invisFound);
+    let msg = `Stripped: ${invisFound.join(", ")}`;
+    LONG_RUN_RE.lastIndex = 0;
+    if (LONG_RUN_RE.test(deAnsi))
+      msg += " [LONG RUN — possible injection payload]";
+    warnings.push(msg);
+  }
+
+  const wellFormed = cleaned.replace(LONE_SURROGATE_RE, "\uFFFD");
+  if (wellFormed !== cleaned) {
+    cleaned = wellFormed;
+    found.push("Lone UTF-16 surrogates");
+    warnings.push("Normalized lone UTF-16 surrogates");
+  }
+
+  if (!html) return { cleaned, found, warnings };
+
+  const { sanitizeHtml, detectExfil } = await import("./html.mjs");
+  // Scan for exfil URLs on the text BEFORE Layer 2 splices anything out — a
+  // beacon URL hidden in a comment or hidden element is more suspicious, not
+  // less, yet Layer 2 would otherwise remove it from view before the scan.
+  const preSplice = cleaned;
+
+  const layer2 = sanitizeHtml(cleaned);
+  if (layer2) {
+    if (layer2.text !== cleaned) {
+      cleaned = layer2.text;
+      if (layer2.removed.comments > 0) found.push("HTML comments");
+      if (layer2.removed.hidden > 0) found.push("hidden HTML");
+      warnings.push(
+        `HTML sanitized: ${describeRemoved(layer2.removed)} replaced with placeholders`,
+      );
+    }
+    const preserved = describeWarned(layer2.warned);
+    if (preserved) warnings.push(preserved);
+  }
+
+  const threats = detectExfil(preSplice);
+  if (threats) {
+    found.push("exfil URLs");
+    const reasons = [
+      ...new Set(
+        threats.map(
+          (threat) =>
+            `${threat.isImage ? "image" : "link"} to ${threat.target}: ${threat.reason}`,
+        ),
+      ),
+    ];
+    warnings.push(`Exfil-shaped URLs detected: ${reasons.join("; ")}`);
+  }
+
+  return { cleaned, found, warnings };
+}

package/src/invisible.mjs

@@ -0,0 +1,248 @@
+/**
+ * Pure, zero-dependency invisible-character + ANSI/SGR primitives.
+ *
+ * Removes payload-capable Unicode (general-category Cf format chars, variation
+ * selectors, blank-rendering fillers, soft hyphens, interior BOMs) while
+ * preserving ZWNJ/ZWJ in genuine linguistic and emoji contexts, and reports
+ * which categories were removed.
+ */
+
+export const VS = [
+  ...Array.from({ length: 16 }, (_, i) => 0xfe00 + i),
+  ...Array.from({ length: 240 }, (_, i) => 0xe0100 + i),
+]
+  .map((codePoint) => String.fromCodePoint(codePoint))
+  .join("");
+
+// Code points that render blank / zero-width but are NOT general category Cf,
+// so the \p{Cf} check below misses them: the Hangul fillers (category Lo,
+// U+115F/U+1160/U+3164/U+FFA0) and the Braille blank pattern (category So,
+// U+2800). A run of these carries a hidden payload exactly as zero-widths do.
+export const BLANK_NON_CF = "\u115F\u1160\u3164\uFFA0\u2800";
+
+const REGEX_FLAGS = "gu";
+
+/** @type {Array<[string, RegExp]>} */
+export const CHECKS = [
+  ["Format chars (Cf)", new RegExp(`\\p{Cf}`, REGEX_FLAGS)],
+  ["Variation selectors", new RegExp(`[${VS}]`, REGEX_FLAGS)],
+  ["Blank-rendering fillers", new RegExp(`[${BLANK_NON_CF}]`, REGEX_FLAGS)],
+];
+
+export const STRIP = new RegExp(
+  CHECKS.map(([, regex]) => regex.source).join("|"),
+  REGEX_FLAGS,
+);
+
+// SGR (Select Graphic Rendition): ESC [ <digits/semicolons> m — colors, bold,
+// reset. The grammar is closed: params are [0-9;]* and the final byte is `m`,
+// so a match can only restyle text, never reposition the cursor, erase, or
+// smuggle an OSC string. Text is "SGR-only" when removing these leaves no ESC
+// byte at all — a lone or partial escape therefore is not SGR-only.
+// eslint-disable-next-line no-control-regex -- matching ESC-led sequences is the point
+export const SGR_RE = /\x1b\[[0-9;]*m/g;
+
+// eslint-disable-next-line no-control-regex -- ESC (U+001B) is exactly what we test for
+const ESC_RE = /\x1b/;
+
+/**
+ * True when every ESC byte in `text` belongs to a display-only SGR color
+ * sequence (so stripping the ANSI removed only cosmetic styling, nothing that
+ * could move the cursor, erase, or carry a payload).
+ * @param {string} text
+ * @returns {boolean}
+ */
+export function isSgrOnly(text) {
+  return !ESC_RE.test(text.replace(SGR_RE, ""));
+}
+
+export const LONG_RUN_THRESHOLD = 10;
+
+/** Total invisible-char count above which a file/prompt is treated as
+ * payload-capable even without a long run (threshold-evasion catch). */
+export const SCATTERED_THRESHOLD = 30;
+
+export const LONG_RUN_RE = new RegExp(
+  `(?:${STRIP.source}){${LONG_RUN_THRESHOLD},}`,
+  REGEX_FLAGS,
+);
+
+// Leading-BOM marker, preserved by stripInvisibleWithReport (see its doc).
+const BOM = "\uFEFF";
+// ─── ZWNJ/ZWJ linguistic carve-out ───────────────────────────────────────────
+// ZWNJ (U+200C) and ZWJ (U+200D) are general category Cf, so the STRIP pass
+// would treat them as hidden-payload bytes. But they are MANDATORY for correct
+// rendering between letters of several scripts (Arabic/Persian and many Indic
+// scripts) and inside emoji ZWJ sequences — blanket stripping corrupts
+// legitimate non-English output. Preserve them ONLY in an unambiguous
+// linguistic context (immediately between two letters of such a script, or
+// between two members of an emoji ZWJ sequence) and strip them as a payload
+// everywhere else: a long run, scattered past SCATTERED_THRESHOLD, a
+// leading/trailing position, or between Latin/ASCII/secret-shaped characters.
+// Over-strip beats under-strip — the carve-out fires only when BOTH neighbors
+// clearly belong to the context.
+const ZWNJ = 0x200c;
+const ZWJ = 0x200d;
+
+// Scripts whose orthography uses ZWNJ/ZWJ between letters as a rendering
+// control. Single source of truth: LINGUISTIC_LETTER is built from this list,
+// and the test suite drives one preserve-case per entry, so adding a script
+// here without a matching test fails.
+export const LINGUISTIC_SCRIPTS = [
+  "Arabic",
+  "Devanagari",
+  "Bengali",
+  "Gurmukhi",
+  "Gujarati",
+  "Oriya",
+  "Tamil",
+  "Telugu",
+  "Kannada",
+  "Malayalam",
+  "Sinhala",
+];
+const LINGUISTIC_LETTER = new RegExp(
+  `[${LINGUISTIC_SCRIPTS.map((script) => `\\p{Script=${script}}`).join("")}]`,
+  "u",
+);
+// Left side of an emoji joiner: a pictograph or a skin-tone modifier (a base
+// emoji may carry a modifier before the joiner, e.g. a health-worker sequence).
+const EMOJI_LEFT = /[\p{Extended_Pictographic}\p{Emoji_Modifier}]/u;
+// Right side of an emoji joiner is always the next component's base pictograph.
+const EMOJI_BASE = /\p{Extended_Pictographic}/u;
+
+// Non-global single-char classifiers (CHECKS carry `g`, whose lastIndex is
+// stateful across `.test`). carveStrip uses these to attribute each removed
+// char to its CHECKS category so `found` names exactly what was stripped.
+const CHECK_ONE = CHECKS.map(
+  ([label, re]) =>
+    /** @type {[string, RegExp]} */ ([label, new RegExp(re.source, "u")]),
+);
+
+/**
+ * The CHECKS category label a single code point belongs to, or null when it is
+ * not payload-capable (an ordinary visible character).
+ * @param {string} ch  one code point
+ * @returns {string | null}
+ */
+function classify(ch) {
+  for (const [label, re] of CHECK_ONE) if (re.test(ch)) return label;
+  return null;
+}
+
+/**
+ * True when `ch` (a ZWNJ/ZWJ) sits in an unambiguous linguistic context and so
+ * must be preserved rather than stripped. `prev`/`next` are the adjacent code
+ * points (single-code-point strings), or "" at a string boundary.
+ * @param {string} ch
+ * @param {string} prev
+ * @param {string} next
+ * @returns {boolean}
+ */
+function isPreservedJoiner(ch, prev, next) {
+  const cp = ch.codePointAt(0);
+  if (cp !== ZWNJ && cp !== ZWJ) return false;
+  // prev/next are "" at a string boundary (see carveStrip), so a leading or
+  // trailing joiner matches neither script nor emoji class and falls through.
+  if (LINGUISTIC_LETTER.test(prev) && LINGUISTIC_LETTER.test(next)) return true;
+  // Emoji ZWJ sequences use ZWJ only, never ZWNJ.
+  if (cp === ZWJ && EMOJI_LEFT.test(prev) && EMOJI_BASE.test(next)) return true;
+  return false;
+}
+
+/**
+ * Bulk strip (the common path: no ZWNJ/ZWJ present, so no carve-out can apply).
+ * A single regex pass removes every payload-capable char; `found` names the
+ * categories present via `.search` (which ignores the `g` lastIndex).
+ * @param {string} body
+ * @returns {{ cleaned: string, found: string[] }}
+ */
+function bulkStrip(body) {
+  const found = CHECKS.filter(([, re]) => body.search(re) !== -1).map(
+    ([label]) => label,
+  );
+  return { cleaned: body.replace(STRIP, ""), found };
+}
+
+/**
+ * Carve-out strip (a ZWNJ/ZWJ is present): walk code points, preserving a join
+ * control only where isPreservedJoiner holds AND the text stays under the
+ * scatter floor — otherwise it is stripped like any other payload byte. `found`
+ * reports only categories actually removed, so a preserved joiner never makes
+ * the caller claim a strip that did not happen.
+ * @param {string} body
+ * @returns {{ cleaned: string, found: string[] }}
+ */
+function carveStrip(body) {
+  // SCATTERED_THRESHOLD is the floor: past it, treat every invisible as payload
+  // regardless of context (threshold-evasion catch — over-strip beats under).
+  // Materialise codepoints once; count invisibles in a first pass so we know
+  // whether the carve-out applies before building the output string.
+  const cps = Array.from(body);
+  let invisCount = 0;
+  for (const ch of cps) if (classify(ch) !== null) invisCount++;
+  const allowCarveOut = invisCount < SCATTERED_THRESHOLD;
+  const foundLabels = new Set();
+  let out = "";
+  for (let i = 0; i < cps.length; i++) {
+    const ch = cps[i];
+    const label = classify(ch);
+    if (label === null) {
+      out += ch; // ordinary visible character
+      continue;
+    }
+    if (
+      allowCarveOut &&
+      isPreservedJoiner(ch, cps[i - 1] ?? "", cps[i + 1] ?? "")
+    ) {
+      out += ch;
+      continue;
+    }
+    foundLabels.add(label);
+  }
+  const found = CHECKS.filter(([label]) => foundLabels.has(label)).map(
+    ([label]) => label,
+  );
+  return { cleaned: out, found };
+}
+
+/**
+ * True when `body` holds at least one ZWNJ/ZWJ (so the carve-out may apply).
+ * @param {string} body
+ * @returns {boolean}
+ */
+function hasJoinControl(body) {
+  return (
+    body.includes(String.fromCodePoint(ZWNJ)) ||
+    body.includes(String.fromCodePoint(ZWJ))
+  );
+}
+
+/**
+ * Strip payload-capable invisible chars and report which categories were
+ * removed. A single leading U+FEFF (BOM) is preserved as a legitimate marker;
+ * interior BOMs and all soft hyphens (U+00AD) are stripped, since either can
+ * encode hidden instructions. ZWNJ/ZWJ survive only in a linguistic context
+ * (see the carve-out above). `found` names exactly the categories stripped, so
+ * a caller never warns about a strip the carve-out skipped.
+ * @param {string} text
+ * @returns {{ cleaned: string, found: string[] }}
+ */
+export function stripInvisibleWithReport(text) {
+  const hasLeadingBom = text.charCodeAt(0) === 0xfeff;
+  const body = hasLeadingBom ? text.slice(1) : text;
+  const { cleaned, found } = hasJoinControl(body)
+    ? carveStrip(body)
+    : bulkStrip(body);
+  return { cleaned: hasLeadingBom ? BOM + cleaned : cleaned, found };
+}
+
+/**
+ * Strip payload-capable invisible chars (cleaned text only). See
+ * stripInvisibleWithReport for the BOM and ZWNJ/ZWJ carve-out semantics.
+ * @param {string} text
+ * @returns {string}
+ */
+export function stripInvisible(text) {
+  return stripInvisibleWithReport(text).cleaned;
+}