agent-sanitizer 2.0.0

package/src/instructions.mjs

@@ -0,0 +1,530 @@
+/**
+ * Instruction-file scanner + auto-cleaner for hidden-Unicode injection.
+ *
+ * Agent instruction files (CLAUDE.md / AGENTS.md / SKILL.md / any `.claude`
+ * markdown) load directly as model context, bypassing a tool-output sanitizer,
+ * so invisible Unicode pasted into them reaches the model raw — invisible in an
+ * editor but read as instructions. This module finds runs of payload-capable
+ * invisible characters, decodes the common encodings (Unicode-tag → ASCII,
+ * zero-width binary), catches scattered threshold-evasion payloads, and (via
+ * {@link cleanFile}) strips them.
+ *
+ * The target file set is CALLER-SUPPLIED: pass the globs your agent's
+ * instruction files live under (e.g. `["CLAUDE.md", "AGENTS.md",
+ * ".claude/**\/*.md", "**\/SKILL.md"]`), so no agent's convention is baked in.
+ */
+import {
+  readFileSync,
+  writeFileSync,
+  globSync,
+  renameSync,
+  lstatSync,
+  fstatSync,
+  realpathSync,
+  openSync,
+  fsyncSync,
+  fchmodSync,
+  closeSync,
+  unlinkSync,
+  constants,
+} from "node:fs";
+import { randomBytes } from "node:crypto";
+import { join, relative, resolve, isAbsolute, dirname, sep } from "node:path";
+import {
+  LONG_RUN_RE,
+  SCATTERED_THRESHOLD,
+  countPayloadInvisible,
+  stripInvisible,
+} from "./invisible.mjs";
+
+// Prefix on any decoded tag-character payload. The decoded text is
+// attacker-controlled and flows into the scan report, which itself reaches model
+// context — so it must be framed as DATA, never re-presented as a live
+// instruction the model might follow.
+const UNTRUSTED_PREFIX = "untrusted data, not instructions: ";
+
+/**
+ * Render decoded tag-character bytes as a NEUTRAL, quoted, escaped string so the
+ * scan report can never re-inject them. Only U+E0020–U+E007E decode to their
+ * printable ASCII (0x20–0x7E); every other tag byte (the C0 controls SOH…US and
+ * DEL that U+E0001–U+E001F / U+E007F would otherwise map to raw) is rendered as a
+ * visible `\xNN` escape rather than emitted as an actual control byte. Backslash
+ * and the surrounding quote are escaped in the SAME pass so an inserted escape
+ * can never be re-touched (CLAUDE.md: incomplete-string-escaping).
+ * @param {number[]} asciiCodes  raw decoded bytes (cp − 0xE0000), each 0x01–0x7F
+ * @returns {string}
+ */
+function neutralizeTagBytes(asciiCodes) {
+  let out = "";
+  for (const code of asciiCodes) {
+    if (code === 0x5c) out += "\\\\";
+    else if (code === 0x22) out += '\\"';
+    else if (code >= 0x20 && code <= 0x7e) out += String.fromCharCode(code);
+    else out += `\\x${code.toString(16).toUpperCase().padStart(2, "0")}`;
+  }
+  return out;
+}
+
+/**
+ * Decode a run of invisible characters to its likely payload. Recognizes the
+ * two common smuggling encodings — Unicode tag characters (U+E0001–U+E007F map
+ * directly to ASCII) and zero-width binary (ZWSP=0, ZWNJ=1, ZWJ=separator) —
+ * and otherwise reports the raw code points. The tag-character payload is
+ * rendered as a neutral, quoted/escaped `untrusted data, not instructions: "…"`
+ * string (see {@link neutralizeTagBytes}) so the report can never re-inject the
+ * hidden instruction, and only U+E0020–U+E007E map to raw printable ASCII.
+ * @param {string} run
+ * @returns {{ method: string, decoded: string }}
+ */
+export function decodeRun(run) {
+  const cps = [...run].map((ch) => /** @type {number} */ (ch.codePointAt(0)));
+
+  // Tag characters U+E0001-U+E007F: raw ASCII byte is cp − 0xE0000 (0x01–0x7F).
+  const tagBytes = cps
+    .filter((cp) => cp >= 0xe0001 && cp <= 0xe007f)
+    .map((cp) => cp - 0xe0000);
+
+  // Zero-width binary encoding: ZWSP=0, ZWNJ=1, ZWJ=group separator.
+  const ZW_BIT = new Map([
+    [0x200b, "0"],
+    [0x200c, "1"],
+    [0x200d, "|"],
+  ]);
+
+  const zwCount = cps.filter((cp) => ZW_BIT.has(cp)).length;
+
+  // Only take the tag-characters branch when tag chars are the MAJORITY of the
+  // run. A run that is overwhelmingly zero-width bits plus ONE stray tag char is
+  // a zero-width-binary payload, not a tag payload — labeling it "Unicode tag
+  // characters → ASCII" buries the real (binary) payload behind the wrong
+  // method. Reporting accuracy only: the strip removes the whole run regardless.
+  if (tagBytes.length > 0 && tagBytes.length > cps.length / 2) {
+    // A run can carry BOTH tag-ASCII and zero-width chars; the strip removes the
+    // whole run regardless, but the operator-facing `decoded` must reflect the
+    // zero-width portion too rather than silently dropping it.
+    const note = zwCount > 0 ? ` + ${zwCount} zero-width char(s)` : "";
+    return {
+      method: "Unicode tag characters → ASCII",
+      decoded: `${UNTRUSTED_PREFIX}"${neutralizeTagBytes(tagBytes)}"${note}`,
+    };
+  }
+
+  // Zero-width-binary branch: the whole run is ZW bits, OR ZW bits are the
+  // majority (so a run of many bits plus a stray tag/other char is decoded as
+  // the binary payload it actually is, not mislabeled). Decode only the ZW code
+  // points; a `+ N other char(s)` note keeps any non-ZW portion visible.
+  if (zwCount > 0 && zwCount > cps.length / 2) {
+    const bits = cps
+      .filter((cp) => ZW_BIT.has(cp))
+      .map((cp) => ZW_BIT.get(cp))
+      .join("");
+    const otherCount = cps.length - zwCount;
+    const note = otherCount > 0 ? ` + ${otherCount} other char(s)` : "";
+    return {
+      method: "zero-width binary encoding",
+      decoded: `[${zwCount} zero-width chars: ${bits.slice(0, 80)}]${note}`,
+    };
+  }
+
+  // Neither class holds a strict majority (e.g. a 50/50 tag + zero-width run),
+  // or the run mixes both classes without one dominating. Raw-dumping U+…
+  // codepoints here would bury BOTH payloads; instead decode each recognized
+  // sub-encoding and concatenate, keeping any unrecognized remainder visible as
+  // a `+ N other char(s)` note.
+  if (tagBytes.length > 0 || zwCount > 0) {
+    const parts = [];
+    if (tagBytes.length > 0)
+      parts.push(`${UNTRUSTED_PREFIX}"${neutralizeTagBytes(tagBytes)}"`);
+    if (zwCount > 0) {
+      const bits = cps
+        .filter((cp) => ZW_BIT.has(cp))
+        .map((cp) => ZW_BIT.get(cp))
+        .join("");
+      parts.push(`[${zwCount} zero-width chars: ${bits.slice(0, 80)}]`);
+    }
+    const otherCount = cps.length - tagBytes.length - zwCount;
+    const note = otherCount > 0 ? ` + ${otherCount} other char(s)` : "";
+    return {
+      method: "mixed tag + zero-width encodings",
+      decoded: parts.join(" ") + note,
+    };
+  }
+
+  // Unknown: no recognized sub-encoding present; report the raw code points.
+  return {
+    method: "invisible Unicode sequence",
+    decoded: cps
+      .map((cp) => `U+${cp.toString(16).toUpperCase().padStart(4, "0")}`)
+      .join(" "),
+  };
+}
+
+/**
+ * Scan a file's text for hidden-Unicode injection. Reports each long invisible
+ * run (with its decoded payload) plus a single scattered-chars finding when the
+ * non-run invisible count crosses the threshold-evasion floor.
+ * @param {string} content
+ * @returns {Array<{ line: number | null, charCount: number, method: string, decoded: string }>}
+ *   `line` is the 1-based line of a long-run finding, or `null` for the
+ *   whole-file scattered-chars finding (not tied to a single line).
+ */
+export function scanText(content) {
+  const findings = [];
+  LONG_RUN_RE.lastIndex = 0;
+  let match;
+  let runChars = 0;
+  while ((match = LONG_RUN_RE.exec(content)) !== null) {
+    const lineNum = content.slice(0, match.index).split("\n").length;
+    const charCount = [...match[0]].length;
+    runChars += charCount;
+    findings.push({ line: lineNum, charCount, ...decodeRun(match[0]) });
+  }
+
+  // Threshold-evasion: scattered invisible chars not in a long run can still be
+  // a payload. Always evaluated; chars already in a run are excluded so they
+  // aren't double-counted. countPayloadInvisible is invisible.mjs's carve-out
+  // counter (the SSOT the stripper itself gates on): it discounts every
+  // invisible that does real rendering work — emoji presentation selectors
+  // (VS16 *and* VS15) on a real pictograph, emoji-sequence ZWJ, and linguistic
+  // ZWNJ/ZWJ between cursive letters or after a virama. A hand-rolled emoji-only
+  // mirror lived here before and over-counted linguistic joiners and VS15, so a
+  // ZWNJ-dense Persian doc or a doc of text-presentation hearts (❤︎) tripped a
+  // scattered false positive on content stripInvisible would preserve.
+  //
+  // Asymmetry (deliberate, benign): the minuend discounts preserved
+  // selectors/joiners EVERYWHERE, while `runChars` is each run's RAW length. A
+  // long run is ≥LONG_RUN_THRESHOLD *consecutive* invisibles, and a preserved
+  // selector/joiner is always flanked by a visible neighbor — so it cannot sit
+  // inside such a run, and the two counts describe disjoint chars in practice.
+  // In the pathological case that they don't (e.g. a run of stacked VS16), the
+  // raw `runChars` subtracts at most a few more than the minuend added, biasing
+  // `scattered` slightly LOW — a false negative, the precision-favoring
+  // direction, never a spurious finding. `scattered` may even go negative; the
+  // `>=` gate treats that as "no scatter", which is correct.
+  const scattered = countPayloadInvisible(content) - runChars;
+  if (scattered >= SCATTERED_THRESHOLD) {
+    findings.push({
+      line: null, // whole-file finding: scattered chars aren't tied to one line
+      charCount: scattered,
+      method: "scattered invisible chars (possible threshold evasion)",
+      decoded: `[${scattered} invisible chars distributed across file]`,
+    });
+  }
+
+  return findings;
+}
+
+/**
+ * True when `realChild` is `realRoot` itself or lives beneath it. Both inputs
+ * must already be realpath-resolved absolute paths. Containment is tested with
+ * `relative(root, child)`: the result is "" when they are the same path, and
+ * for a true descendant it is a forward path with no `..` segment and is not
+ * itself absolute — so a sibling like `/proj-evil` (relative => `../proj-evil`)
+ * is correctly rejected.
+ * @param {string} realRoot
+ * @param {string} realChild
+ * @returns {boolean}
+ */
+function isContained(realRoot, realChild) {
+  const rel = relative(realRoot, realChild);
+  return (
+    rel === "" ||
+    (!rel.startsWith(`..${sep}`) && rel !== ".." && !isAbsolute(rel))
+  );
+}
+
+/**
+ * Classify a glob match for containment: resolve it to a real (symlink-
+ * followed) path and decide whether to keep, drop, or reject it. Three
+ * outcomes are kept distinct:
+ *
+ *   - The realpath is contained in `realRoot` → KEEP.
+ *   - The match is contained LEXICALLY (the glob pattern itself, followed
+ *     literally with no symlink resolution, never leaves `literalRoot`) but its
+ *     REALPATH escapes `realRoot` — an in-tree symlink (e.g. a planted
+ *     `CLAUDE.md -> /etc/passwd`) whose target lives outside the tree → SKIP.
+ *     One planted symlink must not abort scanning every other instruction
+ *     file; treat it like the existing dangling-symlink case.
+ *   - The match ESCAPES LEXICALLY — the glob pattern itself reaches outside
+ *     `literalRoot` via `..` or an absolute path outside the tree, with no
+ *     symlink involved → THROW. That is a caller misconfiguration (a scanner
+ *     pointed outside its own tree) that must surface loudly, not be silently
+ *     skipped.
+ *   - The path cannot be resolved at all (ENOENT/EACCES — a dangling symlink or
+ *     unreadable entry that still lives inside the tree) → return false to SKIP
+ *     it, matching scanInstructionFiles' existing skip-on-unreadable behavior.
+ *
+ * A genuine resolution failure is never allowed to masquerade as a
+ * containment pass: an unresolvable path is skipped, only a successfully
+ * resolved path is classified as kept/skipped/thrown above.
+ * @param {string} absPath  absolute path to a glob match
+ * @param {string} realRoot  realpath of the scan root
+ * @param {string} literalRoot  `cwd`, resolved but NOT symlink-followed
+ * @param {string} pattern  the glob that produced this match
+ * @returns {boolean} true to keep the match, false to skip it
+ */
+function keepContained(absPath, realRoot, literalRoot, pattern) {
+  let real;
+  try {
+    real = realpathSync(absPath);
+  } catch {
+    return false; // dangling/unreadable in-cwd match: skip, do not abort
+  }
+  if (isContained(realRoot, real)) return true;
+  // The glob pattern itself never left the scan root lexically, so the escape
+  // is caused by an in-tree symlink resolving outside the tree, not by the
+  // caller's glob configuration. Skip this one match, do not abort the scan.
+  if (isContained(literalRoot, absPath)) return false;
+  throw new Error(
+    `instruction-file path escapes scan root: pattern ${JSON.stringify(
+      pattern,
+    )} matched ${JSON.stringify(absPath)} which resolves to ${JSON.stringify(
+      real,
+    )} outside ${JSON.stringify(realRoot)}`,
+  );
+}
+
+/**
+ * Expand `globs` (relative to `cwd`) to absolute file paths, skipping
+ * `node_modules`. The glob set is the caller's instruction-file convention.
+ *
+ * Containment is enforced per match (see {@link keepContained}): a match whose
+ * glob pattern itself escapes `cwd` — via `..` or an absolute-path glob
+ * outside the tree — THROWS, since reaching outside the tree is a caller
+ * misconfiguration. A match that lexically stays inside `cwd` but resolves
+ * (via an in-tree symlink) to a target outside the tree, or that simply
+ * cannot be resolved (a dangling symlink or unreadable entry inside the
+ * tree), is SKIPPED, so one bad symlink never aborts scanning the rest of the
+ * project.
+ * @param {string[]} globs
+ * @param {{ cwd?: string }} [options]
+ * @returns {string[]}
+ */
+export function findInstructionFiles(globs, { cwd = process.cwd() } = {}) {
+  const literalRoot = resolve(cwd);
+  const realRoot = realpathSync(literalRoot);
+  const seen = new Set();
+  for (const pattern of globs)
+    for (const name of globSync(pattern, {
+      cwd,
+      exclude: (entry) => entry === "node_modules",
+    })) {
+      // globSync returns absolute paths verbatim for an absolute pattern and
+      // cwd-relative names otherwise; joining an already-absolute name would
+      // double the prefix into a nonexistent path (the absolute-glob miss bug).
+      const absPath = isAbsolute(name) ? name : join(cwd, name);
+      if (keepContained(absPath, realRoot, literalRoot, pattern))
+        seen.add(absPath);
+    }
+  return [...seen];
+}
+
+/**
+ * Scan every instruction file matched by `globs` and return only those with
+ * findings, each path reported relative to `cwd`. Unreadable/missing files are
+ * skipped. Pure scan — no mutation; pair with {@link cleanFile} to strip.
+ * @param {string[]} globs
+ * @param {{ cwd?: string }} [options]
+ * @returns {Array<{ file: string, findings: ReturnType<typeof scanText> }>}
+ */
+export function scanInstructionFiles(globs, { cwd = process.cwd() } = {}) {
+  const out = [];
+  for (const file of findInstructionFiles(globs, { cwd })) {
+    let content;
+    try {
+      content = readFileSync(file, "utf-8");
+    } catch {
+      continue; // missing or unreadable
+    }
+    const findings = scanText(content);
+    if (findings.length > 0) out.push({ file: relative(cwd, file), findings });
+  }
+  return out;
+}
+
+/**
+ * Atomically replace `absPath`'s contents with `data`, preserving `mode`.
+ *
+ * Writes to a sibling temp in the same directory, then `rename`s it over the
+ * original (same dir => same filesystem => the rename is atomic, not a
+ * cross-device copy). The temp name is UNPREDICTABLE (`tmpName()` defaults to
+ * crypto-random) and the temp is created exclusively (O_CREAT|O_EXCL): if the
+ * path already exists — including an attacker-planted symlink at a guessable
+ * temp name — the open fails (EEXIST) and does NOT follow the link to clobber
+ * its target. On the rare collision we fail loud rather than retry into a
+ * different attacker-controlled path.
+ *
+ * Crash-safety (matching the doc claim): the temp fd is `fsync`ed before the
+ * rename and the directory fd is `fsync`ed after it, so a power loss can't leave
+ * the renamed name pointing at unflushed/empty data or lose the rename itself.
+ * The EXACT `mode` is applied with `fchmod` (openSync's create mode is
+ * umask-masked, so it alone would drop bits), and a failed write/sync `unlink`s
+ * the temp before rethrowing so no partial temp leaks. `tmpName` and `remove`
+ * are injectable fault-injection seams for tests (force a known temp path; drive
+ * a cleanup-unlink failure); production callers never pass them.
+ * @param {string} absPath
+ * @param {string} data
+ * @param {number} mode
+ * @param {() => string} [tmpName]
+ * @param {(path: string) => void} [remove]
+ */
+export function atomicReplaceFile(
+  absPath,
+  data,
+  mode,
+  tmpName = () => `.${randomBytes(12).toString("hex")}.tmp`,
+  remove = unlinkSync,
+) {
+  const dir = dirname(absPath);
+  const tmp = join(dir, tmpName());
+  // Exclusive create: EEXIST (incl. a planted symlink at the temp name)
+  // propagates directly — no temp of ours exists yet, so nothing to clean up
+  // and we must never unlink the attacker's pre-existing path.
+  const fd = openSync(
+    tmp,
+    constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL,
+    mode,
+  );
+  try {
+    // writeFileSync(fd, …) loops until every byte is written (a bare writeSync
+    // can short-write a large payload); it does not close the caller's fd.
+    writeFileSync(fd, data);
+    // Preserve the EXACT mode: openSync's create mode is umask-masked, fchmod is
+    // not, so this restores bits (e.g. group-write) the umask would have dropped.
+    fchmodSync(fd, mode);
+    // Flush the temp's bytes before the rename, else a crash can expose the
+    // renamed name pointing at empty/partial content.
+    fsyncSync(fd);
+  } catch (err) {
+    closeSync(fd);
+    try {
+      remove(tmp);
+    } catch {
+      // Best-effort cleanup only: rethrow the ORIGINAL failure below, not a
+      // secondary unlink error, so the real cause stays loud.
+    }
+    throw err;
+  }
+  closeSync(fd);
+  renameSync(tmp, absPath);
+  // fsync the DIRECTORY so the rename (a directory metadata change) is durable
+  // across a crash, not just the file's data blocks.
+  const dirFd = openSync(dir, constants.O_RDONLY);
+  try {
+    fsyncSync(dirFd);
+  } finally {
+    closeSync(dirFd);
+  }
+}
+
+/**
+ * Strip payload-capable invisible characters from `absPath` in place. Returns
+ * `true` when the file's bytes actually changed (a payload {@link scanText}
+ * flags was removed), `false` when {@link scanText} reports nothing, and `null`
+ * when scan flagged a payload but {@link stripInvisible} removes nothing — a
+ * fail-closed signal that the flagged run was PRESERVED (e.g. a well-formed
+ * emoji-tag sequence the stripper keeps), so the caller must not treat it as
+ * cleaned. `true` means and only means "bytes changed".
+ *
+ * Contract (scan/clean coherence): clean strips exactly what scan flags. A
+ * write happens ONLY when `scanText` reports a finding, so the "scan, then
+ * clean what scan flagged" workflow never silently rewrites a file scan called
+ * clean. A handful of sub-threshold invisible chars (which scan ignores) are
+ * left untouched — by design, the scanner's definition of a payload is the
+ * single source of truth for what gets removed.
+ *
+ * Refuses to follow symlinks: instruction files must be regular files. The read
+ * fd is opened with `O_NOFOLLOW`, so a symlinked path (which could redirect the
+ * read/write to a target outside the tree) makes the OPEN itself fail — closing
+ * the lstat→open TOCTOU window a separate stat would leave, in which the path
+ * could be swapped to a symlink between the check and the read.
+ *
+ * Non-UTF-8 safety (O9): the file is read as raw BYTES and required to round-trip
+ * losslessly through UTF-8 before any rewrite. `readFileSync(…, "utf-8")`
+ * silently maps invalid bytes to U+FFFD, which a naive strip-and-rewrite would
+ * then persist file-wide — so a non-UTF-8 file fails loud and is left untouched.
+ *
+ * Lost-update / TOCTOU guard: the on-path file is re-checked against the fstat
+ * snapshot taken right after open (inode, size, mtime, and not-a-symlink) before
+ * the rename; a concurrent write or symlink swap between our read and our write
+ * fails loud rather than silently clobbering the other writer.
+ *
+ * The write is atomic (see {@link atomicReplaceFile}): stripped content goes to
+ * a temp file in the same directory which is then `rename`d over the original
+ * (preserving the original file mode), fsync'd for crash-safety.
+ *
+ * Throws if the file cannot be read or written (the caller decides whether an
+ * unwritable contaminated file is fatal or falls back to alerting).
+ * @param {string} absPath
+ * @param {(path: string) => import("node:fs").Stats} [lstat] injectable
+ *   pre-rename recheck stat (fault-injection seam, mirrors
+ *   {@link atomicReplaceFile}'s `tmpName`): lets a test drive the concurrent
+ *   write/symlink-swap that the TOCTOU guard exists to catch, which is otherwise
+ *   unreachable from this fully-synchronous path. Defaults to `lstatSync`.
+ * @returns {boolean}
+ */
+export function cleanFile(absPath, lstat = lstatSync) {
+  let fd;
+  try {
+    fd = openSync(absPath, constants.O_RDONLY | constants.O_NOFOLLOW);
+  } catch (err) {
+    // O_NOFOLLOW on a symlink fails ELOOP (some libc report EMLINK); surface
+    // the same "regular files only" contract the old lstat check did. Other
+    // errors (ENOENT/EACCES/…) propagate unchanged.
+    const code = /** @type {NodeJS.ErrnoException} */ (err).code;
+    if (code === "ELOOP" || code === "EMLINK")
+      throw new Error(
+        `refusing to clean through a symlink (instruction files must be regular files): ${JSON.stringify(
+          absPath,
+        )}`,
+        { cause: err },
+      );
+    throw err;
+  }
+  try {
+    const before = fstatSync(fd);
+    if (!before.isFile())
+      throw new Error(
+        `refusing to clean a non-regular file (instruction files must be regular files): ${JSON.stringify(
+          absPath,
+        )}`,
+      );
+
+    // Read raw bytes and require a lossless UTF-8 round-trip (see O9 above).
+    const raw = readFileSync(fd);
+    const original = raw.toString("utf-8");
+    if (!Buffer.from(original, "utf-8").equals(raw))
+      throw new Error(
+        `refusing to clean a file that is not valid UTF-8 (round-trip mismatch would corrupt it): ${JSON.stringify(
+          absPath,
+        )}`,
+      );
+
+    // Scan is the SSOT for what counts as a payload: don't rewrite a file scan
+    // would not flag, even if stripInvisible would technically remove a char.
+    if (scanText(original).length === 0) return false;
+
+    const stripped = stripInvisible(original);
+    // Re-verify the on-path file against the open-time snapshot before writing:
+    // an inode/size/mtime change (or a swap to a symlink) means someone modified
+    // it under us, so fail loud rather than clobber their write (lost update).
+    const after = lstat(absPath);
+    if (
+      after.isSymbolicLink() ||
+      after.ino !== before.ino ||
+      after.size !== before.size ||
+      after.mtimeMs !== before.mtimeMs
+    )
+      throw new Error(
+        `instruction file changed between read and write, refusing to clobber (possible concurrent write or symlink swap): ${JSON.stringify(
+          absPath,
+        )}`,
+      );
+    // `before.mode` is the opened regular file's mode. A crash before the rename
+    // leaves the original intact; after it the new content is fully present.
+    atomicReplaceFile(absPath, stripped, before.mode);
+    return true;
+  } finally {
+    closeSync(fd);
+  }
+}