agent-sanitizer 2.19.2 → 2.19.4

package/src/layer1.mjs

@@ -13,98 +13,16 @@
  * point where a lone surrogate would otherwise corrupt a match or a parse.
  */
 import { stripInvisibleWithReport, CATEGORY } from "./invisible.mjs";
+import { CONTROL_INTRODUCER_SOURCE, scanAnsi, TOKEN_KIND } from "./ansi.mjs";
 
-// Raw control introducers that must not survive: 7-bit ESC (U+001B) and the
-// entire 8-bit C1 control block (U+0080–U+009F) — which includes CSI (U+009B),
-// the string introducers DCS/SOS/OSC/PM/APC (U+0090/0098/009D/009E/009F), and ST
-// (U+009C). All are category Cc, so the invisible-char pass (which targets Cf /
-// variation / blank fillers) never removes them; this residual sweep is the
-// guarantee that none survives. Sweeping the whole C1 block — not just the
-// introducers the ANSI grammar above names — fails closed: a DCS/SOS/PM/APC
-// string the grammar does not consume still loses its introducer and terminator
-// here, so no terminal can hide-render its body as a control payload.
-// eslint-disable-next-line no-control-regex -- matching the raw introducers is the point
-const CONTROL_INTRODUCER_RE = /[\u001b\u0080-\u009f]/g;
+// The ANSI grammar and the introducer charset live in ./ansi.mjs so this module
+// and invisible.mjs (which owns the public isSgrOnly / SGR_RE and cannot import
+// this one) scan with the SAME tokenizer.
 
-// An OSC (Operating System Command) string is `<introducer> … <terminator>`.
-// Introducer: 7-bit `ESC]` or 8-bit C1 OSC (U+009D). Terminator: ST (`ESC\` or
-// 8-bit C1 ST U+009C) OR the legacy BEL (U+0007). The body is everything up to
-// the terminator — a title, a clickable-hyperlink URL, a clipboard write — i.e.
-// attacker-controlled PAYLOAD TEXT. Matching the introducer alone (leaving the
-// body) would let that payload survive into the model's view, so the OSC branch
-// consumes the introducer, the whole body, AND the terminator as one unit.
-//
-// Three alternatives, tried in order:
-//   1. a properly TERMINATED string — a body of bytes that no terminator can
-//      start with (the negated class makes that run unambiguous and
-//      backtrack-free), then a terminator (ST or BEL).
-//   2. an ABORTED string — the body runs up to (but does NOT consume) an
-//      interior bare ESC or a nested C1-OSC introducer (U+009D) that is not
-//      itself part of a valid terminator. Per ECMA-48/xterm, a bare ESC (one
-//      not immediately followed by `\` to form ST) aborts the OSC string in
-//      progress — the terminal drops back to processing that ESC as the start
-//      of a NEW sequence, and everything after it is normal text/escapes, not
-//      part of this OSC's payload. Consuming only up to the lookahead (not
-//      the ESC/U+009D itself) leaves it for the next position in this same
-//      `.replace(ANSI_RE, ...)` scan to match as its own sequence (or, if it
-//      doesn't complete one, for the residual C1 sweep in applyLayer1 to
-//      remove just that one introducer byte) — so an interior ESC can no
-//      longer delete the rest of the document (it used to fall through to
-//      alternative 3 below and consume everything to EOS).
-//   3. anything else from the introducer to END-OF-STRING (`[\s\S]*$`) — the
-//      fail-closed catch-all for a GENUINELY unterminated string: no ST, BEL,
-//      interior ESC, or nested OSC intro anywhere in the remainder, so there
-//      is truly nothing left to hand to a later position. Reached only when
-//      alternatives 1 and 2 both fail to find their respective triggers.
-// All three are linear (bounded lookahead / no nested quantifiers), so the
-// branch stays linear.
-const OSC_INTRO = "(?:\\u001b\\]|\\u009d)";
-const OSC_TERM = "(?:\\u001b\\\\|\\u009c|\\u0007)";
-const OSC_BODY = "[^\\u0007\\u001b\\u009c\\u009d]";
-const OSC_BRANCH = `${OSC_INTRO}(?:${OSC_BODY}*${OSC_TERM}|${OSC_BODY}*(?=[\\u001b\\u009d])|${OSC_BODY}*$)`;
-
-// CSI / two-byte ESC sequences (cursor moves, erase, SGR color, charset/DEC
-// selectors): an introducer, a bounded private-intro run, optional numeric
-// params, and a single final byte. Not an enforcement boundary on its own — any
-// introducer this declines to match is still removed by the residual sweep in
-// applyLayer1 — but matching the whole sequence keeps the common case one clean
-// deletion (and avoids a lone-ESC residual on every styled line).
-//
-// The private-intro class is BOUNDED ({0,12}, not *) on purpose: ; and # live
-// in both this class and the parameter group that follows, so an unbounded *
-// here lets a ;#;#... run be split between the two quantifiers — O(n^2)
-// backtracking on an ESC;#;#... string that never completes a sequence
-// (CodeQL js/polynomial-redos). A constant bound makes the intro a constant
-// factor, so the whole match is linear; a real sequence never carries more than
-// a couple of intro bytes.
-//
-// Params allow BOTH `;` (standard parameter separator) and `:` (ITU T.416
-// colon-separated SGR sub-parameters, e.g. truecolor `ESC[38:2:255:0:0m` as
-// emitted by tmux/kitty/mintty) — legitimate, display-only ANSI that must not
-// leave a colon-parameter residue behind.
-//
-// The final-byte class deliberately excludes `\d`: per ECMA-48, CSI final
-// bytes occupy 0x40–0x7E (letters and a handful of punctuation) while digits
-// are PARAMETER bytes and can never terminate a sequence — an unterminated
-// `ESC[` therefore must not be allowed to eat trailing visible digits (e.g.
-// `ESC[2024 report` is NOT `ESC[` + final-byte `2` + literal `024 report`; it
-// is an incomplete CSI intro that the residual sweep in applyLayer1 cleans up,
-// leaving "2024 report" intact). `=`, `<`, `>` are excluded for the same
-// reason: 0x3C–0x3F (`<=>?`) are private PARAMETER-prefix bytes, not final
-// bytes, per ECMA-48 § 5.4 — `?` already lives in the private-intro class
-// above; `<=>` were never valid finals and including them let a private-marker
-// sequence terminate one byte too early. `~` (0x7E) IS a real final byte (vt220
-// function-key sequences, e.g. `ESC[3~` for Delete) and is kept.
-const CSI_BRANCH =
-  "[\\u001b\\u009b][[()#;?]{0,12}(?:(?:\\d{1,4}(?:[;:]\\d{0,4})*)?[A-PR-TZcf-ntqry~])";
-
-// Full ANSI escape grammar (OSC first so `ESC]` / C1-OSC is consumed as a whole
-// string, not split by the CSI branch), not just SGR: the Layer-1 guarantee is
-// that no control introducer and no OSC payload survives, and a cursor-move or
-// erase sequence is as much a display-spoofing hazard as a color one. Built from
-// `\uXXXX`-escaped string parts via `new RegExp`, so no raw control byte sits in
-// the source (no no-control-regex disable needed).
-const ANSI_RE = new RegExp(`(?:${OSC_BRANCH}|${CSI_BRANCH})`, "gu");
+// The residual sweep: every raw control introducer, whatever the grammar made of
+// it. This — not the sequence matching — is the guarantee that no introducer
+// survives Layer 1.
+const CONTROL_INTRODUCER_RE = new RegExp(CONTROL_INTRODUCER_SOURCE, "g");
 
 // Unpaired UTF-16 surrogates (high not followed by low, or low not preceded by
 // high). Normalized before any HTML parser, which throws on a stray byte —
@@ -114,6 +32,29 @@ export const LONE_SURROGATE_RE =
 
 const MAX_ANSI_PASSES = 3;
 
+/**
+ * Splice out every complete ANSI sequence {@link scanAnsi} finds — SGR, other
+ * CSI/two-byte escapes, and whole OSC strings. An ORPHAN introducer (one that
+ * starts no sequence) is deliberately LEFT: deleting it here would drop the
+ * `ESC` out of `ESC`<ZWSP>`[32m` before the invisible pass could reconstitute
+ * the sequence, turning a hidden control into visible `[32m` in the model's
+ * view. Orphans are removed by applyLayer1's residual sweep, once no
+ * reconstitution is possible.
+ * @param {string} text
+ * @returns {string}
+ */
+function stripAnsiOnce(text) {
+  let out = "";
+  let last = 0;
+  for (const token of scanAnsi(text)) {
+    if (token.kind === TOKEN_KIND.ORPHAN) continue;
+    out += text.slice(last, token.start);
+    last = token.end;
+  }
+  if (last === 0) return text;
+  return out + text.slice(last);
+}
+
 /**
  * Strip ANSI escape sequences to a fixed point. Removing one sequence can
  * reconstitute another around it (a lone ESC left of `ESC[32m[0m` gains the
@@ -131,54 +72,98 @@ const MAX_ANSI_PASSES = 3;
  */
 export function stripAnsiFully(input) {
   let prev = input;
-  let out = prev.replace(ANSI_RE, "");
+  let out = stripAnsiOnce(prev);
   for (let pass = 1; pass < MAX_ANSI_PASSES && out !== prev; pass++) {
     prev = out;
-    out = prev.replace(ANSI_RE, "");
+    out = stripAnsiOnce(prev);
   }
   return out;
 }
 
+// How many times the {ANSI strip, invisible strip} composition may re-run before
+// the sweep is forced. Each iteration deletes at least one character, so the
+// loop terminates on its own; the bound is a DoS guard on the same quadratic
+// argument as MAX_ANSI_PASSES (n rounds of O(n) scans on adversarial input).
+// Three rounds is what the deepest REAL reconstitution needs — strip
+// invisibles, remove the ANSI they hid, strip the joiners that ANSI removal
+// made adjacent, confirm — so four leaves a round of margin. Deeper nesting is
+// not reachable through the carve-out: preserving a joiner requires cursive or
+// emoji neighbours on BOTH sides, so a preserved joiner can never sit inside an
+// escape sequence, and every joiner that hides one is therefore stripped by the
+// FIRST invisible pass. Past the bound the composition simply stops early — the
+// final sweep still holds the no-introducer guarantee, and what is left degrades
+// to visible text (the fail-open direction), exactly as with MAX_ANSI_PASSES.
+const MAX_LAYER1_PASSES = 4;
+
 /**
  * Layer 1: ANSI + invisible-char strip with a result guaranteed free of every
  * raw ANSI control introducer (7-bit ESC U+001B and the whole 8-bit C1 control
  * block U+0080–U+009F: CSI, the DCS/SOS/OSC/PM/APC string introducers, and ST).
  *
- * Removing an invisible character can reconstitute an escape its split hid from
- * the ANSI pass (`ESC`<ZWSP>`[32m` → `ESC[32m`), so strip ANSI again after the
- * invisible pass — but only when stripInvisible changed something, since
- * reconstitution is impossible otherwise and the re-strip is a wasted pass on
- * the hot clean path. The ANSI strip still cannot match an *incomplete*
- * reconstituted sequence (a lone `ESC[` left when an inner complete sequence is
- * removed from a nested split), so a final sweep removes every residual raw
- * introducer outright — that sweep, not the regex matching, is the guarantee
- * that no control introducer survives. `deAnsi` is the ANSI strip of the
- * original (invisible runs intact), the scope a LONG_RUN payload check needs.
+ * The two passes FEED each other in both directions, so neither ordering is
+ * enough on its own and the composition is iterated to a fixed point instead:
+ * removing an invisible char reconstitutes an escape its split hid
+ * (`ESC`<ZWSP>`[32m` → `ESC[32m`), and removing an escape makes two invisibles
+ * ADJACENT that were not (`م ZWJ ESC[m ZWJ م` → a joiner run the invisible pass
+ * classifies as a payload channel rather than linguistic). A pipeline with a
+ * fixed number of alternations always leaves one of those unanswered — this used
+ * to re-strip ANSI after the invisible pass and stop, so `applyLayer1` was not
+ * idempotent and the rehydrator's "re-cleaning reproduces the view" assumption
+ * did not hold.
+ *
+ * The residual sweep runs only once the composition is STABLE, because sweeping
+ * an introducer early destroys the sequence a later ANSI pass would have removed
+ * whole, promoting a hidden control to visible text. A final UNCONDITIONAL sweep
+ * follows the loop so the no-raw-introducer guarantee does not depend on the
+ * pass bound.
+ *
+ * `deAnsi` is the ANSI strip of the ORIGINAL text (invisible runs intact), the
+ * scope a LONG_RUN payload check needs — not an intermediate of the loop.
  * @param {string} text
  * @returns {{ cleaned: string, deAnsi: string, found: string[] }}
  */
 export function applyLayer1(text) {
   const deAnsi = stripAnsiFully(text);
-  // stripInvisibleWithReport returns `found` for exactly the categories it
-  // removed — so a ZWNJ/ZWJ the carve-out PRESERVES never registers as a strip,
-  // and the leading-BOM exception is already handled inside it. Pass the ORIGINAL
-  // `text` so a BOM that was interior before the ANSI strip (`ESC[m + interior U+FEFF`, now at
-  // index 0 of `deAnsi`) is treated as interior and stripped, not preserved.
-  const { cleaned: afterInvis, found } = stripInvisibleWithReport(deAnsi, text);
-  let ansiFound = deAnsi.length !== text.length;
+  /** @type {Set<string>} Union of the categories every iteration reported. */
+  const found = new Set();
+  let cleaned = text;
+  let ansiFound = false;
 
-  let cleaned = afterInvis;
-  if (afterInvis !== deAnsi) {
-    const reStripped = stripAnsiFully(afterInvis);
-    if (reStripped.length !== afterInvis.length) ansiFound = true;
-    cleaned = reStripped;
+  for (let pass = 0; pass < MAX_LAYER1_PASSES; pass++) {
+    const afterAnsi = pass === 0 ? deAnsi : stripAnsiFully(cleaned);
+    if (afterAnsi !== cleaned) ansiFound = true;
+    // stripInvisibleWithReport returns `found` for exactly the categories it
+    // removed — so a ZWNJ/ZWJ the carve-out PRESERVES never registers as a
+    // strip. The second argument stays the ORIGINAL `text` on every iteration:
+    // the leading-BOM exception is defined against what the user actually sent,
+    // so a BOM that was interior before an ANSI strip shifted it to index 0 is
+    // treated as interior and stripped, not preserved.
+    const { cleaned: afterInvis, found: passFound } = stripInvisibleWithReport(
+      afterAnsi,
+      text,
+    );
+    for (const category of passFound) found.add(category);
+    if (afterInvis !== cleaned) {
+      cleaned = afterInvis;
+      continue;
+    }
+    // {ANSI, invisible} is stable: nothing left can reconstitute, so the
+    // residual introducers can be swept without hiding a sequence from a later
+    // pass. A sweep that changes the text feeds one more round (removing an
+    // introducer can make invisibles adjacent, exactly as removing a sequence
+    // can); one that changes nothing means the whole composition has converged.
+    const swept = cleaned.replace(CONTROL_INTRODUCER_RE, "");
+    if (swept === cleaned) break;
+    cleaned = swept;
+    ansiFound = true;
   }
+
   const swept = cleaned.replace(CONTROL_INTRODUCER_RE, "");
   if (swept !== cleaned) {
     cleaned = swept;
     ansiFound = true;
   }
 
-  if (ansiFound) found.push(CATEGORY.ANSI);
-  return { cleaned, deAnsi, found };
+  if (ansiFound) found.add(CATEGORY.ANSI);
+  return { cleaned, deAnsi, found: [...found] };
 }

package/src/output.mjs

@@ -27,6 +27,7 @@
 import { CATEGORY, describeStripped, isSgrOnly } from "./invisible.mjs";
 import { HTML_TAG_PRESENT, MD_LINK_HINT } from "./gates.mjs";
 import { applyLayer1, LONE_SURROGATE_RE } from "./layer1.mjs";
+import { orderedMatches, spliceOrdered } from "./view-map.mjs";
 
 /**
  * Closed enum of LIBRARY-OWNED Layer-5 warning codes — the ONLY warning values
@@ -206,22 +207,38 @@ export function describeWarned(warned) {
 /**
  * Delete each verbatim span in `spans` from `text`. The secure Layer-5
  * primitive: a filter can only ask for deletions, so this can never inject
- * bytes. Returns the new text and how many distinct span-occurrences were
- * removed (0 when no span was present).
+ * bytes. Returns the new text and how many span occurrences were removed (0
+ * when no span was present).
+ *
+ * Every occurrence is located in the ORIGINAL `text` and the deletions applied
+ * in one ordered pass ({@link spliceOrdered}), so every removed byte lies inside
+ * a match some span had in the INPUT. Deleting span-by-span with a
+ * chained `split`/`join` would not hold that line: an earlier deletion joins the
+ * bytes on either side of it and can CREATE a match for a later span that never
+ * occurred in the input — `deleteVerbatimSpans("PRE-XX-POST", ["-XX-", "PREPOST"])`
+ * then deletes the whole document. That would widen the Layer-5 seam's blast
+ * radius (see the module doc) from "a compromised filter can at most remove the
+ * content it named" to "it can remove content it never named".
+ *
+ * Overlapping spans are resolved first-match-wins, so `removed` counts the
+ * occurrences actually spliced out, never a double-count of the same bytes.
  * @param {string} text
  * @param {string[]} spans
  * @returns {{ text: string, removed: number }}
  */
 export function deleteVerbatimSpans(text, spans) {
-  let out = text;
-  let removed = 0;
-  for (const span of spans) {
-    if (!span) continue;
-    const parts = out.split(span);
-    removed += parts.length - 1;
-    out = parts.join("");
-  }
-  return { text: out, removed };
+  // Keep only non-empty STRING spans. The filter is untrusted JS, not a
+  // type-checked caller, so the array can hold anything: `indexOf(123)` would
+  // silently match the literal text "123" (deleting content the filter never
+  // named), and `occurrences` steps by `needle.length` — `undefined` for a
+  // number — making `indexOf(needle, NaN)` clamp back to the same index and
+  // loop forever. Fail open on a malformed entry rather than mangle bytes or
+  // hang the pipeline.
+  const usable = spans.filter(
+    (span) => typeof span === "string" && span !== "",
+  );
+  const spliced = spliceOrdered(text, orderedMatches(text, usable), () => "");
+  return { text: spliced.text, removed: spliced.spans.length };
 }
 
 /**

package/src/prompt.mjs

@@ -27,6 +27,7 @@ import {
   isSgrOnly,
 } from "./invisible.mjs";
 import { stripAnsiFully } from "./layer1.mjs";
+import { CONTROL_INTRODUCER_SOURCE } from "./ansi.mjs";
 
 // Every raw ANSI control a prompt can carry: 7-bit ESC (U+001B) and the entire
 // 8-bit C1 block (U+0080-U+009F). Gating on ESC alone -- or on only CSI/OSC --
@@ -34,12 +35,14 @@ import { stripAnsiFully } from "./layer1.mjs";
 // (CSI erase), `U+009D 0;...BEL` (OSC), or the string introducers DCS (U+0090),
 // SOS (U+0098), PM (U+009E), APC (U+009F): Layer 1 strips it, dropping the
 // invisible count to zero, so the prompt reads clean and passes. This gate must
-// match Layer 1's residual sweep (CONTROL_INTRODUCER_RE) exactly -- no raw C1
-// control belongs in a legitimate prompt (the SGR color carve-out is applied
-// separately, after SGR removal), so the whole block is gated, not a hand-picked
-// subset that lets DCS/SOS/PM/APC through.
-// eslint-disable-next-line no-control-regex -- the raw control introducers are exactly what we detect
-const ANSI_INTRODUCER = /[\u001b\u0080-\u009f]/;
+// match Layer 1's residual sweep exactly -- no raw C1 control belongs in a
+// legitimate prompt (the SGR color carve-out is applied separately, after SGR
+// removal), so the whole block is gated, not a hand-picked subset that lets
+// DCS/SOS/PM/APC through. Built from the SHARED charset source instead of a
+// third hand-written copy: keeping the copies in step used to be a prose
+// obligation recorded in this very comment, and two of the three spelled ESC
+// differently, so even a grep-based drift check would have missed a divergence.
+const ANSI_INTRODUCER = new RegExp(CONTROL_INTRODUCER_SOURCE);
 
 /**
  * True when every ANSI introducer in `prompt` belongs to a display-only SGR

package/src/rehydrate.mjs

@@ -51,6 +51,7 @@ import {
   occurrences,
   overlapAwareCount,
   orderedMatches,
+  spliceOrdered,
   alignDeletions,
   resolveSpan,
   rehydrateNewString,
@@ -415,9 +416,9 @@ async function rehydrateWrite(ti, view, io, hint) {
     };
 
   // Resolve each of this file's placeholder texts to its single secret first,
-  // then splice in ONE ordered pass (R6). A chained `out.split(ph).join(secret)`
-  // per placeholder is unsound: an inserted secret whose bytes contain a later
-  // placeholder text would be re-matched and corrupted by the next split.
+  // then splice in ONE ordered pass (R6) via the shared `spliceOrdered` — see
+  // its doc for why a chained `out.split(ph).join(secret)` per placeholder is
+  // unsound.
   const valueByPh = new Map();
   for (const phText of texts) {
     const produced = view.pairs.filter((pair) => pair.placeholder === phText);
@@ -438,23 +439,15 @@ async function rehydrateWrite(ti, view, io, hint) {
       };
     valueByPh.set(phText, values[0]);
   }
-  const matches = orderedMatches(ti.content, texts);
-  let out = "";
-  let last = 0;
-  // Byte ranges in `out` occupied by the substituted secret values. A hint
-  // occurrence inside one of these is a pathological secret whose bytes contain
-  // the hint prefix, NOT a placeholder the model pasted — so it is excluded from
-  // the foreign-placeholder scan below.
-  const secretSpans = [];
-  for (const match of matches) {
-    const secret = valueByPh.get(match.text);
-    out += ti.content.slice(last, match.index);
-    const secretStart = out.length;
-    out += secret;
-    secretSpans.push({ start: secretStart, end: out.length });
-    last = match.index + match.text.length;
-  }
-  out += ti.content.slice(last);
+  // `secretSpans`: byte ranges in `out` occupied by the substituted secret
+  // values. A hint occurrence inside one of these is a pathological secret whose
+  // bytes contain the hint prefix, NOT a placeholder the model pasted — so it is
+  // excluded from the foreign-placeholder scan below.
+  const { text: out, spans: secretSpans } = spliceOrdered(
+    ti.content,
+    orderedMatches(ti.content, texts),
+    (match) => valueByPh.get(match.text),
+  );
   const secrets = [...valueByPh.values()];
 
   // R3: the new content may mix a valid same-file placeholder (substituted

package/src/view-map.mjs

@@ -223,9 +223,15 @@ export function resolveSpan(
 }
 
 /**
- * All occurrences of any needle in `text`, ordered by position. Placeholder
- * texts never substring-overlap one another (each ends in "]" right after its
- * distinguishing label), so the sorted matches are non-overlapping.
+ * All occurrences of any needle in `text`, ordered by position. Every index is
+ * computed against the ORIGINAL `text`, so the caller can splice them in one
+ * pass ({@link spliceOrdered}). Redaction placeholder texts never
+ * substring-overlap one another (each ends in "]" right after its
+ * distinguishing label), so for that caller the sorted matches are also
+ * non-overlapping; needles from an untrusted source (a Layer-5 filter's
+ * removeSpans) can overlap, which spliceOrdered resolves first-match-wins.
+ * Distinct needles matching at the SAME index keep `needles` order (Array#sort
+ * is stable), so first-match-wins is deterministic.
  * @param {string} text
  * @param {string[]} needles
  * @returns {{text: string, index: number}[]}
@@ -238,6 +244,47 @@ export function orderedMatches(text, needles) {
   return out.sort((left, right) => left.index - right.index);
 }
 
+/**
+ * Replace every match in `matches` with `replacementFor(match, i)` in a SINGLE
+ * ordered pass over `text`. THE splice primitive for this codebase — the sole
+ * sound way to substitute several needles at once.
+ *
+ * A chained `text.split(needle).join(value)` per needle is unsound in both
+ * directions, which is why no caller may hand-roll one:
+ *   - substitution: an inserted value whose bytes contain a LATER needle is
+ *     re-matched by the next split and corrupted (or partially exposed);
+ *   - deletion: an earlier deletion joins the bytes on either side of it and
+ *     can CREATE a later needle's match, deleting text that needle never
+ *     matched in the input ("PRE-XX-POST" minus "-XX-" yields "PREPOST").
+ * Because every index in `matches` is measured against the original `text`,
+ * this pass only ever touches bytes the caller actually matched.
+ *
+ * Overlapping matches are resolved first-match-wins: a match starting before
+ * the previous one ended is skipped, never spliced at a shifted offset.
+ * `i` is the match's index in `matches` (stable across skips) so a caller
+ * pairing matches positionally with its own array stays aligned.
+ * @param {string} text
+ * @param {{text: string, index: number}[]} matches ordered by index, indices into `text`
+ * @param {(match: {text: string, index: number}, i: number) => string} replacementFor
+ * @returns {{text: string, spans: {start: number, end: number}[]}} spliced text
+ *   and the [start, end) range each replacement occupies in it
+ */
+export function spliceOrdered(text, matches, replacementFor) {
+  let out = "";
+  let last = 0;
+  /** @type {{start: number, end: number}[]} */
+  const spans = [];
+  matches.forEach((match, i) => {
+    if (match.index < last) return;
+    out += text.slice(last, match.index);
+    const start = out.length;
+    out += replacementFor(match, i);
+    spans.push({ start, end: out.length });
+    last = match.index + match.text.length;
+  });
+  return { text: out + text.slice(last), spans };
+}
+
 /**
  * On-disk [start, end) span of every redaction pair, mapped from its view
  * offset through placeholder expansion (view → cleaned) and stripped invisible
@@ -308,24 +355,16 @@ export function rehydrateNewString(oldS, newS, spanPairs, filePairs) {
     newSeq.length === spanPairs.length &&
     newSeq.every((match, i) => match.text === spanPairs[i].placeholder)
   ) {
-    let out = "";
-    let last = 0;
-    newSeq.forEach((match, i) => {
-      out += newS.slice(last, match.index) + spanPairs[i].original;
-      last = match.index + match.text.length;
-    });
     return {
-      text: out + newS.slice(last),
+      text: spliceOrdered(newS, newSeq, (_match, i) => spanPairs[i].original)
+        .text,
       secrets: spanPairs.map((pair) => pair.original),
     };
   }
 
   // Each placeholder text must name exactly one secret; resolve that mapping
-  // first, then splice in a SINGLE ordered pass. A chained
-  // `out.split(ph).join(secret)` per placeholder is unsound: an inserted secret
-  // whose bytes happen to contain a later placeholder text would be re-matched
-  // and corrupted (or partially exposed) by the next split. One pass over the
-  // ordered match positions only ever touches the original new_string bytes.
+  // first, then splice in a SINGLE ordered pass (see spliceOrdered for why a
+  // chained `out.split(ph).join(secret)` per placeholder is unsound).
   const valueByPh = new Map();
   for (const phText of new Set(newSeq.map((match) => match.text))) {
     const values = [
@@ -344,11 +383,9 @@ export function rehydrateNewString(oldS, newS, spanPairs, filePairs) {
       };
     valueByPh.set(phText, values[0]);
   }
-  let out = "";
-  let last = 0;
-  for (const match of newSeq) {
-    out += newS.slice(last, match.index) + valueByPh.get(match.text);
-    last = match.index + match.text.length;
-  }
-  return { text: out + newS.slice(last), secrets: [...valueByPh.values()] };
+  return {
+    text: spliceOrdered(newS, newSeq, (match) => valueByPh.get(match.text))
+      .text,
+    secrets: [...valueByPh.values()],
+  };
 }

package/types/ansi.d.mts

@@ -0,0 +1,66 @@
+/**
+ * Tokenize every raw control introducer in `text`.
+ *
+ * Every introducer yields exactly one token — an ORPHAN when it starts nothing
+ * the grammar recognizes — so "which introducers are in this text" and "which
+ * sequences are in this text" are answered by the same scan. That is what lets
+ * the stripper (splice every non-orphan token, then sweep) and the SGR-only
+ * predicate (every token is SGR) agree by construction.
+ *
+ * Tokens are disjoint and ordered by `start`; each `end` is strictly greater
+ * than its `start`, so the scan always advances.
+ * @param {string} text
+ * @returns {AnsiToken[]}
+ */
+export function scanAnsi(text: string): AnsiToken[];
+/**
+ * The ONE ANSI grammar: the raw control-introducer charset and the tokenizer
+ * every consumer scans with.
+ *
+ * Two modules need this grammar and they cannot import each other —
+ * `layer1.mjs` imports `invisible.mjs`, so `invisible.mjs` (which owns the
+ * public `isSgrOnly` / `SGR_RE`) must not import back. Before this module the
+ * grammar was therefore written out twice with DIFFERENT param rules
+ * (`invisible.mjs`'s SGR regex accepted any digit run, `layer1.mjs`'s CSI
+ * branch capped each parameter at four digits), and the introducer charset
+ * three times. The looser copy suppressed the operator warning for a sequence
+ * the stripper could not match: `ESC[12345m` read as "display-only colour"
+ * while `[12345m` was spliced into the model's view as visible text. One
+ * tokenizer, one charset, consumed by both — the disagreement cannot recur.
+ *
+ * Same precedent (and same reason) as `cf-charset.mjs`: a dependency-free leaf
+ * module both layers read from.
+ */
+export const CONTROL_INTRODUCER_SOURCE: "[\\u001b\\u0080-\\u009f]";
+/**
+ * Public alias kept for compatibility (re-exported by `invisible.mjs` and the
+ * package root). It is now DERIVED: {@link scanAnsi} classifies a token as SGR
+ * by testing the token's own text against this exact source, so the predicate
+ * and the regex can no longer describe different languages.
+ */
+export const SGR_RE: RegExp;
+/** The four things an introducer can turn out to be. */
+export const TOKEN_KIND: Readonly<{
+    /** A display-only `ESC[…m` / `U+009B…m` colour sequence. */
+    SGR: "sgr";
+    /** Any other complete CSI / two-byte escape (cursor move, erase, charset). */
+    CSI: "csi";
+    /** An OSC string: introducer, body and terminator as one unit. */
+    OSC: "osc";
+    /** An introducer that starts no sequence the grammar recognizes. */
+    ORPHAN: "orphan-introducer";
+}>;
+export type AnsiToken = {
+    /**
+     * Index of the introducer.
+     */
+    start: number;
+    /**
+     * Index one past the last character of the token.
+     */
+    end: number;
+    /**
+     * One of {@link TOKEN_KIND}.
+     */
+    kind: string;
+};