agent-sanitizer 2.31.5 → 2.32.0

package/THREAT-MODEL.md

@@ -78,12 +78,14 @@ introducers) keeps the WARNING. An `ESC` that _opened_ a CSI it never completed
 stays loud too: a terminal's CSI parser is stateful and keeps consuming what
 follows until a final byte arrives, so `ESC[12 world` shows the human `orld`
 while the model reads every word—the same model-sees/human-sees divergence a
-complete sequence buys. OSC strings (titles,
-clickable-hyperlink URLs) are consumed as a
+complete sequence buys. All five ECMA-48 control strings—OSC (titles,
+clickable-hyperlink URLs), DCS, SOS, PM and APC—are consumed as a
 whole, for every terminator form—ST (`ESC\` or 8-bit C1 ST U+009C) and the
-legacy BEL—and for the 8-bit C1 OSC introducer (U+009D); an _unterminated_ OSC
-introducer is dropped through end-of-string (fail-closed), so no OSC body
-survives to carry a payload.
+legacy BEL—and for the 8-bit C1 introducers (U+0090/0098/009D/009E/009F); an
+_unterminated_ introducer is dropped through end-of-string (fail-closed), so no
+string body survives to carry a payload. Every one of those bodies is
+attacker-controlled text, which is why the introducer alone is not enough to
+remove.
 
 ## Layer 2—hidden HTML (remark/rehype)
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-sanitizer",
-  "version": "2.31.5",
+  "version": "2.32.0",
   "description": "Defend an agent against hidden-content injection: strip payload-capable invisible Unicode and ANSI, splice out human-invisible HTML, and flag data-exfil URLs in untrusted text before any model sees it.",
   "type": "module",
   "repository": {

package/src/ansi.mjs

@@ -21,9 +21,9 @@
 // the entire 8-bit C1 control block (U+0080-U+009F) — which includes CSI
 // (U+009B), the string introducers DCS/SOS/OSC/PM/APC
 // (U+0090/0098/009D/009E/009F), and ST (U+009C). Gating the whole C1 block, not
-// just the introducers the sequence grammar below names, fails closed: a
-// DCS/SOS/PM/APC string the grammar does not consume still loses its introducer
-// and terminator, so no terminal can hide-render its body as a control payload.
+// just the introducers the sequence grammar below names, fails closed: a C1 byte
+// the grammar recognizes no sequence for still loses that byte, so no terminal
+// can hide-render what follows it as a control payload.
 //
 // The introducer set as DATA, so a non-JS consumer can share it: the generator
 // (scripts/gen-invisible-charset.mjs) pins these code points into
@@ -91,10 +91,29 @@ const CSI_FINAL_RE = /[A-PR-TZcf-nqrty~]/;
 const ESC = 0x1b;
 const CSI_C1 = 0x9b;
 const ST_C1 = 0x9c;
-const OSC_C1 = 0x9d;
 const BEL = 0x07;
 
-/** The six things an introducer can turn out to be. */
+// PROBLEM CLASS — a control string whose body the grammar leaves as visible
+// text. ECMA-48 opens FIVE strings, not one: OSC (`ESC ]` / U+009D), DCS
+// (`ESC P` / U+0090), SOS (`ESC X` / U+0098), PM (`ESC ^` / U+009E) and APC
+// (`ESC _` / U+009F). All five share one grammar — introducer, body, ST — and
+// every body is attacker-controlled payload text, so all five are consumed as
+// one token. Recognizing OSC alone left the other four's bodies in the model's
+// view: `ESC _ hidden ESC \` reduced to the visible text `_hidden\`, and at the
+// prompt gate the two bare `ESC`s tokenized as inert orphans, which is the
+// benign kind, so the gate answered `note` instead of `block`.
+// The second byte of the 7-bit form, keyed by introducer. `P` also happens to
+// be a CSI final byte, so without this table `ESC P` completed a two-byte CSI
+// and the DCS body survived that way instead.
+const STRING_INTRO_7BIT = "]PX^_";
+const OSC_7BIT = "]";
+const OSC_C1 = 0x9d;
+// The 8-bit C1 introducers for the same five. This doubles as the set a nested
+// introducer aborts on (see scanControlString case 2) — the two ARE the same
+// set: opening a string is exactly what ends the one already open.
+const STRING_INTRO_C1 = new Set([0x90, 0x98, OSC_C1, 0x9e, 0x9f]);
+
+/** The seven things an introducer can turn out to be. */
 export const TOKEN_KIND = Object.freeze({
   /** A display-only `ESC[…m` / `U+009B…m` colour sequence. */
   SGR: "sgr",
@@ -102,6 +121,13 @@ export const TOKEN_KIND = Object.freeze({
   CSI: "csi",
   /** An OSC string: introducer, body and terminator as one unit. */
   OSC: "osc",
+  /**
+   * One of the other four ECMA-48 control strings — DCS, SOS, PM or APC —
+   * introducer, body and terminator as one unit, exactly like
+   * {@link TOKEN_KIND.OSC}. Split from it only so a warning can name what it
+   * found; both are payload-carrying strings and neither is benign.
+   */
+  CONTROL_STRING: "control-string",
   /**
    * A 7-bit `ESC` that starts no sequence the grammar recognizes — a truncated
    * write, a log fragment cut mid-escape, a stray byte living in a file.
@@ -122,11 +148,11 @@ export const TOKEN_KIND = Object.freeze({
    * A RAW C1 byte (U+0080-U+009F) that starts no sequence the grammar
    * recognizes. Split from {@link TOKEN_KIND.ORPHAN} because the two carry very
    * different weight: a lone `ESC` is ordinary debris in terminal output, while
-   * a raw C1 byte is not something legitimate UTF-8 text produces, and the
-   * block includes the string introducers DCS/SOS/PM/APC (U+0090/0098/009E/
-   * 009F) — which this grammar does not consume, so an unrecognized one here
-   * means a terminal WOULD have swallowed the following text as a control
-   * payload. Consumers that downgrade an inert strip to a note (see
+   * a raw C1 byte is not something legitimate UTF-8 text produces. The five
+   * string introducers in the block open a {@link TOKEN_KIND.OSC} or
+   * {@link TOKEN_KIND.CONTROL_STRING} token instead, so a byte that reaches
+   * here is one the grammar recognizes no sequence for at all — and a terminal
+   * may still act on it. Consumers that downgrade an inert strip to a note (see
    * `isBenignAnsiKinds` in ./layer1.mjs) must keep warning on this one.
    */
   ORPHAN_C1: "orphan-c1-introducer",
@@ -153,9 +179,10 @@ export function isOrphanKind(kind) {
  * lone `ESC`. The one place that split is decided, shared by the tokenizer and
  * by Layer 1's residual sweep (which sees bare characters, not tokens).
  *
- * `[` is the only lookahead that matters. `ESC ]` is an OSC, which the scanner
- * consumes to the end of input if unterminated (so it never reaches here), and
- * every other second byte — `ESC (`, `ESC #`, `ESC P` — bounds what a terminal
+ * `[` is the only lookahead that matters. The five string introducers — `ESC ]`,
+ * `ESC P`, `ESC X`, `ESC ^`, `ESC _` — are consumed whole by
+ * {@link scanControlString}, to the end of input if unterminated, so they never
+ * reach here; every other second byte (`ESC (`, `ESC #`) bounds what a terminal
  * swallows to a byte or two rather than running until a final byte arrives.
  * @param {string} ch
  * @param {string} [next]  the following character, or undefined at end of input
@@ -174,40 +201,49 @@ export function orphanKindFor(ch, next) {
  */
 
 /**
- * End index of the OSC string starting at `start`, or -1 if no OSC introducer
- * is there.
+ * The control string starting at `start`, or null when no string introducer is
+ * there.
  *
- * An OSC (Operating System Command) string is `<introducer> body <terminator>`:
- * a title, a clickable-hyperlink URL, a clipboard write — i.e. attacker-
- * controlled PAYLOAD TEXT. Consuming the introducer alone would leave that
- * payload in the model's view, so the whole string is one token. Three ways it
- * can end:
+ * A control string is `<introducer> body <terminator>`: an OSC window title, a
+ * clickable-hyperlink URL, a clipboard write, a DCS device payload, an APC
+ * application command — i.e. attacker-controlled PAYLOAD TEXT in every case.
+ * Consuming the introducer alone would leave that payload in the model's view,
+ * so the whole string is one token. Three ways it can end:
  *   1. a real terminator — ST (`ESC\` or the 8-bit C1 ST U+009C) or the legacy
  *      BEL — which is consumed with the body.
  *   2. an ABORT: per ECMA-48/xterm a bare ESC (one not forming ST) drops the
- *      terminal out of the OSC string, and a nested C1 OSC introducer likewise
+ *      terminal out of the string, and a nested C1 string introducer likewise
  *      starts something new. The token ends BEFORE that byte so the scan
  *      re-reads it as its own sequence — without this, an interior ESC deleted
  *      the rest of the document via case 3.
  *   3. end of input, for a genuinely unterminated string: fail closed and drop
- *      everything from the introducer on, so no OSC body survives.
+ *      everything from the introducer on, so no body survives.
+ *
+ * BEL terminates every arm here, not just OSC. Only xterm's OSC parser accepts
+ * it, so a DCS ending at BEL over-consumes by the width of one body — the
+ * fail-closed direction, and the alternative (a per-introducer terminator set)
+ * makes a terminal that does accept it an under-strip.
  * @param {string} text
  * @param {number} start
- * @returns {number}
+ * @returns {{ end: number, kind: string } | null}
  */
-function scanOsc(text, start) {
+function scanControlString(text, start) {
   const code = text.charCodeAt(start);
-  let i = -1;
-  if (code === OSC_C1) i = start + 1;
-  if (code === ESC && text[start + 1] === "]") i = start + 2;
-  if (i < 0) return -1;
+  const second = code === ESC ? text[start + 1] : undefined;
+  const sevenBit = second !== undefined && STRING_INTRO_7BIT.includes(second);
+  if (!sevenBit && !STRING_INTRO_C1.has(code)) return null;
+  const kind =
+    second === OSC_7BIT || code === OSC_C1
+      ? TOKEN_KIND.OSC
+      : TOKEN_KIND.CONTROL_STRING;
+  let i = sevenBit ? start + 2 : start + 1;
   for (; i < text.length; i++) {
     const byte = text.charCodeAt(i);
-    if (byte === BEL || byte === ST_C1) return i + 1;
-    if (byte === ESC) return text[i + 1] === "\\" ? i + 2 : i;
-    if (byte === OSC_C1) return i;
+    if (byte === BEL || byte === ST_C1) return { end: i + 1, kind };
+    if (byte === ESC) return { end: text[i + 1] === "\\" ? i + 2 : i, kind };
+    if (STRING_INTRO_C1.has(byte)) return { end: i, kind };
   }
-  return text.length;
+  return { end: text.length, kind };
 }
 
 /**
@@ -261,14 +297,17 @@ export function scanAnsi(text) {
   let match;
   while ((match = INTRODUCER_SCAN_RE.exec(text)) !== null) {
     const start = match.index;
-    const oscEnd = scanOsc(text, start);
-    const csiEnd = oscEnd < 0 ? scanCsi(text, start) : -1;
+    // The string arms run FIRST: `ESC P` also completes a two-byte CSI (`P` is a
+    // final byte), so letting scanCsi answer first would take the DCS
+    // introducer alone and leave its body as visible text.
+    const string = scanControlString(text, start);
+    const csiEnd = string ? -1 : scanCsi(text, start);
     let end = start + 1;
     /** @type {string} */
     let kind = orphanKindFor(text[start], text[start + 1]);
-    if (oscEnd >= 0) {
-      end = oscEnd;
-      kind = TOKEN_KIND.OSC;
+    if (string) {
+      end = string.end;
+      kind = string.kind;
     } else if (csiEnd >= 0) {
       end = csiEnd;
       kind = SGR_ANCHORED_RE.test(text.slice(start, csiEnd))

package/src/layer1.mjs

@@ -131,10 +131,9 @@ export const INERT_ANSI_NOTE =
  * mid-escape).
  *
  * The two other orphan kinds are deliberately NOT inert. A raw C1 orphan
- * (TOKEN_KIND.ORPHAN_C1): legit UTF-8 text does not carry raw C1 bytes, and the
- * block holds the DCS/SOS/PM/APC string introducers this grammar does not
- * consume — so an unrecognized one means a terminal would have eaten the
- * following text as a control payload. An incomplete CSI (TOKEN_KIND.ORPHAN_CSI)
+ * (TOKEN_KIND.ORPHAN_C1): legit UTF-8 text does not carry raw C1 bytes, so an
+ * unrecognized one means a terminal may still act on what follows it. An
+ * incomplete CSI (TOKEN_KIND.ORPHAN_CSI)
  * for the same reason at 7 bits: the CSI parser keeps consuming until a final
  * byte, so `hello ESC[12 world` hides ` w` from the human while the model reads
  * the whole prompt.

package/types/ansi.d.mts

@@ -12,9 +12,10 @@ export function isOrphanKind(kind: string): boolean;
  * lone `ESC`. The one place that split is decided, shared by the tokenizer and
  * by Layer 1's residual sweep (which sees bare characters, not tokens).
  *
- * `[` is the only lookahead that matters. `ESC ]` is an OSC, which the scanner
- * consumes to the end of input if unterminated (so it never reaches here), and
- * every other second byte — `ESC (`, `ESC #`, `ESC P` — bounds what a terminal
+ * `[` is the only lookahead that matters. The five string introducers — `ESC ]`,
+ * `ESC P`, `ESC X`, `ESC ^`, `ESC _` — are consumed whole by
+ * {@link scanControlString}, to the end of input if unterminated, so they never
+ * reach here; every other second byte (`ESC (`, `ESC #`) bounds what a terminal
  * swallows to a byte or two rather than running until a final byte arrives.
  * @param {string} ch
  * @param {string} [next]  the following character, or undefined at end of input
@@ -64,7 +65,7 @@ export const CONTROL_INTRODUCER_SOURCE: string;
  * and the regex can no longer describe different languages.
  */
 export const SGR_RE: RegExp;
-/** The six things an introducer can turn out to be. */
+/** The seven things an introducer can turn out to be. */
 export const TOKEN_KIND: Readonly<{
     /** A display-only `ESC[…m` / `U+009B…m` colour sequence. */
     SGR: "sgr";
@@ -72,6 +73,13 @@ export const TOKEN_KIND: Readonly<{
     CSI: "csi";
     /** An OSC string: introducer, body and terminator as one unit. */
     OSC: "osc";
+    /**
+     * One of the other four ECMA-48 control strings — DCS, SOS, PM or APC —
+     * introducer, body and terminator as one unit, exactly like
+     * {@link TOKEN_KIND.OSC}. Split from it only so a warning can name what it
+     * found; both are payload-carrying strings and neither is benign.
+     */
+    CONTROL_STRING: "control-string";
     /**
      * A 7-bit `ESC` that starts no sequence the grammar recognizes — a truncated
      * write, a log fragment cut mid-escape, a stray byte living in a file.
@@ -92,11 +100,11 @@ export const TOKEN_KIND: Readonly<{
      * A RAW C1 byte (U+0080-U+009F) that starts no sequence the grammar
      * recognizes. Split from {@link TOKEN_KIND.ORPHAN} because the two carry very
      * different weight: a lone `ESC` is ordinary debris in terminal output, while
-     * a raw C1 byte is not something legitimate UTF-8 text produces, and the
-     * block includes the string introducers DCS/SOS/PM/APC (U+0090/0098/009E/
-     * 009F) — which this grammar does not consume, so an unrecognized one here
-     * means a terminal WOULD have swallowed the following text as a control
-     * payload. Consumers that downgrade an inert strip to a note (see
+     * a raw C1 byte is not something legitimate UTF-8 text produces. The five
+     * string introducers in the block open a {@link TOKEN_KIND.OSC} or
+     * {@link TOKEN_KIND.CONTROL_STRING} token instead, so a byte that reaches
+     * here is one the grammar recognizes no sequence for at all — and a terminal
+     * may still act on it. Consumers that downgrade an inert strip to a note (see
      * `isBenignAnsiKinds` in ./layer1.mjs) must keep warning on this one.
      */
     ORPHAN_C1: "orphan-c1-introducer";

package/types/layer1.d.mts

@@ -22,10 +22,9 @@ export function stripAnsiFully(input: string, kinds?: Set<string>): string;
  * mid-escape).
  *
  * The two other orphan kinds are deliberately NOT inert. A raw C1 orphan
- * (TOKEN_KIND.ORPHAN_C1): legit UTF-8 text does not carry raw C1 bytes, and the
- * block holds the DCS/SOS/PM/APC string introducers this grammar does not
- * consume — so an unrecognized one means a terminal would have eaten the
- * following text as a control payload. An incomplete CSI (TOKEN_KIND.ORPHAN_CSI)
+ * (TOKEN_KIND.ORPHAN_C1): legit UTF-8 text does not carry raw C1 bytes, so an
+ * unrecognized one means a terminal may still act on what follows it. An
+ * incomplete CSI (TOKEN_KIND.ORPHAN_CSI)
  * for the same reason at 7 bits: the CSI parser keeps consuming until a final
  * byte, so `hello ESC[12 world` hides ` w` from the human while the model reads
  * the whole prompt.