@sema-agent/server 7.133.0 → 7.135.0

package/dist/trace/redact.js

@@ -1,5 +1,5 @@
 import { displayUrlOf, stripFormatCharacters } from "@sema-agent/core";
-import { isHierarchicalUrl } from "@sema-agent/settings-schema";
+import { CONTROL_AND_BIDI_CHARS, isHierarchicalUrl } from "@sema-agent/settings-schema";
 import { recordFailOpen } from "../observability/fail-open.js";
 const normalizeForRedaction = stripFormatCharacters;
 export function mapNormalizedToOriginal(text, normalized) {
@@ -28,6 +28,185 @@ export function mapNormalizedToOriginal(text, normalized) {
     map[normalized.length] = text.length;
     return map;
 }
+const SPELLED_FORMAT = 1, SPELLED_CONTROL = 2, RAW_CONTROL = 4, RAW_WHITESPACE = 8;
+const INVISIBLE_UNIT_CLASS = (() => {
+    const inSet = new RegExp(`[${CONTROL_AND_BIDI_CHARS}]`);
+    const table = new Uint8Array(0x10000);
+    for (let unit = 0; unit < table.length; unit++) {
+        const ch = String.fromCharCode(unit);
+        if (inSet.test(ch))
+            table[unit] = /\s/.test(ch) ? 2 : 1;
+    }
+    return table;
+})();
+const RAW_STRIP_SET_RE = (() => {
+    let body = "";
+    for (let unit = 0; unit < 0x10000; unit++) {
+        if (INVISIBLE_UNIT_CLASS[unit] === 0)
+            continue;
+        let last = unit;
+        while (last + 1 < 0x10000 && INVISIBLE_UNIT_CLASS[last + 1] !== 0)
+            last += 1;
+        const hex = (u) => `\\u${u.toString(16).padStart(4, "0")}`;
+        body += last === unit ? hex(unit) : `${hex(unit)}-${hex(last)}`;
+        unit = last;
+    }
+    return new RegExp(`[${body}]`, "g");
+})();
+function collectRawControlCuts(s, allowed) {
+    const cuts = [];
+    let present = 0;
+    if ((allowed & (RAW_CONTROL | RAW_WHITESPACE)) === 0)
+        return { cuts, present };
+    RAW_STRIP_SET_RE.lastIndex = 0;
+    for (let m = RAW_STRIP_SET_RE.exec(s); m !== null; m = RAW_STRIP_SET_RE.exec(s)) {
+        const cls = INVISIBLE_UNIT_CLASS[s.charCodeAt(m.index)] === 2 ? RAW_WHITESPACE : RAW_CONTROL;
+        if ((cls & allowed) === 0)
+            continue;
+        cuts.push(m.index, m.index + 1, cls);
+        present |= cls;
+    }
+    return { cuts, present };
+}
+function collectSpelledCuts(s) {
+    const cuts = [];
+    let present = 0;
+    let i = s.indexOf("\\");
+    if (i < 0)
+        return { cuts, present };
+    const n = s.length;
+    const read = { end: 0 };
+    while (i >= 0) {
+        let body = i + 1;
+        while (body < n && s.charCodeAt(body) === CH_BACKSLASH)
+            body += 1;
+        const cp = readSpelledCodePoint(s, body, read);
+        const cls = cp < 0 ? 0 : isFormatCodePoint(cp) ? SPELLED_FORMAT : cp < 0x10000 && INVISIBLE_UNIT_CLASS[cp] === 1 ? SPELLED_CONTROL : 0;
+        if (cls !== 0) {
+            cuts.push(i, read.end, cls);
+            present |= cls;
+            body = read.end;
+        }
+        i = s.indexOf("\\", body);
+    }
+    return { cuts, present };
+}
+function deleteInvisibleCharacters(s, all, mask) {
+    const atList = [];
+    const cumList = [];
+    const parts = [];
+    let from = 0;
+    let removed = 0;
+    const { cuts } = all;
+    for (let k = 0; k < cuts.length; k += 3) {
+        if ((cuts[k + 2] & mask) === 0)
+            continue;
+        const a = cuts[k];
+        const b = cuts[k + 1];
+        parts.push(s.slice(from, a));
+        atList.push(a - removed);
+        removed += b - a;
+        cumList.push(removed);
+        from = b;
+    }
+    if (atList.length === 0)
+        return { text: s, at: undefined, cum: undefined };
+    parts.push(s.slice(from));
+    return { text: parts.join(""), at: Int32Array.from(atList), cum: Int32Array.from(cumList) };
+}
+function viewIndexInInput(fold, v) {
+    const at = fold.at;
+    let lo = 0;
+    let hi = at.length;
+    while (lo < hi) {
+        const mid = (lo + hi) >>> 1;
+        if (at[mid] <= v)
+            lo = mid + 1;
+        else
+            hi = mid;
+    }
+    return lo === 0 ? v : v + fold.cum[lo - 1];
+}
+function readHexRun(s, at, count) {
+    let v = 0;
+    for (let k = at; k < at + count; k++) {
+        const d = hexDigitValue(s.charCodeAt(k));
+        if (d < 0)
+            return -1;
+        v = v * 16 + d;
+    }
+    return v;
+}
+function readEscapeElement(s, at, out) {
+    const kind = s.charCodeAt(at);
+    if (kind === CH_x) {
+        out.end = at + 3;
+        return readHexRun(s, at + 1, 2);
+    }
+    if (kind !== CH_u)
+        return -1;
+    if (s.charCodeAt(at + 1) === CH_LBRACE) {
+        let v = 0;
+        let k = at + 2;
+        for (; k < s.length; k++) {
+            const d = hexDigitValue(s.charCodeAt(k));
+            if (d < 0)
+                break;
+            v = v * 16 + d;
+            if (v > 0x10ffff)
+                return -1;
+        }
+        if (k === at + 2 || s.charCodeAt(k) !== CH_RBRACE)
+            return -1;
+        out.end = k + 1;
+        return v;
+    }
+    out.end = at + 5;
+    return readHexRun(s, at + 1, 4);
+}
+function readSpelledCodePoint(s, at, out) {
+    const v = readEscapeElement(s, at, out);
+    if (v < 0xd800 || v > 0xdfff)
+        return v;
+    if (v > 0xdbff)
+        return -1;
+    let k = out.end;
+    if (s.charCodeAt(k) !== CH_BACKSLASH)
+        return -1;
+    while (s.charCodeAt(k) === CH_BACKSLASH)
+        k += 1;
+    const lo = readEscapeElement(s, k, out);
+    return lo >= 0xdc00 && lo <= 0xdfff ? 0x10000 + ((v - 0xd800) << 10) + (lo - 0xdc00) : -1;
+}
+const FORMAT_CODEPOINT_TABLE = (() => {
+    const table = new Uint8Array(0x10000);
+    const units = [];
+    const codes = [];
+    for (let c = 0; c < 0x10000; c++) {
+        if (c >= 0xd800 && c <= 0xdfff)
+            continue;
+        units.push(String.fromCharCode(c));
+        codes.push(c);
+    }
+    const kept = normalizeForRedaction(units.join(""));
+    let j = 0;
+    for (let i = 0; i < codes.length; i++) {
+        if (j < kept.length && kept.charCodeAt(j) === codes[i])
+            j += 1;
+        else
+            table[codes[i]] = 1;
+    }
+    if (j !== kept.length)
+        throw new Error("redact: format-character stripping is not a code-unit deletion over the BMP — refusing to derive the escape-spelled format-character table");
+    return table;
+})();
+function isFormatCodePoint(cp) {
+    if (cp > 0x10ffff || (cp >= 0xd800 && cp <= 0xdfff))
+        return false;
+    if (cp >= 0x10000)
+        return normalizeForRedaction(String.fromCodePoint(cp)) === "";
+    return FORMAT_CODEPOINT_TABLE[cp] === 1;
+}
 export const REDACTION_TOKEN_RE_SRC = String.raw `«redacted(?::[a-z-]{1,16}){0,2}»`;
 const ARM_VALUE_CLASSES = [];
 const atLeast = (cls, min) => {
@@ -42,7 +221,7 @@ const WORD_INTERIOR_CLASS = "[A-Za-z0-9]";
 const CH_TAB = 9, CH_LF = 10, CH_CR = 13, CH_ESC = 27, CH_SPACE = 32, CH_PLUS = 43, CH_MINUS = 45, CH_SLASH = 47;
 const CH_0 = 48, CH_9 = 57, CH_COLON = 58, CH_EQ = 61, CH_AT = 64, CH_A = 65, CH_Z = 90;
 const CH_LBRACKET = 91, CH_BACKSLASH = 92, CH_UNDERSCORE = 95;
-const CH_a = 97, CH_b = 98, CH_f = 102, CH_n = 110, CH_r = 114, CH_t = 116, CH_u = 117, CH_z = 122, CH_TILDE = 126;
+const CH_a = 97, CH_b = 98, CH_f = 102, CH_n = 110, CH_r = 114, CH_t = 116, CH_u = 117, CH_x = 120, CH_z = 122, CH_TILDE = 126;
 const CH_QUOTE = 34;
 const CH_CSI8 = 155;
 const CH_LAQUO = 171;
@@ -282,8 +461,9 @@ let urlSegmentRuns = 0;
 let urlSegmentReplays = 0;
 let ssh2ScanBytes = 0;
 let mappedUnits = 0;
+let readings = 0;
 export function armPoolBudgetForTest() {
-    return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, forensicRuns, forensicSkips, urlSegmentRuns, urlSegmentReplays, ssh2ScanBytes, mappedUnits };
+    return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, forensicRuns, forensicSkips, urlSegmentRuns, urlSegmentReplays, ssh2ScanBytes, mappedUnits, readings };
 }
 export function resetArmPoolBudgetForTest() {
     poolSpanCount = 0;
@@ -297,6 +477,7 @@ export function resetArmPoolBudgetForTest() {
     urlSegmentReplays = 0;
     ssh2ScanBytes = 0;
     mappedUnits = 0;
+    readings = 0;
 }
 const unitBit = (t, c) => (t[c >>> 5] >>> (c & 31)) & 1;
 let codeUnitDomain;
@@ -632,9 +813,15 @@ function collectStickySpans(a, text, at, out) {
     }
 }
 const WORD_SEGMENT_CLASS = "[A-Za-z0-9._~+/-]";
+const SCHEME_DELIMITER_CHARS = "\\\"'`“”|";
+if (/[\]^-]/.test(SCHEME_DELIMITER_CHARS))
+    throw new Error("redact: scheme delimiter table holds a character-class metacharacter — the class source is built without escaping it");
+const SCHEME_DELIMITER_CODES = new Set([...SCHEME_DELIMITER_CHARS].map((c) => c.charCodeAt(0)));
+const isSchemeDelimiterCode = (c) => SCHEME_DELIMITER_CODES.has(c);
+const SCHEME_SEPARATOR_CLASS = `[\\s${SCHEME_DELIMITER_CHARS.replace(/\\/g, "\\\\")}]`;
 const LEXICAL_KEYWORD_ARMS = new Set();
-const keywordArm = (keywordSrc, valueClass, min, tailSrc = "") => {
-    const re = new RegExp(`(?:${keywordSrc}\\s+${atLeast(valueClass, min)}${tailSrc}` +
+const keywordArm = (keywordSrc, valueClass, min, tailSrc = "", separatorClass = "\\s") => {
+    const re = new RegExp(`(?:${keywordSrc}${separatorClass}+${atLeast(valueClass, min)}${tailSrc}` +
         `|(?<!${WORD_SEGMENT_CLASS})${keywordSrc}(?:${twoStrongAhead(valueClass)}|${noConnectorAhead(valueClass)})${atLeast(valueClass, min)}${tailSrc})`, "g");
     LEXICAL_KEYWORD_ARMS.add(re);
     return re;
@@ -657,6 +844,12 @@ const CREDENTIAL_NAME_ALT = (() => {
     }
     return [...CREDENTIAL_NAME_WORDS].sort((a, b) => b.length - a.length || (a < b ? -1 : 1)).join("|");
 })();
+const CREDENTIAL_NAME_MAX = Math.max(...CREDENTIAL_NAME_WORDS.map((w) => w.length));
+const asciiLower = (s) => s.replace(/[A-Z]/g, (c) => c.toLowerCase());
+function credentialNameAnchor(name) {
+    return CREDENTIAL_ANCHOR_OF.get(asciiLower(name));
+}
+const CREDENTIAL_ANCHOR_OF = new Map(Object.entries(CREDENTIAL_NAME_ANCHORS));
 const STRUCTURAL_SECRET_PATTERNS = [
     [/-----BEGIN(?:[^-](?:[^-](?:[^-]|-(?!-))*)?)?PRIVATE KEY(?: BLOCK)?-----[\s\S]{0,8192}?-----END(?:[^-](?:[^-](?:[^-]|-(?!-))*)?)?PRIVATE KEY(?: BLOCK)?-----/g, "«redacted:private-key»"],
     [/---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----[\s\S]{0,8192}?---- END SSH2 ENCRYPTED PRIVATE KEY ----/g, "«redacted:private-key»"],
@@ -668,8 +861,8 @@ const PEM_FRAGMENT_TOKEN = "«redacted:private-key:partial»";
 const HEADLESS_PPK_TOKEN = "«redacted:private-key:partial»";
 const ANCHORED_VALUE_TOKEN = "«redacted»";
 const KEYWORD_SECRET_PATTERNS = [
-    [keywordArm("[Bb]earer", "[A-Za-z0-9._~+/-]", 20, "=*"), "Bearer «redacted»"],
-    [keywordArm("Basic", "[A-Za-z0-9+/]", 16, "={0,2}"), "Basic «redacted»"],
+    [keywordArm("[Bb]earer", "[A-Za-z0-9._~+/-]", 20, "=*", SCHEME_SEPARATOR_CLASS), "Bearer «redacted»"],
+    [keywordArm("Basic", "[A-Za-z0-9+/]", 16, "={0,2}", SCHEME_SEPARATOR_CLASS), "Basic «redacted»"],
     [keywordArm("[Tt]oken", "[A-Za-z0-9._~+/-]", 20), "token «redacted»"],
 ];
 const SK_FULL_ARM_MIN = 20;
@@ -1249,9 +1442,11 @@ const isRegexSpace = (c) => c === 32 || (c >= 9 && c <= 13) || c === 0xa0 || c =
 const isLineTerminator = (c) => c === 10 || c === 13 || c === 0x2028 || c === 0x2029;
 const isControlCode = (c) => c < 0x20 || c === 0x7f;
 const TOKEN_UNIT_Y = new RegExp(REDACTION_TOKEN_RE_SRC, "y");
-const CREDENTIAL_PARAM_Y = new RegExp(`(?:${CREDENTIAL_NAME_ALT})=`, "iy");
-const CREDENTIAL_NAME_SET = new Set(CREDENTIAL_NAME_WORDS);
-const CREDENTIAL_NAME_SCAN_MAX = Math.max(...CREDENTIAL_NAME_WORDS.map((w) => w.length)) * 3 + 1;
+export const CREDENTIAL_PARAM_NAMES = Object.freeze([...CREDENTIAL_NAME_WORDS]);
+export function isCredentialParamName(decodedName) {
+    return credentialNameAnchor(decodedName) !== undefined;
+}
+const CREDENTIAL_NAME_SCAN_MAX = CREDENTIAL_NAME_MAX * 3 + 1;
 function decodeAsciiName(name) {
     let out = "";
     for (let i = 0; i < name.length; i++) {
@@ -1272,9 +1467,6 @@ function decodeAsciiName(name) {
     return out;
 }
 function credentialParamValueStart(text, p) {
-    CREDENTIAL_PARAM_Y.lastIndex = p;
-    if (CREDENTIAL_PARAM_Y.test(text))
-        return CREDENTIAL_PARAM_Y.lastIndex;
     const limit = Math.min(text.length, p + CREDENTIAL_NAME_SCAN_MAX);
     let hasPct = false;
     for (let i = p; i < limit; i++) {
@@ -1283,12 +1475,14 @@ function credentialParamValueStart(text, p) {
             hasPct = true;
             continue;
         }
-        if (c !== CH_EQ)
-            continue;
-        if (!hasPct)
+        if (c === CH_EQ) {
+            if (i === p)
+                return -1;
+            const name = hasPct ? decodeAsciiName(text.slice(p, i)) : text.slice(p, i);
+            return name !== undefined && credentialNameAnchor(name) !== undefined ? i + 1 : -1;
+        }
+        if (!(isAsciiAlnumCode(c) || c === CH_UNDERSCORE || c === CH_MINUS))
             return -1;
-        const decoded = decodeAsciiName(text.slice(p, i));
-        return decoded !== undefined && CREDENTIAL_NAME_SET.has(decoded.toLowerCase()) ? i + 1 : -1;
     }
     return -1;
 }
@@ -1697,24 +1891,19 @@ function collectUrlCredentialSpans(scan, text, json, span) {
     scan.footHi = armHi;
 }
 const JSON_FIELD_ANCHOR = '"';
-const CREDENTIAL_NAME_MAX = Math.max(...CREDENTIAL_NAME_WORDS.map((w) => w.length));
 function jsonSecretFieldValueQuoteAt(text, p) {
     const n = text.length;
-    let name = "";
     let i = p + 1;
     for (; i < n; i++) {
-        let c = text.charCodeAt(i);
+        const c = text.charCodeAt(i);
         if (c === CH_QUOTE)
             break;
         if (i - p > CREDENTIAL_NAME_MAX)
             return -1;
-        if (c >= CH_A && c <= CH_Z)
-            c += 32;
-        if (!((c >= CH_a && c <= CH_z) || (c >= CH_0 && c <= CH_9) || c === CH_UNDERSCORE || c === CH_MINUS))
+        if (!(isAsciiAlnumCode(c) || c === CH_UNDERSCORE || c === CH_MINUS))
             return -1;
-        name += String.fromCharCode(c);
     }
-    if (i >= n || !CREDENTIAL_NAME_SET.has(name))
+    if (i >= n || i === p + 1 || credentialNameAnchor(text.slice(p + 1, i)) === undefined)
         return -1;
     i += 1;
     while (i < n && isRegexSpace(text.charCodeAt(i)))
@@ -1759,7 +1948,6 @@ const STRONG_AUTH_SCHEMES = new Set(["basic", "bearer"]);
 const STRONG_AUTH_SCHEME_MAX = Math.max(...[...STRONG_AUTH_SCHEMES].map((w) => w.length));
 const ABSENT_OR_BOOLEAN_WORDS = new Set(["null", "nil", "none", "undefined", "true", "false", "yes", "no"]);
 const ABSENT_OR_BOOLEAN_MAX = Math.max(...[...ABSENT_OR_BOOLEAN_WORDS].map((w) => w.length));
-const CREDENTIAL_ANCHOR_OF = new Map(Object.entries(CREDENTIAL_NAME_ANCHORS));
 const CH_GRAVE = 96, CH_APOS = 39, CH_DOT = 46, CH_COMMA = 44, CH_SEMI = 59, CH_DOLLAR = 36;
 const CH_LPAREN = 40, CH_RPAREN = 41, CH_RBRACKET = 93, CH_LBRACE = 123, CH_RBRACE = 125, CH_LT = 60, CH_GT = 62;
 const CH_QMARK = 63, CH_AMP = 38, CH_HASH = 35;
@@ -1772,7 +1960,7 @@ const isValueCloser = (ch) => ch === CH_DOT || ch === CH_COMMA || ch === CH_SEMI
 const isInlineSpaceUnit = (ch) => ch === CH_SPACE || ch === CH_TAB;
 const isToken68Code = (c) => isAsciiAlnumCode(c) || c === CH_DOT || c === CH_MINUS || c === CH_UNDERSCORE || c === CH_TILDE || c === CH_PLUS || c === CH_SLASH;
 const endsValueUnit = (ch) => ch === JSON_STRING_END || isRegexSpace(ch) || isQuoteUnit(ch) || isValueCloser(ch);
-const endsHeaderValueUnit = (ch) => ch === JSON_STRING_END || isLineTerminator(ch) || isQuoteUnit(ch) || ch === CH_COMMA || ch === CH_SEMI || ch === CH_RPAREN || ch === CH_RBRACKET || ch === CH_RBRACE || ch === CH_GT;
+const endsHeaderValueUnit = (ch) => ch === JSON_STRING_END || isLineTerminator(ch) || isSchemeDelimiterCode(ch) || ch === CH_COMMA || ch === CH_SEMI || ch === CH_RPAREN || ch === CH_RBRACKET || ch === CH_RBRACE || ch === CH_GT;
 const bracketOf = (c) => c === CH_LPAREN ? 1 : c === CH_RPAREN ? -1 : c === CH_LBRACKET ? 2 : c === CH_RBRACKET ? -2 : c === CH_LBRACE ? 3 : c === CH_RBRACE ? -3 : c === CH_LT ? 4 : c === CH_GT ? -4 : 0;
 let anchoredScanUnits = 0;
 export function anchoredScanUnitsForTest() {
@@ -1884,7 +2072,7 @@ function anchorHeadAt(text, op, json) {
             return undefined;
         before = j - openLen;
     }
-    const strength = CREDENTIAL_ANCHOR_OF.get(text.slice(nameStart, nameEnd).toLowerCase());
+    const strength = credentialNameAnchor(text.slice(nameStart, nameEnd));
     if (strength === undefined)
         return undefined;
     if (!afterEscape && runStart === j + 1 && quote === 0 && j >= 0 && text.charCodeAt(j) === CH_SLASH)
@@ -2016,15 +2204,16 @@ function schemeValueAt(text, s, json) {
     anchoredScanUnits += i - w;
     if (i - w > STRONG_AUTH_SCHEME_MAX || !STRONG_AUTH_SCHEMES.has(text.slice(w, i).toLowerCase()))
         return undefined;
-    let spaced = false;
+    let separated = false;
     while (i < n) {
         const unit = readAnchorUnit(text, i, json);
-        if (!isInlineSpaceUnit(unitChOf(unit)))
+        const ch = unitChOf(unit);
+        if (!isInlineSpaceUnit(ch) && !isSchemeDelimiterCode(ch))
             break;
         i += unitLenOf(unit);
-        spaced = true;
+        separated = true;
     }
-    if (!spaced)
+    if (!separated)
         return undefined;
     const vs = i;
     if (i >= n || !isAsciiAlnumCode(text.charCodeAt(i)))
@@ -2118,7 +2307,7 @@ export function structuralScannerForTest(arm) {
 let ppkScanBytes = 0;
 export function ppkScanBytesForTest() { return ppkScanBytes; }
 export function resetPpkScanBytesForTest() { ppkScanBytes = 0; }
-function collectPpkContainerSpans(scan, text, record, span) {
+function collectPpkContainerSpans(scan, text, eat, span) {
     const HEAD = "PuTTY-User-Key-File-";
     const MAC = "Private-MAC: ";
     const HASH = "Private-Hash: ";
@@ -2204,8 +2393,10 @@ function collectPpkContainerSpans(scan, text, record, span) {
             end = ateToEnd ? text.length : hexEnd;
             break;
         }
-        if (ateToEnd && record)
-            recordFailOpen("server.redact.ppk-eat-to-end", `from=${at} len=${text.length}`);
+        if (ateToEnd && eat !== undefined && eat.from < 0) {
+            eat.from = at;
+            eat.len = text.length;
+        }
         span(at, end);
         at = end >= text.length ? Number.POSITIVE_INFINITY : nextVersionedHead(end);
     }
@@ -2238,8 +2429,8 @@ const armPool = (() => {
             phase: 1,
             lexical: LEXICAL_KEYWORD_ARMS.has(re),
             collect: boundary === undefined
-                ? (text, _json, _observe, out) => collectRegexSpans(arm, text, out)
-                : (text, _json, _observe, out) => collectValueArmSpans(boundary, text, out),
+                ? (text, _json, _eat, out) => collectRegexSpans(arm, text, out)
+                : (text, _json, _eat, out) => collectValueArmSpans(boundary, text, out),
             retry: boundary === undefined ? undefined : (text, at, out) => collectStickySpans(boundary, text, at, out),
             headSrc: boundary === undefined ? undefined : headSrc,
             headFirst: boundary === undefined ? undefined : headFirst,
@@ -2253,7 +2444,7 @@ const armPool = (() => {
             order,
             phase: 1,
             lexical,
-            collect: (text, json, observe, out) => arm(text, json, observe, (start, end) => out.push({ start, end, repl, rank, order })),
+            collect: (text, json, eat, out) => arm(text, json, eat, (start, end) => out.push({ start, end, repl, rank, order })),
             retry: undefined,
             headSrc: undefined,
             headFirst: undefined,
@@ -2268,7 +2459,7 @@ const armPool = (() => {
             order,
             phase: 2,
             lexical: false,
-            collect: (scan, text, json, observe, out) => arm(scan, text, json, observe, (start, end) => out.push({ start, end, repl, rank, order })),
+            collect: (scan, text, json, eat, out) => arm(scan, text, json, eat, (start, end) => out.push({ start, end, repl, rank, order })),
             retry: undefined,
             headSrc: undefined,
             headFirst: undefined,
@@ -2276,22 +2467,22 @@ const armPool = (() => {
     };
     const structuralScannerMember = (row, order) => {
         if (row === PEM_FENCE_PATTERN)
-            return scannerMember(row[1], order, (text, _json, _observe, span) => collectPemFenceSpans(text, span));
+            return scannerMember(row[1], order, (text, _json, _eat, span) => collectPemFenceSpans(text, span));
         if (row === SSH2_FENCE_PATTERN)
-            return forensicMember(row[1], order, (scan, text, _json, _observe, span) => collectSsh2FenceSpans(scan, text, span));
+            return forensicMember(row[1], order, (scan, text, _json, _eat, span) => collectSsh2FenceSpans(scan, text, span));
         if (row === URL_CREDENTIAL_PATTERN)
-            return forensicMember(row[1], order, (scan, text, json, _observe, span) => collectUrlCredentialSpans(scan, text, json, span));
+            return forensicMember(row[1], order, (scan, text, json, _eat, span) => collectUrlCredentialSpans(scan, text, json, span));
         if (row === JSON_SECRET_FIELD_PATTERN)
-            return scannerMember(row[1], order, (text, _json, _observe, span) => collectJsonSecretFieldSpans(text, span));
+            return scannerMember(row[1], order, (text, _json, _eat, span) => collectJsonSecretFieldSpans(text, span));
         throw new Error(`redact: structural row /${row[0].source}/ has no hand-written scanner — structural grammars are declared as regexes but never executed as one (S-102 / S-265)`);
     };
     const members = [];
     for (const row of STRUCTURAL_SECRET_PATTERNS)
         members.push(structuralScannerMember(row, members.length));
-    members.push(forensicMember(PPK_CONTAINER_TOKEN, members.length, (scan, text, _json, observe, span) => collectPpkContainerSpans(scan, text, observe, span)));
-    members.push(forensicMember(PEM_FRAGMENT_TOKEN, members.length, (scan, text, json, _observe, span) => collectPemFragmentSpans(scan, text, json, span)));
-    members.push(forensicMember(HEADLESS_PPK_TOKEN, members.length, (scan, text, _json, _observe, span) => collectHeadlessPpkSpans(scan, text, span)));
-    members.push(scannerMember(ANCHORED_VALUE_TOKEN, members.length, (text, json, _observe, span) => collectAnchoredValueSpans(text, json, span), ARM_RANK.keyword, true));
+    members.push(forensicMember(PPK_CONTAINER_TOKEN, members.length, (scan, text, _json, eat, span) => collectPpkContainerSpans(scan, text, eat, span)));
+    members.push(forensicMember(PEM_FRAGMENT_TOKEN, members.length, (scan, text, json, _eat, span) => collectPemFragmentSpans(scan, text, json, span)));
+    members.push(forensicMember(HEADLESS_PPK_TOKEN, members.length, (scan, text, _json, _eat, span) => collectHeadlessPpkSpans(scan, text, span)));
+    members.push(scannerMember(ANCHORED_VALUE_TOKEN, members.length, (text, json, _eat, span) => collectAnchoredValueSpans(text, json, span), ARM_RANK.keyword, true));
     for (const row of KEYWORD_SECRET_PATTERNS)
         members.push(regexMember(row, ARM_RANK.keyword, members.length));
     for (const row of SHAPE_SECRET_PATTERNS)
@@ -2322,7 +2513,8 @@ const armPool = (() => {
     const headFirstUnit = unionMembers(retryMembers.map((m) => m.headFirst)) ?? new Uint32Array(0x10000 >>> 5).fill(0xffffffff);
     poolArmCount = members.length;
     poolBoundaryArmCount = retryMembers.length;
-    return (text, wantCount, json, observe, scope) => {
+    return (text, wantCount, json, eat, scope) => {
+        readings += 1;
         const lane = LANES[scope];
         const spans = [];
         let tried;
@@ -2358,7 +2550,7 @@ const armPool = (() => {
             }
         };
         for (const m of lane.phase1)
-            m.collect(text, json, observe, spans);
+            m.collect(text, json, eat, spans);
         closeCoveragePositions(0);
         if (lane.phase2.length > 0) {
             const scan = buildForensicScan(credentialSpans(text), lane.phase2.length);
@@ -2375,7 +2567,7 @@ const armPool = (() => {
                     }
                     scan.footLo = FOOT_NONE;
                     scan.footHi = -1;
-                    lane.phase2[j].collect(scan, text, json, observe && level === 0, spans);
+                    lane.phase2[j].collect(scan, text, json, level === 0 ? eat : undefined, spans);
                     foot[2 * j] = scan.footLo;
                     foot[2 * j + 1] = scan.footHi;
                     forensicRuns += 1;
@@ -2396,6 +2588,7 @@ const armPool = (() => {
         const tokenOf = (sp) => members[sp.order].token;
         const strong = spans.filter((sp) => armOfToken(tokenOf(sp)) === "full").sort((a, b) => a.start - b.start);
         const byArmedToken = new Map();
+        const stamped = [];
         let si = 0;
         let strongMaxEnd = -1;
         for (const sp of kept) {
@@ -2411,16 +2604,17 @@ const armPool = (() => {
                 byArmedToken.set(key, { token, arm, count: 1 });
             else
                 cur.count += 1;
+            stamped.push({ ...sp, hit: { token, arm } });
         }
-        return { hits: [...byArmedToken.values()].map(({ token, arm, count }) => ({ label: labelOfToken(token), count, arm })), kept };
+        return { hits: [...byArmedToken.values()].map(({ token, arm, count }) => ({ label: labelOfToken(token), count, arm })), kept: stamped };
     };
 })();
 let onRedaction;
 export function setRedactionObserver(fn) {
     onRedaction = fn;
 }
-function applyRules(text, wantCount, json, observe, scope) {
-    return armPool(text, wantCount, json, observe, scope);
+function applyRules(text, wantCount, json, eat, scope) {
+    return armPool(text, wantCount, json, eat, scope);
 }
 function reportHits(hits) {
     if (onRedaction === undefined)
@@ -2433,35 +2627,111 @@ function reportHits(hits) {
         }
     }
 }
-function runRedaction(text, observe, json, wantHits, scope) {
+function invisibleClassesOf(layout) {
+    switch (layout) {
+        case "multi-line":
+            return SPELLED_FORMAT | SPELLED_CONTROL | RAW_CONTROL;
+        case "single-line":
+            return SPELLED_FORMAT | SPELLED_CONTROL | RAW_CONTROL | RAW_WHITESPACE;
+        default: {
+            const unhandled = layout;
+            throw new Error(`redact: unhandled layout ${String(unhandled)}`);
+        }
+    }
+}
+function runRedaction(text, observe, json, wantHits, scope, layout) {
     const wantCount = wantHits || (onRedaction !== undefined && observe);
-    const normalized = normalizeForRedaction(text);
-    const r = applyRules(normalized, wantCount, json, observe, scope);
-    forgetEscapedSeparatorTails();
+    const eat = observe ? { from: -1, len: 0 } : undefined;
+    let spans = [];
+    let hits = NO_HITS;
+    let readsSoFar = 0;
+    let unioned = false;
+    const read = (layers, view) => {
+        const r = applyRules(view, wantCount, json, eat, scope);
+        forgetEscapedSeparatorTails();
+        readsSoFar += 1;
+        if (readsSoFar === 1) {
+            spans = spansInOriginal(layers, r.kept);
+            hits = r.hits;
+            return;
+        }
+        const extra = spansInOriginal(layers, r.kept);
+        if (extra.length === 0)
+            return;
+        spans = resolveSpanOverlaps([...spans, ...extra]);
+        unioned = true;
+    };
+    const raw = collectRawControlCuts(text, invisibleClassesOf(layout));
+    for (let rmask = 0;; rmask = (rmask - raw.present) & raw.present) {
+        const rfold = deleteInvisibleCharacters(text, raw, rmask);
+        const normalized = normalizeForRedaction(rfold.text);
+        const spelled = collectSpelledCuts(normalized);
+        for (let smask = 0;; smask = (smask - spelled.present) & spelled.present) {
+            const sfold = deleteInvisibleCharacters(normalized, spelled, smask);
+            read([{ fold: sfold }, { input: rfold.text, view: normalized }, { fold: rfold }], sfold.text);
+            if (smask === spelled.present)
+                break;
+        }
+        if (rmask === raw.present)
+            break;
+    }
+    if (unioned && wantCount)
+        hits = hitsOfSpans(spans);
+    if (eat !== undefined && eat.from >= 0)
+        recordFailOpen("server.redact.ppk-eat-to-end", `from=${eat.from} len=${eat.len}`);
     if (observe)
-        reportHits(r.hits);
-    const spans = spansInOriginal(text, normalized, r.kept);
-    return { text: applySpans(text, spans).out, hits: r.hits, spans };
+        reportHits(hits);
+    return { text: applySpans(text, spans).out, hits, spans };
+}
+function hitsOfSpans(spans) {
+    const by = new Map();
+    for (const sp of spans) {
+        if (sp.hit === undefined)
+            continue;
+        const key = `${sp.hit.token}\u0000${sp.hit.arm}`;
+        const cur = by.get(key);
+        if (cur === undefined)
+            by.set(key, { token: sp.hit.token, arm: sp.hit.arm, count: 1 });
+        else
+            cur.count += 1;
+    }
+    return [...by.values()].map(({ token, arm, count }) => ({ label: labelOfToken(token), count, arm }));
 }
-function spansInOriginal(text, normalized, kept) {
-    if (kept.length === 0 || normalized === text)
-        return kept;
-    const map = mapNormalizedToOriginal(text, normalized);
-    return kept.map((s) => ({ ...s, start: map[s.start], end: map[s.end - 1] + 1 }));
+function spansInOriginal(layers, kept) {
+    let out = kept;
+    for (const layer of layers) {
+        if (out.length === 0)
+            return out;
+        let at;
+        if ("fold" in layer) {
+            const fold = layer.fold;
+            if (fold.at === undefined)
+                continue;
+            at = (v) => viewIndexInInput(fold, v);
+        }
+        else {
+            if (layer.view === layer.input)
+                continue;
+            const map = mapNormalizedToOriginal(layer.input, layer.view);
+            at = (v) => map[v];
+        }
+        out = out.map((s) => ({ ...s, start: at(s.start), end: at(s.end - 1) + 1 }));
+    }
+    return out;
 }
 export function redactSecrets(text, opts) {
-    return runRedaction(text, opts?.observe ?? true, opts?.json === true, false, "free-text").text;
+    return runRedaction(text, opts?.observe ?? true, opts?.json === true, false, "free-text", "multi-line").text;
 }
 export function redactSecretsWithArms(text, opts) {
-    const r = runRedaction(text, opts?.observe ?? true, opts?.json === true, true, "free-text");
+    const r = runRedaction(text, opts?.observe ?? true, opts?.json === true, true, "free-text", "multi-line");
     return { text: r.text, arms: new Set(r.hits.map((h) => h.arm)) };
 }
 export function redactionSpansForTest(text, opts) {
-    const r = runRedaction(text, false, opts?.json === true, false, "free-text");
+    const r = runRedaction(text, false, opts?.json === true, false, "free-text", "multi-line");
     return { text: r.text, spans: r.spans.map((sp) => ({ start: sp.start, end: sp.end })) };
 }
-export function proposeRedactionSpans(text, scope) {
-    return runRedaction(text, true, false, false, scope).spans.map((sp) => ({ start: sp.start, end: sp.end, repl: sp.repl }));
+export function proposeRedactionSpans(text, scope, layout) {
+    return runRedaction(text, true, false, false, scope, layout).spans.map((sp) => ({ start: sp.start, end: sp.end, repl: sp.repl }));
 }
 const REDACT_MAX_DEPTH = 64;
 export function redactDeep(value) {
@@ -2563,7 +2833,7 @@ function unitCarriesCredential(unit) {
         for (const v of armValueClassViews())
             views.add(boundedUnitView(spelling, v.inClass, v.filler));
     views.delete("");
-    const hit = [...views].some((view) => runRedaction(view, true, false, false, "bounded-unit").text !== view);
+    const hit = [...views].some((view) => runRedaction(view, true, false, false, "bounded-unit", "multi-line").text !== view);
     return { hit, unreadable: decoded.residue };
 }
 function displaySafePathname(pathname) {
@@ -2593,7 +2863,7 @@ function shapeLawCredentialInQueryOrFragment(canonical) {
     const boundary = canonical.indexOf("?");
     if (boundary < 0)
         return false;
-    if (runRedaction(canonical, false, false, false, "free-text").spans.some((sp) => sp.end > boundary))
+    if (runRedaction(canonical, false, false, false, "free-text", "multi-line").spans.some((sp) => sp.end > boundary))
         return true;
     for (const unit of canonical.slice(boundary + 1).split("&").flatMap((sg) => {
         const eq = sg.indexOf("=");