@sema-agent/server 7.134.0 → 7.136.0

package/dist/trace/redact.js

@@ -1,5 +1,5 @@
 import { displayUrlOf, stripFormatCharacters } from "@sema-agent/core";
-import { isHierarchicalUrl } from "@sema-agent/settings-schema";
+import { CONTROL_AND_BIDI_CHARS, CREDENTIAL_PARAM_NAMES, isHierarchicalUrl } from "@sema-agent/settings-schema";
 import { recordFailOpen } from "../observability/fail-open.js";
 const normalizeForRedaction = stripFormatCharacters;
 export function mapNormalizedToOriginal(text, normalized) {
@@ -28,43 +28,91 @@ export function mapNormalizedToOriginal(text, normalized) {
     map[normalized.length] = text.length;
     return map;
 }
-function deleteSpelledFormatCharacters(s) {
+const SPELLED_FORMAT = 1, SPELLED_CONTROL = 2, RAW_CONTROL = 4, RAW_WHITESPACE = 8;
+const INVISIBLE_UNIT_CLASS = (() => {
+    const inSet = new RegExp(`[${CONTROL_AND_BIDI_CHARS}]`);
+    const table = new Uint8Array(0x10000);
+    for (let unit = 0; unit < table.length; unit++) {
+        const ch = String.fromCharCode(unit);
+        if (inSet.test(ch))
+            table[unit] = /\s/.test(ch) ? 2 : 1;
+    }
+    return table;
+})();
+const RAW_STRIP_SET_RE = (() => {
+    let body = "";
+    for (let unit = 0; unit < 0x10000; unit++) {
+        if (INVISIBLE_UNIT_CLASS[unit] === 0)
+            continue;
+        let last = unit;
+        while (last + 1 < 0x10000 && INVISIBLE_UNIT_CLASS[last + 1] !== 0)
+            last += 1;
+        const hex = (u) => `\\u${u.toString(16).padStart(4, "0")}`;
+        body += last === unit ? hex(unit) : `${hex(unit)}-${hex(last)}`;
+        unit = last;
+    }
+    return new RegExp(`[${body}]`, "g");
+})();
+function collectRawControlCuts(s, allowed) {
+    const cuts = [];
+    let present = 0;
+    if ((allowed & (RAW_CONTROL | RAW_WHITESPACE)) === 0)
+        return { cuts, present };
+    RAW_STRIP_SET_RE.lastIndex = 0;
+    for (let m = RAW_STRIP_SET_RE.exec(s); m !== null; m = RAW_STRIP_SET_RE.exec(s)) {
+        const cls = INVISIBLE_UNIT_CLASS[s.charCodeAt(m.index)] === 2 ? RAW_WHITESPACE : RAW_CONTROL;
+        if ((cls & allowed) === 0)
+            continue;
+        cuts.push(m.index, m.index + 1, cls);
+        present |= cls;
+    }
+    return { cuts, present };
+}
+function collectSpelledCuts(s) {
+    const cuts = [];
+    let present = 0;
     let i = s.indexOf("\\");
     if (i < 0)
-        return { text: s, at: undefined, cum: undefined };
+        return { cuts, present };
     const n = s.length;
     const read = { end: 0 };
-    let cuts;
     while (i >= 0) {
         let body = i + 1;
         while (body < n && s.charCodeAt(body) === CH_BACKSLASH)
             body += 1;
         const cp = readSpelledCodePoint(s, body, read);
-        if (cp >= 0 && isFormatCodePoint(cp)) {
-            (cuts ??= []).push(i, read.end);
+        const cls = cp < 0 ? 0 : isFormatCodePoint(cp) ? SPELLED_FORMAT : cp < 0x10000 && INVISIBLE_UNIT_CLASS[cp] === 1 ? SPELLED_CONTROL : 0;
+        if (cls !== 0) {
+            cuts.push(i, read.end, cls);
+            present |= cls;
             body = read.end;
         }
         i = s.indexOf("\\", body);
     }
-    if (cuts === undefined)
-        return { text: s, at: undefined, cum: undefined };
-    const count = cuts.length / 2;
-    const at = new Int32Array(count);
-    const cum = new Int32Array(count);
+    return { cuts, present };
+}
+function deleteInvisibleCharacters(s, all, mask) {
+    const atList = [];
+    const cumList = [];
     const parts = [];
     let from = 0;
     let removed = 0;
-    for (let k = 0; k < count; k++) {
-        const a = cuts[2 * k];
-        const b = cuts[2 * k + 1];
+    const { cuts } = all;
+    for (let k = 0; k < cuts.length; k += 3) {
+        if ((cuts[k + 2] & mask) === 0)
+            continue;
+        const a = cuts[k];
+        const b = cuts[k + 1];
         parts.push(s.slice(from, a));
-        at[k] = a - removed;
+        atList.push(a - removed);
         removed += b - a;
-        cum[k] = removed;
+        cumList.push(removed);
         from = b;
     }
+    if (atList.length === 0)
+        return { text: s, at: undefined, cum: undefined };
     parts.push(s.slice(from));
-    return { text: parts.join(""), at, cum };
+    return { text: parts.join(""), at: Int32Array.from(atList), cum: Int32Array.from(cumList) };
 }
 function viewIndexInInput(fold, v) {
     const at = fold.at;
@@ -413,8 +461,9 @@ let urlSegmentRuns = 0;
 let urlSegmentReplays = 0;
 let ssh2ScanBytes = 0;
 let mappedUnits = 0;
+let readings = 0;
 export function armPoolBudgetForTest() {
-    return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, forensicRuns, forensicSkips, urlSegmentRuns, urlSegmentReplays, ssh2ScanBytes, mappedUnits };
+    return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, forensicRuns, forensicSkips, urlSegmentRuns, urlSegmentReplays, ssh2ScanBytes, mappedUnits, readings };
 }
 export function resetArmPoolBudgetForTest() {
     poolSpanCount = 0;
@@ -428,6 +477,7 @@ export function resetArmPoolBudgetForTest() {
     urlSegmentReplays = 0;
     ssh2ScanBytes = 0;
     mappedUnits = 0;
+    readings = 0;
 }
 const unitBit = (t, c) => (t[c >>> 5] >>> (c & 31)) & 1;
 let codeUnitDomain;
@@ -786,14 +836,19 @@ const CREDENTIAL_NAME_ANCHORS = {
     "x-amz-signature": "structure-only", "x-amz-security-token": "structure-only", "x-amz-credential": "structure-only",
     sas: "structure-only",
 };
-const CREDENTIAL_NAME_WORDS = Object.keys(CREDENTIAL_NAME_ANCHORS);
 const CREDENTIAL_NAME_ALT = (() => {
-    for (const w of CREDENTIAL_NAME_WORDS) {
+    for (const w of CREDENTIAL_PARAM_NAMES) {
         if (!/^[a-z][a-z0-9_-]*$/.test(w))
             throw new Error(`redact: credential name ${JSON.stringify(w)} must be lowercase [a-z0-9_-] (the alternation is built without escaping)`);
     }
-    return [...CREDENTIAL_NAME_WORDS].sort((a, b) => b.length - a.length || (a < b ? -1 : 1)).join("|");
+    return [...CREDENTIAL_PARAM_NAMES].sort((a, b) => b.length - a.length || (a < b ? -1 : 1)).join("|");
 })();
+const CREDENTIAL_NAME_MAX = Math.max(...CREDENTIAL_PARAM_NAMES.map((w) => w.length));
+const asciiLower = (s) => s.replace(/[A-Z]/g, (c) => c.toLowerCase());
+function credentialNameAnchor(name) {
+    return CREDENTIAL_ANCHOR_OF.get(asciiLower(name));
+}
+const CREDENTIAL_ANCHOR_OF = new Map(Object.entries(CREDENTIAL_NAME_ANCHORS));
 const STRUCTURAL_SECRET_PATTERNS = [
     [/-----BEGIN(?:[^-](?:[^-](?:[^-]|-(?!-))*)?)?PRIVATE KEY(?: BLOCK)?-----[\s\S]{0,8192}?-----END(?:[^-](?:[^-](?:[^-]|-(?!-))*)?)?PRIVATE KEY(?: BLOCK)?-----/g, "«redacted:private-key»"],
     [/---- BEGIN SSH2 ENCRYPTED PRIVATE KEY ----[\s\S]{0,8192}?---- END SSH2 ENCRYPTED PRIVATE KEY ----/g, "«redacted:private-key»"],
@@ -1029,14 +1084,17 @@ function isPemFenceLabel(s, from, to) {
         return false;
     if (!s.startsWith(PEM_KEY_WORD, end - PEM_KEY_WORD.length))
         return false;
+    return !pemLabelDashRuleBroken(s, from, to);
+}
+function pemLabelDashRuleBroken(s, from, to) {
     if (s.charCodeAt(from) === 45)
-        return false;
+        return true;
     if (s.charCodeAt(from + 1) === 45)
-        return false;
+        return true;
     for (let i = from + 1; i < to; i++)
         if (s.charCodeAt(i) === 45 && s.charCodeAt(i + 1) === 45)
-            return false;
-    return true;
+            return true;
+    return false;
 }
 const PEM_FENCE_OPEN = "-----BEGIN";
 const PEM_FENCE_CLOSE = "-----";
@@ -1081,7 +1139,7 @@ function unitCountWithin(scan, from, to, max) {
     }
     return n <= max;
 }
-function collectSsh2FenceSpans(scan, text, span) {
+function collectSsh2FenceSpans(scan, text, span, isHead) {
     let cachedFrom = 0;
     let cachedAt;
     const nextClose = (from) => {
@@ -1107,6 +1165,10 @@ function collectSsh2FenceSpans(scan, text, span) {
             continue;
         }
         const bodyFrom = fence + SSH2_FENCE_OPEN.length;
+        if (isHead && unitCountWithin(scan, bodyFrom, text.length, SSH2_BODY_MAX)) {
+            span(fence, text.length);
+            return;
+        }
         const close = nextClose(bodyFrom);
         if (close < 0)
             return;
@@ -1121,11 +1183,20 @@ function collectSsh2FenceSpans(scan, text, span) {
 }
 const PEM_END_OPEN = "-----END";
 const PEM_FENCE_BODY_MAX = 8192;
+const PEM_LINE_INVALID = -1;
+const PEM_LINE_OPEN = -2;
 function pemFenceLineEnd(text, labelFrom) {
     const close = text.indexOf(PEM_FENCE_CLOSE, labelFrom);
-    return close >= 0 && isPemFenceLabel(text, labelFrom, close) ? close + PEM_FENCE_CLOSE.length : -1;
+    if (close < 0) {
+        let to = text.length;
+        while (to > labelFrom && text.charCodeAt(to - 1) === 45)
+            to -= 1;
+        const stillPossible = text.length - to >= 2 ? isPemFenceLabel(text, labelFrom, to) : !pemLabelDashRuleBroken(text, labelFrom, to);
+        return stillPossible ? PEM_LINE_OPEN : PEM_LINE_INVALID;
+    }
+    return isPemFenceLabel(text, labelFrom, close) ? close + PEM_FENCE_CLOSE.length : PEM_LINE_INVALID;
 }
-function collectPemFenceSpans(text, span) {
+function collectPemFenceSpans(text, span, isHead) {
     let endFrom = -1;
     let endAt = -1;
     let endAfter = -1;
@@ -1145,13 +1216,17 @@ function collectPemFenceSpans(text, span) {
         if (fence < 0)
             return;
         const bodyFrom = pemFenceLineEnd(text, fence + PEM_FENCE_OPEN.length);
-        if (bodyFrom < 0) {
+        if (bodyFrom === PEM_LINE_INVALID) {
             from = fence + 1;
             continue;
         }
-        earliestEndFence(bodyFrom);
-        if (endAt < 0)
+        if (bodyFrom !== PEM_LINE_OPEN)
+            earliestEndFence(bodyFrom);
+        if (bodyFrom === PEM_LINE_OPEN || endAt < 0) {
+            if (isHead)
+                span(fence, text.length);
             return;
+        }
         if (endAt - bodyFrom > PEM_FENCE_BODY_MAX) {
             from = fence + 1;
             continue;
@@ -1192,9 +1267,19 @@ function collectPemFragmentSpans(scan, text, json, span) {
         cur = next;
     }
 }
-function collectHeadlessPpkSpans(scan, text, span) {
+function collectHeadlessPpkSpans(scan, text, span, isHead) {
     const KEY = "Private-Lines: ";
     const n = text.length;
+    const LINE_NONE = -1;
+    const LINE_OPEN = -2;
+    const openAt = (i) => {
+        if (!isHead)
+            return false;
+        let k = i;
+        while (k < n && opaqueUnitEnd(scan, k) < 0 && text.charCodeAt(k) === 92)
+            k += 1;
+        return k >= n;
+    };
     const isB64 = (c) => (c >= 48 && c <= 57) || (c >= 65 && c <= 90) || (c >= 97 && c <= 122) || c === 43 || c === 47 || c === 61;
     const b64At = (i) => opaqueUnitEnd(scan, i) < 0 && isB64(text.charCodeAt(i));
     const skipHs = (i) => { while (i < n && opaqueUnitEnd(scan, i) < 0 && (text.charCodeAt(i) === 32 || text.charCodeAt(i) === 9))
@@ -1233,7 +1318,9 @@ function collectHeadlessPpkSpans(scan, text, span) {
             seen = true;
             q = skipHs(q);
         }
-        return seen ? q : -1;
+        if (openAt(q))
+            return LINE_OPEN;
+        return seen ? q : LINE_NONE;
     };
     const escapedNewlineLen = (i) => {
         if (opaqueUnitEnd(scan, i) >= 0)
@@ -1248,13 +1335,35 @@ function collectHeadlessPpkSpans(scan, text, span) {
     };
     const atLineEnd = (i) => {
         const b = skipHs(i);
-        if (b >= n)
+        if (b >= n || openAt(b))
             return true;
         if (opaqueUnitEnd(scan, b) >= 0)
             return false;
         const c = text.charCodeAt(b);
         return c === 10 || c === 13 || c === 34 || escapedNewlineLen(b) > 0;
     };
+    const bodyLineAt = (from) => {
+        const q0 = skipNewlines(from);
+        if (q0 < 0)
+            return q0;
+        let q = q0;
+        const mk = opaqueUnitEnd(scan, q) < 0 ? text.charCodeAt(q) : -1;
+        if (mk === 43 || mk === 45)
+            q = skipHs(q + 1);
+        let r = q;
+        while (r < n && b64At(r))
+            r += 1;
+        if (r - q < 4) {
+            if (isHead && r >= n)
+                return LINE_OPEN;
+            r = q0;
+            while (r < n && b64At(r))
+                r += 1;
+            if (r - q0 < 4)
+                return LINE_NONE;
+        }
+        return atLineEnd(r) ? r : LINE_NONE;
+    };
     let search = 0;
     for (;;) {
         const at = text.indexOf(KEY, search);
@@ -1268,36 +1377,21 @@ function collectHeadlessPpkSpans(scan, text, span) {
         const d0 = p;
         while (p < n && opaqueUnitEnd(scan, p) < 0 && text.charCodeAt(p) >= 48 && text.charCodeAt(p) <= 57)
             p += 1;
-        if (p === d0) {
+        if (p === d0 && p < n) {
             search = at + KEY.length;
             continue;
         }
-        let end = p;
-        let lines = 0;
-        for (;;) {
-            const q0 = skipNewlines(end);
-            if (q0 === -1)
+        let end = -1;
+        for (let from = p;;) {
+            const r = bodyLineAt(from);
+            if (r === LINE_NONE)
                 break;
-            let q = q0;
-            const mk = opaqueUnitEnd(scan, q) < 0 ? text.charCodeAt(q) : -1;
-            if (mk === 43 || mk === 45)
-                q = skipHs(q + 1);
-            let r = q;
-            while (r < n && b64At(r))
-                r += 1;
-            if (r - q < 4) {
-                r = q0;
-                while (r < n && b64At(r))
-                    r += 1;
-                if (r - q0 < 4)
-                    break;
-            }
-            if (!atLineEnd(r))
+            end = r === LINE_OPEN ? n : r;
+            if (r === LINE_OPEN)
                 break;
-            lines += 1;
-            end = r;
+            from = r;
         }
-        if (lines === 0) {
+        if (end < 0) {
             search = p;
             continue;
         }
@@ -1386,9 +1480,7 @@ const isRegexSpace = (c) => c === 32 || (c >= 9 && c <= 13) || c === 0xa0 || c =
 const isLineTerminator = (c) => c === 10 || c === 13 || c === 0x2028 || c === 0x2029;
 const isControlCode = (c) => c < 0x20 || c === 0x7f;
 const TOKEN_UNIT_Y = new RegExp(REDACTION_TOKEN_RE_SRC, "y");
-const CREDENTIAL_PARAM_Y = new RegExp(`(?:${CREDENTIAL_NAME_ALT})=`, "iy");
-const CREDENTIAL_NAME_SET = new Set(CREDENTIAL_NAME_WORDS);
-const CREDENTIAL_NAME_SCAN_MAX = Math.max(...CREDENTIAL_NAME_WORDS.map((w) => w.length)) * 3 + 1;
+const CREDENTIAL_NAME_SCAN_MAX = CREDENTIAL_NAME_MAX * 3 + 1;
 function decodeAsciiName(name) {
     let out = "";
     for (let i = 0; i < name.length; i++) {
@@ -1409,9 +1501,6 @@ function decodeAsciiName(name) {
     return out;
 }
 function credentialParamValueStart(text, p) {
-    CREDENTIAL_PARAM_Y.lastIndex = p;
-    if (CREDENTIAL_PARAM_Y.test(text))
-        return CREDENTIAL_PARAM_Y.lastIndex;
     const limit = Math.min(text.length, p + CREDENTIAL_NAME_SCAN_MAX);
     let hasPct = false;
     for (let i = p; i < limit; i++) {
@@ -1420,12 +1509,14 @@ function credentialParamValueStart(text, p) {
             hasPct = true;
             continue;
         }
-        if (c !== CH_EQ)
-            continue;
-        if (!hasPct)
+        if (c === CH_EQ) {
+            if (i === p)
+                return -1;
+            const name = hasPct ? decodeAsciiName(text.slice(p, i)) : text.slice(p, i);
+            return name !== undefined && credentialNameAnchor(name) !== undefined ? i + 1 : -1;
+        }
+        if (!(isAsciiAlnumCode(c) || c === CH_UNDERSCORE || c === CH_MINUS))
             return -1;
-        const decoded = decodeAsciiName(text.slice(p, i));
-        return decoded !== undefined && CREDENTIAL_NAME_SET.has(decoded.toLowerCase()) ? i + 1 : -1;
     }
     return -1;
 }
@@ -1700,10 +1791,12 @@ function scanUrlRun(scan, text, i, kind, json) {
     noteEvidenceRead(scan, i);
     return i;
 }
-function matchUserinfoAt(scan, text, p, json) {
+function matchUserinfoAt(scan, text, p, json, isHead) {
     const probeEnd = scanUrlRun(scan, text, p, "plain-user", json);
     const keyId = probeEnd > p && text.charCodeAt(probeEnd) === 58 && credentialSpansCover(scan, p, probeEnd);
-    scanUrlRun(scan, text, p, keyId ? "token-pass" : "plain-pass", json);
+    const runEnd = scanUrlRun(scan, text, p, keyId ? "token-pass" : "plain-pass", json);
+    if (isHead && runEnd >= text.length && runEnd > p)
+        return runEnd;
     const at = scan.lastAt;
     if (at <= p)
         return null;
@@ -1721,7 +1814,7 @@ function growUrlSegments(list) {
     list.a = bigger;
     return true;
 }
-function collectUrlCredentialSpans(scan, text, json, span) {
+function collectUrlCredentialSpans(scan, text, json, span, isHead) {
     const dirty = scan.dirty;
     const hr = scan.headsRead;
     const hw = scan.headsWrite;
@@ -1757,7 +1850,7 @@ function collectUrlCredentialSpans(scan, text, json, span) {
         else {
             scan.footLo = FOOT_NONE;
             scan.footHi = -1;
-            at = matchUserinfoAt(scan, text, start, json) ?? -1;
+            at = matchUserinfoAt(scan, text, start, json, isHead) ?? -1;
             lo = scan.footLo;
             hi = scan.footHi;
             urlSegmentRuns += 1;
@@ -1834,24 +1927,19 @@ function collectUrlCredentialSpans(scan, text, json, span) {
     scan.footHi = armHi;
 }
 const JSON_FIELD_ANCHOR = '"';
-const CREDENTIAL_NAME_MAX = Math.max(...CREDENTIAL_NAME_WORDS.map((w) => w.length));
 function jsonSecretFieldValueQuoteAt(text, p) {
     const n = text.length;
-    let name = "";
     let i = p + 1;
     for (; i < n; i++) {
-        let c = text.charCodeAt(i);
+        const c = text.charCodeAt(i);
         if (c === CH_QUOTE)
             break;
         if (i - p > CREDENTIAL_NAME_MAX)
             return -1;
-        if (c >= CH_A && c <= CH_Z)
-            c += 32;
-        if (!((c >= CH_a && c <= CH_z) || (c >= CH_0 && c <= CH_9) || c === CH_UNDERSCORE || c === CH_MINUS))
+        if (!(isAsciiAlnumCode(c) || c === CH_UNDERSCORE || c === CH_MINUS))
             return -1;
-        name += String.fromCharCode(c);
     }
-    if (i >= n || !CREDENTIAL_NAME_SET.has(name))
+    if (i >= n || i === p + 1 || credentialNameAnchor(text.slice(p + 1, i)) === undefined)
         return -1;
     i += 1;
     while (i < n && isRegexSpace(text.charCodeAt(i)))
@@ -1863,7 +1951,7 @@ function jsonSecretFieldValueQuoteAt(text, p) {
         i += 1;
     return text.charCodeAt(i) === CH_QUOTE ? i : -1;
 }
-function jsonStringEndAt(text, q) {
+function jsonStringEndAt(text, q, isHead) {
     const n = text.length;
     for (let i = q + 1; i < n;) {
         const c = text.charCodeAt(i);
@@ -1874,16 +1962,16 @@ function jsonStringEndAt(text, q) {
                 i += 2;
                 continue;
             }
-            return -1;
+            return isHead && i + 1 >= n ? n : -1;
         }
         i += 1;
     }
-    return -1;
+    return isHead ? n : -1;
 }
-function collectJsonSecretFieldSpans(text, span) {
+function collectJsonSecretFieldSpans(text, span, isHead) {
     for (let p = text.indexOf(JSON_FIELD_ANCHOR); p >= 0;) {
         const q = jsonSecretFieldValueQuoteAt(text, p);
-        const end = q < 0 ? -1 : jsonStringEndAt(text, q);
+        const end = q < 0 ? -1 : jsonStringEndAt(text, q, isHead);
         if (end < 0) {
             p = text.indexOf(JSON_FIELD_ANCHOR, p + 1);
             continue;
@@ -1896,7 +1984,6 @@ const STRONG_AUTH_SCHEMES = new Set(["basic", "bearer"]);
 const STRONG_AUTH_SCHEME_MAX = Math.max(...[...STRONG_AUTH_SCHEMES].map((w) => w.length));
 const ABSENT_OR_BOOLEAN_WORDS = new Set(["null", "nil", "none", "undefined", "true", "false", "yes", "no"]);
 const ABSENT_OR_BOOLEAN_MAX = Math.max(...[...ABSENT_OR_BOOLEAN_WORDS].map((w) => w.length));
-const CREDENTIAL_ANCHOR_OF = new Map(Object.entries(CREDENTIAL_NAME_ANCHORS));
 const CH_GRAVE = 96, CH_APOS = 39, CH_DOT = 46, CH_COMMA = 44, CH_SEMI = 59, CH_DOLLAR = 36;
 const CH_LPAREN = 40, CH_RPAREN = 41, CH_RBRACKET = 93, CH_LBRACE = 123, CH_RBRACE = 125, CH_LT = 60, CH_GT = 62;
 const CH_QMARK = 63, CH_AMP = 38, CH_HASH = 35;
@@ -2021,7 +2108,7 @@ function anchorHeadAt(text, op, json) {
             return undefined;
         before = j - openLen;
     }
-    const strength = CREDENTIAL_ANCHOR_OF.get(text.slice(nameStart, nameEnd).toLowerCase());
+    const strength = credentialNameAnchor(text.slice(nameStart, nameEnd));
     if (strength === undefined)
         return undefined;
     if (!afterEscape && runStart === j + 1 && quote === 0 && j >= 0 && text.charCodeAt(j) === CH_SLASH)
@@ -2037,7 +2124,7 @@ function anchorHeadAt(text, op, json) {
     }
     return { strength, quotedName: quote !== 0, spacedLeft, listSep };
 }
-function quotedValueEnd(text, cs, quote, json) {
+function quotedValueEnd(text, cs, quote, json, isHead) {
     const n = text.length;
     let i = cs;
     while (i < n) {
@@ -2046,18 +2133,18 @@ function quotedValueEnd(text, cs, quote, json) {
         if (ch === quote)
             return i;
         if (ch === JSON_STRING_END)
-            break;
+            return -1;
         i += unitLenOf(unit);
         if (ch === CH_BACKSLASH && quote !== JSON_STRING_END) {
             if (i >= n)
                 break;
             const escaped = readAnchorUnit(text, i, json);
             if (unitChOf(escaped) === JSON_STRING_END)
-                break;
+                return -1;
             i += unitLenOf(escaped);
         }
     }
-    return -1;
+    return isHead ? n : -1;
 }
 function bareValueEnd(text, s, json, listSep) {
     const n = text.length;
@@ -2102,7 +2189,7 @@ function bareValueEnd(text, s, json, listSep) {
     }
     return e;
 }
-function literalValueAt(text, s, json, declaredIfBare, listSep) {
+function literalValueAt(text, s, json, declaredIfBare, listSep, isHead) {
     const n = text.length;
     if (s >= n)
         return undefined;
@@ -2115,7 +2202,7 @@ function literalValueAt(text, s, json, declaredIfBare, listSep) {
             return undefined;
         if (!isAsciiAlnumCode(unitChOf(readAnchorUnit(text, vs, json))))
             return undefined;
-        const close = quotedValueEnd(text, vs, first, json);
+        const close = quotedValueEnd(text, vs, first, json, isHead);
         if (close >= 0)
             return { start: vs, end: close, quoted: true };
     }
@@ -2183,7 +2270,7 @@ function schemeValueAt(text, s, json) {
         return undefined;
     return { start: vs, end: e, quoted: false };
 }
-function collectAnchoredValueSpans(text, json, span) {
+function collectAnchoredValueSpans(text, json, span, isHead) {
     const n = text.length;
     let resume = 0;
     let nextEq = text.indexOf("=");
@@ -2224,10 +2311,10 @@ function collectAnchoredValueSpans(text, json, span) {
                 v = schemeValueAt(text, s, json);
                 break;
             case "declarative":
-                v = literalValueAt(text, s, json, head.quotedName || (!head.spacedLeft && !spacedRight), head.listSep);
+                v = literalValueAt(text, s, json, head.quotedName || (!head.spacedLeft && !spacedRight), head.listSep, isHead);
                 break;
             case "any-form":
-                v = literalValueAt(text, s, json, true, head.listSep);
+                v = literalValueAt(text, s, json, true, head.listSep, isHead);
                 break;
             default: {
                 const unhandled = head.strength;
@@ -2243,7 +2330,7 @@ function collectAnchoredValueSpans(text, json, span) {
 export function structuralScannerForTest(arm) {
     const collect = (scan) => (text) => {
         const out = [];
-        scan(text, (start, end) => out.push([start, end]));
+        scan(text, (start, end) => out.push([start, end]), false);
         return out;
     };
     switch (arm) {
@@ -2256,7 +2343,7 @@ export function structuralScannerForTest(arm) {
 let ppkScanBytes = 0;
 export function ppkScanBytesForTest() { return ppkScanBytes; }
 export function resetPpkScanBytesForTest() { ppkScanBytes = 0; }
-function collectPpkContainerSpans(scan, text, record, span) {
+function collectPpkContainerSpans(scan, text, eat, span, isHead) {
     const HEAD = "PuTTY-User-Key-File-";
     const MAC = "Private-MAC: ";
     const HASH = "Private-Hash: ";
@@ -2272,9 +2359,8 @@ function collectPpkContainerSpans(scan, text, record, span) {
         cache.set(needle, v);
         return v;
     };
-    const isVersionedHead = (at) => {
+    const versionEnd = (at) => {
         let p = at + HEAD.length;
-        const start = p;
         while (p < text.length && opaqueUnitEnd(scan, p) < 0) {
             const c = text.charCodeAt(p);
             if (c >= 48 && c <= 57)
@@ -2282,7 +2368,15 @@ function collectPpkContainerSpans(scan, text, record, span) {
             else
                 break;
         }
-        return p > start && text.startsWith(": ", p);
+        return p;
+    };
+    const isVersionedHead = (at) => {
+        const p = versionEnd(at);
+        return p > at + HEAD.length && text.startsWith(": ", p);
+    };
+    const isOpenHead = (at) => {
+        const p = versionEnd(at);
+        return p >= text.length || (p === text.length - 1 && p > at + HEAD.length && text.charCodeAt(p) === 58);
     };
     const nextVersionedHead = (from) => {
         let p = find(HEAD, from);
@@ -2305,7 +2399,7 @@ function collectPpkContainerSpans(scan, text, record, span) {
     while (at !== Number.POSITIVE_INFINITY) {
         let pos = at + HEAD.length + 1;
         let end = text.length;
-        let ateToEnd = true;
+        let ateToEnd = !isHead;
         for (;;) {
             const mac = find(MAC, pos);
             const hash = find(HASH, pos);
@@ -2338,15 +2432,21 @@ function collectPpkContainerSpans(scan, text, record, span) {
                     }
                 }
             }
+            const undecided = isHead && limit === Number.POSITIVE_INFINITY;
             ateToEnd = straggler < limit || !gapBlank;
-            end = ateToEnd ? text.length : hexEnd;
+            end = ateToEnd || undecided ? text.length : hexEnd;
             break;
         }
-        if (ateToEnd && record)
-            recordFailOpen("server.redact.ppk-eat-to-end", `from=${at} len=${text.length}`);
+        if (ateToEnd && eat !== undefined && eat.from < 0) {
+            eat.from = at;
+            eat.len = text.length;
+        }
         span(at, end);
         at = end >= text.length ? Number.POSITIVE_INFINITY : nextVersionedHead(end);
     }
+    const last = isHead ? text.lastIndexOf(HEAD) : -1;
+    if (last >= 0 && !inOpaqueUnit(scan, last) && isOpenHead(last))
+        span(last, text.length);
 }
 const armPool = (() => {
     const spanRepl = (token) => {
@@ -2376,8 +2476,8 @@ const armPool = (() => {
             phase: 1,
             lexical: LEXICAL_KEYWORD_ARMS.has(re),
             collect: boundary === undefined
-                ? (text, _json, _observe, out) => collectRegexSpans(arm, text, out)
-                : (text, _json, _observe, out) => collectValueArmSpans(boundary, text, out),
+                ? (text, _json, _eat, out) => collectRegexSpans(arm, text, out)
+                : (text, _json, _eat, out) => collectValueArmSpans(boundary, text, out),
             retry: boundary === undefined ? undefined : (text, at, out) => collectStickySpans(boundary, text, at, out),
             headSrc: boundary === undefined ? undefined : headSrc,
             headFirst: boundary === undefined ? undefined : headFirst,
@@ -2391,7 +2491,7 @@ const armPool = (() => {
             order,
             phase: 1,
             lexical,
-            collect: (text, json, observe, out) => arm(text, json, observe, (start, end) => out.push({ start, end, repl, rank, order })),
+            collect: (text, json, eat, out, isHead) => arm(text, json, eat, (start, end) => out.push({ start, end, repl, rank, order }), isHead),
             retry: undefined,
             headSrc: undefined,
             headFirst: undefined,
@@ -2406,7 +2506,7 @@ const armPool = (() => {
             order,
             phase: 2,
             lexical: false,
-            collect: (scan, text, json, observe, out) => arm(scan, text, json, observe, (start, end) => out.push({ start, end, repl, rank, order })),
+            collect: (scan, text, json, eat, out, isHead) => arm(scan, text, json, eat, (start, end) => out.push({ start, end, repl, rank, order }), isHead),
             retry: undefined,
             headSrc: undefined,
             headFirst: undefined,
@@ -2414,22 +2514,22 @@ const armPool = (() => {
     };
     const structuralScannerMember = (row, order) => {
         if (row === PEM_FENCE_PATTERN)
-            return scannerMember(row[1], order, (text, _json, _observe, span) => collectPemFenceSpans(text, span));
+            return scannerMember(row[1], order, (text, _json, _eat, span, isHead) => collectPemFenceSpans(text, span, isHead));
         if (row === SSH2_FENCE_PATTERN)
-            return forensicMember(row[1], order, (scan, text, _json, _observe, span) => collectSsh2FenceSpans(scan, text, span));
+            return forensicMember(row[1], order, (scan, text, _json, _eat, span, isHead) => collectSsh2FenceSpans(scan, text, span, isHead));
         if (row === URL_CREDENTIAL_PATTERN)
-            return forensicMember(row[1], order, (scan, text, json, _observe, span) => collectUrlCredentialSpans(scan, text, json, span));
+            return forensicMember(row[1], order, (scan, text, json, _eat, span, isHead) => collectUrlCredentialSpans(scan, text, json, span, isHead));
         if (row === JSON_SECRET_FIELD_PATTERN)
-            return scannerMember(row[1], order, (text, _json, _observe, span) => collectJsonSecretFieldSpans(text, span));
+            return scannerMember(row[1], order, (text, _json, _eat, span, isHead) => collectJsonSecretFieldSpans(text, span, isHead));
         throw new Error(`redact: structural row /${row[0].source}/ has no hand-written scanner — structural grammars are declared as regexes but never executed as one (S-102 / S-265)`);
     };
     const members = [];
     for (const row of STRUCTURAL_SECRET_PATTERNS)
         members.push(structuralScannerMember(row, members.length));
-    members.push(forensicMember(PPK_CONTAINER_TOKEN, members.length, (scan, text, _json, observe, span) => collectPpkContainerSpans(scan, text, observe, span)));
-    members.push(forensicMember(PEM_FRAGMENT_TOKEN, members.length, (scan, text, json, _observe, span) => collectPemFragmentSpans(scan, text, json, span)));
-    members.push(forensicMember(HEADLESS_PPK_TOKEN, members.length, (scan, text, _json, _observe, span) => collectHeadlessPpkSpans(scan, text, span)));
-    members.push(scannerMember(ANCHORED_VALUE_TOKEN, members.length, (text, json, _observe, span) => collectAnchoredValueSpans(text, json, span), ARM_RANK.keyword, true));
+    members.push(forensicMember(PPK_CONTAINER_TOKEN, members.length, (scan, text, _json, eat, span, isHead) => collectPpkContainerSpans(scan, text, eat, span, isHead)));
+    members.push(forensicMember(PEM_FRAGMENT_TOKEN, members.length, (scan, text, json, _eat, span) => collectPemFragmentSpans(scan, text, json, span)));
+    members.push(forensicMember(HEADLESS_PPK_TOKEN, members.length, (scan, text, _json, _eat, span, isHead) => collectHeadlessPpkSpans(scan, text, span, isHead)));
+    members.push(scannerMember(ANCHORED_VALUE_TOKEN, members.length, (text, json, _eat, span, isHead) => collectAnchoredValueSpans(text, json, span, isHead), ARM_RANK.keyword, true));
     for (const row of KEYWORD_SECRET_PATTERNS)
         members.push(regexMember(row, ARM_RANK.keyword, members.length));
     for (const row of SHAPE_SECRET_PATTERNS)
@@ -2460,7 +2560,8 @@ const armPool = (() => {
     const headFirstUnit = unionMembers(retryMembers.map((m) => m.headFirst)) ?? new Uint32Array(0x10000 >>> 5).fill(0xffffffff);
     poolArmCount = members.length;
     poolBoundaryArmCount = retryMembers.length;
-    return (text, wantCount, json, observe, scope) => {
+    return (text, wantCount, json, eat, scope, isHead) => {
+        readings += 1;
         const lane = LANES[scope];
         const spans = [];
         let tried;
@@ -2496,7 +2597,7 @@ const armPool = (() => {
             }
         };
         for (const m of lane.phase1)
-            m.collect(text, json, observe, spans);
+            m.collect(text, json, eat, spans, isHead);
         closeCoveragePositions(0);
         if (lane.phase2.length > 0) {
             const scan = buildForensicScan(credentialSpans(text), lane.phase2.length);
@@ -2513,7 +2614,7 @@ const armPool = (() => {
                     }
                     scan.footLo = FOOT_NONE;
                     scan.footHi = -1;
-                    lane.phase2[j].collect(scan, text, json, observe && level === 0, spans);
+                    lane.phase2[j].collect(scan, text, json, level === 0 ? eat : undefined, spans, isHead);
                     foot[2 * j] = scan.footLo;
                     foot[2 * j + 1] = scan.footHi;
                     forensicRuns += 1;
@@ -2559,8 +2660,8 @@ let onRedaction;
 export function setRedactionObserver(fn) {
     onRedaction = fn;
 }
-function applyRules(text, wantCount, json, observe, scope) {
-    return armPool(text, wantCount, json, observe, scope);
+function applyRules(text, wantCount, json, eat, scope, isHead) {
+    return armPool(text, wantCount, json, eat, scope, isHead);
 }
 function reportHits(hits) {
     if (onRedaction === undefined)
@@ -2573,24 +2674,151 @@ function reportHits(hits) {
         }
     }
 }
-function runRedaction(text, observe, json, wantHits, scope, muteCount = false) {
-    const wantCount = wantHits || (onRedaction !== undefined && observe && !muteCount);
-    const normalized = normalizeForRedaction(text);
-    const first = applyRules(normalized, wantCount, json, observe, scope);
-    forgetEscapedSeparatorTails();
-    let spans = spansInOriginal(text, normalized, undefined, first.kept);
-    let hits = first.hits;
-    const second = deleteSpelledFormatCharacters(normalized);
-    if (second.at !== undefined) {
-        const r2 = applyRules(second.text, wantCount, json, false, scope);
+function invisibleClassesOf(layout) {
+    switch (layout) {
+        case "multi-line":
+            return SPELLED_FORMAT | SPELLED_CONTROL | RAW_CONTROL;
+        case "single-line":
+            return SPELLED_FORMAT | SPELLED_CONTROL | RAW_CONTROL | RAW_WHITESPACE;
+        default: {
+            const unhandled = layout;
+            throw new Error(`redact: unhandled layout ${String(unhandled)}`);
+        }
+    }
+}
+function trailingPartialEscapeStart(s) {
+    const n = s.length;
+    const runStart = (end) => {
+        let k = end;
+        while (k > 0 && s.charCodeAt(k - 1) === CH_BACKSLASH)
+            k -= 1;
+        return k;
+    };
+    const highSurrogateEscapeStartBefore = (end) => {
+        let j = end;
+        let value = -1;
+        let bodyStart = -1;
+        if (j >= 1 && s.charCodeAt(j - 1) === CH_RBRACE) {
+            let k = j - 1;
+            while (k > 0 && hexDigitValue(s.charCodeAt(k - 1)) >= 0)
+                k -= 1;
+            if (k >= 2 && s.charCodeAt(k - 1) === CH_LBRACE && s.charCodeAt(k - 2) === CH_u && j - 1 - k >= 1) {
+                value = readHexRunValue(s, k, j - 1);
+                bodyStart = k - 2;
+            }
+        }
+        else if (j >= 5 && s.charCodeAt(j - 5) === CH_u) {
+            value = readHexRun(s, j - 4, 4);
+            bodyStart = j - 5;
+        }
+        if (value < 0xd800 || value > 0xdbff)
+            return -1;
+        const k = runStart(bodyStart);
+        return k < bodyStart ? k : -1;
+    };
+    let j = n;
+    while (j > 0 && hexDigitValue(s.charCodeAt(j - 1)) >= 0)
+        j -= 1;
+    const hexRun = n - j;
+    let element = -1;
+    if (j >= 2 && s.charCodeAt(j - 1) === CH_LBRACE && s.charCodeAt(j - 2) === CH_u)
+        element = j - 2;
+    else if (j >= 1 && s.charCodeAt(j - 1) === CH_u && hexRun <= 3)
+        element = j - 1;
+    else if (j >= 1 && s.charCodeAt(j - 1) === CH_x && hexRun <= 1)
+        element = j - 1;
+    else if (hexRun === 0 && n >= 1 && s.charCodeAt(n - 1) === CH_BACKSLASH)
+        element = n;
+    if (element >= 0) {
+        const k = runStart(element);
+        if (k >= element)
+            return -1;
+        const high = highSurrogateEscapeStartBefore(k);
+        return high >= 0 ? high : k;
+    }
+    return highSurrogateEscapeStartBefore(n);
+}
+function readHexRunValue(s, from, to) {
+    let v = 0;
+    for (let k = from; k < to; k++) {
+        const d = hexDigitValue(s.charCodeAt(k));
+        if (d < 0)
+            return -1;
+        v = v * 16 + d;
+        if (v > 0x10ffff)
+            return -1;
+    }
+    return v;
+}
+function shortestReading(text, layout) {
+    const raw = collectRawControlCuts(text, invisibleClassesOf(layout));
+    const normalized = normalizeForRedaction(deleteInvisibleCharacters(text, raw, raw.present).text);
+    const open = trailingPartialEscapeStart(normalized);
+    const spelled = collectSpelledCuts(normalized);
+    const length = deleteInvisibleCharacters(normalized, spelled, spelled.present).text.length;
+    return open < 0 ? { visible: length, partial: false } : { visible: length - (normalized.length - open), partial: true };
+}
+export function readingHeadEnd(text, layout, units, from = 0) {
+    const rest = text.length - from;
+    let len = Math.min(rest, units);
+    let first = true;
+    let grow = 64;
+    while (len < rest) {
+        const { visible, partial } = shortestReading(text.slice(from, from + len), layout);
+        if (visible >= units) {
+            if (!partial)
+                return from + len;
+            len = Math.min(rest, len + grow);
+            grow *= 2;
+            continue;
+        }
+        const step = first && visible * 2 >= len ? Math.ceil((units - visible) / (visible / len)) + 1 : Math.max(units - visible, len);
+        first = false;
+        len = Math.min(rest, len + step);
+    }
+    return text.length;
+}
+function runRedaction(text, observe, json, wantHits, scope, layout, extent) {
+    const wantCount = wantHits || (onRedaction !== undefined && observe);
+    const eat = observe ? { from: -1, len: 0 } : undefined;
+    let spans = [];
+    let hits = NO_HITS;
+    let readsSoFar = 0;
+    let unioned = false;
+    const read = (layers, view) => {
+        const r = applyRules(view, wantCount, json, eat, scope, extent === "head");
         forgetEscapedSeparatorTails();
-        const extra = spansInOriginal(text, normalized, second, r2.kept);
-        if (extra.length > 0) {
-            spans = resolveSpanOverlaps([...spans, ...extra]);
-            hits = wantCount ? hitsOfSpans(spans) : NO_HITS;
+        readsSoFar += 1;
+        if (readsSoFar === 1) {
+            spans = spansInOriginal(layers, r.kept);
+            hits = r.hits;
+            return;
         }
+        const extra = spansInOriginal(layers, r.kept);
+        if (extra.length === 0)
+            return;
+        spans = resolveSpanOverlaps([...spans, ...extra]);
+        unioned = true;
+    };
+    const raw = collectRawControlCuts(text, invisibleClassesOf(layout));
+    for (let rmask = 0;; rmask = (rmask - raw.present) & raw.present) {
+        const rfold = deleteInvisibleCharacters(text, raw, rmask);
+        const normalized = normalizeForRedaction(rfold.text);
+        const spelled = collectSpelledCuts(normalized);
+        for (let smask = 0;; smask = (smask - spelled.present) & spelled.present) {
+            const sfold = deleteInvisibleCharacters(normalized, spelled, smask);
+            read([{ fold: sfold }, { input: rfold.text, view: normalized }, { fold: rfold }], sfold.text);
+            if (smask === spelled.present)
+                break;
+        }
+        if (rmask === raw.present)
+            break;
     }
-    if (observe && !muteCount)
+    if (unioned && wantCount)
+        hits = hitsOfSpans(spans);
+    if (eat !== undefined && eat.from >= 0)
+        recordFailOpen("server.redact.ppk-eat-to-end", `from=${eat.from} len=${eat.len}`);
+    if (observe)
         reportHits(hits);
     return { text: applySpans(text, spans).out, hits, spans };
 }
@@ -2608,28 +2836,41 @@ function hitsOfSpans(spans) {
     }
     return [...by.values()].map(({ token, arm, count }) => ({ label: labelOfToken(token), count, arm }));
 }
-function spansInOriginal(text, normalized, fold, kept) {
-    if (kept.length === 0)
-        return kept;
-    const inNormalized = fold?.at === undefined ? kept : kept.map((s) => ({ ...s, start: viewIndexInInput(fold, s.start), end: viewIndexInInput(fold, s.end - 1) + 1 }));
-    if (normalized === text)
-        return inNormalized;
-    const map = mapNormalizedToOriginal(text, normalized);
-    return inNormalized.map((s) => ({ ...s, start: map[s.start], end: map[s.end - 1] + 1 }));
+function spansInOriginal(layers, kept) {
+    let out = kept;
+    for (const layer of layers) {
+        if (out.length === 0)
+            return out;
+        let at;
+        if ("fold" in layer) {
+            const fold = layer.fold;
+            if (fold.at === undefined)
+                continue;
+            at = (v) => viewIndexInInput(fold, v);
+        }
+        else {
+            if (layer.view === layer.input)
+                continue;
+            const map = mapNormalizedToOriginal(layer.input, layer.view);
+            at = (v) => map[v];
+        }
+        out = out.map((s) => ({ ...s, start: at(s.start), end: at(s.end - 1) + 1 }));
+    }
+    return out;
 }
 export function redactSecrets(text, opts) {
-    return runRedaction(text, opts?.observe ?? true, opts?.json === true, false, "free-text").text;
+    return runRedaction(text, opts?.observe ?? true, opts?.json === true, false, "free-text", "multi-line", "whole").text;
 }
 export function redactSecretsWithArms(text, opts) {
-    const r = runRedaction(text, opts?.observe ?? true, opts?.json === true, true, "free-text");
+    const r = runRedaction(text, opts?.observe ?? true, opts?.json === true, true, "free-text", "multi-line", "whole");
     return { text: r.text, arms: new Set(r.hits.map((h) => h.arm)) };
 }
 export function redactionSpansForTest(text, opts) {
-    const r = runRedaction(text, false, opts?.json === true, false, "free-text");
+    const r = runRedaction(text, false, opts?.json === true, false, "free-text", "multi-line", "whole");
     return { text: r.text, spans: r.spans.map((sp) => ({ start: sp.start, end: sp.end })) };
 }
-export function proposeRedactionSpans(text, scope, opts) {
-    return runRedaction(text, true, false, false, scope, opts?.countHits === false).spans.map((sp) => ({ start: sp.start, end: sp.end, repl: sp.repl }));
+export function proposeRedactionSpans(text, scope, layout, extent) {
+    return runRedaction(text, true, false, false, scope, layout, extent).spans.map((sp) => ({ start: sp.start, end: sp.end, repl: sp.repl }));
 }
 const REDACT_MAX_DEPTH = 64;
 export function redactDeep(value) {
@@ -2731,7 +2972,7 @@ function unitCarriesCredential(unit) {
         for (const v of armValueClassViews())
             views.add(boundedUnitView(spelling, v.inClass, v.filler));
     views.delete("");
-    const hit = [...views].some((view) => runRedaction(view, true, false, false, "bounded-unit").text !== view);
+    const hit = [...views].some((view) => runRedaction(view, true, false, false, "bounded-unit", "multi-line", "whole").text !== view);
     return { hit, unreadable: decoded.residue };
 }
 function displaySafePathname(pathname) {
@@ -2761,7 +3002,7 @@ function shapeLawCredentialInQueryOrFragment(canonical) {
     const boundary = canonical.indexOf("?");
     if (boundary < 0)
         return false;
-    if (runRedaction(canonical, false, false, false, "free-text").spans.some((sp) => sp.end > boundary))
+    if (runRedaction(canonical, false, false, false, "free-text", "multi-line", "whole").spans.some((sp) => sp.end > boundary))
         return true;
     for (const unit of canonical.slice(boundary + 1).split("&").flatMap((sg) => {
         const eq = sg.indexOf("=");