@maudecode/talaria-web 1.17.2 → 1.19.0

package/dist/redact.js

@@ -1,6 +1,8 @@
 import { str } from './util.js';
+import { diffHunks } from './text/diff.js';
+import { isRasterDataUri } from './raster-image.js';
 import { snapshotArgs, toolArgs, toolDisplay, toolName } from './sessions/tool-display.js';
-import { messageText } from './sessions/merge.js';
+import { hasNativeImages, messageText, normalizeUserDisplay } from './sessions/merge.js';
 import { stripAttachedFilesMarker } from './sessions/session.js';
 /**
  * Credential redaction and the public session projection (Python
@@ -258,39 +260,56 @@ const ANSI_PAYLOAD_GAPS_RE = new RegExp([SEQ_CSI, String.raw `${SEQ_ESC}[\]PX^_]
  * terminal title can hold a token), and without control characters only (an escape's final byte can be a token's).
  */
 const SPLIT_VIEWS = [ANSI_GAPS_RE, ANSI_PAYLOAD_GAPS_RE, CONTROL_CHARS_RE];
+/** `ANSI_GAPS_RE` without the whitespace controls, so a view keeps its lines and words apart. */
+const DISPLAY_GAPS_RE = new RegExp(String.raw `(?![\t-\r])(?:${ANSI_GAPS_RE.source})`, 'g');
 const ANSI_GAP_AT_RE = new RegExp(ANSI_GAPS_RE.source, 'y');
 const TOKEN_CHAR_RE = /[A-Za-z0-9_-]/;
+/** A character a credential's prefix or body can hold (`SG.`, `glrt-a.b`, `gAAAA…=`): what a fail-closed run walks. */
+const RUN_CHAR_RE = /[A-Za-z0-9_.=-]/;
 /** A string sequence's opener, with an OSC's command number (`8;`), which is no payload. */
 const STRING_OPENER_RE = new RegExp(String.raw `^(?:(?:${SEQ_ESC}\]|\x9d)(?:\d*;)?|${SEQ_ESC}[PX^_]|[\x90\x98\x9e\x9f])`);
 const PAYLOAD_RUN_CAP = 65_536;
-const CRED_TEST_RE = new RegExp(CRED_RE.source);
+/**
+ * The token families a split token is joined for, each with its own boundaries: `CRED_RE`'s prefixes, and `JWT_RE`, whose
+ * `\b` lets a JWT start after a `-` and which has no right boundary.
+ */
+const TOKEN_TEST_RE = new RegExp(`${CRED_RE.source}|${JWT_RE.source}`);
 /** `CRED_RE`'s prefix and body from a position, without its boundaries (a split token checks the original ones). */
 const CRED_RUN_RE = new RegExp(CRED_RE.source.replace(/^\(\?<!\[A-Za-z0-9_-\]\)/, '').replace(/\(\?!\[A-Za-z0-9_-\]\)$/, ''), 'y');
+const JWT_RUN_SOURCE = JWT_RE.source.replace(/^\\b/, '');
+/** A credential's or JWT's prefix and body from a position, without its boundaries. */
+const TOKEN_RUN_RE = new RegExp(`${CRED_RUN_RE.source}|${JWT_RUN_SOURCE}`, 'y');
 const SPLIT_TOKEN_CAP = 1024;
 /** Where a `CRED_RE` prefix and body start at a boundary, however they end. */
 const CRED_START_RE = new RegExp(CRED_RE.source.replace(/\(\?!\[A-Za-z0-9_-\]\)$/, ''), 'g');
-const CRED_WHOLE_RE = new RegExp(`^${CRED_RE.source}$`);
-/** `CRED_START_RE` and `CRED_RUN_RE` without their body: a credential's prefix, however little of its body follows. */
-const CRED_PREFIX_RE = new RegExp(CRED_START_RE.source.replace(/\{\d+,?\}/g, '{0}'), 'g');
-const CRED_PREFIX_RUN_RE = new RegExp(CRED_RUN_RE.source.replace(/\{\d+,?\}/g, '{0}'), 'y');
+const TOKEN_START_RE = new RegExp(`${CRED_START_RE.source}|${JWT_RE.source}`, 'g');
+const TOKEN_WHOLE_RE = new RegExp(`^(?:${CRED_RE.source}|${JWT_RE.source})$`);
+/** `TOKEN_START_RE` and `TOKEN_RUN_RE` without their body: a token's prefix, however little of its body follows. */
+const TOKEN_PREFIX_RE = new RegExp(`${CRED_START_RE.source.replace(/\{\d+,?\}/g, '{0}')}|\\beyJ`, 'g');
+const TOKEN_PREFIX_RUN_RE = new RegExp(`${CRED_RUN_RE.source.replace(/\{\d+,?\}/g, '{0}')}|eyJ`, 'y');
 /** A URL query parameter's value, up to the next `&` or `#`. */
 const QUERY_VALUE_RE = /[^&#]*/y;
-const PRIVKEY_RE = /-----BEGIN[A-Z ]*PRIVATE KEY-----[\s\S]*?-----END[A-Z ]*PRIVATE KEY-----/g;
-/** A private key whose end marker is missing (a display cap cut it off): masked to the end of the text. */
-const PRIVKEY_OPEN_RE = /-----BEGIN[A-Z ]*PRIVATE KEY-----[\s\S]*$/;
+const PEM_OPENER = '-----BEGIN';
+/**
+ * A private key through its end marker (captured) or, when that is missing (a display cap cut it off), to the end of the
+ * text.
+ */
+const PRIVKEY_RE = /-----BEGIN[A-Z ]*PRIVATE KEY-----(?:[\s\S]*?(-----END[A-Z ]*PRIVATE KEY-----)|[\s\S]*$)/g;
+/** `PRIVKEY_RE` after its opener, from a position. */
+const PRIVKEY_REST_RE = new RegExp(PRIVKEY_RE.source.slice(PEM_OPENER.length), 'y');
 const CODE_ENV_KEY_LITERAL_RE = /([A-Z0-9_]{0,50}(?:API_?KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|AUTH)[A-Z0-9_]{0,50}=)(["'][)\]:,]+|[)\]:,]+)/y;
 const ENV_KEY_PREFIX_RE = /([A-Z0-9_]{0,50}(?:API_?KEY|TOKEN|SECRET|PASSWORD|PASSWD|CREDENTIAL|AUTH)[A-Z0-9_]{0,50}=)/g;
 const REDACTED_ENV_VALUE_RE = /(?:\*{3,}|[A-Za-z0-9][A-Za-z0-9_.:/+-]{0,32}\.\.\.[A-Za-z0-9_.:/+-]{1,16})/y;
 /**
- * Where a credential split by control or zero-width characters (`ghp_abc\x1bdef`, `ghp_abc\ndef…`) starting at `i` of the
+ * Where a credential or JWT split by control or zero-width characters (`ghp_abc\x1bdef`, `ghp_abc\ndef…`) starting at `i` of the
  * stripped text ends there, or -1. The run of its prefix and body is read once (capped), then the longest candidate is
- * taken that ends at a piece boundary of the original, is a whole credential, is not the name of a `KEY=` that follows,
+ * taken that ends at a piece boundary of the original, is a whole token, is not the name of a `KEY=` that follows,
  * and never reaches a line holding a whole token itself (that line is the prefix pass's and joins nothing).
  */
 function splitTokenEnd(text, stripped, kept, i) {
-    CRED_RUN_RE.lastIndex = 0;
+    TOKEN_RUN_RE.lastIndex = 0;
     // ponytail: a split credential longer than SPLIT_TOKEN_CAP is cut at the cap; raise it if longer tokens appear.
-    const run = CRED_RUN_RE.exec(stripped.slice(i, i + SPLIT_TOKEN_CAP))?.[0];
+    const run = TOKEN_RUN_RE.exec(stripped.slice(i, i + SPLIT_TOKEN_CAP))?.[0];
     if (!run)
         return -1;
     const start = kept[i];
@@ -298,7 +317,7 @@ function splitTokenEnd(text, stripped, kept, i) {
     for (let lineStart = start; lineStart < limit;) {
         const lineEnd = text.slice(lineStart, limit).search(/[\r\n]/);
         const end = lineEnd === -1 ? limit : lineStart + lineEnd;
-        if (CRED_TEST_RE.test(text.slice(lineStart, end))) {
+        if (TOKEN_TEST_RE.test(text.slice(lineStart, end))) {
             limit = lineStart === start ? end : lineStart;
             break;
         }
@@ -309,23 +328,17 @@ function splitTokenEnd(text, stripped, kept, i) {
     for (let e = i + run.length; e > i; e -= 1) {
         const atBoundary = e === stripped.length || kept[e] - kept[e - 1] > 1 || (e === i + run.length && !/[A-Za-z0-9_-]/.test(stripped[e]));
         const originalEnd = kept[e - 1] + 1;
-        if (!atBoundary || originalEnd > limit || !CRED_WHOLE_RE.test(stripped.slice(i, e)) || /^[ \t]*=/.test(text.slice(originalEnd, originalEnd + 64)))
+        if (!atBoundary || originalEnd > limit || !TOKEN_WHOLE_RE.test(stripped.slice(i, e)) || /^[ \t]*=/.test(text.slice(originalEnd, originalEnd + 64)))
             continue;
         return e;
     }
     return -1;
 }
-/**
- * The original spans, and the joined text, of prefixed credentials whose body a gap of one of `SPLIT_VIEWS` splits,
- * matched on the text without those gaps, plus `payloadRunSpan`'s from each prefix. A token starts at a boundary of the
- * stripped text, or where a stripped gap hid the original one (`note\nghp_…`); one starting inside a token already
- * masked is part of it.
- */
-function splitTokenSpans(text, gaps) {
+/** The text without the gaps of `gaps`, and the original index of each kept character, or null when it has none. */
+function splitView(text, gaps) {
     const stripped = text.replace(gaps, '');
     if (stripped.length === text.length)
-        return [];
-    // The original index of each kept character.
+        return null;
     const kept = [];
     let at = 0;
     for (const m of text.matchAll(gaps)) {
@@ -335,14 +348,27 @@ function splitTokenSpans(text, gaps) {
     }
     while (at < text.length)
         kept.push(at++);
-    const starts = [...stripped.matchAll(CRED_START_RE)].map((m) => m.index);
-    const prefixStarts = [...stripped.matchAll(CRED_PREFIX_RE)].map((m) => m.index);
+    return { stripped, kept };
+}
+/**
+ * The original spans, and the joined text, of prefixed credentials and JWTs whose body a gap of one of `SPLIT_VIEWS` splits,
+ * matched on the text without those gaps, plus `payloadRunSpan`'s from each prefix. A token starts at a boundary of the
+ * stripped text, or where a stripped gap hid the original one (`note\nghp_…`); one starting inside a token already
+ * masked is part of it.
+ */
+function splitTokenSpans(text, gaps) {
+    const view = splitView(text, gaps);
+    if (!view)
+        return [];
+    const { stripped, kept } = view;
+    const starts = [...stripped.matchAll(TOKEN_START_RE)].map((m) => m.index);
+    const prefixStarts = [...stripped.matchAll(TOKEN_PREFIX_RE)].map((m) => m.index);
     for (let i = 1; i < kept.length; i += 1) {
         if (kept[i] - kept[i - 1] <= 1 || !TOKEN_CHAR_RE.test(stripped[i - 1]))
             continue;
         starts.push(i);
-        CRED_PREFIX_RUN_RE.lastIndex = i;
-        if (CRED_PREFIX_RUN_RE.test(stripped))
+        TOKEN_PREFIX_RUN_RE.lastIndex = i;
+        if (TOKEN_PREFIX_RUN_RE.test(stripped))
             prefixStarts.push(i);
     }
     const spans = [];
@@ -356,17 +382,103 @@ function splitTokenSpans(text, gaps) {
         spans.push([kept[i], kept[e - 1] + 1, stripped.slice(i, e)]);
         scanned = e;
     }
+    return [...spans, ...payloadRunSpans(text, prefixStarts.map((i) => kept[i]))];
+}
+/** `payloadRunSpan`'s spans from credential prefixes at these original indices, each run walked once. */
+function payloadRunSpans(text, starts) {
+    const spans = [];
     let walked = 0;
-    for (const i of prefixStarts.sort((a, b) => a - b)) {
-        if (kept[i] < walked)
+    for (const start of starts.sort((a, b) => a - b)) {
+        if (start < walked)
             continue;
-        const [span, reached] = payloadRunSpan(text, kept[i]);
+        const [span, reached] = payloadRunSpan(text, start);
         if (span)
             spans.push(span);
         walked = reached;
     }
     return spans;
 }
+/** A string sequence's terminator: BEL (OSC only), ST, or CAN or SUB cancelling it. */
+const STRING_END_RE = /(?:\x07|\x1b\\|\x9c|[\x18\x1a])$/;
+/** Tags a candidate that can grow only into a JWT prefix: one starting after a `-`, a boundary for `JWT_RE`'s `\b` alone. */
+const JWT_ONLY = '\0';
+/**
+ * What a piece after `prev` (undefined after a gap) seeds: any prefix after a non-token character, only a JWT's after a
+ * `-`, none after a word character.
+ */
+const seedAfter = (prev) => (prev === undefined || !TOKEN_CHAR_RE.test(prev) ? '' : prev === '-' ? JWT_ONLY : null);
+/**
+ * Where a credential or JWT prefix or a private key's opener starts, where it ends, and which marker it is, when its
+ * pieces lie apart in a run of token characters and escape sequences, with a string payload between them taken whole or
+ * skipped (`g\x1b]0;x\x07h\x1b]0;p_\x07…`), so that no view holds it whole.
+ * Display characters are mandatory and each payload's optional. A piece starts on the display after a gap or a non-token
+ * character (`.`), or in a payload's trailing run after its start or a non-token character, where `payloadRunSpan` can
+ * walk from it; a JWT's also after a `-`.
+ * Candidates live only while they can grow into a prefix, which bounds them and keeps the scan linear.
+ */
+function crossPayloadPrefixStarts(text) {
+    const starts = [];
+    if (!/[\x1b\x90\x98\x9d-\x9f]/.test(text))
+        return starts;
+    // Each candidate string, with the original index of its earliest start.
+    let candidates = new Map();
+    // Grows a candidate by the run characters of `chars`, which starts at original index `at`.
+    const grow = (into, candidate, start, chars, at) => {
+        for (let j = 0; j < chars.length; j += 1) {
+            if (!RUN_CHAR_RE.test(chars[j]))
+                continue;
+            candidate += chars[j];
+            if (PAYLOAD_MARKERS.has(candidate))
+                return void starts.push([start, at + j + 1, candidate]);
+            if (!PAYLOAD_MARKER_STEMS.has(candidate))
+                return;
+        }
+        const seen = into.get(candidate);
+        if (seen === undefined || start < seen)
+            into.set(candidate, start);
+    };
+    // The last display character, unless a gap came after it: what a piece starting here may seed.
+    let prev;
+    for (let k = 0; k < text.length;) {
+        ANSI_GAP_AT_RE.lastIndex = k;
+        const gap = ANSI_GAP_AT_RE.exec(text)?.[0];
+        if (gap) {
+            const from = STRING_OPENER_RE.exec(gap)?.[0].length ?? gap.length;
+            const next = new Map(candidates);
+            for (const [candidate, start] of candidates)
+                grow(next, candidate, start, gap.slice(from), k + from);
+            const end = Math.max(from, gap.length - (STRING_END_RE.exec(gap)?.[0].length ?? 0));
+            let tail = end;
+            while (tail > from && RUN_CHAR_RE.test(gap[tail - 1]))
+                tail -= 1;
+            for (let j = tail; j < end; j += 1) {
+                const seed = seedAfter(j === tail ? undefined : gap[j - 1]);
+                if (seed !== null)
+                    grow(next, seed, k + j, gap.slice(j, end), k + j);
+            }
+            candidates = next;
+            prev = undefined;
+            k += gap.length;
+        }
+        else if (RUN_CHAR_RE.test(text[k])) {
+            const next = new Map();
+            for (const [candidate, start] of candidates)
+                grow(next, candidate, start, text[k], k);
+            const seed = seedAfter(prev);
+            if (seed !== null)
+                grow(next, seed, k, text[k], k);
+            candidates = next;
+            prev = text[k];
+            k += 1;
+        }
+        else {
+            candidates.clear();
+            prev = undefined;
+            k += 1;
+        }
+    }
+    return starts;
+}
 /**
  * Fail-closed for a token split across string payloads: from a credential prefix at `start`, the span through the run of
  * token characters and whole escape sequences after it, when a string payload in the run holds token characters, and
@@ -385,7 +497,7 @@ function payloadRunSpan(text, start) {
         if (gap) {
             const from = STRING_OPENER_RE.exec(gap)?.[0].length ?? gap.length;
             for (let j = from; j < gap.length; j += 1) {
-                if (!TOKEN_CHAR_RE.test(gap[j]))
+                if (!RUN_CHAR_RE.test(gap[j]))
                     continue;
                 token += gap[j];
                 end = k + j + 1;
@@ -393,7 +505,7 @@ function payloadRunSpan(text, start) {
             }
             k += gap.length;
         }
-        else if (TOKEN_CHAR_RE.test(text[k])) {
+        else if (RUN_CHAR_RE.test(text[k])) {
             token += text[k];
             end = k += 1;
         }
@@ -402,9 +514,51 @@ function payloadRunSpan(text, start) {
     }
     return [hidden ? [start, end, token] : null, k];
 }
+/**
+ * The original spans of private keys a gap of one of `SPLIT_VIEWS` splits (`-----BE\x1b[1mGIN …`), matched on each view,
+ * and of those whose opener lies apart across string payloads (`-----B\x1b]0;x\x07E\x1b]0;GIN\x07 …`), matched on each
+ * view from where the opener ends. A whole key is `PRIVKEY_RE`'s on the text itself. A key one view leaves open runs to
+ * the end of the text only when no view closes one there (`-----END PRI\x1b[1mVATE KEY-----`).
+ */
+function privateKeySpans(text) {
+    const openers = crossPayloadPrefixStarts(text).filter(([, , marker]) => marker === PEM_OPENER).sort((a, b) => a[1] - b[1]);
+    const closed = [];
+    const open = [];
+    for (const gaps of SPLIT_VIEWS) {
+        const view = splitView(text, gaps);
+        if (!view)
+            continue;
+        const { stripped, kept } = view;
+        // A key running to the end of the view runs to the end of the text, past any trailing gap.
+        const end = (e) => (e === stripped.length ? text.length : kept[e - 1] + 1);
+        for (const m of stripped.matchAll(PRIVKEY_RE))
+            (m[1] ? closed : open).push([kept[m.index], end(m.index + m[0].length), '']);
+        let i = 0;
+        let scanned = 0;
+        for (const [start, opened] of openers) {
+            if (opened < scanned)
+                continue;
+            while (i < kept.length && kept[i] < opened)
+                i += 1;
+            PRIVKEY_REST_RE.lastIndex = i;
+            const m = PRIVKEY_REST_RE.exec(stripped);
+            if (!m)
+                continue;
+            scanned = end(PRIVKEY_REST_RE.lastIndex);
+            const spans = m[1] ? closed : open;
+            spans.push([start, scanned, '']);
+        }
+    }
+    return [...closed, ...open.filter(([start]) => !closed.some(([from, to]) => from <= start && start < to))];
+}
 /** Masks each union of the split-token spans every view finds, so no view masks a piece of a token another one joins. */
 function maskSplitTokens(text) {
-    const spans = SPLIT_VIEWS.flatMap((gaps) => splitTokenSpans(text, gaps)).sort((a, b) => a[0] - b[0]);
+    const starts = crossPayloadPrefixStarts(text).flatMap(([start, , marker]) => (marker === PEM_OPENER ? [] : [start]));
+    return maskSpans(text, [...SPLIT_VIEWS.flatMap((gaps) => splitTokenSpans(text, gaps)), ...payloadRunSpans(text, starts)], mask);
+}
+/** Replaces each union of overlapping original spans, given the longest of their texts. */
+function maskSpans(text, found, replace) {
+    const spans = found.sort((a, b) => a[0] - b[0]);
     let out = '';
     let last = 0;
     for (let k = 0; k < spans.length;) {
@@ -416,7 +570,7 @@ function maskSplitTokens(text) {
             if (nextToken.length > token.length)
                 token = nextToken;
         }
-        out += text.slice(last, start) + mask(token);
+        out += text.slice(last, start) + replace(token);
         last = end;
     }
     return out + text.slice(last);
@@ -938,7 +1092,10 @@ const REDIRECT_RE = /[0-9]*(?:&>>?|<<<|<<-?|<>|>>|>&|<&|>\||[<>])/y;
  */
 function commandWords(text, from, enclosing, list) {
     const spans = [];
-    const [gap, close, stop] = list ? [/[ \t,<>]/, /[\n;&|()\]}]/, /[\s;&|()<>,\]}]/] : [/[ \t]/, /[\n;&|()]/, /[\s;&|()<>]/];
+    const [gap, close] = list ? [/[ \t,<>]/, /[\n;&|()\]}]/] : [/[ \t]/, /[\n;&|()]/];
+    // A word ends only where the scan can step on (a gap, a redirection) or the command ends, so it always advances. Other
+    // whitespace (`\r`, a no-break space) is part of the word, as the shell reads it, so a secret holding one is masked whole.
+    const stop = new RegExp(`${gap.source}|${close.source}|[<>]`);
     let i = from;
     let redirect = false;
     for (;;) {
@@ -1606,6 +1763,69 @@ function secretTokenCounts(text) {
 export function redactSensitive(text) {
     if (!text)
         return text;
+    const redacted = redactShellText(text);
+    // An escape sequence in a key or between a key and its value (`\x1b[1;31mAPI_KEY\x1b[m=…`, `pass\x1b[1mword …`) hides
+    // it from the rules. A unit of the text the view without sequences masks anywhere is masked in the text's redaction,
+    // wherever it still shows, removing a sequence inside one and keeping the rest.
+    const stripped = text.replace(DISPLAY_GAPS_RE, '');
+    if (stripped.length === text.length)
+        return redacted;
+    const shown = redactShellText(stripped);
+    const base = unitCounts(stripped);
+    const shownCounts = unitCounts(shown);
+    const view = splitView(redacted, DISPLAY_GAPS_RE);
+    const plain = view?.stripped ?? redacted;
+    const at = (i) => (view ? view.kept[i] : i);
+    const raw = unitCounts(plain);
+    // Hidden by a redaction showing `counts`: fewer than the text holds, or fewer than another redaction shows (a value
+    // equal to a mask made elsewhere, `ghp_ab...cdef`, fails closed). Only a unit of the text is masked, never a mask.
+    const hides = (counts, other, unit) => {
+        const n = counts.get(unit) ?? 0;
+        return base.has(unit) && (n < base.get(unit) || n < (other.get(unit) ?? 0));
+    };
+    // Masked units only blanks apart are one value (`'correct horse battery'`), masked whole so its word count stays hidden.
+    const spans = [];
+    let last = -1;
+    for (const m of plain.matchAll(MASK_OR_UNIT_RE)) {
+        if (!hides(shownCounts, raw, m[0]))
+            continue;
+        const end = at(m.index + m[0].length - 1) + 1;
+        if (last !== -1 && /^[ \t]+$/.test(plain.slice(last, m.index)))
+            spans.at(-1)[1] = end;
+        else
+            spans.push([at(m.index), end, m[0]]);
+        last = m.index + m[0].length;
+    }
+    const out = spans.length ? maskSpans(redacted, spans, () => '***') : redacted;
+    // A credential of delimiters only (`Bear\x1b[1mer $$$$`) has no unit to mask: when the view shows fewer of a delimiter
+    // than the text's redaction, the view's redaction is returned, without escape sequences, with each unit the text's
+    // redaction hides masked in it too.
+    const plainOut = out.replace(DISPLAY_GAPS_RE, '');
+    const outCounts = delimiterCounts(plainOut);
+    const viewCounts = delimiterCounts(shown);
+    if (![...outCounts].some(([char, count]) => (viewCounts.get(char) ?? 0) < count))
+        return out;
+    const hidden = unitCounts(plainOut);
+    return shown.replace(MASK_OR_UNIT_RE, (unit) => (hides(hidden, shownCounts, unit) ? '***' : unit));
+}
+const PRIVATE_KEY_MASK = '[REDACTED PRIVATE KEY]';
+/** A private key's mask, as one unit (its words are no units of their own), or a `SECRET_UNIT_RE` unit. */
+const MASK_OR_UNIT_RE = new RegExp(String.raw `\[REDACTED PRIVATE KEY\]|${SECRET_UNIT_RE.source}`, 'gu');
+/** A text's `MASK_OR_UNIT_RE` units, counted. */
+function unitCounts(text) {
+    const counts = new Map();
+    for (const [unit] of text.matchAll(MASK_OR_UNIT_RE))
+        counts.set(unit, (counts.get(unit) ?? 0) + 1);
+    return counts;
+}
+/** A text's delimiters, `SECRET_UNIT_RE`'s but whitespace, counted outside the masks `***` and a private key's. */
+function delimiterCounts(text) {
+    const counts = new Map();
+    for (const [char] of text.replaceAll(PRIVATE_KEY_MASK, '').replace(/(?<!\*)\*{3}(?!\*)/g, '').matchAll(/['"`$\\=:@/;&|<>(){}[\],*]/g))
+        counts.set(char, (counts.get(char) ?? 0) + 1);
+    return counts;
+}
+function redactShellText(text) {
     // Line continuations join their lines as the shell runs them (`--user \⏎ bob:pw`, `--pass\⏎word=x`): that view is
     // redacted, and a text with nothing to mask keeps its lines as written.
     const joined = text.replace(/\\\r?\n/g, '');
@@ -1783,7 +2003,7 @@ export function redactSensitive(text) {
 function redactRules(text) {
     if (!text)
         return text;
-    let out = maskSplitTokens(text).replace(CRED_RE, (_, t) => mask(t));
+    let out = maskSplitTokens(maskSpans(text, privateKeySpans(text), () => '[REDACTED PRIVATE KEY]')).replace(CRED_RE, (_, t) => mask(t));
     out = out.replace(EMBEDDED_AWS_RE, (t) => mask(t));
     out = redactHeaderCredentials(out, AUTH_HDR_RE);
     out = out.replace(JWT_RE, (t) => mask(t));
@@ -1813,7 +2033,7 @@ function redactRules(text) {
     out = redactUserFlags(out);
     out = redactCommandFlags(out);
     out = out.replace(QUERY_KEY_RE, (whole, head, value) => (/[A-Za-z0-9]/.test(value) ? head + mask(value) : whole));
-    out = out.replace(PRIVKEY_RE, '[REDACTED PRIVATE KEY]').replace(PRIVKEY_OPEN_RE, '[REDACTED PRIVATE KEY]');
+    out = out.replace(PRIVKEY_RE, '[REDACTED PRIVATE KEY]');
     return restoreCodeEnvKeyLiterals(text, out);
 }
 const CASE_MARKERS = [
@@ -1822,6 +2042,11 @@ const CASE_MARKERS = [
     'gsk_', 'syt_', 'retaindb_', 'hsk-', 'mem0_', 'brv_', 'xapp-', 'xai-', 'ntn_', 'fw-', 'fw_', 'fpk_', 'glpat-', 'gloas-', 'gldt-', 'glrt-', 'glrtr-',
     'glcbt-', 'glptt-', 'glft-', 'glimt-', 'glagent-', 'glsoat-', 'glffct-', 'glwt-', 'GR1348941', 'pk-lf-', 'eyJ', '-----BEGIN',
 ];
+/** The literal prefixes of `CRED_RE` (`xapp-` stands for `xapp-1-`) and `JWT_RE`, the JWT's also as `JWT_ONLY`'s. */
+const TOKEN_MARKERS = new Set([...CASE_MARKERS.filter((m) => ((TOKEN_PREFIX_RUN_RE.lastIndex = 0), TOKEN_PREFIX_RUN_RE.test(`${m}0-`))), `${JWT_ONLY}eyJ`]);
+/** What a cross-payload scan assembles: `TOKEN_MARKERS` and a private key's opener, and their proper prefixes. */
+const PAYLOAD_MARKERS = new Set([...TOKEN_MARKERS, PEM_OPENER]);
+const PAYLOAD_MARKER_STEMS = new Set([...PAYLOAD_MARKERS].flatMap((m) => Array.from({ length: m.length - 1 }, (_, n) => m.slice(0, n + 1))));
 const LOWER_MARKERS = [
     'authorization: bearer ', 'authorization: bot ', 'private key', 'postgres://', 'postgresql://', 'mysql://', 'mongodb://', 'redis://', 'amqp://', '://',
     'access_token', 'refresh_token', 'id_token', 'api_key', 'apikey', 'client_secret', 'auth_token', 'raw_secret', 'secret_input', 'key_material',
@@ -1838,6 +2063,12 @@ export function mightContainSensitiveText(text) {
     const views = SPLIT_VIEWS.map((gaps) => text.replace(gaps, ''));
     if (CASE_MARKERS.some((m) => views.some((view) => view.includes(m))))
         return true;
+    if (crossPayloadPrefixStarts(text).length)
+        return true;
+    // So is a key or token an escape sequence splits (`pass\x1b[1mword`, `123456789\x1b[0m:…`).
+    return mightContainKeyword(text) || (views[0] !== text && mightContainKeyword(views[0]));
+}
+function mightContainKeyword(text) {
     const lower = text.toLowerCase();
     if (LOWER_MARKERS.some((m) => lower.includes(m)))
         return true;
@@ -1901,11 +2132,15 @@ export function redactValue(v, enabled) {
         return redactText(v, enabled);
     if (Array.isArray(v))
         return v.map((item) => redactValue(item, enabled));
-    if (v && typeof v === 'object')
-        return Object.fromEntries(Object.entries(v).map(([k, val]) => [k, redactValue(val, enabled)]));
-    return v;
+    if (!v || typeof v !== 'object')
+        return v;
+    const out = Object.fromEntries(Object.entries(v).map(([k, val]) => [k, redactValue(val, enabled)]));
+    // TAL-604: a diff's `hunks` are re-parsed from its redacted text, which masks spans (a private key) that no single line shows.
+    if (enabled && Array.isArray(out.hunks) && typeof out.diff === 'string')
+        out.hunks = diffHunks(out.diff);
+    return out;
 }
-export const PUBLIC_MESSAGE_INTERNAL_FIELDS = new Set(['api_content', '_row_id', '_state_db_row_id', '_state_db_generation', '_db_row_id', 'state_db_row_id', '_active_turn_token', '_active_turn_user', '_fork_child_turn']);
+export const PUBLIC_MESSAGE_INTERNAL_FIELDS = new Set(['_background_review_terminal', 'api_content', '_absorbed_message_uids', '_row_id', '_state_db_row_id', '_state_db_message_uid', '_state_db_generation', '_state_db_authored_change', '_state_db_source_key', '_db_row_id', 'state_db_row_id', '_active_turn_token', '_active_turn_user', '_fork_child_turn']);
 export function copyJson(value) {
     if (Array.isArray(value))
         return value.map((v) => copyJson(v));
@@ -2016,25 +2251,68 @@ export function buildActiveTurnToken(streamId, pendingStartedAt) {
 const COLLAPSIBLE_USER_MESSAGE_LINES = 20;
 const COLLAPSIBLE_USER_MESSAGE_CHARS = 2000;
 /**
- * Whether a public (redacted) row is a typed user message (no steer, wakeup or marker) longer than either limit, measured
+ * Whether a public (redacted) row is a typed user message (no steer, wakeup, marker or delivery) longer than either limit, measured
  * without the `[Attached files: ...]` line clients hide.
  */
 function isCollapsibleUserMessage(item) {
-    if (item.role !== 'user' || item._steer !== undefined || item._background_update !== undefined || item._marker_kind !== undefined)
+    if (item.role !== 'user' || item._steer !== undefined || item._background_update !== undefined || item._marker_kind !== undefined || item._delivery !== undefined)
         return false;
     const text = stripAttachedFilesMarker(messageText(item.content));
     return text.length > COLLAPSIBLE_USER_MESSAGE_CHARS || text.split('\n').length > COLLAPSIBLE_USER_MESSAGE_LINES;
 }
+/**
+ * TAL-600: how clients show a public row. A tool result or consumed steer shows only inside its turn's activity scene
+ * (`in_scene`); a silent background reply, or a row with nothing to show, not at all (`hidden`); every other row on its own.
+ */
+function messageDisplay(item) {
+    if (item.role === 'tool' || item._steer)
+        return 'in_scene';
+    if (!item.role || item._background_silent || item.display_kind === 'hidden')
+        return 'hidden';
+    if (item._marker_kind)
+        return 'row';
+    const shown = Boolean(messageText(item.content).trim()) || (Array.isArray(item.attachments) && item.attachments.length > 0) || (item.role === 'user' && hasNativeImages([item]));
+    if (item.role !== 'assistant')
+        return shown ? 'row' : 'hidden';
+    const hasTools = Array.isArray(item.tool_calls) && item.tool_calls.length > 0;
+    return shown || hasTools || Boolean(item.reasoning || item._anchor_activity_scene) ? 'row' : 'hidden';
+}
+/**
+ * One user `content[*]` part (Python `_redact_message_content_part`): an `image_url.url` raster data URI passes through
+ * unscanned (TAL-583). The exemption exists only at this schema position; image-shaped values elsewhere are redacted.
+ */
+function redactUserContentPart(part, enabled) {
+    if (!enabled || !part || typeof part !== 'object' || Array.isArray(part))
+        return redactValue(part, enabled);
+    const p = part;
+    const image = p.image_url;
+    if (p.type !== 'image_url' || !image || typeof image !== 'object' || Array.isArray(image))
+        return redactValue(part, enabled);
+    const url = image.url;
+    if (!isRasterDataUri(url))
+        return redactValue(part, enabled);
+    const rest = Object.fromEntries(Object.entries(image).filter(([k]) => k !== 'url'));
+    const redacted = redactValue({ ...p, image_url: rest }, enabled);
+    redacted.image_url = { ...redacted.image_url, url };
+    return redacted;
+}
 function publicMessageProjection(message, enabled, activeTurnToken) {
     const isActive = Boolean(message && typeof message === 'object' && message.role === 'user' && activeTurnToken !== null && message._active_turn_token === activeTurnToken);
-    const scrubbed = scrubInternalReplayFields([message], { messageRecords: true })[0];
+    // Exports and nested public documents bypass scene hydration; project their user content at this boundary too.
+    const scrubbed = normalizeUserDisplay(scrubInternalReplayFields([message], { messageRecords: true })[0]);
     if (!scrubbed || typeof scrubbed !== 'object' || Array.isArray(scrubbed))
         return redactValue(scrubbed, enabled);
     const item = {};
     for (const [key, value] of Object.entries(scrubbed)) {
         if (PUBLIC_MESSAGE_INTERNAL_FIELDS.has(key))
             continue;
-        item[key] = redactValue(value, enabled);
+        // TAL-909: a tool row's metadata carries the whole edit diff, which ships capped and redacted as its call's `edit_diff`.
+        if (key === 'display_metadata' && scrubbed.role === 'tool')
+            continue;
+        const userParts = key === 'content' && Array.isArray(value) && scrubbed.role === 'user';
+        // Preserve the string/parts/null contract and redact JSON credential fields with their key context intact.
+        const publicValue = key === 'content' && value != null && typeof value !== 'string' && !Array.isArray(value) ? JSON.stringify(value) : value;
+        item[key] = userParts ? value.map((part) => redactUserContentPart(part, enabled)) : redactValue(publicValue, enabled);
     }
     if (isActive)
         item._active_turn_user = true;
@@ -2043,6 +2321,24 @@ function publicMessageProjection(message, enabled, activeTurnToken) {
         item._collapsible = true;
     else
         delete item._collapsible;
+    item._display = messageDisplay(item);
+    // TAL-598: stored and Agent rows may carry these as numeric strings or mixed arrays; ship the contract types or nothing.
+    for (const key of ['_ts', '_turnDuration', '_turnTps']) {
+        if (!(key in item))
+            continue;
+        const n = typeof item[key] === 'string' && item[key] !== '' ? Number(item[key]) : item[key];
+        if (typeof n === 'number' && Number.isFinite(n))
+            item[key] = n;
+        else
+            Reflect.deleteProperty(item, key);
+    }
+    if ('reasoning_titles' in item) {
+        const titles = Array.isArray(item.reasoning_titles) ? item.reasoning_titles.filter((t) => typeof t === 'string' && Boolean(t.trim())) : [];
+        if (titles.length)
+            item.reasoning_titles = titles;
+        else
+            delete item.reasoning_titles;
+    }
     return withMessageToolDisplay(scrubbed, item, enabled);
 }
 /**