@sema-agent/server 7.136.0 → 7.138.0

package/dist/trace/redact.js

@@ -1,4 +1,4 @@
-import { displayUrlOf, stripFormatCharacters } from "@sema-agent/core";
+import { REDACTION_MARK, displayUrlOf, stripFormatCharacters } from "@sema-agent/core";
 import { CONTROL_AND_BIDI_CHARS, CREDENTIAL_PARAM_NAMES, isHierarchicalUrl } from "@sema-agent/settings-schema";
 import { recordFailOpen } from "../observability/fail-open.js";
 const normalizeForRedaction = stripFormatCharacters;
@@ -819,13 +819,57 @@ if (/[\]^-]/.test(SCHEME_DELIMITER_CHARS))
 const SCHEME_DELIMITER_CODES = new Set([...SCHEME_DELIMITER_CHARS].map((c) => c.charCodeAt(0)));
 const isSchemeDelimiterCode = (c) => SCHEME_DELIMITER_CODES.has(c);
 const SCHEME_SEPARATOR_CLASS = `[\\s${SCHEME_DELIMITER_CHARS.replace(/\\/g, "\\\\")}]`;
-const LEXICAL_KEYWORD_ARMS = new Set();
+function expandKeywordForms(src) {
+    let forms = [""];
+    for (let i = 0; i < src.length;) {
+        let alts;
+        const pair = /^\[([A-Za-z])([A-Za-z])\]/.exec(src.slice(i));
+        if (pair !== null) {
+            if (pair[1].toLowerCase() !== pair[2].toLowerCase())
+                throw new Error(`redact: keyword source ${src} holds a class that is not a case pair`);
+            alts = [pair[1], pair[2]];
+            i += 4;
+        }
+        else if (/[A-Za-z0-9]/.test(src.charAt(i))) {
+            alts = [src.charAt(i)];
+            i += 1;
+        }
+        else {
+            throw new Error(`redact: keyword source ${src} holds an atom the head reading cannot enumerate (only letters, digits and [Xx] case pairs)`);
+        }
+        forms = forms.flatMap((f) => alts.map((a) => f + a));
+    }
+    const whole = new RegExp(`^(?:${src})$`);
+    for (const f of forms)
+        if (!whole.test(f))
+            throw new Error(`redact: keyword source ${src} does not match its own enumerated form ${f}`);
+    return forms;
+}
+const LEXICAL_KEYWORD_ARMS = new Map();
 const keywordArm = (keywordSrc, valueClass, min, tailSrc = "", separatorClass = "\\s") => {
     const re = new RegExp(`(?:${keywordSrc}${separatorClass}+${atLeast(valueClass, min)}${tailSrc}` +
         `|(?<!${WORD_SEGMENT_CLASS})${keywordSrc}(?:${twoStrongAhead(valueClass)}|${noConnectorAhead(valueClass)})${atLeast(valueClass, min)}${tailSrc})`, "g");
-    LEXICAL_KEYWORD_ARMS.add(re);
+    const separator = new RegExp(`^${separatorClass}$`);
+    const value = new RegExp(`^${valueClass}$`);
+    LEXICAL_KEYWORD_ARMS.set(re, { forms: expandKeywordForms(keywordSrc), min, isSeparator: (u) => separator.test(u), isValue: (u) => value.test(u) });
     return re;
 };
+function undecidedKeywordStart(text, t) {
+    const n = text.length;
+    let valueRun = 0;
+    while (valueRun < t.min && valueRun < n && t.isValue(text.charAt(n - 1 - valueRun)))
+        valueRun += 1;
+    if (valueRun >= t.min)
+        return -1;
+    let sepStart = n - valueRun;
+    while (sepStart > 0 && t.isSeparator(text.charAt(sepStart - 1)))
+        sepStart -= 1;
+    for (const f of t.forms) {
+        if (sepStart >= f.length && text.startsWith(f, sepStart - f.length))
+            return sepStart - f.length;
+    }
+    return -1;
+}
 const CREDENTIAL_NAME_ANCHORS = {
     api_key: "structure-only", apikey: "structure-only", "api-key": "structure-only", key: "structure-only",
     token: "structure-only", access_token: "structure-only", refresh_token: "structure-only", id_token: "structure-only",
@@ -859,8 +903,9 @@ const PPK_CONTAINER_TOKEN = "«redacted:private-key»";
 const PEM_FRAGMENT_TOKEN = "«redacted:private-key:partial»";
 const HEADLESS_PPK_TOKEN = "«redacted:private-key:partial»";
 const ANCHORED_VALUE_TOKEN = "«redacted»";
+const BEARER_VALUE_MIN = 20;
 const KEYWORD_SECRET_PATTERNS = [
-    [keywordArm("[Bb]earer", "[A-Za-z0-9._~+/-]", 20, "=*", SCHEME_SEPARATOR_CLASS), "Bearer «redacted»"],
+    [keywordArm("[Bb]earer", "[A-Za-z0-9._~+/-]", BEARER_VALUE_MIN, "=*", SCHEME_SEPARATOR_CLASS), "Bearer «redacted»"],
     [keywordArm("Basic", "[A-Za-z0-9+/]", 16, "={0,2}", SCHEME_SEPARATOR_CLASS), "Basic «redacted»"],
     [keywordArm("[Tt]oken", "[A-Za-z0-9._~+/-]", 20), "token «redacted»"],
 ];
@@ -1166,7 +1211,7 @@ function collectSsh2FenceSpans(scan, text, span, isHead) {
         }
         const bodyFrom = fence + SSH2_FENCE_OPEN.length;
         if (isHead && unitCountWithin(scan, bodyFrom, text.length, SSH2_BODY_MAX)) {
-            span(fence, text.length);
+            span(fence, text.length, true);
             return;
         }
         const close = nextClose(bodyFrom);
@@ -1224,7 +1269,7 @@ function collectPemFenceSpans(text, span, isHead) {
             earliestEndFence(bodyFrom);
         if (bodyFrom === PEM_LINE_OPEN || endAt < 0) {
             if (isHead)
-                span(fence, text.length);
+                span(fence, text.length, true);
             return;
         }
         if (endAt - bodyFrom > PEM_FENCE_BODY_MAX) {
@@ -1382,20 +1427,23 @@ function collectHeadlessPpkSpans(scan, text, span, isHead) {
             continue;
         }
         let end = -1;
+        let open = false;
         for (let from = p;;) {
             const r = bodyLineAt(from);
             if (r === LINE_NONE)
                 break;
             end = r === LINE_OPEN ? n : r;
-            if (r === LINE_OPEN)
+            if (r === LINE_OPEN) {
+                open = true;
                 break;
+            }
             from = r;
         }
         if (end < 0) {
             search = p;
             continue;
         }
-        span(at, end);
+        span(at, end, open);
         search = end;
     }
 }
@@ -1855,7 +1903,7 @@ function collectUrlCredentialSpans(scan, text, json, span, isHead) {
             hi = scan.footHi;
             urlSegmentRuns += 1;
             if (at >= 0 && at !== (known ? ph[pc + 1] : -2))
-                span(start, at);
+                span(start, at, isHead && at >= text.length);
         }
         if (hw.n + URL_SEG_STRIDE <= hw.a.length || growUrlSegments(hw)) {
             const w = hw.a;
@@ -1951,6 +1999,7 @@ function jsonSecretFieldValueQuoteAt(text, p) {
         i += 1;
     return text.charCodeAt(i) === CH_QUOTE ? i : -1;
 }
+const STRING_OPEN = -2;
 function jsonStringEndAt(text, q, isHead) {
     const n = text.length;
     for (let i = q + 1; i < n;) {
@@ -1962,25 +2011,36 @@ function jsonStringEndAt(text, q, isHead) {
                 i += 2;
                 continue;
             }
-            return isHead && i + 1 >= n ? n : -1;
+            return isHead && i + 1 >= n ? STRING_OPEN : -1;
         }
         i += 1;
     }
-    return isHead ? n : -1;
+    return isHead ? STRING_OPEN : -1;
 }
 function collectJsonSecretFieldSpans(text, span, isHead) {
     for (let p = text.indexOf(JSON_FIELD_ANCHOR); p >= 0;) {
         const q = jsonSecretFieldValueQuoteAt(text, p);
-        const end = q < 0 ? -1 : jsonStringEndAt(text, q, isHead);
-        if (end < 0) {
+        const found = q < 0 ? -1 : jsonStringEndAt(text, q, isHead);
+        if (found === -1) {
             p = text.indexOf(JSON_FIELD_ANCHOR, p + 1);
             continue;
         }
-        span(q, end);
+        const open = found === STRING_OPEN;
+        const end = open ? text.length : found;
+        span(q, end, open);
         p = text.indexOf(JSON_FIELD_ANCHOR, scanResumeAfterHit(p, end, JSON_FIELD_ANCHOR.length));
     }
 }
 const STRONG_AUTH_SCHEMES = new Set(["basic", "bearer"]);
+function isEngineMarkedCandidate(text, start, end, repl) {
+    const open = repl.indexOf("«");
+    const close = repl.lastIndexOf("»");
+    if (open <= 0 || close < open || close === repl.length - 1)
+        return false;
+    const lead = repl.slice(0, open);
+    const trail = repl.slice(close + 1);
+    return end - start === lead.length + REDACTION_MARK.length + trail.length && text.startsWith(lead, start) && text.startsWith(REDACTION_MARK, start + lead.length) && text.startsWith(trail, end - trail.length);
+}
 const STRONG_AUTH_SCHEME_MAX = Math.max(...[...STRONG_AUTH_SCHEMES].map((w) => w.length));
 const ABSENT_OR_BOOLEAN_WORDS = new Set(["null", "nil", "none", "undefined", "true", "false", "yes", "no"]);
 const ABSENT_OR_BOOLEAN_MAX = Math.max(...[...ABSENT_OR_BOOLEAN_WORDS].map((w) => w.length));
@@ -2144,7 +2204,7 @@ function quotedValueEnd(text, cs, quote, json, isHead) {
             i += unitLenOf(escaped);
         }
     }
-    return isHead ? n : -1;
+    return isHead ? STRING_OPEN : -1;
 }
 function bareValueEnd(text, s, json, listSep) {
     const n = text.length;
@@ -2203,6 +2263,8 @@ function literalValueAt(text, s, json, declaredIfBare, listSep, isHead) {
         if (!isAsciiAlnumCode(unitChOf(readAnchorUnit(text, vs, json))))
             return undefined;
         const close = quotedValueEnd(text, vs, first, json, isHead);
+        if (close === STRING_OPEN)
+            return { start: vs, end: n, quoted: true, open: true };
         if (close >= 0)
             return { start: vs, end: close, quoted: true };
     }
@@ -2235,11 +2297,18 @@ function schemeValueAt(text, s, json) {
     if (isQuoteUnit(unitChOf(opening)) || unitChOf(opening) === JSON_STRING_END)
         i += unitLenOf(opening);
     const w = i;
-    while (i < n && i - w <= STRONG_AUTH_SCHEME_MAX && (isIdentStartCode(text.charCodeAt(i))))
-        i += 1;
-    anchoredScanUnits += i - w;
-    if (i - w > STRONG_AUTH_SCHEME_MAX || !STRONG_AUTH_SCHEMES.has(text.slice(w, i).toLowerCase()))
-        return undefined;
+    const engineMarked = text.startsWith(REDACTION_MARK, i);
+    if (engineMarked) {
+        i += REDACTION_MARK.length;
+        anchoredScanUnits += REDACTION_MARK.length;
+    }
+    else {
+        while (i < n && i - w <= STRONG_AUTH_SCHEME_MAX && (isIdentStartCode(text.charCodeAt(i))))
+            i += 1;
+        anchoredScanUnits += i - w;
+        if (i - w > STRONG_AUTH_SCHEME_MAX || !STRONG_AUTH_SCHEMES.has(text.slice(w, i).toLowerCase()))
+            return undefined;
+    }
     let separated = false;
     while (i < n) {
         const unit = readAnchorUnit(text, i, json);
@@ -2252,6 +2321,17 @@ function schemeValueAt(text, s, json) {
     if (!separated)
         return undefined;
     const vs = i;
+    if (engineMarked) {
+        let k = i;
+        while (k < n && isToken68Code(text.charCodeAt(k)))
+            k += 1;
+        while (k < n && text.charCodeAt(k) === CH_EQ)
+            k += 1;
+        if (k - vs >= BEARER_VALUE_MIN) {
+            anchoredScanUnits += k - vs;
+            return { start: vs, end: k, quoted: false };
+        }
+    }
     if (i >= n || !isAsciiAlnumCode(text.charCodeAt(i)))
         return undefined;
     while (i < n && isToken68Code(text.charCodeAt(i)))
@@ -2323,7 +2403,7 @@ function collectAnchoredValueSpans(text, json, span, isHead) {
         }
         if (v === undefined)
             continue;
-        span(v.start, v.end);
+        span(v.start, v.end, v.open === true);
         resume = v.quoted ? v.start : v.end;
     }
 }
@@ -2400,12 +2480,15 @@ function collectPpkContainerSpans(scan, text, eat, span, isHead) {
         let pos = at + HEAD.length + 1;
         let end = text.length;
         let ateToEnd = !isHead;
+        let open = false;
         for (;;) {
             const mac = find(MAC, pos);
             const hash = find(HASH, pos);
             const term = Math.min(mac, hash);
-            if (term === Number.POSITIVE_INFINITY)
+            if (term === Number.POSITIVE_INFINITY) {
+                open = isHead;
                 break;
+            }
             const hexStart = term + (term === mac ? MAC.length : HASH.length);
             const hexEnd = hexRunEnd(hexStart);
             if (hexEnd - hexStart < 32) {
@@ -2435,18 +2518,19 @@ function collectPpkContainerSpans(scan, text, eat, span, isHead) {
             const undecided = isHead && limit === Number.POSITIVE_INFINITY;
             ateToEnd = straggler < limit || !gapBlank;
             end = ateToEnd || undecided ? text.length : hexEnd;
+            open = undecided;
             break;
         }
         if (ateToEnd && eat !== undefined && eat.from < 0) {
             eat.from = at;
             eat.len = text.length;
         }
-        span(at, end);
+        span(at, end, open);
         at = end >= text.length ? Number.POSITIVE_INFINITY : nextVersionedHead(end);
     }
     const last = isHead ? text.lastIndexOf(HEAD) : -1;
     if (last >= 0 && !inOpaqueUnit(scan, last) && isOpenHead(last))
-        span(last, text.length);
+        span(last, text.length, true);
 }
 const armPool = (() => {
     const spanRepl = (token) => {
@@ -2468,6 +2552,7 @@ const armPool = (() => {
             throw new Error(`redact: boundary arm /${re.source}/ has a zero-length structural head — the bounded head screen would admit every candidate position`);
         }
         const arm = { g: re, y, back, headSrc, headFirst, repl, rank, order };
+        const keywordTail = LEXICAL_KEYWORD_ARMS.get(re);
         const boundary = y === undefined || headSrc === undefined || headFirst === undefined ? undefined : { ...arm, y, headSrc, headFirst };
         return {
             token,
@@ -2475,9 +2560,17 @@ const armPool = (() => {
             order,
             phase: 1,
             lexical: LEXICAL_KEYWORD_ARMS.has(re),
-            collect: boundary === undefined
-                ? (text, _json, _eat, out) => collectRegexSpans(arm, text, out)
-                : (text, _json, _eat, out) => collectValueArmSpans(boundary, text, out),
+            collect: (text, _json, _eat, out, isHead) => {
+                if (boundary === undefined)
+                    collectRegexSpans(arm, text, out);
+                else
+                    collectValueArmSpans(boundary, text, out);
+                if (isHead && keywordTail !== undefined) {
+                    const at = undecidedKeywordStart(text, keywordTail);
+                    if (at >= 0)
+                        out.push({ start: at, end: text.length, repl, rank, order, undecided: true });
+                }
+            },
             retry: boundary === undefined ? undefined : (text, at, out) => collectStickySpans(boundary, text, at, out),
             headSrc: boundary === undefined ? undefined : headSrc,
             headFirst: boundary === undefined ? undefined : headFirst,
@@ -2491,7 +2584,7 @@ const armPool = (() => {
             order,
             phase: 1,
             lexical,
-            collect: (text, json, eat, out, isHead) => arm(text, json, eat, (start, end) => out.push({ start, end, repl, rank, order }), isHead),
+            collect: (text, json, eat, out, isHead) => arm(text, json, eat, (start, end, undecided) => out.push(undecided === true ? { start, end, repl, rank, order, undecided } : { start, end, repl, rank, order }), isHead),
             retry: undefined,
             headSrc: undefined,
             headFirst: undefined,
@@ -2506,7 +2599,7 @@ const armPool = (() => {
             order,
             phase: 2,
             lexical: false,
-            collect: (scan, text, json, eat, out, isHead) => arm(scan, text, json, eat, (start, end) => out.push({ start, end, repl, rank, order }), isHead),
+            collect: (scan, text, json, eat, out, isHead) => arm(scan, text, json, eat, (start, end, undecided) => out.push(undecided === true ? { start, end, repl, rank, order, undecided } : { start, end, repl, rank, order }), isHead),
             retry: undefined,
             headSrc: undefined,
             headFirst: undefined,
@@ -2560,8 +2653,9 @@ const armPool = (() => {
     const headFirstUnit = unionMembers(retryMembers.map((m) => m.headFirst)) ?? new Uint32Array(0x10000 >>> 5).fill(0xffffffff);
     poolArmCount = members.length;
     poolBoundaryArmCount = retryMembers.length;
-    return (text, wantCount, json, eat, scope, isHead) => {
+    return (text, wantCount, json, eat, scope, reach, engineMarked) => {
         readings += 1;
+        const isHead = reach !== undefined;
         const lane = LANES[scope];
         const spans = [];
         let tried;
@@ -2629,11 +2723,12 @@ const armPool = (() => {
             }
         }
         poolSpanCount += spans.length;
-        const kept = resolveSpanOverlaps(spans);
+        const survives = (sp) => !engineMarked(sp.start, sp.end, sp.repl) && (sp.undecided !== true || reach === undefined || reach(sp.start));
+        const kept = resolveSpanOverlaps(spans.filter(survives));
         if (!wantCount || kept.length === 0)
             return { hits: NO_HITS, kept };
         const tokenOf = (sp) => members[sp.order].token;
-        const strong = spans.filter((sp) => armOfToken(tokenOf(sp)) === "full").sort((a, b) => a.start - b.start);
+        const strong = spans.filter((sp) => armOfToken(tokenOf(sp)) === "full" && survives(sp)).sort((a, b) => a.start - b.start);
         const byArmedToken = new Map();
         const stamped = [];
         let si = 0;
@@ -2660,8 +2755,8 @@ let onRedaction;
 export function setRedactionObserver(fn) {
     onRedaction = fn;
 }
-function applyRules(text, wantCount, json, eat, scope, isHead) {
-    return armPool(text, wantCount, json, eat, scope, isHead);
+function applyRules(text, wantCount, json, eat, scope, reach, engineMarked) {
+    return armPool(text, wantCount, json, eat, scope, reach, engineMarked);
 }
 function reportHits(hits) {
     if (onRedaction === undefined)
@@ -2674,6 +2769,7 @@ function reportHits(hits) {
         }
     }
 }
+export const WHOLE_EXTENT = { kind: "whole" };
 function invisibleClassesOf(layout) {
     switch (layout) {
         case "multi-line":
@@ -2786,15 +2882,18 @@ function runRedaction(text, observe, json, wantHits, scope, layout, extent) {
     let readsSoFar = 0;
     let unioned = false;
     const read = (layers, view) => {
-        const r = applyRules(view, wantCount, json, eat, scope, extent === "head");
+        const toOriginal = createViewToOriginal(layers);
+        const reach = extent.kind === "head" ? (viewStart) => toOriginal.start(viewStart) < extent.emitEnd : undefined;
+        const engineMarked = (viewStart, viewEnd, repl) => isEngineMarkedCandidate(text, toOriginal.start(viewStart), toOriginal.start(viewEnd - 1) + 1, repl);
+        const r = applyRules(view, wantCount, json, eat, scope, reach, engineMarked);
         forgetEscapedSeparatorTails();
         readsSoFar += 1;
         if (readsSoFar === 1) {
-            spans = spansInOriginal(layers, r.kept);
+            spans = toOriginal.spans(r.kept);
             hits = r.hits;
             return;
         }
-        const extra = spansInOriginal(layers, r.kept);
+        const extra = toOriginal.spans(r.kept);
         if (extra.length === 0)
             return;
         spans = resolveSpanOverlaps([...spans, ...extra]);
@@ -2836,37 +2935,32 @@ function hitsOfSpans(spans) {
     }
     return [...by.values()].map(({ token, arm, count }) => ({ label: labelOfToken(token), count, arm }));
 }
-function spansInOriginal(layers, kept) {
-    let out = kept;
-    for (const layer of layers) {
-        if (out.length === 0)
-            return out;
-        let at;
+function createViewToOriginal(layers) {
+    let ats;
+    const layerMaps = () => (ats ??= layers.flatMap((layer) => {
         if ("fold" in layer) {
             const fold = layer.fold;
-            if (fold.at === undefined)
-                continue;
-            at = (v) => viewIndexInInput(fold, v);
-        }
-        else {
-            if (layer.view === layer.input)
-                continue;
-            const map = mapNormalizedToOriginal(layer.input, layer.view);
-            at = (v) => map[v];
+            return fold.at === undefined ? [] : [(v) => viewIndexInInput(fold, v)];
         }
-        out = out.map((s) => ({ ...s, start: at(s.start), end: at(s.end - 1) + 1 }));
-    }
-    return out;
+        if (layer.view === layer.input)
+            return [];
+        const map = mapNormalizedToOriginal(layer.input, layer.view);
+        return [(v) => map[v]];
+    }));
+    return {
+        start: (viewIndex) => layerMaps().reduce((p, at) => at(p), viewIndex),
+        spans: (kept) => (kept.length === 0 ? kept : layerMaps().reduce((out, at) => out.map((s) => ({ ...s, start: at(s.start), end: at(s.end - 1) + 1 })), kept)),
+    };
 }
 export function redactSecrets(text, opts) {
-    return runRedaction(text, opts?.observe ?? true, opts?.json === true, false, "free-text", "multi-line", "whole").text;
+    return runRedaction(text, opts?.observe ?? true, opts?.json === true, false, "free-text", "multi-line", WHOLE_EXTENT).text;
 }
 export function redactSecretsWithArms(text, opts) {
-    const r = runRedaction(text, opts?.observe ?? true, opts?.json === true, true, "free-text", "multi-line", "whole");
+    const r = runRedaction(text, opts?.observe ?? true, opts?.json === true, true, "free-text", "multi-line", WHOLE_EXTENT);
     return { text: r.text, arms: new Set(r.hits.map((h) => h.arm)) };
 }
 export function redactionSpansForTest(text, opts) {
-    const r = runRedaction(text, false, opts?.json === true, false, "free-text", "multi-line", "whole");
+    const r = runRedaction(text, false, opts?.json === true, false, "free-text", "multi-line", WHOLE_EXTENT);
     return { text: r.text, spans: r.spans.map((sp) => ({ start: sp.start, end: sp.end })) };
 }
 export function proposeRedactionSpans(text, scope, layout, extent) {
@@ -2972,7 +3066,7 @@ function unitCarriesCredential(unit) {
         for (const v of armValueClassViews())
             views.add(boundedUnitView(spelling, v.inClass, v.filler));
     views.delete("");
-    const hit = [...views].some((view) => runRedaction(view, true, false, false, "bounded-unit", "multi-line", "whole").text !== view);
+    const hit = [...views].some((view) => runRedaction(view, true, false, false, "bounded-unit", "multi-line", WHOLE_EXTENT).text !== view);
     return { hit, unreadable: decoded.residue };
 }
 function displaySafePathname(pathname) {
@@ -3002,7 +3096,7 @@ function shapeLawCredentialInQueryOrFragment(canonical) {
     const boundary = canonical.indexOf("?");
     if (boundary < 0)
         return false;
-    if (runRedaction(canonical, false, false, false, "free-text", "multi-line", "whole").spans.some((sp) => sp.end > boundary))
+    if (runRedaction(canonical, false, false, false, "free-text", "multi-line", WHOLE_EXTENT).spans.some((sp) => sp.end > boundary))
         return true;
     for (const unit of canonical.slice(boundary + 1).split("&").flatMap((sg) => {
         const eq = sg.indexOf("=");