@sema-agent/server 7.143.0 → 7.145.0

package/dist/trace/redact.js

@@ -73,7 +73,7 @@ function collectSpelledCuts(s) {
     let present = 0;
     let i = s.indexOf("\\");
     if (i < 0)
-        return { cuts, present };
+        return { cuts, present, open: -1 };
     const n = s.length;
     const read = { end: 0 };
     while (i >= 0) {
@@ -81,6 +81,8 @@ function collectSpelledCuts(s) {
         while (body < n && s.charCodeAt(body) === CH_BACKSLASH)
             body += 1;
         const cp = readSpelledCodePoint(s, body, read);
+        if (cp === ESCAPE_TRUNCATED)
+            return { cuts, present, open: i };
         const cls = cp < 0 ? 0 : isFormatCodePoint(cp) ? SPELLED_FORMAT : cp < 0x10000 && INVISIBLE_UNIT_CLASS[cp] === 1 ? SPELLED_CONTROL : 0;
         if (cls !== 0) {
             cuts.push(i, read.end, cls);
@@ -89,7 +91,7 @@ function collectSpelledCuts(s) {
         }
         i = s.indexOf("\\", body);
     }
-    return { cuts, present };
+    return { cuts, present, open: -1 };
 }
 function deleteInvisibleCharacters(s, all, mask) {
     const atList = [];
@@ -127,9 +129,12 @@ function viewIndexInInput(fold, v) {
     }
     return lo === 0 ? v : v + fold.cum[lo - 1];
 }
+const ESCAPE_TRUNCATED = -2;
 function readHexRun(s, at, count) {
     let v = 0;
     for (let k = at; k < at + count; k++) {
+        if (k >= s.length)
+            return ESCAPE_TRUNCATED;
         const d = hexDigitValue(s.charCodeAt(k));
         if (d < 0)
             return -1;
@@ -138,6 +143,8 @@ function readHexRun(s, at, count) {
     return v;
 }
 function readEscapeElement(s, at, out) {
+    if (at >= s.length)
+        return ESCAPE_TRUNCATED;
     const kind = s.charCodeAt(at);
     if (kind === CH_x) {
         out.end = at + 3;
@@ -145,6 +152,8 @@ function readEscapeElement(s, at, out) {
     }
     if (kind !== CH_u)
         return -1;
+    if (at + 1 >= s.length)
+        return ESCAPE_TRUNCATED;
     if (s.charCodeAt(at + 1) === CH_LBRACE) {
         let v = 0;
         let k = at + 2;
@@ -156,6 +165,8 @@ function readEscapeElement(s, at, out) {
             if (v > 0x10ffff)
                 return -1;
         }
+        if (k >= s.length)
+            return ESCAPE_TRUNCATED;
         if (k === at + 2 || s.charCodeAt(k) !== CH_RBRACE)
             return -1;
         out.end = k + 1;
@@ -171,13 +182,20 @@ function readSpelledCodePoint(s, at, out) {
     if (v > 0xdbff)
         return -1;
     let k = out.end;
+    if (k >= s.length)
+        return ESCAPE_TRUNCATED;
     if (s.charCodeAt(k) !== CH_BACKSLASH)
         return -1;
     while (s.charCodeAt(k) === CH_BACKSLASH)
         k += 1;
     const lo = readEscapeElement(s, k, out);
+    if (lo === ESCAPE_TRUNCATED)
+        return ESCAPE_TRUNCATED;
     return lo >= 0xdc00 && lo <= 0xdfff ? 0x10000 + ((v - 0xd800) << 10) + (lo - 0xdc00) : -1;
 }
+function isEscapeBodyUnit(c) {
+    return c === CH_BACKSLASH || c === CH_u || c === CH_x || c === CH_LBRACE || c === CH_RBRACE || hexDigitValue(c) >= 0;
+}
 const FORMAT_CODEPOINT_TABLE = (() => {
     const table = new Uint8Array(0x10000);
     const units = [];
@@ -463,9 +481,13 @@ let ssh2ScanBytes = 0;
 let mappedUnits = 0;
 let coverageVisits = 0;
 let rowVisits = 0;
+let scannedUnits = 0;
+let windowAdvances = 0;
+let windowRefusals = 0;
+let windowEvaluations = 0;
 let readings = 0;
 export function armPoolBudgetForTest() {
-    return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, forensicRuns, forensicSkips, urlSegmentRuns, urlSegmentReplays, ssh2ScanBytes, mappedUnits, readings, coverageVisits, rowVisits };
+    return { spanCount: poolSpanCount, coveredPositions, headScreenHits, fullYCalls, armCount: poolArmCount, boundaryArmCount: poolBoundaryArmCount, forensicLevels, forensicRuns, forensicSkips, urlSegmentRuns, urlSegmentReplays, ssh2ScanBytes, mappedUnits, readings, coverageVisits, rowVisits, scannedUnits, windowAdvances, windowRefusals, windowEvaluations };
 }
 export function resetArmPoolBudgetForTest() {
     poolSpanCount = 0;
@@ -482,6 +504,10 @@ export function resetArmPoolBudgetForTest() {
     readings = 0;
     coverageVisits = 0;
     rowVisits = 0;
+    scannedUnits = 0;
+    windowAdvances = 0;
+    windowRefusals = 0;
+    windowEvaluations = 0;
 }
 const unitBit = (t, c) => (t[c >>> 5] >>> (c & 31)) & 1;
 let codeUnitDomain;
@@ -3169,7 +3195,7 @@ function anchorHeadAt(scan, text, op, json) {
         if (i < 0)
             return undefined;
         const c = text.charCodeAt(i);
-        if (c === CH_SPACE || c === CH_TAB) {
+        if (isInlineSpaceUnit(c)) {
             i -= 1;
             spacedLeft = true;
             anchoredScanUnits += 1;
@@ -3237,7 +3263,7 @@ function anchorHeadAt(scan, text, op, json) {
     let listSep = 0;
     if (!afterEscape) {
         let k = before;
-        while (k >= 0 && (text.charCodeAt(k) === CH_SPACE || text.charCodeAt(k) === CH_TAB))
+        while (k >= 0 && isInlineSpaceUnit(text.charCodeAt(k)))
             k -= 1;
         anchoredScanUnits += before - k;
         const left = k >= 0 ? text.charCodeAt(k) : 0;
@@ -3695,6 +3721,7 @@ function collectPpkContainerSpans(scan, text, eat, span, isHead) {
     if (last >= 0 && isOpenHead(last) && !lineHeld(scan, last, text.length, readingKey(fam, last)))
         span(last, text.length, last, NOT_CLOSED, true);
 }
+const NO_GAP = (_unit) => false;
 const armPool = (() => {
     const spanRepl = (token) => {
         const body = token.startsWith("$1") ? token.slice(2) : token;
@@ -3739,9 +3766,10 @@ const armPool = (() => {
             headFirst: boundary === undefined ? undefined : headFirst,
             family: "prose",
             reach: regexDecisionReach(re, LEXICAL_KEYWORD_ARMS.has(re)),
+            prior: { kind: "bounded", gap: keywordTail === undefined ? NO_GAP : (c) => keywordTail.isSeparator(String.fromCharCode(c)) },
         };
     };
-    const scannerMember = (token, order, arm, reach, family) => {
+    const scannerMember = (token, order, arm, reach, family, prior) => {
         const repl = spanRepl(token);
         const rank = ARM_RANK.structural;
         return {
@@ -3756,9 +3784,10 @@ const armPool = (() => {
             headFirst: undefined,
             family,
             reach,
+            prior,
         };
     };
-    const forensicMember = (token, order, arm, reach, family, rank = ARM_RANK.structural, lexical = false) => {
+    const forensicMember = (token, order, arm, reach, family, prior, rank = ARM_RANK.structural, lexical = false) => {
         const repl = spanRepl(token);
         return {
             token,
@@ -3773,26 +3802,27 @@ const armPool = (() => {
             family,
             repl,
             reach,
+            prior,
         };
     };
     const structuralScannerMember = (row, order) => {
         if (row === PEM_FENCE_PATTERN)
-            return forensicMember(row[1], order, (scan, text, _json, _eat, span, isHead) => collectPemFenceSpans(scan, text, span, isHead), PEM_FENCE_OPEN.length, "pem-armor");
+            return forensicMember(row[1], order, (scan, text, _json, _eat, span, isHead) => collectPemFenceSpans(scan, text, span, isHead), PEM_FENCE_OPEN.length, "pem-armor", { kind: "anchored", anchors: [PEM_FENCE_OPEN, PEM_END_OPEN] });
         if (row === SSH2_FENCE_PATTERN)
-            return forensicMember(row[1], order, (scan, text, json, _eat, span, isHead) => collectSsh2FenceSpans(scan, text, json, span, isHead), SSH2_FENCE_OPEN.length, "ssh2-armor");
+            return forensicMember(row[1], order, (scan, text, json, _eat, span, isHead) => collectSsh2FenceSpans(scan, text, json, span, isHead), SSH2_FENCE_OPEN.length, "ssh2-armor", { kind: "anchored", anchors: [SSH2_FENCE_OPEN, SSH2_FENCE_CLOSE] });
         if (row === URL_CREDENTIAL_PATTERN)
-            return forensicMember(row[1], order, (scan, text, json, _eat, span, isHead) => collectUrlCredentialSpans(scan, text, json, span, isHead), "://".length, "url");
+            return forensicMember(row[1], order, (scan, text, json, _eat, span, isHead) => collectUrlCredentialSpans(scan, text, json, span, isHead), "://".length, "url", { kind: "bounded", gap: NO_GAP });
         if (row === JSON_SECRET_FIELD_PATTERN)
-            return scannerMember(row[1], order, (text, _json, _eat, span, isHead) => collectJsonSecretFieldSpans(text, span, isHead), JSON_FIELD_BACK, "json-field");
+            return scannerMember(row[1], order, (text, _json, _eat, span, isHead) => collectJsonSecretFieldSpans(text, span, isHead), JSON_FIELD_BACK, "json-field", { kind: "bounded", gap: isRegexSpace });
         throw new Error(`redact: structural row /${row[0].source}/ has no hand-written scanner — structural grammars are declared as regexes but never executed as one (S-102 / S-265)`);
     };
     const members = [];
     for (const row of STRUCTURAL_SECRET_PATTERNS)
         members.push(structuralScannerMember(row, members.length));
-    members.push(forensicMember(PPK_CONTAINER_TOKEN, members.length, (scan, text, _json, eat, span, isHead) => collectPpkContainerSpans(scan, text, eat, span, isHead), PPK_FILE_HEAD.length, "putty-ppk"));
-    members.push(forensicMember(PEM_FRAGMENT_TOKEN, members.length, (scan, text, json, _eat, span, isHead) => collectPemFragmentSpans(scan, text, json, span, isHead), PEM_FENCE_OPEN.length, "pem-armor"));
-    members.push(forensicMember(HEADLESS_PPK_TOKEN, members.length, (scan, text, _json, _eat, span, isHead) => collectHeadlessPpkSpans(scan, text, span, isHead), PPK_LINES_FIELD.length, "putty-ppk"));
-    members.push(forensicMember(ANCHORED_VALUE_TOKEN, members.length, (scan, text, json, _eat, span, isHead) => collectAnchoredValueSpans(scan, text, json, span, isHead), Math.max(BEARER_VALUE_MIN, ABSENT_OR_BOOLEAN_MAX + 1, STRONG_AUTH_SCHEME_MAX + 1), "anchor", ARM_RANK.keyword, true));
+    members.push(forensicMember(PPK_CONTAINER_TOKEN, members.length, (scan, text, _json, eat, span, isHead) => collectPpkContainerSpans(scan, text, eat, span, isHead), PPK_FILE_HEAD.length, "putty-ppk", { kind: "anchored", anchors: [PPK_FILE_HEAD] }));
+    members.push(forensicMember(PEM_FRAGMENT_TOKEN, members.length, (scan, text, json, _eat, span, isHead) => collectPemFragmentSpans(scan, text, json, span, isHead), PEM_FENCE_OPEN.length, "pem-armor", { kind: "anchored", anchors: [PEM_FENCE_OPEN, PEM_END_OPEN] }));
+    members.push(forensicMember(HEADLESS_PPK_TOKEN, members.length, (scan, text, _json, _eat, span, isHead) => collectHeadlessPpkSpans(scan, text, span, isHead), PPK_LINES_FIELD.length, "putty-ppk", { kind: "anchored", anchors: [PPK_LINES_FIELD] }));
+    members.push(forensicMember(ANCHORED_VALUE_TOKEN, members.length, (scan, text, json, _eat, span, isHead) => collectAnchoredValueSpans(scan, text, json, span, isHead), Math.max(BEARER_VALUE_MIN, ABSENT_OR_BOOLEAN_MAX + 1, STRONG_AUTH_SCHEME_MAX + 1), "anchor", { kind: "bounded", gap: (c) => isInlineSpaceUnit(c) || isSchemeDelimiterCode(c) }, ARM_RANK.keyword, true));
     for (const row of KEYWORD_SECRET_PATTERNS)
         members.push(regexMember(row, ARM_RANK.keyword, members.length));
     for (const row of SHAPE_SECRET_PATTERNS)
@@ -3963,7 +3993,7 @@ const armPool = (() => {
         }
         return { hits: [...byArmedToken.values()].map(({ token, arm, count }) => ({ label: labelOfToken(token), count, arm })), kept: stamped };
     };
-    return Object.assign(run, { reaches, pendingTailUnits: Math.max(...reaches.map((r) => r.reach)) });
+    return Object.assign(run, { reaches, pendingTailUnits: Math.max(...reaches.map((r) => r.reach)), priors: members.map((m) => m.prior) });
 })();
 let onRedaction;
 export function setRedactionObserver(fn) {
@@ -3996,70 +4026,6 @@ function invisibleClassesOf(layout) {
         }
     }
 }
-function trailingPartialEscapeStart(s) {
-    const n = s.length;
-    const runStart = (end) => {
-        let k = end;
-        while (k > 0 && s.charCodeAt(k - 1) === CH_BACKSLASH)
-            k -= 1;
-        return k;
-    };
-    const highSurrogateEscapeStartBefore = (end) => {
-        let j = end;
-        let value = -1;
-        let bodyStart = -1;
-        if (j >= 1 && s.charCodeAt(j - 1) === CH_RBRACE) {
-            let k = j - 1;
-            while (k > 0 && hexDigitValue(s.charCodeAt(k - 1)) >= 0)
-                k -= 1;
-            if (k >= 2 && s.charCodeAt(k - 1) === CH_LBRACE && s.charCodeAt(k - 2) === CH_u && j - 1 - k >= 1) {
-                value = readHexRunValue(s, k, j - 1);
-                bodyStart = k - 2;
-            }
-        }
-        else if (j >= 5 && s.charCodeAt(j - 5) === CH_u) {
-            value = readHexRun(s, j - 4, 4);
-            bodyStart = j - 5;
-        }
-        if (value < 0xd800 || value > 0xdbff)
-            return -1;
-        const k = runStart(bodyStart);
-        return k < bodyStart ? k : -1;
-    };
-    let j = n;
-    while (j > 0 && hexDigitValue(s.charCodeAt(j - 1)) >= 0)
-        j -= 1;
-    const hexRun = n - j;
-    let element = -1;
-    if (j >= 2 && s.charCodeAt(j - 1) === CH_LBRACE && s.charCodeAt(j - 2) === CH_u)
-        element = j - 2;
-    else if (j >= 1 && s.charCodeAt(j - 1) === CH_u && hexRun <= 3)
-        element = j - 1;
-    else if (j >= 1 && s.charCodeAt(j - 1) === CH_x && hexRun <= 1)
-        element = j - 1;
-    else if (hexRun === 0 && n >= 1 && s.charCodeAt(n - 1) === CH_BACKSLASH)
-        element = n;
-    if (element >= 0) {
-        const k = runStart(element);
-        if (k >= element)
-            return -1;
-        const high = highSurrogateEscapeStartBefore(k);
-        return high >= 0 ? high : k;
-    }
-    return highSurrogateEscapeStartBefore(n);
-}
-function readHexRunValue(s, from, to) {
-    let v = 0;
-    for (let k = from; k < to; k++) {
-        const d = hexDigitValue(s.charCodeAt(k));
-        if (d < 0)
-            return -1;
-        v = v * 16 + d;
-        if (v > 0x10ffff)
-            return -1;
-    }
-    return v;
-}
 function shortestReading(text, layout) {
     const r = readShortest(text, layout);
     return { visible: r.visible, partial: r.partial };
@@ -4068,8 +4034,8 @@ function readShortest(text, layout) {
     const raw = collectRawControlCuts(text, invisibleClassesOf(layout));
     const rfold = deleteInvisibleCharacters(text, raw, raw.present);
     const normalized = normalizeForRedaction(rfold.text);
-    const open = trailingPartialEscapeStart(normalized);
     const spelled = collectSpelledCuts(normalized);
+    const open = spelled.open;
     const sfold = deleteInvisibleCharacters(normalized, spelled, spelled.present);
     const layers = [{ fold: sfold }, { input: rfold.text, view: normalized }, { fold: rfold }];
     return open < 0 ? { visible: sfold.text.length, partial: false, layers } : { visible: sfold.text.length - (normalized.length - open), partial: true, layers };
@@ -4094,7 +4060,8 @@ export function readingHeadEnd(text, layout, units, from = 0) {
     }
     return text.length;
 }
-function runRedaction(text, observe, json, wantHits, scope, layout, extent) {
+function runRedaction(text, observe, json, wantHits, scope, layout, extent, observeFrom = 0) {
+    scannedUnits += text.length;
     const wantCount = wantHits || (onRedaction !== undefined && observe);
     const eat = observe ? { from: -1, len: 0, capFrom: -1, capLen: 0 } : undefined;
     let spans = [];
@@ -4140,7 +4107,7 @@ function runRedaction(text, observe, json, wantHits, scope, layout, extent) {
     if (eat !== undefined && eat.capFrom >= 0)
         recordFailOpen("server.redact.forensic-level-cap", `from=${eat.capFrom} len=${eat.capLen}`);
     if (observe)
-        reportHits(hits);
+        reportHits(observeFrom === 0 ? hits : hitsOfSpans(spans.filter((sp) => sp.end > observeFrom)));
     return { text: applySpans(text, spans).out, hits, spans };
 }
 function hitsOfSpans(spans) {
@@ -4175,7 +4142,12 @@ function createViewToOriginal(layers) {
     };
 }
 export function redactSecrets(text, opts) {
-    return runRedaction(text, opts?.observe ?? true, opts?.json === true, false, "free-text", "multi-line", WHOLE_EXTENT).text;
+    const before = opts?.before ?? "";
+    if (before === "")
+        return runRedaction(text, opts?.observe ?? true, opts?.json === true, false, "free-text", "multi-line", WHOLE_EXTENT).text;
+    const whole = before + text;
+    const r = runRedaction(whole, opts?.observe ?? true, opts?.json === true, false, "free-text", "multi-line", WHOLE_EXTENT, before.length);
+    return renderRange(whole, r.spans, before.length, whole.length);
 }
 export function redactSecretsWithArms(text, opts) {
     const r = runRedaction(text, opts?.observe ?? true, opts?.json === true, true, "free-text", "multi-line", WHOLE_EXTENT);
@@ -4220,56 +4192,219 @@ function renderRange(text, spans, from, until) {
 }
 const STREAM_RESCAN_MIN = 64;
 const streamRescanStep = (n) => Math.max(STREAM_RESCAN_MIN, n >>> 5);
+const STREAM_WINDOW_HORIZON = 2;
+const STREAM_WINDOW_MIN = 2048;
+function deriveStreamWindowAlphabets(priors) {
+    const blank = new Uint8Array(0x10000);
+    const gaps = new Set();
+    const containerDelimiters = [];
+    for (const prior of priors) {
+        switch (prior.kind) {
+            case "bounded":
+                if (prior.gap !== NO_GAP)
+                    gaps.add(prior.gap);
+                break;
+            case "anchored":
+                if (prior.anchors.length === 0)
+                    throw new Error("redact: an anchored arm declares no anchor literal — the stream window cannot tell where its reading of prior text starts");
+                for (const anchor of prior.anchors) {
+                    if (anchor.length === 0)
+                        throw new Error("redact: an anchored arm declares an empty anchor literal");
+                    for (let i = 0; i < anchor.length; i++) {
+                        const c = anchor.charCodeAt(i);
+                        if (INVISIBLE_UNIT_CLASS[c] !== 0 || FORMAT_CODEPOINT_TABLE[c] === 1 || (c & 0xf800) === 0xd800 || c === CH_BACKSLASH) {
+                            throw new Error(`redact: anchor literal ${JSON.stringify(anchor)} holds a unit some reading deletes or reads as an escape (index ${i}) — its presence is not monotone under deletion, so the stream window's delimiter check on the most-deleted reading would be incomplete`);
+                        }
+                    }
+                    if (isEscapeBodyUnit(anchor.charCodeAt(0))) {
+                        throw new Error(`redact: anchor literal ${JSON.stringify(anchor)} starts with a unit an escape spelling's body can read — an escape begun just before it deletes that unit in the most-deleted reading, so its presence is not monotone under deletion and the stream window's delimiter check would be incomplete`);
+                    }
+                    if (!containerDelimiters.includes(anchor))
+                        containerDelimiters.push(anchor);
+                }
+                break;
+            default: {
+                const unhandled = prior;
+                throw new Error(`redact: unhandled prior reading ${JSON.stringify(unhandled)}`);
+            }
+        }
+    }
+    for (const gap of gaps)
+        for (let u = 0; u < 0x10000; u++)
+            if (gap(u))
+                blank[u] = 1;
+    return { blank, containerDelimiters };
+}
+const STREAM_WINDOW_ALPHABETS = deriveStreamWindowAlphabets(armPool.priors);
+export function streamWindowAlphabetsForTest(extra = []) {
+    const a = extra.length === 0 ? STREAM_WINDOW_ALPHABETS : deriveStreamWindowAlphabets([...armPool.priors, ...extra]);
+    return { isBlank: (unit) => a.blank[unit] === 1, containerDelimiters: a.containerDelimiters };
+}
+export function escapeGrammarForTest() {
+    return {
+        read: (s) => {
+            let body = 0;
+            while (body < s.length && s.charCodeAt(body) === CH_BACKSLASH)
+                body += 1;
+            const out = { end: -1 };
+            const value = readSpelledCodePoint(s, body, out);
+            return { value, end: value >= 0 ? out.end : -1 };
+        },
+        isBodyUnit: isEscapeBodyUnit,
+    };
+}
+function windowAdvancePoint(text, units, alphabets = STREAM_WINDOW_ALPHABETS) {
+    const r = readShortest(text, "multi-line");
+    if (r.partial)
+        return -1;
+    const view = r.layers[0].fold.text;
+    let k = r.visible;
+    let n = 0;
+    while (k > 0 && n < units) {
+        k -= 1;
+        if (alphabets.blank[view.charCodeAt(k)] !== 1)
+            n += 1;
+    }
+    if (n < units)
+        return 0;
+    for (const d of alphabets.containerDelimiters) {
+        const at = view.indexOf(d);
+        if (at >= 0 && at < k)
+            k = at;
+    }
+    const at = createViewToOriginal(r.layers).start(k);
+    return at <= 0 ? 0 : text.lastIndexOf("\n", at - 1) + 1;
+}
+function windowIndependentAt(text, q) {
+    const full = runRedaction(text, false, false, false, "free-text", "multi-line", { kind: "head", emitEnd: text.length }).spans;
+    if (full.some((s) => s.start < q && s.end > q))
+        return false;
+    const after = full.filter((s) => s.start >= q);
+    const tail = text.slice(q);
+    const alone = runRedaction(tail, false, false, false, "free-text", "multi-line", { kind: "head", emitEnd: tail.length }).spans;
+    if (after.length !== alone.length)
+        return false;
+    return after.every((a, i) => a.start - q === alone[i].start && a.end - q === alone[i].end && a.repl === alone[i].repl);
+}
+export function windowAdvancePointForTest(text, units, extra = []) {
+    return windowAdvancePoint(text, units, extra.length === 0 ? STREAM_WINDOW_ALPHABETS : deriveStreamWindowAlphabets([...armPool.priors, ...extra]));
+}
+export function windowIndependentAtForTest(text, q) {
+    return windowIndependentAt(text, q);
+}
 export function createRedactionStream() {
     return openRedactionStream(PENDING_TAIL_UNITS);
 }
+export function createRedactionStreamWithoutWindowForTest() {
+    return openRedactionStream(PENDING_TAIL_UNITS, false);
+}
+export function createRedactionStreamWithCertifierForTest(certify) {
+    return openRedactionStream(PENDING_TAIL_UNITS, true, certify);
+}
 export function createRedactionStreamWithTailForTest(units) {
     return openRedactionStream(units);
 }
-function openRedactionStream(pendingTail) {
-    let buf = "";
+function openRedactionStream(pendingTail, windowed = true, certify = windowIndependentAt) {
+    let past = [];
+    let total = 0;
+    let win = "";
+    let base = 0;
+    let blockFrom = 0;
     let released = 0;
     let deliveredFrom = 0;
     let sent = "";
     let scannedAt = 0;
+    let evaluatedAt = -1;
+    let retryAt = 0;
+    const whole = () => (past.length === 0 ? win : past.join("") + win);
+    const advanceWindow = () => {
+        if (!windowed || win.length < Math.max(STREAM_WINDOW_MIN, retryAt) || released < total || total === evaluatedAt)
+            return;
+        evaluatedAt = total;
+        retryAt = win.length + (win.length >> 1);
+        windowEvaluations += 1;
+        const q = windowAdvancePoint(win, STREAM_WINDOW_HORIZON * pendingTail);
+        if (q < 0) {
+            retryAt = 0;
+            return;
+        }
+        if (q === 0 || q * 2 < win.length)
+            return;
+        if (!certify(win, q)) {
+            windowRefusals += 1;
+            return;
+        }
+        past.push(win.slice(0, q));
+        win = win.slice(q);
+        base += q;
+        retryAt = 0;
+        windowAdvances += 1;
+    };
     return {
         feed(chunk) {
-            buf += chunk;
+            win += chunk;
+            total += chunk.length;
         },
         takeSettled() {
-            if (buf.length - released <= pendingTail)
+            if (total - released <= pendingTail)
                 return "";
-            if (buf.length - scannedAt < streamRescanStep(buf.length))
+            if (total - scannedAt < streamRescanStep(total))
                 return "";
-            scannedAt = buf.length;
-            const emitEnd = pendingTailStart(buf, pendingTail);
-            if (emitEnd <= released)
+            scannedAt = total;
+            const emitEnd = pendingTailStart(win, pendingTail);
+            const from = released - base;
+            if (emitEnd <= from)
                 return "";
-            const spans = runRedaction(buf, false, false, false, "free-text", "multi-line", { kind: "head", emitEnd }).spans;
+            const spans = runRedaction(win, false, false, false, "free-text", "multi-line", { kind: "head", emitEnd }).spans;
             let cut = emitEnd;
             for (const s of spans)
                 if (s.start < cut && s.end > cut)
                     cut = s.start;
-            if (cut > 0 && cut < buf.length && (buf.charCodeAt(cut - 1) & 0xfc00) === 0xd800 && (buf.charCodeAt(cut) & 0xfc00) === 0xdc00)
+            if (cut > 0 && cut < win.length && (win.charCodeAt(cut - 1) & 0xfc00) === 0xd800 && (win.charCodeAt(cut) & 0xfc00) === 0xdc00)
                 cut -= 1;
-            if (cut <= released)
+            if (cut <= from)
                 return "";
-            const out = renderRange(buf, spans, released, cut);
-            released = cut;
+            const out = renderRange(win, spans, from, cut);
+            released = base + cut;
+            sent += out;
+            return out;
+        },
+        takeAll() {
+            if (released === total)
+                return "";
+            const spans = runRedaction(win, false, false, false, "free-text", "multi-line", WHOLE_EXTENT).spans;
+            const out = renderRange(win, spans, released - base, win.length);
+            released = scannedAt = total;
             sent += out;
             return out;
         },
         flush() {
+            if (deliveredFrom === total)
+                return { rest: "", diverged: false };
+            const buf = whole();
             const spans = runRedaction(buf, false, false, false, "free-text", "multi-line", WHOLE_EXTENT).spans;
             const split = released > deliveredFrom && spans.some((s) => s.start < released && s.end > released);
             return { rest: renderRange(buf, spans, released, buf.length), diverged: split || renderRange(buf, spans, deliveredFrom, released) !== sent };
         },
         skip() {
-            released = deliveredFrom = scannedAt = buf.length;
+            released = deliveredFrom = scannedAt = total;
             sent = "";
         },
+        endBlock() {
+            blockFrom = total;
+            advanceWindow();
+        },
+        blockContext() {
+            return win.slice(0, blockFrom - base);
+        },
+        blockBytes() {
+            return win.slice(blockFrom - base);
+        },
         get fed() {
-            return buf;
+            return whole();
+        },
+        get length() {
+            return total;
         },
     };
 }