@bli-cockpit/cli 0.2.270 → 0.2.272

package/dist/adapters/raw-evidence-sanitize-held.js

@@ -2,7 +2,7 @@ import { containsSecretLikeContent, redactSecretLikeContent, scannedCleanRedacti
 import { createHash } from "node:crypto";
 import { WHOLE_FILE_MASK_LINE, fallbackRedactionMetadata, logRedactionCrash, maskSecretBearingLines, } from "./raw-evidence-sanitize.js";
 /** What `scannedCleanRedactionMetadata` gives for these bytes, from their hash and size (the same record, key for key). */
-function scannedCleanFromFacts(facts) {
+export function scannedCleanFromFacts(facts) {
     return {
         ...scannedCleanRedactionMetadata(Buffer.alloc(0)),
         original_content_hash_sha256: facts.sha256,

package/dist/adapters/raw-evidence-sanitize-lines.js

@@ -0,0 +1,257 @@
+/**
+ * Exact line-by-line redaction (BLI-23344): the sanitizer's decision table, run over a transcript that
+ * is never held whole.
+ *
+ * The redactor (`redactSecretLikeContent`, `containsSecretLikeContent`) is the UNCHANGED exported one.
+ * It is run on a CHUNK, and a chunk is one physical line, extended across a line boundary only when a
+ * secret rule could cross that boundary:
+ *
+ *  - a newline can sit inside a match only in a run of `\s` that follows a name, a quote, `:`, `=`, `-`,
+ *    `_` or `Bearer` (credential and password assignments, the bearer rule), so a boundary is UNSAFE
+ *    exactly when the last non-`\s` character so far is in `[A-Za-z0-9_\-"':=]`;
+ *  - or inside the `[^{]*?` body of a private key block, so a boundary is UNSAFE while a BEGIN header is
+ *    open: no `{` and no END footer since it.
+ *
+ * Everywhere else a boundary is safe: no match of any of the 30 patterns contains it, and the lookbehind
+ * and `\b` context at the start of the next chunk is what the start of a string gives. So the matches are
+ * the same ones, in the same order, with the same counts. Proof, scripts and the field census (3,652
+ * real transcripts, 0 unsafe boundaries, longest chunk one line): ticket 23317 `equiv/`; the fixtures of
+ * `raw-evidence-sanitize-lines.test.ts` compare old and new byte for byte, metadata JSON included.
+ *
+ * Two passes with bounded state. Pass one reads the lines and keeps only: whether the content guard fired,
+ * the match count over all chunks, the rule counts, the first 200 ranges as ABSOLUTE UTF-16 offsets, the
+ * original hash and size, and two residual flags. The residual flag is the whole-file fallback: after
+ * line masking (or, for a secret-like file name, after inline replacement) is the text STILL secret-like?
+ * It is computed by feeding the masked text, line exactly, through a second chunker, so it needs no copy
+ * of the masked text. Pass two writes the decided output a chunk at a time; a clean file needs no pass two
+ * (the input already is the output). Memory is a function of the longest chunk, never of the file.
+ */
+import { RawEvidenceRedactionMetadataSchema, containsSecretLikeContent, redactSecretLikeContent, } from "@bli-cockpit/telemetry-core";
+import { WHOLE_FILE_MASK_LINE, fallbackRedactionMetadata, maskSecretBearingLines } from "./raw-evidence-sanitize.js";
+import { createHash } from "node:crypto";
+const GUARD_LINE = "[REDACTED_LINE:secret_like_content_guard]\n";
+/** The redactor lists at most this many ranges per call, and so does the whole-file metadata. */
+const RANGE_CAP = 200;
+const PRECURSOR = /[A-Za-z0-9_\-"':=]/;
+const WHITESPACE = /\s/;
+const BEGIN_HEADER = /-----BEGIN [A-Z ]*PRIVATE KEY-----/g;
+const END_FOOTER = /-----END [A-Z ]*PRIVATE KEY-----/g;
+/** Physical lines of an in-memory string, cut exactly where `maskSecretBearingLines` cuts them. */
+export function* linesOfString(text) {
+    for (let position = 0; position < text.length;) {
+        const newline = text.indexOf("\n", position);
+        const end = newline === -1 ? text.length : newline + 1;
+        yield text.slice(position, end);
+        position = end;
+    }
+}
+/** Joins lines into chunks at safe boundaries. */
+export class Chunker {
+    parts = [];
+    length = 0;
+    lastNonSpace = "";
+    keyOpen = false;
+    maxChunk = 0;
+    chunks = 0;
+    /** Feed one physical line; returns the chunk it completes when the boundary after it is safe. */
+    feed(line) {
+        this.parts.push(line);
+        this.length += line.length;
+        for (let i = line.length - 1; i >= 0; i -= 1) {
+            const character = line[i];
+            if (!WHITESPACE.test(character)) {
+                this.lastNonSpace = character;
+                break;
+            }
+        }
+        let lastAt = -1;
+        let open = this.keyOpen;
+        const brace = line.lastIndexOf("{");
+        if (brace !== -1) {
+            lastAt = brace;
+            open = false;
+        }
+        for (const match of line.matchAll(BEGIN_HEADER)) {
+            if (match.index > lastAt) {
+                lastAt = match.index;
+                open = true;
+            }
+        }
+        for (const match of line.matchAll(END_FOOTER)) {
+            if (match.index > lastAt) {
+                lastAt = match.index;
+                open = false;
+            }
+        }
+        this.keyOpen = open;
+        return !this.keyOpen && !PRECURSOR.test(this.lastNonSpace) ? this.take() : null;
+    }
+    /** What is held so far (the unfinished last chunk at the end of the input), or null. */
+    take() {
+        if (this.length === 0)
+            return null;
+        const chunk = this.parts.length === 1 ? this.parts[0] : this.parts.join("");
+        this.parts = [];
+        this.length = 0;
+        this.maxChunk = Math.max(this.maxChunk, chunk.length);
+        this.chunks += 1;
+        return chunk;
+    }
+}
+export async function scanLines(source, options) {
+    const redact = options.redact ?? redactSecretLikeContent;
+    const chunker = new Chunker();
+    const lineMasked = new Chunker();
+    const inlineReplaced = new Chunker();
+    const scan = {
+        secretLike: false, matchCount: 0, ruleCounts: new Map(), ranges: [], residualAfterLineMask: false, residualAfterInline: false,
+        originalSha256: "", originalBytes: 0, textLength: 0, chunks: 0, maxChunkUnits: 0, maxResidualChunkUnits: 0,
+    };
+    const originalHash = createHash("sha256");
+    let offset = 0;
+    const feedResidual = (residual, text, flag) => {
+        for (const line of linesOfString(text)) {
+            const joined = residual.feed(line);
+            if (joined !== null && !scan[flag] && containsSecretLikeContent(joined))
+                scan[flag] = true;
+        }
+    };
+    const scanChunk = (chunk) => {
+        const encoded = Buffer.from(chunk, "utf8");
+        originalHash.update(encoded);
+        scan.originalBytes += encoded.byteLength;
+        if (!scan.secretLike && containsSecretLikeContent(chunk))
+            scan.secretLike = true;
+        const result = redact(chunk, { appliedBy: "local_collector", redactedFields: options.redactedFields });
+        const local = [];
+        if (result.redacted && result.metadata) {
+            for (const count of result.metadata.rule_counts) {
+                const known = scan.ruleCounts.get(count.rule_id) ?? { rule_id: count.rule_id, match_count: 0, redacted_char_count: 0 };
+                known.match_count += count.match_count;
+                known.redacted_char_count += count.redacted_char_count;
+                scan.ruleCounts.set(count.rule_id, known);
+            }
+            const room = Math.max(0, Math.min(RANGE_CAP - scan.ranges.length, result.metadata.redacted_ranges.length));
+            for (let i = 0; i < room; i += 1) {
+                const range = result.metadata.redacted_ranges[i];
+                local.push(range);
+                scan.ranges.push({ start: range.start + offset, end: range.end + offset, rule_id: range.rule_id });
+            }
+            scan.matchCount += result.metadata.secret_like_match_count;
+        }
+        feedResidual(lineMasked, local.length > 0 ? maskLocal(chunk, local) : chunk, "residualAfterLineMask");
+        if (options.secretLikeFileName)
+            feedResidual(inlineReplaced, result.text, "residualAfterInline");
+        offset += chunk.length;
+    };
+    for await (const line of source.lines()) {
+        const chunk = chunker.feed(line);
+        if (chunk !== null)
+            scanChunk(chunk);
+    }
+    const tail = chunker.take();
+    if (tail !== null)
+        scanChunk(tail);
+    const lineMaskedTail = lineMasked.take();
+    if (lineMaskedTail !== null && !scan.residualAfterLineMask && containsSecretLikeContent(lineMaskedTail))
+        scan.residualAfterLineMask = true;
+    const inlineTail = inlineReplaced.take();
+    if (inlineTail !== null && !scan.residualAfterInline && containsSecretLikeContent(inlineTail))
+        scan.residualAfterInline = true;
+    scan.originalSha256 = originalHash.digest("hex");
+    scan.textLength = offset;
+    scan.chunks = chunker.chunks;
+    scan.maxChunkUnits = chunker.maxChunk;
+    scan.maxResidualChunkUnits = Math.max(lineMasked.maxChunk, inlineReplaced.maxChunk);
+    return scan;
+}
+/** `maskSecretBearingLines` over ranges local to `chunk`; it reads nothing of the metadata but the ranges. */
+function maskLocal(chunk, ranges) {
+    return maskSecretBearingLines(chunk, { redacted_ranges: ranges });
+}
+/** The whole-file metadata the redactor would have returned for the whole text, from the aggregate. */
+function aggregateMetadata(scan, redactedFields) {
+    if (scan.matchCount === 0)
+        return null;
+    return RawEvidenceRedactionMetadataSchema.parse({
+        schema_version: "raw-evidence-redaction.v1",
+        status: "sanitized",
+        mode: "deterministic_text_replacement",
+        applied_by: ["local_collector"],
+        rule_counts: [...scan.ruleCounts.values()].sort((a, b) => a.rule_id.localeCompare(b.rule_id)),
+        secret_like_match_count: scan.matchCount,
+        redacted_fields: redactedFields,
+        redacted_ranges: scan.ranges,
+    });
+}
+/** Same decision table, in the same order, as `sanitizeTextEvidenceForUpload`. */
+export function decideLinePlan(scan, options) {
+    const aggregate = aggregateMetadata(scan, options.redactedFields);
+    if (options.secretLikeFileName) {
+        const metadata = aggregate ?? fallbackRedactionMetadata({
+            ruleId: "secret_like_file_name", originalLength: scan.textLength, redactedFields: options.redactedFields, fullContentRedacted: false,
+        });
+        // Still secret-like after the redactor's own replacement (or, with no match at all, as it came): one marker line.
+        const residual = scan.matchCount > 0 ? scan.residualAfterInline : scan.secretLike;
+        return residual ? { kind: "whole_mask", label: "secret_like_name_masked", metadata } : { kind: "inline", label: "secret_like_name_masked", metadata };
+    }
+    if (scan.secretLike) {
+        const metadata = aggregate ?? fallbackRedactionMetadata({
+            ruleId: "secret_like_content_guard", originalLength: scan.textLength, redactedFields: options.redactedFields,
+        });
+        if (metadata.redacted_ranges.length === 0)
+            return { kind: "guard_line", metadata };
+        // The guard fired and the redactor found nothing: the fallback range is the whole text, every line is masked,
+        // and a text made only of marker lines holds nothing secret-like.
+        if (aggregate === null)
+            return { kind: "line_mask", metadata, maskEveryLine: true };
+        if (scan.residualAfterLineMask)
+            return { kind: "whole_mask", label: "secret_content_masked", metadata };
+        return { kind: "line_mask", metadata, maskEveryLine: false };
+    }
+    if (scan.matchCount > 0)
+        return { kind: "inline", label: "secret_content_masked", metadata: aggregate };
+    return { kind: "clean" };
+}
+/** The completeness label of a plan that rewrites the file; the content guard's two plans share one label. */
+export function labelOfPlan(plan) {
+    return plan.kind === "line_mask" || plan.kind === "guard_line" ? "secret_content_masked" : plan.label;
+}
+/** Pass two: write the decided output a chunk at a time, on the same boundaries pass one used. */
+export async function writeLinePlan(source, plan, options, write) {
+    if (plan.kind === "whole_mask")
+        return write(WHOLE_FILE_MASK_LINE);
+    if (plan.kind === "guard_line")
+        return write(GUARD_LINE);
+    const redact = options.redact ?? redactSecretLikeContent;
+    const ranges = plan.kind === "line_mask" ? plan.metadata.redacted_ranges : [];
+    const chunker = new Chunker();
+    let offset = 0;
+    let next = 0;
+    const emit = async (chunk) => {
+        if (plan.kind === "inline") {
+            await write(redact(chunk, { appliedBy: "local_collector", redactedFields: options.redactedFields }).text);
+        }
+        else if (plan.maskEveryLine) {
+            await write(maskLocal(chunk, [{ start: 0, end: chunk.length, rule_id: plan.metadata.redacted_ranges[0].rule_id }]));
+        }
+        else {
+            const local = [];
+            const end = offset + chunk.length;
+            for (; next < ranges.length && ranges[next].start < end; next += 1) {
+                const range = ranges[next];
+                local.push({ start: range.start - offset, end: range.end - offset, rule_id: range.rule_id });
+            }
+            await write(local.length > 0 ? maskLocal(chunk, local) : chunk);
+        }
+        offset += chunk.length;
+    };
+    for await (const line of source.lines()) {
+        const chunk = chunker.feed(line);
+        if (chunk !== null)
+            await emit(chunk);
+    }
+    const tail = chunker.take();
+    if (tail !== null)
+        await emit(tail);
+}

package/dist/adapters/raw-evidence-sanitize-staged.js

@@ -0,0 +1,97 @@
+import { createHash } from "node:crypto";
+import path from "node:path";
+import { fallbackRedactionMetadata, logRedactionCrash } from "./raw-evidence-sanitize.js";
+import { decideLinePlan, labelOfPlan, scanLines, writeLinePlan, } from "./raw-evidence-sanitize-lines.js";
+import { scannedCleanFromFacts } from "./raw-evidence-sanitize-held.js";
+import { linesOfStagedFile } from "./staged-lines.js";
+import { openStagedOutput } from "./staged-output.js";
+import { TranscriptReadError } from "./transcript-text-stream.js";
+const TAG = "[raw-evidence-sanitize-lines]";
+const STUB_LINE = "[REDACTED_LINE:redaction_crashed_stubbed]\n";
+export async function sanitizeStagedTranscript(spool, options) {
+    const source = { lines: () => linesOfStagedFile(spool) };
+    const scanOptions = { redactedFields: options.redactedFields, secretLikeFileName: options.secretLikeFileName === true, redact: options.redact };
+    let output;
+    try {
+        const scan = await scanLines(source, scanOptions);
+        const plan = decideLinePlan(scan, scanOptions);
+        if (plan.kind === "clean") {
+            console.error(TAG + " transcript is clean, the spool is the upload", JSON.stringify({ reason: "clean", byte_size: spool.byteLength, chunks: scan.chunks, longest_chunk_units: scan.maxChunkUnits }));
+            return { status: "clean", staged: spool, redaction: scannedCleanFromFacts(spool) };
+        }
+        output = await openStagedOutput(path.dirname(spool.path));
+        await writeLinePlan(source, plan, scanOptions, (text) => output.write(text));
+        const written = await output.finish();
+        const redaction = {
+            ...plan.metadata,
+            original_content_hash_sha256: spool.sha256,
+            sanitized_content_hash_sha256: written.sha256,
+            original_byte_size: spool.byteLength,
+            sanitized_byte_size: written.byteLength,
+        };
+        console.error(TAG + " transcript rewritten line by line", JSON.stringify({
+            reason: plan.kind,
+            label: labelOfPlan(plan),
+            byte_size: spool.byteLength,
+            uploaded_bytes: written.byteLength,
+            match_count: scan.matchCount,
+            listed_ranges: scan.ranges.length,
+            chunks: scan.chunks,
+            longest_chunk_units: scan.maxChunkUnits,
+            longest_residual_chunk_units: scan.maxResidualChunkUnits,
+        }));
+        return { status: "redacted", staged: written, redaction, completenessLabel: labelOfPlan(plan) };
+    }
+    catch (error) {
+        await output?.abandon();
+        if (error instanceof TranscriptReadError)
+            throw error;
+        logRedactionCrash(error, spool.byteLength, options.redactedFields.length);
+        return stubForCrash(source, spool, options.redactedFields);
+    }
+}
+/** The stub the whole-text path builds: its metadata names the text's length in UTF-16 units, which a counting pass gives. */
+async function stubForCrash(source, spool, redactedFields) {
+    let units = 0;
+    for await (const line of source.lines())
+        units += line.length;
+    const bytes = Buffer.from(STUB_LINE, "utf8");
+    const sanitizedSha = createHash("sha256").update(bytes).digest("hex");
+    return {
+        status: "redacted",
+        bytes,
+        redaction: {
+            ...fallbackRedactionMetadata({ ruleId: "redaction_crashed_stubbed", originalLength: units, redactedFields }),
+            original_content_hash_sha256: spool.sha256,
+            sanitized_content_hash_sha256: sanitizedSha,
+            original_byte_size: spool.byteLength,
+            sanitized_byte_size: bytes.byteLength,
+        },
+        completenessLabel: "redaction_crashed_stubbed",
+    };
+}
+/**
+ * Mask secret-like values in a spooled transcript and record that it happened (the twin of
+ * `maskSecretsInHeldTranscript`: same ledger entry, same log line).
+ */
+export async function maskSecretsInStagedTranscript(collection, spool, options) {
+    const sanitized = await sanitizeStagedTranscript(spool, {
+        redactedFields: [`${options.kind}.body`],
+        secretLikeFileName: options.secretLikeFileName,
+    });
+    if (sanitized.status === "redacted") {
+        collection.redacted.push({
+            kind: options.kind,
+            label: options.evidenceLabel,
+            redaction: sanitized.redaction,
+            completenessLabel: sanitized.completenessLabel,
+        });
+        console.error("[raw-evidence] text evidence sanitized", JSON.stringify({
+            kind: options.kind,
+            mode: sanitized.completenessLabel,
+            original_bytes: sanitized.redaction.original_byte_size,
+            uploaded_bytes: sanitized.redaction.sanitized_byte_size,
+        }));
+    }
+    return sanitized;
+}

package/dist/adapters/raw-evidence-transcript-file.js

@@ -22,6 +22,7 @@ import { describeError } from "../health-detail.js";
 import { DELIVERY_BACKOFF_HOLDING_REASON, evidenceSourceKey, } from "../raw-evidence-staging.js";
 import { isSecretLikePath, sha256, } from "./raw-evidence-keys.js";
 import { maskSecretsInHeldTranscript } from "./raw-evidence-sanitize-held.js";
+import { maskSecretsInStagedTranscript } from "./raw-evidence-sanitize-staged.js";
 import { discardStagedOutput } from "./staged-output.js";
 import { byteLengthOf, dropStagedBody, stageOneTranscript } from "./raw-evidence-transcript-stage.js";
 import { agentOfKind, prepareTranscriptBody } from "./raw-evidence-transcript-body.js";
@@ -129,21 +130,20 @@ async function prepareTranscriptForUpload(collection, options, names, source) {
     const body = await prepareTranscriptBody(collection, { source, kind: options.kind, sessionId: options.sessionId });
     const redactedBefore = collection.redacted.length;
     let sanitized;
+    const masking = { kind: options.kind, evidenceLabel: names.evidenceLabel, secretLikeFileName: names.secretLikeFileName };
     try {
-        sanitized = maskSecretsInHeldTranscript(collection, body.held, {
-            kind: options.kind,
-            evidenceLabel: names.evidenceLabel,
-            secretLikeFileName: names.secretLikeFileName,
-        });
+        // BLI-23344: a spooled transcript is read back a line at a time, never as one string.
+        sanitized = body.held.staged && body.held.text === null
+            ? await maskSecretsInStagedTranscript(collection, body.held.staged, masking)
+            : maskSecretsInHeldTranscript(collection, body.held, masking);
     }
     catch (error) {
         if (body.held.staged)
             await discardStagedOutput(body.held.staged, "mask_threw");
         throw error;
     }
-    // BLI-23317 stage D: only a clean file IS its staged bytes. Anything the mask rewrote has new bytes, built
-    // from the sanitizer's text, so the streamed partial is of no use.
-    if (body.held.staged && !("staged" in sanitized))
+    // Only a clean file IS the spool. Anything the mask rewrote has new bytes (a new staged file, or the crash stub's), so the spool is of no use.
+    if (body.held.staged && !("staged" in sanitized && sanitized.staged === body.held.staged))
         await discardStagedOutput(body.held.staged, "redaction_rewrote_the_bytes");
     // The ledger entry this mask pushed (if any), kept so a later skip can replay it.
     const redactedEntry = collection.redacted.length > redactedBefore ? collection.redacted[collection.redacted.length - 1] : null;

package/dist/adapters/raw-evidence-transcript-stream.js

@@ -35,10 +35,11 @@ export async function streamTranscriptBody(session, agent, file) {
         return null;
     }
     try {
-        const prepared = await session.prepareTranscriptFile({ filePath: file.filePath, source: agent, sessionId: file.sessionId, maxFileBytes: file.maxFileBytes, stageDir: file.stageDir });
+        const prepared = await session.prepareTranscriptFile({ filePath: file.filePath, source: agent, sessionId: file.sessionId, maxFileBytes: file.maxFileBytes, stageDir: file.stageDir, omitText: file.stageDir !== undefined });
         const { volatile, pendingImages } = volatilityOf(session.mode, prepared);
         console.error(TAG + " transcript streamed through image swap and slim", JSON.stringify({ kind: file.kind, byte_size: prepared.readBytes, changed: prepared.changed, images: prepared.imageCount, volatile, staged_bytes: prepared.staged?.byteLength ?? null }));
-        return { held: { text: prepared.text, exactBytes: null, ...(prepared.staged ? { staged: prepared.staged } : {}) }, volatile, pendingImages, readBytes: prepared.readBytes };
+        // BLI-23344: with a spool the text is not read back; the sanitizer reads the spool line by line.
+        return { held: { text: prepared.staged ? null : prepared.text, exactBytes: null, ...(prepared.staged ? { staged: prepared.staged } : {}) }, volatile, pendingImages, readBytes: prepared.readBytes };
     }
     catch (error) {
         if (error instanceof TranscriptReadError || error instanceof TranscriptTooLargeError)

package/dist/adapters/staged-lines.js

@@ -0,0 +1,67 @@
+/**
+ * A staged file read back as physical lines, a block at a time (BLI-23344).
+ *
+ * The line-by-line redactor (`raw-evidence-sanitize-lines.ts`) reads the slimmed spool twice and never
+ * holds more than the longest chunk. A line ends just after its `\n`; the last one may not. The bytes are
+ * decoded with a streaming UTF-8 decoder, so a character split across two blocks is whole in the line, and
+ * a byte-order mark stays a character, exactly as `Buffer.toString("utf8")` of the whole file gives it. The
+ * file is valid UTF-8 (the streamed pass refuses anything else), so no byte is replaced.
+ *
+ * A read that fails, or a file shorter than its recorded length, is a `TranscriptReadError`: a local disk
+ * problem is reported as one, never mistaken for a redaction crash that would stub a good session.
+ */
+import fsp from "node:fs/promises";
+import { StringDecoder } from "node:string_decoder";
+import { describeError } from "../health-detail.js";
+import { TranscriptReadError } from "./transcript-text-stream.js";
+const BLOCK_BYTES = 1024 * 1024;
+export async function* linesOfStagedFile(staged) {
+    if (staged.byteLength === 0)
+        return;
+    let handle;
+    try {
+        handle = await fsp.open(staged.path, "r");
+    }
+    catch (error) {
+        throw readFailure("open", staged, error);
+    }
+    try {
+        const decoder = new StringDecoder("utf8");
+        const block = Buffer.allocUnsafeSlow(BLOCK_BYTES);
+        let pending = [];
+        for (let position = 0; position < staged.byteLength;) {
+            let bytesRead;
+            try {
+                ({ bytesRead } = await handle.read(block, 0, Math.min(BLOCK_BYTES, staged.byteLength - position), position));
+            }
+            catch (error) {
+                throw readFailure("read", staged, error);
+            }
+            if (bytesRead === 0)
+                throw readFailure("short", staged, new Error("staged_file_shorter_than_written"));
+            position += bytesRead;
+            const text = decoder.write(block.subarray(0, bytesRead));
+            let from = 0;
+            for (let newline = text.indexOf("\n", from); newline !== -1; newline = text.indexOf("\n", from)) {
+                pending.push(text.slice(from, newline + 1));
+                from = newline + 1;
+                yield pending.length === 1 ? pending[0] : pending.join("");
+                pending = [];
+            }
+            if (from < text.length)
+                pending.push(text.slice(from));
+        }
+        const rest = decoder.end();
+        if (rest.length > 0)
+            pending.push(rest);
+        if (pending.length > 0)
+            yield pending.length === 1 ? pending[0] : pending.join("");
+    }
+    finally {
+        await handle.close().catch(() => undefined);
+    }
+}
+function readFailure(stage, staged, error) {
+    console.error("[staged-lines] staged file could not be read back", JSON.stringify({ reason: "staged_read_failed", stage, byte_size: staged.byteLength, ...describeError(error) }));
+    return new TranscriptReadError(`staged_${stage}`, error);
+}

package/dist/commands/public-root.js

@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
   }
 
   if (command === "--version" || command === "-V" || command === "version") {
-    writeLine(io?.stdout ?? process.stdout, "0.2.270");
+    writeLine(io?.stdout ?? process.stdout, "0.2.272");
     return 0;
   }
 

package/dist/image-census-stream.js

@@ -17,11 +17,17 @@ import { isUtf8 } from "node:buffer";
 import { formatImageMarker } from "@bli-cockpit/telemetry-core";
 import { TranscriptNotStreamable } from "./adapters/transcript-text-stream.js";
 import { bytesMayHoldImage, scanImagesInLine } from "./image-byte-scan.js";
+import { bytesMayHoldPdf, scanPdfsInLine } from "./pdf-byte-scan.js";
 export async function censusOfLines(lines) {
-    const census = { total: 0, occurrences: [], unique: new Map(), tally: new Map() };
+    const census = { total: 0, occurrences: [], unique: new Map(), tally: new Map(), pdfs: [] };
     for await (const { start, bytes } of lines) {
         if (!isUtf8(bytes))
             throw new TranscriptNotStreamable("invalid_utf8");
+        if (bytesMayHoldPdf(bytes)) {
+            for (const pdf of scanPdfsInLine(bytes)) {
+                census.pdfs.push({ ...pdf, replaceStart: pdf.replaceStart + start, replaceEnd: pdf.replaceEnd + start, base64Start: pdf.base64Start + start, base64End: pdf.base64End + start });
+            }
+        }
         if (!bytesMayHoldImage(bytes))
             continue;
         for (const found of scanImagesInLine(bytes)) {

package/dist/image-description-session.js

@@ -11,6 +11,7 @@ import { sendPendingImages } from "./image-description-send.js";
 import { resolveImageHandlingMode } from "./image-mode-state.js";
 import { censusOfLines } from "./image-census-stream.js";
 import { sha256OfBase64Run } from "./image-byte-scan.js";
+import { addPdfsToFinds, mergeOccurrences, pdfsOfText } from "./pdf-transcript-step.js";
 import { openPinnedTranscript } from "./adapters/pinned-transcript.js";
 import { openStagedOutput, readStagedText } from "./adapters/staged-output.js";
 import { TranscriptNotStreamable } from "./adapters/transcript-text-stream.js";
@@ -47,7 +48,12 @@ export async function openImageDescriptions(options) {
         isDescribed: (sha256) => described.has(imageStateKey(sha256)),
         hasDescribeRoom: (sha256s) => describeRoomLeft(run, sha256s),
         async prepareTranscript(input) {
-            const { replace, finds, ...afterImages } = await describeAndReplace(run, input, findsOfText(input.text));
+            // BLI-23347: in drop mode each base64 PDF becomes a marker block (its text, or its described page images).
+            const imageFinds = findsOfText(input.text);
+            const finds0 = mode === "describe_and_drop"
+                ? await addPdfsToFinds(imageFinds, pdfsOfText(input.text), pdfContext(run, { stateDir: options.stateDir, read: async (pdf) => input.text.slice(pdf.base64Start, pdf.base64End), changed: () => new Error("pdf_changed_in_text") }))
+                : imageFinds;
+            const { replace, finds, ...afterImages } = await describeAndReplace(run, input, finds0);
             if (mode !== "describe_and_drop")
                 return { ...afterImages, text: input.text, changed: false, slim: null };
             // BLI-23317: the image swap and the slim rules are ONE pass over the lines (one joined string, no
@@ -116,17 +122,23 @@ async function prepareFileInTwoPasses(run, input) {
         // Pass one: discover. Nothing of the text is kept; only where each image is and what it is.
         const census = await censusOfLines(source.lines());
         // Describe attempts finish and the swap decisions are settled before a single line is swapped, as with the whole text.
-        const { replace, finds: _finds, ...afterImages } = await describeAndReplace(run, input, findsOfCensus(census, source));
+        // BLI-23347: the census also found the PDFs; they are converted here and swapped in the same pass as the images.
+        const pdfFinds = await addPdfsToFinds(findsOfCensus(census, source), census.pdfs, pdfContext(run, {
+            stateDir: run.options.stateDir,
+            read: (pdf) => source.read(pdf.base64Start, pdf.base64End - pdf.base64Start),
+            changed: () => new TranscriptNotStreamable("file_changed_during_pass"),
+        }));
+        const { replace, finds, ...afterImages } = await describeAndReplace(run, input, pdfFinds);
         // Pass two: swap and slim over exactly the bytes pass one saw, writing the result to staging as it goes.
         writer = input.stageDir ? await openStagedOutput(input.stageDir) : undefined;
         const memory = writer ? undefined : new LineBytesSink(source.length);
-        const slim = await slimLineStream(source.lines(), input.source, census.occurrences, replace, writer ?? memory);
+        const slim = await slimLineStream(source.lines(), input.source, mergeOccurrences(census.occurrences, finds.extraOccurrences), replace, writer ?? memory);
         if (slim.replacedCount !== afterImages.replacedCount)
             throw new TranscriptNotStreamable("images_changed_between_passes");
         await source.verifyUnchanged();
         const staged = await writer?.finish();
         // The sanitizer reads the whole text: from the staged file when one was written, else from the sink's buffer.
-        const text = staged ? await readStagedText(staged) : memory.text();
+        const text = staged ? (input.omitText ? "" : await readStagedText(staged)) : memory.text();
         logSlimmed(input.source, input.sessionId, slim.stats, startedAt);
         return { ...afterImages, text, changed: slim.changed, slim: slim.stats, readBytes: source.length, ...(staged ? { staged } : {}) };
     }
@@ -202,14 +214,24 @@ function countImageSwaps(swap) {
     let replacedCount = 0;
     let charsSaved = 0;
     for (const image of swap.images) {
-        if (!swap.replace(image))
+        if (image.replacement === undefined && !swap.replace(image))
             continue;
-        const marker = formatImageMarker({ sha256: image.sha256, mediaType: image.mediaType, byteSize: image.byteSize });
+        const marker = image.replacement ?? formatImageMarker({ sha256: image.sha256, mediaType: image.mediaType, byteSize: image.byteSize });
         replacedCount += 1;
         charsSaved += image.replaceEnd - image.replaceStart - marker.length;
     }
     return { replacedCount, charsSaved };
 }
+/** What the PDF step needs from this run (BLI-23347); `described` is the same set the image step reads. */
+function pdfContext(run, parts) {
+    return {
+        key: run,
+        stateDir: parts.stateDir,
+        isDescribed: (sha256) => run.described.has(imageStateKey(sha256)),
+        readBase64: parts.read,
+        sourceChanged: parts.changed,
+    };
+}
 function selectImagesToDescribe(run, unique, counts) {
     const { described, settledThisRun, failedImages, now } = run;
     const toSend = [];