@bli-cockpit/cli 0.2.264 → 0.2.265

package/dist/adapters/raw-evidence-transcript-body.js

@@ -12,6 +12,11 @@
 import { isUtf8 } from "node:buffer";
 import { mightHoldEmbeddedImages } from "@bli-cockpit/telemetry-core";
 import { describeError } from "../health-detail.js";
+import { readTranscriptWhole, streamTranscriptBody, volatilityOf } from "./raw-evidence-transcript-stream.js";
+/** Which agent's transcript rules apply to this evidence kind; null for anything the image step does not know. */
+export function agentOfKind(kind) {
+    return kind.startsWith("codex") ? "codex" : kind.startsWith("claude") ? "claude_code" : null;
+}
 /** Decode once and release the bytes, unless the decoding was lossy and the bytes are the only exact copy. */
 export function holdDecodedText(source) {
     const bytes = source.bytes;
@@ -27,12 +32,23 @@ export function holdDecodedText(source) {
  * BLI-16977: describe each distinct embedded image once and, when the one switch says so, swap the
  * base64 for a short marker BEFORE masking, so the masker and the content hash see the final bytes.
  * The text is unchanged (the very same decoding) unless this sync has an image session AND the switch
- * is `describe_and_drop`; in `describe_and_keep` it always comes back untouched. Never throws: an
- * image step that breaks hands back the original text and says so.
+ * is `describe_and_drop`; in `describe_and_keep` it always comes back untouched. An image step
+ * that breaks hands back the original text and says so. The one throw: a streamed source (BLI-23317)
+ * whose file cannot be read to the end raises `TranscriptReadError` or `TranscriptTooLargeError`, which
+ * the caller turns into the `file_read_failed` and `file_too_large` skips.
  */
 export async function prepareTranscriptBody(collection, file) {
     const session = collection.budget.images;
-    const agent = file.kind.startsWith("codex") ? "codex" : file.kind.startsWith("claude") ? "claude_code" : null;
+    const agent = agentOfKind(file.kind);
+    if (file.source.stream) {
+        const streamed = await streamTranscriptBody(session, agent, { ...file.source.stream, sessionId: file.sessionId, kind: file.kind });
+        if (streamed) {
+            file.source.readBytes = streamed.readBytes;
+            return { held: streamed.held, volatile: streamed.volatile, pendingImages: streamed.pendingImages };
+        }
+        file.source.bytes = await readTranscriptWhole(file.source.stream);
+        file.source.readBytes = file.source.bytes.byteLength;
+    }
     const untouched = (volatile, pendingImages) => ({ held: holdDecodedText(file.source), volatile, pendingImages });
     if (!session || !agent)
         return untouched(false, null);
@@ -46,14 +62,9 @@ export async function prepareTranscriptBody(collection, file) {
         // BLI-3936: "volatile" = the next tick could build different bytes, or do image work, from
         // this very file: an image still without a stored description (keep mode never replaces, so
         // any image counts) or a slim step that failed and should be retried.
-        const imagesPending = session.mode === "describe_and_drop"
-            ? prepared.imageCount > prepared.replacedCount
-            : prepared.imageCount > 0;
-        const slimFailed = session.mode === "describe_and_drop" && prepared.slim === null;
-        const volatile = imagesPending || slimFailed;
         // BLI-22794: volatile ONLY because of images without a stored description (the slim step held
         // up): the bytes change only when one of these gains one, so the memo may keep the entry.
-        const pendingImages = volatile && !slimFailed ? prepared.pendingImages : null;
+        const { volatile, pendingImages } = volatilityOf(session.mode, prepared);
         if (prepared.changed) {
             // The old path encoded the new text and read it back; a text with no lone surrogate (slim output
             // re-escapes them) is the same either way, so the new text is the exact decoding of the new bytes.

package/dist/adapters/raw-evidence-transcript-file.js

@@ -22,7 +22,8 @@ import { describeError } from "../health-detail.js";
 import { DELIVERY_BACKOFF_HOLDING_REASON, evidenceSourceKey, } from "../raw-evidence-staging.js";
 import { isSecretLikePath, sha256, shortHash, } from "./raw-evidence-keys.js";
 import { maskSecretsInHeldTranscript } from "./raw-evidence-sanitize-held.js";
-import { prepareTranscriptBody } from "./raw-evidence-transcript-body.js";
+import { agentOfKind, prepareTranscriptBody } from "./raw-evidence-transcript-body.js";
+import { TranscriptReadError, TranscriptTooLargeError } from "./transcript-text-stream.js";
 import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
 import { evidenceEntry } from "./raw-evidence-manifest.js";
 import { prepareRecipe, statSourceForMemo, } from "./raw-evidence-prepare-memo.js";
@@ -57,16 +58,30 @@ export async function collectOneEvidenceFile(collection, options) {
     const recipe = prepareRecipe({ imageMode: collection.budget.images?.mode, kind: options.kind });
     if (await reuseUnchangedTranscript(collection, options, names, stamp, recipe))
         return true;
-    const read = await readEvidenceFileWithinCap(options.filePath, options.maxFileBytes);
+    const read = await readEvidenceFileWithinCap(options.filePath, options.maxFileBytes, streamMinBytesFor(collection, options.kind));
     if (read.status === "read_failed")
         return skip("file_read_failed");
     if (read.status === "too_large") {
         markCapApplied(collection, options.kind, "max_file_bytes");
         return skip("file_too_large");
     }
-    const { sanitized, contentHash, stubClaim, volatile, pendingImages, redactedEntry } = await prepareTranscriptForUpload(collection, options, names, read.source);
+    let prepared;
+    try {
+        prepared = await prepareTranscriptForUpload(collection, options, names, read.source);
+    }
+    catch (error) {
+        // BLI-23317: a streamed or fallback read that fails ends exactly as a whole-file read that failed.
+        if (error instanceof TranscriptReadError)
+            return skip("file_read_failed");
+        if (error instanceof TranscriptTooLargeError) {
+            markCapApplied(collection, options.kind, "max_file_bytes");
+            return skip("file_too_large");
+        }
+        throw error;
+    }
+    const { sanitized, contentHash, stubClaim, volatile, pendingImages, redactedEntry } = prepared;
     await rememberPreparedTranscript(collection, options, names, {
-        stamp, recipe, readBytes: read.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, pendingImages, redactedEntry,
+        stamp, recipe, readBytes: read.source.readBytes ?? read.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, pendingImages, redactedEntry,
     });
     if (collection.skipContentHashes.has(contentHash) && !stubClaim) {
         collection.reused.push({
@@ -264,12 +279,19 @@ async function stageOneTranscript(collection, file) {
         }),
     }));
 }
+/** The size from which this pass streams a transcript instead of reading it whole; null when its image session cannot (BLI-23317). */
+function streamMinBytesFor(collection, kind) {
+    const session = collection.budget.images;
+    if (!session?.prepareTranscriptFile || !agentOfKind(kind))
+        return null;
+    return session.streamMinBytes ?? null;
+}
 /**
  * Size is checked twice on purpose: once by `stat` so an oversized transcript
  * is never buffered at all, and once on the bytes actually read, because a live
  * session can grow between the two.
  */
-async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
+async function readEvidenceFileWithinCap(filePath, maxFileBytes, streamMinBytes) {
     if (maxFileBytes) {
         let stat;
         try {
@@ -290,6 +312,14 @@ async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
         if (stat.size > maxFileBytes)
             return { status: "too_large" };
     }
+    if (streamMinBytes !== null) {
+        // BLI-23317: a big transcript is streamed by the prepare step, not buffered here. A stat that fails
+        // falls through to the read below, which reports the failure itself.
+        const size = await fs.stat(filePath).then((stat) => stat.size, () => null);
+        if (size !== null && size >= streamMinBytes) {
+            return { status: "ok", source: { bytes: null, stream: { filePath, maxFileBytes } }, byteLength: size };
+        }
+    }
     let bytes;
     try {
         bytes = await fs.readFile(filePath);

package/dist/adapters/raw-evidence-transcript-stream.js

@@ -0,0 +1,63 @@
+/**
+ * The transcript that is never read whole (BLI-23317): its path goes to the image session's two
+ * streaming passes, and what comes back is the one slimmed string the sanitizer takes.
+ *
+ * Three endings, each named:
+ *  - a body: the text is the slim form of the file (valid UTF-8 was checked across every chunk, so
+ *    `exactBytes` is null: the text encodes back to the bytes a clean file would upload);
+ *  - null: the streamed path cannot give the whole-file answer (invalid UTF-8, a file that changed
+ *    between the passes, an image step that broke), and the caller reads the file whole, which is
+ *    what ran before this existed and still keeps the original bytes for the clean branch;
+ *  - a throw of `TranscriptReadError` or `TranscriptTooLargeError`: the same outcomes a whole-file
+ *    read gives (`file_read_failed`, `file_too_large`), never a shorter file.
+ */
+import fs from "node:fs/promises";
+import { describeError } from "../health-detail.js";
+import { TranscriptNotStreamable, TranscriptReadError, TranscriptTooLargeError, } from "./transcript-text-stream.js";
+const TAG = "[raw-evidence]";
+/**
+ * BLI-3936: "volatile" = the next tick could build different bytes, or do image work, from this very
+ * file: an image still without a stored description (keep mode never replaces, so any image counts) or
+ * a slim step that failed and should be retried. BLI-22794: volatile ONLY because of images without a
+ * stored description means the bytes change only when one of them gains one, so the memo may keep the entry.
+ */
+export function volatilityOf(mode, prepared) {
+    const imagesPending = mode === "describe_and_drop"
+        ? prepared.imageCount > prepared.replacedCount
+        : prepared.imageCount > 0;
+    const slimFailed = mode === "describe_and_drop" && prepared.slim === null;
+    const volatile = imagesPending || slimFailed;
+    return { volatile, pendingImages: volatile && !slimFailed ? prepared.pendingImages : null };
+}
+export async function streamTranscriptBody(session, agent, file) {
+    if (!session?.prepareTranscriptFile || !agent) {
+        console.error(TAG + " streamed read not available, reading the transcript whole", JSON.stringify({ reason: "stream_unavailable", kind: file.kind }));
+        return null;
+    }
+    try {
+        const prepared = await session.prepareTranscriptFile({ filePath: file.filePath, source: agent, sessionId: file.sessionId, maxFileBytes: file.maxFileBytes });
+        const { volatile, pendingImages } = volatilityOf(session.mode, prepared);
+        console.error(TAG + " transcript streamed through image swap and slim", JSON.stringify({ kind: file.kind, byte_size: prepared.readBytes, changed: prepared.changed, images: prepared.imageCount, volatile }));
+        return { held: { text: prepared.text, exactBytes: null }, volatile, pendingImages, readBytes: prepared.readBytes };
+    }
+    catch (error) {
+        if (error instanceof TranscriptReadError || error instanceof TranscriptTooLargeError)
+            throw error;
+        console.error(TAG + " streamed read set aside, reading the transcript whole", JSON.stringify({ reason: error instanceof TranscriptNotStreamable ? error.reason : "stream_step_failed", kind: file.kind, ...(error instanceof TranscriptNotStreamable ? {} : describeError(error)) }));
+        return null;
+    }
+}
+/** The whole-file read the streamed path falls back to: same failures, same labels, as the reader that did not stream. */
+export async function readTranscriptWhole(file) {
+    let bytes;
+    try {
+        bytes = await fs.readFile(file.filePath);
+    }
+    catch (error) {
+        console.error(TAG + " evidence file could not be read", JSON.stringify({ reason: "file_read_failed", stage: "read", ...describeError(error) }));
+        throw new TranscriptReadError("read", error);
+    }
+    if (file.maxFileBytes && bytes.byteLength > file.maxFileBytes)
+        throw new TranscriptTooLargeError(file.maxFileBytes);
+    return bytes;
+}

package/dist/adapters/transcript-text-stream.js

@@ -0,0 +1,124 @@
+/**
+ * A transcript's lines as text, straight from disk, for the pass that never holds the whole file (BLI-23317).
+ *
+ * Unlike `transcript-line-stream.ts` (the image reader's, which logs a failed read and ends the lines
+ * early), this one tells the two endings apart. A read that fails is a `TranscriptReadError`, never a
+ * shorter file: the caller maps it to the same `file_read_failed` skip a whole-file read failure gets.
+ * Text that is not valid UTF-8 is a `TranscriptNotStreamable`: the streamed path replaces nothing, so
+ * the caller falls back to the whole-file path, which keeps the original bytes for the clean branch.
+ *
+ * One `TextDecoder` (fatal, BOM kept, `stream: true`) lives across all chunks, so a character split
+ * between two chunks is valid input, and `decode()` at EOF rejects a file that ends mid-character.
+ * For valid UTF-8 it yields exactly the string `Buffer.toString("utf8")` makes, cut at "\n" like
+ * `split("\n")`: the last line is whatever follows the final newline, empty or not.
+ *
+ * A line is held as pieces until its newline arrives, so the peak is the longest line (a single very
+ * long line is still concatenated whole; that bound is not claimed to be small).
+ */
+import fsp from "node:fs/promises";
+import { TextDecoder } from "node:util";
+import { describeError } from "../health-detail.js";
+/** Bytes read per step, into one reused buffer. */
+export const TRANSCRIPT_CHUNK_BYTES = 64 * 1024;
+const CHUNK_BYTES = TRANSCRIPT_CHUNK_BYTES;
+/** The file could not be read to its end. Not EOF: whoever catches it must not treat the lines so far as the file. */
+export class TranscriptReadError extends Error {
+    stage;
+    underlying;
+    constructor(stage, underlying) {
+        super(`transcript read failed at ${stage}`);
+        this.name = "TranscriptReadError";
+        this.stage = stage;
+        this.underlying = underlying;
+    }
+}
+/** The file is bigger than this pass's cap. */
+export class TranscriptTooLargeError extends Error {
+    maxFileBytes;
+    constructor(maxFileBytes) {
+        super("transcript over the file cap");
+        this.name = "TranscriptTooLargeError";
+        this.maxFileBytes = maxFileBytes;
+    }
+}
+/** The streamed path cannot give the whole-file answer for this file; read it whole instead. `reason` is a label. */
+export class TranscriptNotStreamable extends Error {
+    reason;
+    constructor(reason) {
+        super(`transcript not streamable: ${reason}`);
+        this.name = "TranscriptNotStreamable";
+        this.reason = reason;
+    }
+}
+/**
+ * The lines of `filePath`. `byteLimit` reads exactly that many bytes (the second pass reads the first
+ * pass's count, so a file that grew between the passes is not half-seen) and fails with
+ * `file_changed_during_pass` when fewer arrive. `stopAfter` is a ceiling only (the first pass, bounded
+ * by the file cap): fewer is fine.
+ */
+export function openTranscriptLines(filePath, options = {}) {
+    let bytes = 0;
+    const { byteLimit } = options;
+    const readTo = byteLimit ?? options.stopAfter;
+    async function* lines() {
+        if (readTo === 0) {
+            yield "";
+            return;
+        }
+        const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+        let handle;
+        let held = [];
+        try {
+            handle = await fsp.open(filePath, "r");
+            // One buffer for the whole pass: the decoder copies out of it, so nothing waits on the collector to free a chunk.
+            const buffer = Buffer.allocUnsafe(CHUNK_BYTES);
+            for (let position = 0;;) {
+                const want = readTo === undefined ? CHUNK_BYTES : Math.min(CHUNK_BYTES, readTo - position);
+                if (want <= 0)
+                    break;
+                const { bytesRead } = await handle.read(buffer, 0, want, position);
+                if (bytesRead === 0)
+                    break;
+                position += bytesRead;
+                bytes += bytesRead;
+                const text = decodeOrThrow(decoder, buffer.subarray(0, bytesRead));
+                let start = 0;
+                for (let newline = text.indexOf("\n", start); newline !== -1; newline = text.indexOf("\n", start)) {
+                    held.push(text.slice(start, newline));
+                    const line = held.length === 1 ? held[0] : held.join("");
+                    held = [];
+                    start = newline + 1;
+                    yield line;
+                }
+                if (start < text.length)
+                    held.push(text.slice(start));
+            }
+            const tail = decodeOrThrow(decoder, undefined);
+            if (tail)
+                held.push(tail);
+            if (byteLimit !== undefined && bytes < byteLimit)
+                throw new TranscriptNotStreamable("file_changed_during_pass");
+            yield held.join("");
+        }
+        catch (error) {
+            if (error instanceof TranscriptNotStreamable)
+                throw error;
+            console.error("[transcript-text-stream] transcript could not be read to its end", JSON.stringify({ reason: "file_read_failed", stage: "stream", bytes_read: bytes, ...describeError(error) }));
+            throw new TranscriptReadError("stream", error);
+        }
+        finally {
+            await handle?.close().catch((error) => {
+                console.error("[transcript-text-stream] file handle did not close", JSON.stringify({ reason: "file_close_failed", ...describeError(error) }));
+            });
+        }
+    }
+    return { lines: lines(), bytesRead: () => bytes };
+}
+function decodeOrThrow(decoder, chunk) {
+    try {
+        return chunk ? decoder.decode(chunk, { stream: true }) : decoder.decode();
+    }
+    catch {
+        throw new TranscriptNotStreamable("invalid_utf8");
+    }
+}

package/dist/commands/public-root.js

@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
   }
 
   if (command === "--version" || command === "-V" || command === "version") {
-    writeLine(io?.stdout ?? process.stdout, "0.2.264");
+    writeLine(io?.stdout ?? process.stdout, "0.2.265");
     return 0;
   }
 

package/dist/image-census-stream.js

@@ -0,0 +1,37 @@
+/**
+ * Pass one of the streamed transcript: find every embedded image line by line and keep only what the
+ * describe step needs, never the text (BLI-23317).
+ *
+ * `findEmbeddedImages` over the whole text and over each line give the same images, because an image's
+ * characters never include "\n" and the finder's quote and data-URL anchors sit inside one line
+ * (`slimPass` already throws when one does not). So the census is the whole-text finder run per line,
+ * and what outlives a line is small: one record per distinct image, a tally per image of how often it
+ * occurs and how many characters swapping each occurrence saves, and the base64 of only those images
+ * the describe step may send (a copy, so no line or chunk stays alive behind it).
+ */
+import { findEmbeddedImages, formatImageMarker, } from "@bli-cockpit/telemetry-core";
+/** Could this line hold an embedded image? A superset of what the finder matches: its anchors are `base64` and the four image magics. */
+export function lineMayHoldImage(line) {
+    return line.includes("base64") || line.includes("iVBOR") || line.includes("/9j/") || line.includes("R0lGO") || line.includes("UklGR");
+}
+export async function censusOfLines(lines, keepBase64) {
+    const census = { total: 0, unique: new Map(), tally: new Map(), base64: new Map() };
+    for await (const line of lines) {
+        if (!lineMayHoldImage(line))
+            continue;
+        for (const image of findEmbeddedImages(line)) {
+            census.total += 1;
+            const marker = formatImageMarker({ sha256: image.sha256, mediaType: image.mediaType, byteSize: image.byteSize });
+            const tally = census.tally.get(image.sha256) ?? { count: 0, savedChars: 0 };
+            tally.count += 1;
+            tally.savedChars += image.replaceEnd - image.replaceStart - marker.length;
+            census.tally.set(image.sha256, tally);
+            if (census.unique.has(image.sha256))
+                continue;
+            census.unique.set(image.sha256, image);
+            if (keepBase64(image))
+                census.base64.set(image.sha256, Buffer.from(line.slice(image.base64Start, image.base64End), "latin1"));
+        }
+    }
+    return census;
+}

package/dist/image-description-session.js

@@ -8,7 +8,10 @@
 import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, base64Of, findEmbeddedImages, formatImageMarker, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
 import { postImageToDescribeDoor } from "./image-describe-client.js";
 import { resolveImageHandlingMode } from "./image-mode-state.js";
-import { slimOrKeep, slimPassOrNull, slimTranscript } from "./slim/slim-transcript.js";
+import { censusOfLines } from "./image-census-stream.js";
+import { openTranscriptLines, TranscriptNotStreamable, TranscriptTooLargeError } from "./adapters/transcript-text-stream.js";
+import { slimLineStream } from "./slim/slim-stream.js";
+import { logSlimmed, slimOrKeep, slimPassOrNull, slimTranscript } from "./slim/slim-transcript.js";
 import { imageStateKey, imageRetryDelayMs, readDescribedImageKeys, readFailedImages, writeImageDescriptionState, } from "./image-description-state.js";
 /** New images sent to the door in one sync (all worktrees together); the count never refills within a tick. */
 export const IMAGE_DESCRIBE_MAX_PER_RUN = 100;
@@ -34,16 +37,19 @@ export async function openImageDescriptions(options) {
     const maxRunMs = options.maxRunMs ?? IMAGE_DESCRIBE_MAX_RUN_MS;
     const sentThisRun = 0;
     const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now, describingMs: 0, maxPerRun, maxRunMs, sentThisRun };
+    const streamMinBytes = options.streamMinBytes ?? STREAM_MIN_BYTES;
     return {
         mode,
+        streamMinBytes,
         isDescribed: (sha256) => described.has(imageStateKey(sha256)),
         hasDescribeRoom: (sha256s) => describeRoomLeft(run, sha256s),
         async prepareTranscript(input) {
-            const { swap, ...afterImages } = await describeAndReplace(run, input);
+            const { replace, finds, ...afterImages } = await describeAndReplace(run, input, findsOfText(input.text));
             if (mode !== "describe_and_drop")
                 return { ...afterImages, text: input.text, changed: false, slim: null };
             // BLI-23317: the image swap and the slim rules are ONE pass over the lines (one joined string, no
             // intermediate copy). A pass that cannot run falls back to the two-step order, which is the same bytes.
+            const swap = { images: finds.images, replace };
             const pass = slimPassOrNull(input.text, input.source, { sessionId: input.sessionId }, swap);
             if (pass)
                 return { ...afterImages, text: pass.text, changed: pass.changed, slim: pass.stats };
@@ -51,8 +57,79 @@ export async function openImageDescriptions(options) {
             const slim = slimOrKeep(replaced.text, input.source, { sessionId: input.sessionId });
             return { ...afterImages, text: slim.text, changed: slim.text !== input.text, slim: slim.stats };
         },
+        prepareTranscriptFile: mode === "describe_and_drop" ? (input) => prepareFileInTwoPasses(run, input) : undefined,
     };
 }
+/** Files smaller than this are cheaper to read whole; the streamed path pays a second read. */
+const STREAM_MIN_BYTES = 8 * 1024 * 1024;
+function findsOfText(text) {
+    const images = findEmbeddedImages(text);
+    const unique = new Map();
+    for (const image of images)
+        if (!unique.has(image.sha256))
+            unique.set(image.sha256, image);
+    return {
+        total: images.length,
+        unique,
+        images,
+        base64For: (image) => base64Of(text, image),
+        swapped: (replace) => countImageSwaps({ images, replace }),
+    };
+}
+function findsOfCensus(census) {
+    return {
+        total: census.total,
+        unique: census.unique,
+        base64For: (image) => {
+            const kept = census.base64.get(image.sha256);
+            if (!kept)
+                throw new Error("image_base64_not_kept");
+            return kept.toString("latin1");
+        },
+        swapped: (replace) => {
+            let replacedCount = 0;
+            let charsSaved = 0;
+            for (const [sha256, image] of census.unique) {
+                if (!replace(image))
+                    continue;
+                const tally = census.tally.get(sha256);
+                replacedCount += tally.count;
+                charsSaved += tally.savedChars;
+            }
+            return { replacedCount, charsSaved };
+        },
+    };
+}
+async function prepareFileInTwoPasses(run, input) {
+    const startedAt = Date.now();
+    // Pass one: discover. Nothing of the text is kept; only the images and, for those that may still be sent, their base64.
+    const room = Math.max(0, run.maxPerRun - run.sentThisRun);
+    let wanted = 0;
+    const first = openTranscriptLines(input.filePath, { stopAfter: input.maxFileBytes ? input.maxFileBytes + 1 : undefined });
+    const census = await censusOfLines(first.lines, (image) => {
+        // The same filters `selectImagesToDescribe` applies, so what is kept is what may be sent.
+        const key = imageStateKey(image.sha256);
+        const failure = run.failedImages[key];
+        if (wanted >= room || run.described.has(key) || run.settledThisRun.has(image.sha256) || image.byteSize > IMAGE_DESCRIBE_MAX_BYTES)
+            return false;
+        if (failure && failure.retry_after > run.now())
+            return false;
+        wanted += 1;
+        return true;
+    });
+    const readBytes = first.bytesRead();
+    if (input.maxFileBytes && readBytes > input.maxFileBytes)
+        throw new TranscriptTooLargeError(input.maxFileBytes);
+    // Describe attempts finish and the swap decisions are settled before a single line is swapped, as with the whole text.
+    const { replace, finds: _finds, ...afterImages } = await describeAndReplace(run, input, findsOfCensus(census));
+    // Pass two: swap and slim over exactly the bytes pass one saw.
+    const second = openTranscriptLines(input.filePath, { byteLimit: readBytes });
+    const slim = await slimLineStream(second.lines, input.source, replace, readBytes);
+    if (slim.replacedCount !== afterImages.replacedCount)
+        throw new TranscriptNotStreamable("images_changed_between_passes");
+    logSlimmed(input.source, input.sessionId, slim.stats, startedAt);
+    return { ...afterImages, text: slim.text, changed: slim.changed, slim: slim.stats, readBytes };
+}
 function describeRoomLeft(run, sha256s) {
     if (run.sentThisRun >= run.maxPerRun)
         return false;
@@ -74,7 +151,7 @@ function markDescribed(run, sha256) {
     described.add(key);
     orderedKeys.push(key);
 }
-async function sendOne(run, image, text, input) {
+async function sendOne(run, image, base64, input) {
     const { options, fetchImpl, settledThisRun, failedImages } = run;
     run.sentThisRun += 1;
     const outcome = await postImageToDescribeDoor({ fetchImpl, dashboardUrl: options.dashboardUrl, deviceToken: options.deviceToken, sleep: options.sleep }, {
@@ -82,7 +159,7 @@ async function sendOne(run, image, text, input) {
         sha256: image.sha256,
         media_type: image.mediaType,
         byte_size: image.byteSize,
-        content_base64: base64Of(text, image),
+        content_base64: base64,
         source: input.source,
         session_id: input.sessionId.slice(0, 200),
     });
@@ -113,19 +190,15 @@ function rememberFailure(run, key, outcome) {
     run.stateDirty = true;
     console.error(TAG + " image not described, will not be re-sent until later", JSON.stringify({ reason: outcome.reason, http_status: outcome.httpStatus, attempts, retry_in_hours: Math.round(delay / 3_600_000), sha256_prefix: key.slice(0, 16) }));
 }
-async function describeAndReplace(run, input) {
+async function describeAndReplace(run, input, finds) {
     const { mode, described, options, orderedKeys, failedImages } = run;
-    const images = findEmbeddedImages(input.text);
-    const swap = { images, replace: (image) => described.has(imageStateKey(image.sha256)) };
-    if (images.length === 0)
-        return { swap, pendingImages: [], imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
-    const unique = new Map();
-    for (const image of images)
-        if (!unique.has(image.sha256))
-            unique.set(image.sha256, image);
+    const replace = (image) => described.has(imageStateKey(image.sha256));
+    const { unique } = finds;
+    if (finds.total === 0)
+        return { replace, finds, pendingImages: [], imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
     const counts = { settled_earlier: 0, already: 0, described_now: 0, known: 0, too_large: 0, over_budget: 0, failed: 0, backing_off: 0 };
     const toSend = selectImagesToDescribe(run, unique, counts);
-    await sendPendingImages(run, toSend, counts, input);
+    await sendPendingImages(run, toSend, counts, input, finds.base64For);
     const failedReasons = countImageOutcomes(run, toSend, counts);
     if (counts.described_now + counts.known > 0 || run.stateDirty) {
         await writeImageDescriptionState(options.stateDir, { described: orderedKeys, failed: failedImages });
@@ -133,24 +206,25 @@ async function describeAndReplace(run, input) {
     }
     const deferredOnly = counts.over_budget + counts.backing_off > 0
         && counts.described_now + counts.known + counts.failed + counts.too_large + counts.settled_earlier === 0;
-    const replaced = mode === "describe_and_drop" ? countImageSwaps(swap) : { replacedCount: 0, charsSaved: 0 };
+    const replaced = mode === "describe_and_drop" ? finds.swapped(replace) : { replacedCount: 0, charsSaved: 0 };
     const pendingImages = [...unique.keys()].filter((sha256) => !described.has(imageStateKey(sha256)));
     console.error(TAG + " transcript prepared", JSON.stringify({
         mode,
         source: input.source,
-        images: images.length,
+        images: finds.total,
         unique: unique.size,
         ...counts,
         deferred_only: deferredOnly,
         failed_reasons: failedReasons,
         replaced: replaced.replacedCount,
         chars_saved: replaced.charsSaved,
-        kept_without_description: mode === "describe_and_drop" ? images.length - replaced.replacedCount : 0,
+        kept_without_description: mode === "describe_and_drop" ? finds.total - replaced.replacedCount : 0,
     }));
     return {
-        swap,
+        replace,
+        finds,
         pendingImages,
-        imageCount: images.length,
+        imageCount: finds.total,
         uniqueCount: unique.size,
         replacedCount: replaced.replacedCount,
         charsSaved: replaced.charsSaved,
@@ -197,7 +271,7 @@ function selectImagesToDescribe(run, unique, counts) {
     }
     return toSend;
 }
-async function sendPendingImages(run, toSend, counts, input) {
+async function sendPendingImages(run, toSend, counts, input, base64For) {
     const { now, maxPerRun, maxRunMs } = run;
     const callStartedAt = now();
     let next = 0;
@@ -209,7 +283,7 @@ async function sendPendingImages(run, toSend, counts, input) {
                 counts.over_budget += 1;
                 continue;
             }
-            await sendOne(run, image, input.text, input);
+            await sendOne(run, image, base64For(image), input);
         }
     };
     await Promise.all(Array.from({ length: Math.min(CONCURRENCY, toSend.length) }, worker));

package/dist/slim/slim-stream.js

@@ -0,0 +1,77 @@
+/**
+ * Pass two of the streamed transcript: image swap and slimming over lines as they arrive (BLI-23317).
+ *
+ * The same per-line work as `slimPass` (swap the chosen images for markers inside the line, then the
+ * slim rules), with the lines coming from a stream instead of a decoded string, so the original text
+ * and the slimmed text are never both whole in memory. Output is byte-identical to `slimPass` on the
+ * joined input; `raw-evidence-transcript-stream.test.ts` and the frozen goldens hold it to that.
+ *
+ * The result is ONE string, because the sanitizer after it takes one. Slimmed lines are written as
+ * UTF-8 into one byte buffer sized to the file (off the JS heap, one byte per ASCII character, no
+ * per-line garbage waiting on the collector) and decoded once at the end, so the last step holds the
+ * bytes beside the string they make, not a second copy of the string. Text that came from valid
+ * UTF-8 survives encode and decode unchanged; the slim rules re-escape lone surrogates, so none is
+ * ever written.
+ */
+import { findEmbeddedImages, replaceImagesWithMarkers } from "@bli-cockpit/telemetry-core";
+import { lineMayHoldImage } from "../image-census-stream.js";
+import { createSlimmer } from "./slim-transcript.js";
+const NEWLINE = 0x0a;
+/** UTF-8 bytes of the lines so far, in a buffer that starts at the input's size and grows only if the output outgrows it. */
+class LineBytes {
+    buffer;
+    length = 0;
+    constructor(capacity) {
+        this.buffer = Buffer.allocUnsafe(Math.max(capacity, 1024));
+    }
+    reserve(extra) {
+        const needed = this.length + extra;
+        if (needed <= this.buffer.length)
+            return;
+        const grown = Buffer.allocUnsafe(Math.max(needed, Math.ceil(this.buffer.length * 1.25)));
+        this.buffer.copy(grown, 0, 0, this.length);
+        this.buffer = grown;
+    }
+    newline() {
+        this.reserve(1);
+        this.buffer[this.length++] = NEWLINE;
+    }
+    line(text) {
+        const size = Buffer.byteLength(text);
+        this.reserve(size);
+        this.length += this.buffer.write(text, this.length, size, "utf8");
+    }
+    text() {
+        return this.buffer.toString("utf8", 0, this.length);
+    }
+}
+export async function slimLineStream(lines, kind, replace, inputBytes) {
+    const slimmer = createSlimmer(kind);
+    const output = new LineBytes(inputBytes);
+    let replacedCount = 0;
+    let charsSaved = 0;
+    let changed = false;
+    let first = true;
+    for await (const original of lines) {
+        let line = original;
+        if (lineMayHoldImage(line)) {
+            const found = findEmbeddedImages(line);
+            if (found.length > 0) {
+                const replaced = replaceImagesWithMarkers(line, found, replace);
+                if (replaced.replacedCount > 0) {
+                    line = replaced.text;
+                    replacedCount += replaced.replacedCount;
+                    charsSaved += replaced.charsSaved;
+                }
+            }
+        }
+        const out = slimmer.slimLine(line);
+        if (line !== original || out !== original)
+            changed = true;
+        if (!first)
+            output.newline();
+        first = false;
+        output.line(out);
+    }
+    return { text: output.text(), stats: slimmer.stats(), replacedCount, charsSaved, changed };
+}

package/dist/slim/slim-transcript.js

@@ -170,7 +170,7 @@ export function slimTranscript(text, kind) {
     return { text: slim.text, stats: slim.stats };
 }
 const TAG = "[slim-transcript]";
-function logSlimmed(kind, sessionId, stats, startedAt) {
+export function logSlimmed(kind, sessionId, stats, startedAt) {
     console.error(TAG + (stats.linesModified === 0 ? " nothing to strip" : " transcript slimmed"), JSON.stringify({
         source: kind,
         session_id: sessionId.slice(0, 80),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.264",
+  "version": "0.2.265",
   "private": false,
   "type": "module",
   "bin": {