@bli-cockpit/cli 0.2.263 → 0.2.265

package/dist/adapters/agent-image-evidence.js

@@ -1,4 +1,5 @@
 import { SECRET_FILE_SEGMENT_PATTERN } from "@bli-cockpit/telemetry-core";
+import { linesOfFile } from "./transcript-line-stream.js";
 import crypto from "node:crypto";
 import fs from "node:fs/promises";
 import path from "node:path";
@@ -14,13 +15,12 @@ export async function collectAgentImageEvidenceFromJsonlFile(options) {
             skipped: [{ label: "transcript", reason: "file_too_large" }],
         };
     }
-    const content = await fs.readFile(options.filePath, "utf8").catch(() => null);
-    if (content === null)
-        return { images: [], skipped: [] };
     const images = [];
     const skipped = [];
     let lineIndex = 0;
-    for (const line of content.split("\n")) {
+    // BLI-23317: lines stream off the disk (the same lines `readFile` + `split("\n")` gave), so a 200 MB
+    // transcript is never held as bytes, string and lines at once; only the longest line is in memory.
+    for await (const line of linesOfFile(options.filePath)) {
         if (!line.trim())
             continue;
         lineIndex += 1;

package/dist/adapters/raw-evidence-collection-state.js

@@ -29,6 +29,7 @@ import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core"
 import crypto from "node:crypto";
 import fs from "node:fs/promises";
 import path from "node:path";
+import { describeError } from "../health-detail.js";
 import { DELIVERY_BACKOFF_BYPASS_REASON, deliveryBackoffApplies, heldSourceKeys, readRawEvidenceStagingState, resolveStagedObject, } from "../raw-evidence-staging.js";
 import { openStagingAllowance, } from "../staging-gate.js";
 import { chmodPrivate } from "./raw-evidence-pack-store.js";
@@ -201,8 +202,19 @@ export async function stageEvidenceBytes(collection, options) {
         return { local_path: existing.local_path, staged_in_pack: false };
     }
     const destination = path.join(collection.filesDir, options.fileName);
-    await fs.writeFile(destination, options.bytes, { mode: 0o600 });
-    await chmodPrivate(destination, 0o600);
+    // BLI-23317: written beside the destination and renamed in, so a crash or a full disk mid-write
+    // leaves a `.partial` file and never a truncated object under the name the manifest points at.
+    const partial = `${destination}.partial-${process.pid}`;
+    try {
+        await fs.writeFile(partial, options.bytes, { mode: 0o600 });
+        await chmodPrivate(partial, 0o600);
+        await fs.rename(partial, destination);
+    }
+    catch (error) {
+        await fs.rm(partial, { force: true }).catch(() => undefined);
+        console.error("[raw-evidence] staging write failed, nothing was published", JSON.stringify({ reason: "staging_write_failed", kind: options.kind, byte_size: options.bytes.byteLength, ...describeError(error) }));
+        throw error;
+    }
     collection.stagedNewCount += 1;
     return { local_path: destination, staged_in_pack: true };
 }

package/dist/adapters/raw-evidence-sanitize-held.js

@@ -0,0 +1,191 @@
+import { containsSecretLikeContent, redactSecretLikeContent, scannedCleanRedactionMetadata, } from "@bli-cockpit/telemetry-core";
+import { createHash } from "node:crypto";
+import { WHOLE_FILE_MASK_LINE, fallbackRedactionMetadata, logRedactionCrash, maskSecretBearingLines, } from "./raw-evidence-sanitize.js";
+/** Code units per encode window: 4 Mi units encode to at most 12 MiB, and the window is garbage right after. */
+const WINDOW_UNITS = 4 * 1024 * 1024;
+/**
+ * The sha256 and byte length of `Buffer.from(text, "utf8")`, without that buffer: the text is encoded
+ * a window at a time. A window never ends between the halves of a surrogate pair, so the bytes fed to
+ * the hash are the bytes a whole-string encode produces (a lone surrogate encodes the same either way).
+ */
+export function factsOfText(text, windowUnits = WINDOW_UNITS) {
+    const hash = createHash("sha256");
+    let byteLength = 0;
+    for (let at = 0; at < text.length;) {
+        let end = Math.min(text.length, at + windowUnits);
+        if (end < text.length && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)))
+            end += 1;
+        const chunk = Buffer.from(text.slice(at, end), "utf8");
+        hash.update(chunk);
+        byteLength += chunk.byteLength;
+        at = end;
+    }
+    return { sha256: hash.digest("hex"), byteLength };
+}
+function isHighSurrogate(unit) {
+    return unit >= 0xd800 && unit <= 0xdbff;
+}
+function isLowSurrogate(unit) {
+    return unit >= 0xdc00 && unit <= 0xdfff;
+}
+/**
+ * Same decision table as `sanitizeTextEvidenceForUpload`. The holder's text is cleared before the
+ * bytes of the answer are built, so the answer is the only large thing left alive when this returns.
+ */
+export function sanitizeHeldTextForUpload(held, options) {
+    // No local in this frame points at the text: only `held` does, so clearing it frees the transcript.
+    let facts;
+    const scan = {
+        textLength: held.text.length,
+        facts: () => (facts ??= held.exactBytes ? { sha256: sha256Of(held.exactBytes), byteLength: held.exactBytes.byteLength } : factsOfText(held.text)),
+        exactBytes: held.exactBytes,
+        redactedFields: options.redactedFields,
+        secretLikeFileName: options.secretLikeFileName === true,
+    };
+    try {
+        const found = scanHeldText(held, options.redactedFields, options.redact ?? redactSecretLikeContent);
+        if (scan.secretLikeFileName)
+            return maskNamedFile(held, scan, found.attempt);
+        if (found.secretLikeContent) {
+            // This branch needs the redactor's receipt, never its text; dropping the rest lets the rope of
+            // slices (which points into the transcript) go with the transcript.
+            const metadata = found.attempt.metadata;
+            found.attempt = { redacted: false, text: "", metadata: null };
+            return maskSecretBearingLinesOnly(held, scan, metadata);
+        }
+        if (found.attempt.redacted)
+            return keepDeterministicRedaction(held, scan, found.attempt);
+        const bytes = held.exactBytes ?? Buffer.from(held.text, "utf8");
+        held.text = null;
+        return { status: "clean", bytes, redaction: scannedCleanRedactionMetadata(bytes) };
+    }
+    catch (error) {
+        // Reads the facts while the text is still held; a crash inside them lands on the same stub.
+        const stubFacts = safeFacts(scan);
+        held.text = null;
+        logRedactionCrash(error, stubFacts?.byteLength ?? 0, options.redactedFields.length);
+        return stubForCrashedRedaction(scan, stubFacts);
+    }
+}
+/** The content guard's verdict and the redactor's answer, in one frame that ends before any bytes are built. */
+function scanHeldText(held, redactedFields, redact) {
+    // BLI-11094: inside the caller's try. The content guard runs regexes over the whole file.
+    const secretLikeContent = containsSecretLikeContent(held.text);
+    const attempt = redact(held.text, { appliedBy: "local_collector", redactedFields });
+    return { secretLikeContent, attempt };
+}
+function safeFacts(scan) {
+    try {
+        return scan.facts();
+    }
+    catch {
+        return undefined;
+    }
+}
+function sha256Of(bytes) {
+    return createHash("sha256").update(bytes).digest("hex");
+}
+function withFacts(metadata, original, sanitizedBytes) {
+    if (!metadata)
+        throw new Error("redacted evidence is missing redaction metadata");
+    return {
+        ...metadata,
+        original_content_hash_sha256: original.sha256,
+        sanitized_content_hash_sha256: sha256Of(sanitizedBytes),
+        original_byte_size: original.byteLength,
+        sanitized_byte_size: sanitizedBytes.byteLength,
+    };
+}
+/** The name itself is the signal: whatever the redactor left is re-checked, and a still-secret result becomes one marker line. */
+function maskNamedFile(held, scan, attempt) {
+    const original = scan.facts();
+    const sanitizedText = attempt.redacted ? attempt.text : held.text;
+    const safeText = containsSecretLikeContent(sanitizedText) ? WHOLE_FILE_MASK_LINE : sanitizedText;
+    const metadata = attempt.metadata ?? fallbackRedactionMetadata({
+        ruleId: "secret_like_file_name",
+        originalLength: scan.textLength,
+        redactedFields: scan.redactedFields,
+        fullContentRedacted: false,
+    });
+    held.text = null;
+    const sanitizedBytes = Buffer.from(safeText, "utf8");
+    return {
+        status: "redacted",
+        bytes: sanitizedBytes,
+        redaction: withFacts(metadata, original, sanitizedBytes),
+        completenessLabel: "secret_like_name_masked",
+    };
+}
+/** Only the lines that matched are replaced; every other line uploads intact. */
+function maskSecretBearingLinesOnly(held, scan, metadata) {
+    const original = scan.facts();
+    const redaction = metadata ?? fallbackRedactionMetadata({
+        ruleId: "secret_like_content_guard",
+        originalLength: scan.textLength,
+        redactedFields: scan.redactedFields,
+    });
+    let maskedText = maskSecretBearingLines(held.text, redaction);
+    held.text = null;
+    if (containsSecretLikeContent(maskedText))
+        maskedText = WHOLE_FILE_MASK_LINE;
+    const sanitizedBytes = Buffer.from(maskedText, "utf8");
+    return {
+        status: "redacted",
+        bytes: sanitizedBytes,
+        redaction: withFacts(redaction, original, sanitizedBytes),
+        completenessLabel: "secret_content_masked",
+    };
+}
+/** The redactor found and replaced something the content guard did not flag. */
+function keepDeterministicRedaction(held, scan, attempt) {
+    const original = scan.facts();
+    const metadata = attempt.metadata ?? fallbackRedactionMetadata({
+        ruleId: "secret_redaction_failed",
+        originalLength: scan.textLength,
+        redactedFields: scan.redactedFields,
+    });
+    held.text = null;
+    const sanitizedBytes = Buffer.from(attempt.text, "utf8");
+    return {
+        status: "redacted",
+        bytes: sanitizedBytes,
+        redaction: withFacts(metadata, original, sanitizedBytes),
+        completenessLabel: "secret_content_masked",
+    };
+}
+function stubForCrashedRedaction(scan, facts) {
+    const sanitizedBytes = Buffer.from("[REDACTED_LINE:redaction_crashed_stubbed]\n", "utf8");
+    return {
+        status: "redacted",
+        bytes: sanitizedBytes,
+        redaction: withFacts(fallbackRedactionMetadata({ ruleId: "redaction_crashed_stubbed", originalLength: scan.textLength, redactedFields: scan.redactedFields }), 
+        // The old path would have thrown here (hashing the original) only if the original were unreadable; it never is.
+        facts ?? { sha256: sha256Of(Buffer.alloc(0)), byteLength: 0 }, sanitizedBytes),
+        completenessLabel: "redaction_crashed_stubbed",
+    };
+}
+/**
+ * Mask secret-like values in a transcript and record that it happened (see `maskSecretsInTranscript`
+ * in raw-evidence-sanitize.ts, whose answer this one matches), from text the caller holds once.
+ */
+export function maskSecretsInHeldTranscript(collection, held, options) {
+    const sanitized = sanitizeHeldTextForUpload(held, {
+        redactedFields: [`${options.kind}.body`],
+        secretLikeFileName: options.secretLikeFileName,
+    });
+    if (sanitized.status === "redacted") {
+        collection.redacted.push({
+            kind: options.kind,
+            label: options.evidenceLabel,
+            redaction: sanitized.redaction,
+            completenessLabel: sanitized.completenessLabel,
+        });
+        console.error("[raw-evidence] text evidence sanitized", JSON.stringify({
+            kind: options.kind,
+            mode: sanitized.completenessLabel,
+            original_bytes: sanitized.redaction.original_byte_size,
+            uploaded_bytes: sanitized.bytes.byteLength,
+        }));
+    }
+    return sanitized;
+}

package/dist/adapters/raw-evidence-sanitize.js

@@ -64,7 +64,7 @@ export function sanitizeTextEvidenceForUpload(options) {
         });
     }
 }
-function logRedactionCrash(error, byteSize, redactedFieldCount) {
+export function logRedactionCrash(error, byteSize, redactedFieldCount) {
     // The redactor itself crashed. The stub below is the safe answer — a
     // placeholder goes up instead of unredacted bytes — but it is also
     // indistinguishable downstream from a file that was legitimately masked,
@@ -97,7 +97,7 @@ function maskFileWithSecretLikeName(input) {
         redaction: withRedactionContentMetadata(input.redactionResult.metadata ??
             fallbackRedactionMetadata({
                 ruleId: "secret_like_file_name",
-                originalText: input.text,
+                originalLength: input.text.length,
                 redactedFields: input.redactedFields,
                 fullContentRedacted: false,
             }), { originalBytes: input.originalBytes, sanitizedBytes }),
@@ -109,7 +109,7 @@ function maskSecretBearingLinesOnly(input) {
     const redaction = input.redactionResult.metadata ??
         fallbackRedactionMetadata({
             ruleId: "secret_like_content_guard",
-            originalText: input.text,
+            originalLength: input.text.length,
             redactedFields: input.redactedFields,
         });
     let maskedText = maskSecretBearingLines(input.text, redaction);
@@ -136,7 +136,7 @@ function keepDeterministicRedaction(input) {
         redaction: withRedactionContentMetadata(input.redactionResult.metadata ??
             fallbackRedactionMetadata({
                 ruleId: "secret_redaction_failed",
-                originalText: input.text,
+                originalLength: input.text.length,
                 redactedFields: input.redactedFields,
             }), { originalBytes: input.originalBytes, sanitizedBytes }),
         completenessLabel: "secret_content_masked",
@@ -149,7 +149,7 @@ function stubForCrashedRedaction(input) {
         bytes: sanitizedBytes,
         redaction: withRedactionContentMetadata(fallbackRedactionMetadata({
             ruleId: "redaction_crashed_stubbed",
-            originalText: input.text,
+            originalLength: input.text.length,
             redactedFields: input.redactedFields,
         }), { originalBytes: input.originalBytes, sanitizedBytes }),
         completenessLabel: "redaction_crashed_stubbed",
@@ -174,7 +174,7 @@ function stubForCrashedRedaction(input) {
  * offsets are UTF-16 and file-absolute, and a line takes the rule of the FIRST
  * range in array order that overlaps it.
  */
-function maskSecretBearingLines(text, redaction) {
+export function maskSecretBearingLines(text, redaction) {
     if (redaction.redacted_ranges.length === 0) {
         return "[REDACTED_LINE:secret_like_content_guard]\n";
     }
@@ -240,7 +240,7 @@ function lineEndingOf(text, start, end) {
         return "\n";
     return "";
 }
-function fallbackRedactionMetadata(options) {
+export function fallbackRedactionMetadata(options) {
     const fullContentRedacted = options.fullContentRedacted !== false;
     return {
         schema_version: "raw-evidence-redaction.v1",
@@ -252,17 +252,17 @@ function fallbackRedactionMetadata(options) {
                 rule_id: options.ruleId,
                 match_count: 1,
                 redacted_char_count: fullContentRedacted
-                    ? options.originalText.length
+                    ? options.originalLength
                     : 0,
             },
         ],
         secret_like_match_count: 1,
         redacted_fields: options.redactedFields,
-        redacted_ranges: fullContentRedacted && options.originalText.length > 0
+        redacted_ranges: fullContentRedacted && options.originalLength > 0
             ? [
                 {
                     start: 0,
-                    end: options.originalText.length,
+                    end: options.originalLength,
                     rule_id: options.ruleId,
                 },
             ]

package/dist/adapters/raw-evidence-transcript-body.js

@@ -0,0 +1,80 @@
+/**
+ * The transcript text that the mask step reads, and the image and slim step that produce it
+ * (BLI-16977 for the steps, BLI-23317 for how few copies of a 200 MB file they leave behind).
+ *
+ * The file's bytes are decoded ONCE and handed on in a holder (`HeldText`); the bytes themselves
+ * are let go the moment the text exists, because for valid UTF-8 the text encodes back to exactly
+ * those bytes (`Buffer.from(text)` is the original, so nothing needs the buffer to stay). A file
+ * that is not valid UTF-8 keeps its bytes in the holder: the decoding replaced characters, so the
+ * bytes cannot be rebuilt, and the old path uploaded the original bytes when nothing changed.
+ * The order stays image replacement, slim, redact.
+ */
+import { isUtf8 } from "node:buffer";
+import { mightHoldEmbeddedImages } from "@bli-cockpit/telemetry-core";
+import { describeError } from "../health-detail.js";
+import { readTranscriptWhole, streamTranscriptBody, volatilityOf } from "./raw-evidence-transcript-stream.js";
+/** Which agent's transcript rules apply to this evidence kind; null for anything the image step does not know. */
+export function agentOfKind(kind) {
+    return kind.startsWith("codex") ? "codex" : kind.startsWith("claude") ? "claude_code" : null;
+}
+/** Decode once and release the bytes, unless the decoding was lossy and the bytes are the only exact copy. */
+export function holdDecodedText(source) {
+    const bytes = source.bytes;
+    const text = bytes.toString("utf8");
+    const lossless = isUtf8(bytes);
+    source.bytes = null;
+    if (!lossless) {
+        console.error("[raw-evidence] transcript is not valid UTF-8; keeping its bytes beside the text so an unchanged file uploads as it is", JSON.stringify({ reason: "invalid_utf8_bytes_kept", byte_size: bytes.byteLength }));
+    }
+    return { text, exactBytes: lossless ? null : bytes };
+}
+/**
+ * BLI-16977: describe each distinct embedded image once and, when the one switch says so, swap the
+ * base64 for a short marker BEFORE masking, so the masker and the content hash see the final bytes.
+ * The text is unchanged (the very same decoding) unless this sync has an image session AND the switch
+ * is `describe_and_drop`; in `describe_and_keep` it always comes back untouched. An image step
+ * that breaks hands back the original text and says so. The one throw: a streamed source (BLI-23317)
+ * whose file cannot be read to the end raises `TranscriptReadError` or `TranscriptTooLargeError`, which
+ * the caller turns into the `file_read_failed` and `file_too_large` skips.
+ */
+export async function prepareTranscriptBody(collection, file) {
+    const session = collection.budget.images;
+    const agent = agentOfKind(file.kind);
+    if (file.source.stream) {
+        const streamed = await streamTranscriptBody(session, agent, { ...file.source.stream, sessionId: file.sessionId, kind: file.kind });
+        if (streamed) {
+            file.source.readBytes = streamed.readBytes;
+            return { held: streamed.held, volatile: streamed.volatile, pendingImages: streamed.pendingImages };
+        }
+        file.source.bytes = await readTranscriptWhole(file.source.stream);
+        file.source.readBytes = file.source.bytes.byteLength;
+    }
+    const untouched = (volatile, pendingImages) => ({ held: holdDecodedText(file.source), volatile, pendingImages });
+    if (!session || !agent)
+        return untouched(false, null);
+    // Drop mode slims every transcript, image-bearing or not; keep mode only describes the ones that can hold an image.
+    if (session.mode !== "describe_and_drop" && !mightHoldEmbeddedImages(file.source.bytes))
+        return untouched(false, null);
+    let held;
+    try {
+        held = holdDecodedText(file.source);
+        const prepared = await session.prepareTranscript({ text: held.text, source: agent, sessionId: file.sessionId });
+        // BLI-3936: "volatile" = the next tick could build different bytes, or do image work, from
+        // this very file: an image still without a stored description (keep mode never replaces, so
+        // any image counts) or a slim step that failed and should be retried.
+        // BLI-22794: volatile ONLY because of images without a stored description (the slim step held
+        // up): the bytes change only when one of these gains one, so the memo may keep the entry.
+        const { volatile, pendingImages } = volatilityOf(session.mode, prepared);
+        if (prepared.changed) {
+            // The old path encoded the new text and read it back; a text with no lone surrogate (slim output
+            // re-escapes them) is the same either way, so the new text is the exact decoding of the new bytes.
+            held.text = prepared.text;
+            held.exactBytes = null;
+        }
+        return { held, volatile, pendingImages };
+    }
+    catch (error) {
+        console.error("[raw-evidence] image or slim step failed, uploading the transcript as it was", JSON.stringify({ reason: "image_step_failed", kind: file.kind, ...describeError(error) }));
+        return { held: held ?? holdDecodedText(file.source), volatile: true, pendingImages: null };
+    }
+}

package/dist/adapters/raw-evidence-transcript-file.js

@@ -21,8 +21,9 @@ import path from "node:path";
 import { describeError } from "../health-detail.js";
 import { DELIVERY_BACKOFF_HOLDING_REASON, evidenceSourceKey, } from "../raw-evidence-staging.js";
 import { isSecretLikePath, sha256, shortHash, } from "./raw-evidence-keys.js";
-import { maskSecretsInTranscript, } from "./raw-evidence-sanitize.js";
-import { mightHoldEmbeddedImages } from "@bli-cockpit/telemetry-core";
+import { maskSecretsInHeldTranscript } from "./raw-evidence-sanitize-held.js";
+import { agentOfKind, prepareTranscriptBody } from "./raw-evidence-transcript-body.js";
+import { TranscriptReadError, TranscriptTooLargeError } from "./transcript-text-stream.js";
 import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
 import { evidenceEntry } from "./raw-evidence-manifest.js";
 import { prepareRecipe, statSourceForMemo, } from "./raw-evidence-prepare-memo.js";
@@ -57,16 +58,30 @@ export async function collectOneEvidenceFile(collection, options) {
     const recipe = prepareRecipe({ imageMode: collection.budget.images?.mode, kind: options.kind });
     if (await reuseUnchangedTranscript(collection, options, names, stamp, recipe))
         return true;
-    const read = await readEvidenceFileWithinCap(options.filePath, options.maxFileBytes);
+    const read = await readEvidenceFileWithinCap(options.filePath, options.maxFileBytes, streamMinBytesFor(collection, options.kind));
     if (read.status === "read_failed")
         return skip("file_read_failed");
     if (read.status === "too_large") {
         markCapApplied(collection, options.kind, "max_file_bytes");
         return skip("file_too_large");
     }
-    const { sanitized, contentHash, stubClaim, volatile, pendingImages, redactedEntry } = await prepareTranscriptForUpload(collection, options, names, read.bytes);
+    let prepared;
+    try {
+        prepared = await prepareTranscriptForUpload(collection, options, names, read.source);
+    }
+    catch (error) {
+        // BLI-23317: a streamed or fallback read that fails ends exactly as a whole-file read that failed.
+        if (error instanceof TranscriptReadError)
+            return skip("file_read_failed");
+        if (error instanceof TranscriptTooLargeError) {
+            markCapApplied(collection, options.kind, "max_file_bytes");
+            return skip("file_too_large");
+        }
+        throw error;
+    }
+    const { sanitized, contentHash, stubClaim, volatile, pendingImages, redactedEntry } = prepared;
     await rememberPreparedTranscript(collection, options, names, {
-        stamp, recipe, readBytes: read.bytes.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, pendingImages, redactedEntry,
+        stamp, recipe, readBytes: read.source.readBytes ?? read.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, pendingImages, redactedEntry,
     });
     if (collection.skipContentHashes.has(contentHash) && !stubClaim) {
         collection.reused.push({
@@ -95,17 +110,14 @@ function recordEvidenceSkip(collection, kind, label, reason) {
     collection.skipped.push({ kind, label, reason });
     return false;
 }
-async function prepareTranscriptForUpload(collection, options, names, bytes) {
+async function prepareTranscriptForUpload(collection, options, names, source) {
     // BLI-16977: describe each distinct embedded image once and, when the one switch says so, swap the
     // base64 for a short marker BEFORE masking, so the masker and the content hash see the final bytes.
-    const images = await prepareEmbeddedImages(collection, {
-        bytes,
-        kind: options.kind,
-        sessionId: options.sessionId,
-    });
+    // BLI-23317: the text is decoded once and travels in a holder the mask step clears, so the file's
+    // bytes, the decoded text and the masked text are never all alive together.
+    const body = await prepareTranscriptBody(collection, { source, kind: options.kind, sessionId: options.sessionId });
     const redactedBefore = collection.redacted.length;
-    const sanitized = maskSecretsInTranscript(collection, {
-        raw: images.bytes,
+    const sanitized = maskSecretsInHeldTranscript(collection, body.held, {
         kind: options.kind,
         evidenceLabel: names.evidenceLabel,
         secretLikeFileName: names.secretLikeFileName,
@@ -122,47 +134,7 @@ async function prepareTranscriptForUpload(collection, options, names, bytes) {
         byteSize: sanitized.bytes.byteLength,
         alreadyDurable: collection.skipContentHashes.has(contentHash),
     });
-    return { sanitized, contentHash, stubClaim, volatile: images.volatile, pendingImages: images.pendingImages, redactedEntry };
-}
-/**
- * The transcript bytes after the image step. Unchanged (the very same buffer) unless this sync has
- * an image session AND the switch is `describe_and_drop` (images become markers where a description
- * is stored, then the slim rules run, see src/slim/). In `describe_and_keep` the bytes always come
- * back untouched. Never throws: an image step that breaks hands back the original bytes and says so.
- */
-async function prepareEmbeddedImages(collection, file) {
-    const session = collection.budget.images;
-    const source = file.kind.startsWith("codex") ? "codex" : file.kind.startsWith("claude") ? "claude_code" : null;
-    if (!session || !source)
-        return { bytes: file.bytes, volatile: false, pendingImages: null };
-    // Drop mode slims every transcript, image-bearing or not; keep mode only describes the ones that can hold an image.
-    if (session.mode !== "describe_and_drop" && !mightHoldEmbeddedImages(file.bytes))
-        return { bytes: file.bytes, volatile: false, pendingImages: null };
-    try {
-        const prepared = await session.prepareTranscript({
-            text: file.bytes.toString("utf8"),
-            source,
-            sessionId: file.sessionId,
-        });
-        // BLI-3936: "volatile" = the next tick could build different bytes, or do image work, from
-        // this very file: an image still without a stored description (keep mode never replaces, so
-        // any image counts) or a slim step that failed and should be retried.
-        const imagesPending = session.mode === "describe_and_drop"
-            ? prepared.imageCount > prepared.replacedCount
-            : prepared.imageCount > 0;
-        const slimFailed = session.mode === "describe_and_drop" && prepared.slim === null;
-        const volatile = imagesPending || slimFailed;
-        // BLI-22794: volatile ONLY because of images without a stored description (the slim step held
-        // up): the bytes change only when one of these gains one, so the memo may keep the entry.
-        const pendingImages = volatile && !slimFailed ? prepared.pendingImages : null;
-        if (!prepared.changed)
-            return { bytes: file.bytes, volatile, pendingImages };
-        return { bytes: Buffer.from(prepared.text, "utf8"), volatile, pendingImages };
-    }
-    catch (error) {
-        console.error("[raw-evidence] image or slim step failed, uploading the transcript as it was", JSON.stringify({ reason: "image_step_failed", kind: file.kind, ...describeError(error) }));
-        return { bytes: file.bytes, volatile: true, pendingImages: null };
-    }
+    return { sanitized, contentHash, stubClaim, volatile: body.volatile, pendingImages: body.pendingImages, redactedEntry };
 }
 /**
  * BLI-3936: the answer for a transcript whose file has not changed since a tick that found its
@@ -307,12 +279,19 @@ async function stageOneTranscript(collection, file) {
         }),
     }));
 }
+/** The size from which this pass streams a transcript instead of reading it whole; null when its image session cannot (BLI-23317). */
+function streamMinBytesFor(collection, kind) {
+    const session = collection.budget.images;
+    if (!session?.prepareTranscriptFile || !agentOfKind(kind))
+        return null;
+    return session.streamMinBytes ?? null;
+}
 /**
  * Size is checked twice on purpose: once by `stat` so an oversized transcript
  * is never buffered at all, and once on the bytes actually read, because a live
  * session can grow between the two.
  */
-async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
+async function readEvidenceFileWithinCap(filePath, maxFileBytes, streamMinBytes) {
     if (maxFileBytes) {
         let stat;
         try {
@@ -333,6 +312,14 @@ async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
         if (stat.size > maxFileBytes)
             return { status: "too_large" };
     }
+    if (streamMinBytes !== null) {
+        // BLI-23317: a big transcript is streamed by the prepare step, not buffered here. A stat that fails
+        // falls through to the read below, which reports the failure itself.
+        const size = await fs.stat(filePath).then((stat) => stat.size, () => null);
+        if (size !== null && size >= streamMinBytes) {
+            return { status: "ok", source: { bytes: null, stream: { filePath, maxFileBytes } }, byteLength: size };
+        }
+    }
     let bytes;
     try {
         bytes = await fs.readFile(filePath);
@@ -348,5 +335,5 @@ async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
     if (maxFileBytes && bytes.byteLength > maxFileBytes) {
         return { status: "too_large" };
     }
-    return { status: "ok", bytes };
+    return { status: "ok", source: { bytes }, byteLength: bytes.byteLength };
 }

package/dist/adapters/raw-evidence-transcript-stream.js

@@ -0,0 +1,63 @@
+/**
+ * The transcript that is never read whole (BLI-23317): its path goes to the image session's two
+ * streaming passes, and what comes back is the one slimmed string the sanitizer takes.
+ *
+ * Three endings, each named:
+ *  - a body: the text is the slim form of the file (valid UTF-8 was checked across every chunk, so
+ *    `exactBytes` is null: the text encodes back to the bytes a clean file would upload);
+ *  - null: the streamed path cannot give the whole-file answer (invalid UTF-8, a file that changed
+ *    between the passes, an image step that broke), and the caller reads the file whole, which is
+ *    what ran before this existed and still keeps the original bytes for the clean branch;
+ *  - a throw of `TranscriptReadError` or `TranscriptTooLargeError`: the same outcomes a whole-file
+ *    read gives (`file_read_failed`, `file_too_large`), never a shorter file.
+ */
+import fs from "node:fs/promises";
+import { describeError } from "../health-detail.js";
+import { TranscriptNotStreamable, TranscriptReadError, TranscriptTooLargeError, } from "./transcript-text-stream.js";
+const TAG = "[raw-evidence]";
+/**
+ * BLI-3936: "volatile" = the next tick could build different bytes, or do image work, from this very
+ * file: an image still without a stored description (keep mode never replaces, so any image counts) or
+ * a slim step that failed and should be retried. BLI-22794: volatile ONLY because of images without a
+ * stored description means the bytes change only when one of them gains one, so the memo may keep the entry.
+ */
+export function volatilityOf(mode, prepared) {
+    const imagesPending = mode === "describe_and_drop"
+        ? prepared.imageCount > prepared.replacedCount
+        : prepared.imageCount > 0;
+    const slimFailed = mode === "describe_and_drop" && prepared.slim === null;
+    const volatile = imagesPending || slimFailed;
+    return { volatile, pendingImages: volatile && !slimFailed ? prepared.pendingImages : null };
+}
+export async function streamTranscriptBody(session, agent, file) {
+    if (!session?.prepareTranscriptFile || !agent) {
+        console.error(TAG + " streamed read not available, reading the transcript whole", JSON.stringify({ reason: "stream_unavailable", kind: file.kind }));
+        return null;
+    }
+    try {
+        const prepared = await session.prepareTranscriptFile({ filePath: file.filePath, source: agent, sessionId: file.sessionId, maxFileBytes: file.maxFileBytes });
+        const { volatile, pendingImages } = volatilityOf(session.mode, prepared);
+        console.error(TAG + " transcript streamed through image swap and slim", JSON.stringify({ kind: file.kind, byte_size: prepared.readBytes, changed: prepared.changed, images: prepared.imageCount, volatile }));
+        return { held: { text: prepared.text, exactBytes: null }, volatile, pendingImages, readBytes: prepared.readBytes };
+    }
+    catch (error) {
+        if (error instanceof TranscriptReadError || error instanceof TranscriptTooLargeError)
+            throw error;
+        console.error(TAG + " streamed read set aside, reading the transcript whole", JSON.stringify({ reason: error instanceof TranscriptNotStreamable ? error.reason : "stream_step_failed", kind: file.kind, ...(error instanceof TranscriptNotStreamable ? {} : describeError(error)) }));
+        return null;
+    }
+}
+/** The whole-file read the streamed path falls back to: same failures, same labels, as the reader that did not stream. */
+export async function readTranscriptWhole(file) {
+    let bytes;
+    try {
+        bytes = await fs.readFile(file.filePath);
+    }
+    catch (error) {
+        console.error(TAG + " evidence file could not be read", JSON.stringify({ reason: "file_read_failed", stage: "read", ...describeError(error) }));
+        throw new TranscriptReadError("read", error);
+    }
+    if (file.maxFileBytes && bytes.byteLength > file.maxFileBytes)
+        throw new TranscriptTooLargeError(file.maxFileBytes);
+    return bytes;
+}