@bli-cockpit/cli 0.2.263 → 0.2.264

package/dist/adapters/agent-image-evidence.js

@@ -1,4 +1,5 @@
 import { SECRET_FILE_SEGMENT_PATTERN } from "@bli-cockpit/telemetry-core";
+import { linesOfFile } from "./transcript-line-stream.js";
 import crypto from "node:crypto";
 import fs from "node:fs/promises";
 import path from "node:path";
@@ -14,13 +15,12 @@ export async function collectAgentImageEvidenceFromJsonlFile(options) {
             skipped: [{ label: "transcript", reason: "file_too_large" }],
         };
     }
-    const content = await fs.readFile(options.filePath, "utf8").catch(() => null);
-    if (content === null)
-        return { images: [], skipped: [] };
     const images = [];
     const skipped = [];
     let lineIndex = 0;
-    for (const line of content.split("\n")) {
+    // BLI-23317: lines stream off the disk (the same lines `readFile` + `split("\n")` gave), so a 200 MB
+    // transcript is never held as bytes, string and lines at once; only the longest line is in memory.
+    for await (const line of linesOfFile(options.filePath)) {
         if (!line.trim())
             continue;
         lineIndex += 1;

package/dist/adapters/raw-evidence-collection-state.js

@@ -29,6 +29,7 @@ import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES } from "@bli-cockpit/telemetry-core"
 import crypto from "node:crypto";
 import fs from "node:fs/promises";
 import path from "node:path";
+import { describeError } from "../health-detail.js";
 import { DELIVERY_BACKOFF_BYPASS_REASON, deliveryBackoffApplies, heldSourceKeys, readRawEvidenceStagingState, resolveStagedObject, } from "../raw-evidence-staging.js";
 import { openStagingAllowance, } from "../staging-gate.js";
 import { chmodPrivate } from "./raw-evidence-pack-store.js";
@@ -201,8 +202,19 @@ export async function stageEvidenceBytes(collection, options) {
         return { local_path: existing.local_path, staged_in_pack: false };
     }
     const destination = path.join(collection.filesDir, options.fileName);
-    await fs.writeFile(destination, options.bytes, { mode: 0o600 });
-    await chmodPrivate(destination, 0o600);
+    // BLI-23317: written beside the destination and renamed in, so a crash or a full disk mid-write
+    // leaves a `.partial` file and never a truncated object under the name the manifest points at.
+    const partial = `${destination}.partial-${process.pid}`;
+    try {
+        await fs.writeFile(partial, options.bytes, { mode: 0o600 });
+        await chmodPrivate(partial, 0o600);
+        await fs.rename(partial, destination);
+    }
+    catch (error) {
+        await fs.rm(partial, { force: true }).catch(() => undefined);
+        console.error("[raw-evidence] staging write failed, nothing was published", JSON.stringify({ reason: "staging_write_failed", kind: options.kind, byte_size: options.bytes.byteLength, ...describeError(error) }));
+        throw error;
+    }
     collection.stagedNewCount += 1;
     return { local_path: destination, staged_in_pack: true };
 }

package/dist/adapters/raw-evidence-sanitize-held.js

@@ -0,0 +1,191 @@
+import { containsSecretLikeContent, redactSecretLikeContent, scannedCleanRedactionMetadata, } from "@bli-cockpit/telemetry-core";
+import { createHash } from "node:crypto";
+import { WHOLE_FILE_MASK_LINE, fallbackRedactionMetadata, logRedactionCrash, maskSecretBearingLines, } from "./raw-evidence-sanitize.js";
+/** Code units per encode window: 4 Mi units encode to at most 12 MiB, and the window is garbage right after. */
+const WINDOW_UNITS = 4 * 1024 * 1024;
+/**
+ * The sha256 and byte length of `Buffer.from(text, "utf8")`, without that buffer: the text is encoded
+ * a window at a time. A window never ends between the halves of a surrogate pair, so the bytes fed to
+ * the hash are the bytes a whole-string encode produces (a lone surrogate encodes the same either way).
+ */
+export function factsOfText(text, windowUnits = WINDOW_UNITS) {
+    const hash = createHash("sha256");
+    let byteLength = 0;
+    for (let at = 0; at < text.length;) {
+        let end = Math.min(text.length, at + windowUnits);
+        if (end < text.length && isHighSurrogate(text.charCodeAt(end - 1)) && isLowSurrogate(text.charCodeAt(end)))
+            end += 1;
+        const chunk = Buffer.from(text.slice(at, end), "utf8");
+        hash.update(chunk);
+        byteLength += chunk.byteLength;
+        at = end;
+    }
+    return { sha256: hash.digest("hex"), byteLength };
+}
+function isHighSurrogate(unit) {
+    return unit >= 0xd800 && unit <= 0xdbff;
+}
+function isLowSurrogate(unit) {
+    return unit >= 0xdc00 && unit <= 0xdfff;
+}
+/**
+ * Same decision table as `sanitizeTextEvidenceForUpload`. The holder's text is cleared before the
+ * bytes of the answer are built, so the answer is the only large thing left alive when this returns.
+ */
+export function sanitizeHeldTextForUpload(held, options) {
+    // No local in this frame points at the text: only `held` does, so clearing it frees the transcript.
+    let facts;
+    const scan = {
+        textLength: held.text.length,
+        facts: () => (facts ??= held.exactBytes ? { sha256: sha256Of(held.exactBytes), byteLength: held.exactBytes.byteLength } : factsOfText(held.text)),
+        exactBytes: held.exactBytes,
+        redactedFields: options.redactedFields,
+        secretLikeFileName: options.secretLikeFileName === true,
+    };
+    try {
+        const found = scanHeldText(held, options.redactedFields, options.redact ?? redactSecretLikeContent);
+        if (scan.secretLikeFileName)
+            return maskNamedFile(held, scan, found.attempt);
+        if (found.secretLikeContent) {
+            // This branch needs the redactor's receipt, never its text; dropping the rest lets the rope of
+            // slices (which points into the transcript) go with the transcript.
+            const metadata = found.attempt.metadata;
+            found.attempt = { redacted: false, text: "", metadata: null };
+            return maskSecretBearingLinesOnly(held, scan, metadata);
+        }
+        if (found.attempt.redacted)
+            return keepDeterministicRedaction(held, scan, found.attempt);
+        const bytes = held.exactBytes ?? Buffer.from(held.text, "utf8");
+        held.text = null;
+        return { status: "clean", bytes, redaction: scannedCleanRedactionMetadata(bytes) };
+    }
+    catch (error) {
+        // Reads the facts while the text is still held; a crash inside them lands on the same stub.
+        const stubFacts = safeFacts(scan);
+        held.text = null;
+        logRedactionCrash(error, stubFacts?.byteLength ?? 0, options.redactedFields.length);
+        return stubForCrashedRedaction(scan, stubFacts);
+    }
+}
+/** The content guard's verdict and the redactor's answer, in one frame that ends before any bytes are built. */
+function scanHeldText(held, redactedFields, redact) {
+    // BLI-11094: inside the caller's try. The content guard runs regexes over the whole file.
+    const secretLikeContent = containsSecretLikeContent(held.text);
+    const attempt = redact(held.text, { appliedBy: "local_collector", redactedFields });
+    return { secretLikeContent, attempt };
+}
+function safeFacts(scan) {
+    try {
+        return scan.facts();
+    }
+    catch {
+        return undefined;
+    }
+}
+function sha256Of(bytes) {
+    return createHash("sha256").update(bytes).digest("hex");
+}
+function withFacts(metadata, original, sanitizedBytes) {
+    if (!metadata)
+        throw new Error("redacted evidence is missing redaction metadata");
+    return {
+        ...metadata,
+        original_content_hash_sha256: original.sha256,
+        sanitized_content_hash_sha256: sha256Of(sanitizedBytes),
+        original_byte_size: original.byteLength,
+        sanitized_byte_size: sanitizedBytes.byteLength,
+    };
+}
+/** The name itself is the signal: whatever the redactor left is re-checked, and a still-secret result becomes one marker line. */
+function maskNamedFile(held, scan, attempt) {
+    const original = scan.facts();
+    const sanitizedText = attempt.redacted ? attempt.text : held.text;
+    const safeText = containsSecretLikeContent(sanitizedText) ? WHOLE_FILE_MASK_LINE : sanitizedText;
+    const metadata = attempt.metadata ?? fallbackRedactionMetadata({
+        ruleId: "secret_like_file_name",
+        originalLength: scan.textLength,
+        redactedFields: scan.redactedFields,
+        fullContentRedacted: false,
+    });
+    held.text = null;
+    const sanitizedBytes = Buffer.from(safeText, "utf8");
+    return {
+        status: "redacted",
+        bytes: sanitizedBytes,
+        redaction: withFacts(metadata, original, sanitizedBytes),
+        completenessLabel: "secret_like_name_masked",
+    };
+}
+/** Only the lines that matched are replaced; every other line uploads intact. */
+function maskSecretBearingLinesOnly(held, scan, metadata) {
+    const original = scan.facts();
+    const redaction = metadata ?? fallbackRedactionMetadata({
+        ruleId: "secret_like_content_guard",
+        originalLength: scan.textLength,
+        redactedFields: scan.redactedFields,
+    });
+    let maskedText = maskSecretBearingLines(held.text, redaction);
+    held.text = null;
+    if (containsSecretLikeContent(maskedText))
+        maskedText = WHOLE_FILE_MASK_LINE;
+    const sanitizedBytes = Buffer.from(maskedText, "utf8");
+    return {
+        status: "redacted",
+        bytes: sanitizedBytes,
+        redaction: withFacts(redaction, original, sanitizedBytes),
+        completenessLabel: "secret_content_masked",
+    };
+}
+/** The redactor found and replaced something the content guard did not flag. */
+function keepDeterministicRedaction(held, scan, attempt) {
+    const original = scan.facts();
+    const metadata = attempt.metadata ?? fallbackRedactionMetadata({
+        ruleId: "secret_redaction_failed",
+        originalLength: scan.textLength,
+        redactedFields: scan.redactedFields,
+    });
+    held.text = null;
+    const sanitizedBytes = Buffer.from(attempt.text, "utf8");
+    return {
+        status: "redacted",
+        bytes: sanitizedBytes,
+        redaction: withFacts(metadata, original, sanitizedBytes),
+        completenessLabel: "secret_content_masked",
+    };
+}
+function stubForCrashedRedaction(scan, facts) {
+    const sanitizedBytes = Buffer.from("[REDACTED_LINE:redaction_crashed_stubbed]\n", "utf8");
+    return {
+        status: "redacted",
+        bytes: sanitizedBytes,
+        redaction: withFacts(fallbackRedactionMetadata({ ruleId: "redaction_crashed_stubbed", originalLength: scan.textLength, redactedFields: scan.redactedFields }), 
+        // The old path would have thrown here (hashing the original) only if the original were unreadable; it never is.
+        facts ?? { sha256: sha256Of(Buffer.alloc(0)), byteLength: 0 }, sanitizedBytes),
+        completenessLabel: "redaction_crashed_stubbed",
+    };
+}
+/**
+ * Mask secret-like values in a transcript and record that it happened (see `maskSecretsInTranscript`
+ * in raw-evidence-sanitize.ts, whose answer this one matches), from text the caller holds once.
+ */
+export function maskSecretsInHeldTranscript(collection, held, options) {
+    const sanitized = sanitizeHeldTextForUpload(held, {
+        redactedFields: [`${options.kind}.body`],
+        secretLikeFileName: options.secretLikeFileName,
+    });
+    if (sanitized.status === "redacted") {
+        collection.redacted.push({
+            kind: options.kind,
+            label: options.evidenceLabel,
+            redaction: sanitized.redaction,
+            completenessLabel: sanitized.completenessLabel,
+        });
+        console.error("[raw-evidence] text evidence sanitized", JSON.stringify({
+            kind: options.kind,
+            mode: sanitized.completenessLabel,
+            original_bytes: sanitized.redaction.original_byte_size,
+            uploaded_bytes: sanitized.bytes.byteLength,
+        }));
+    }
+    return sanitized;
+}

package/dist/adapters/raw-evidence-sanitize.js

@@ -64,7 +64,7 @@ export function sanitizeTextEvidenceForUpload(options) {
         });
     }
 }
-function logRedactionCrash(error, byteSize, redactedFieldCount) {
+export function logRedactionCrash(error, byteSize, redactedFieldCount) {
     // The redactor itself crashed. The stub below is the safe answer — a
     // placeholder goes up instead of unredacted bytes — but it is also
     // indistinguishable downstream from a file that was legitimately masked,
@@ -97,7 +97,7 @@ function maskFileWithSecretLikeName(input) {
         redaction: withRedactionContentMetadata(input.redactionResult.metadata ??
             fallbackRedactionMetadata({
                 ruleId: "secret_like_file_name",
-                originalText: input.text,
+                originalLength: input.text.length,
                 redactedFields: input.redactedFields,
                 fullContentRedacted: false,
             }), { originalBytes: input.originalBytes, sanitizedBytes }),
@@ -109,7 +109,7 @@ function maskSecretBearingLinesOnly(input) {
     const redaction = input.redactionResult.metadata ??
         fallbackRedactionMetadata({
             ruleId: "secret_like_content_guard",
-            originalText: input.text,
+            originalLength: input.text.length,
             redactedFields: input.redactedFields,
         });
     let maskedText = maskSecretBearingLines(input.text, redaction);
@@ -136,7 +136,7 @@ function keepDeterministicRedaction(input) {
         redaction: withRedactionContentMetadata(input.redactionResult.metadata ??
             fallbackRedactionMetadata({
                 ruleId: "secret_redaction_failed",
-                originalText: input.text,
+                originalLength: input.text.length,
                 redactedFields: input.redactedFields,
             }), { originalBytes: input.originalBytes, sanitizedBytes }),
         completenessLabel: "secret_content_masked",
@@ -149,7 +149,7 @@ function stubForCrashedRedaction(input) {
         bytes: sanitizedBytes,
         redaction: withRedactionContentMetadata(fallbackRedactionMetadata({
             ruleId: "redaction_crashed_stubbed",
-            originalText: input.text,
+            originalLength: input.text.length,
             redactedFields: input.redactedFields,
         }), { originalBytes: input.originalBytes, sanitizedBytes }),
         completenessLabel: "redaction_crashed_stubbed",
@@ -174,7 +174,7 @@ function stubForCrashedRedaction(input) {
  * offsets are UTF-16 and file-absolute, and a line takes the rule of the FIRST
  * range in array order that overlaps it.
  */
-function maskSecretBearingLines(text, redaction) {
+export function maskSecretBearingLines(text, redaction) {
     if (redaction.redacted_ranges.length === 0) {
         return "[REDACTED_LINE:secret_like_content_guard]\n";
     }
@@ -240,7 +240,7 @@ function lineEndingOf(text, start, end) {
         return "\n";
     return "";
 }
-function fallbackRedactionMetadata(options) {
+export function fallbackRedactionMetadata(options) {
     const fullContentRedacted = options.fullContentRedacted !== false;
     return {
         schema_version: "raw-evidence-redaction.v1",
@@ -252,17 +252,17 @@ function fallbackRedactionMetadata(options) {
                 rule_id: options.ruleId,
                 match_count: 1,
                 redacted_char_count: fullContentRedacted
-                    ? options.originalText.length
+                    ? options.originalLength
                     : 0,
             },
         ],
         secret_like_match_count: 1,
         redacted_fields: options.redactedFields,
-        redacted_ranges: fullContentRedacted && options.originalText.length > 0
+        redacted_ranges: fullContentRedacted && options.originalLength > 0
             ? [
                 {
                     start: 0,
-                    end: options.originalText.length,
+                    end: options.originalLength,
                     rule_id: options.ruleId,
                 },
             ]

package/dist/adapters/raw-evidence-transcript-body.js

@@ -0,0 +1,69 @@
+/**
+ * The transcript text that the mask step reads, and the image and slim step that produce it
+ * (BLI-16977 for the steps, BLI-23317 for how few copies of a 200 MB file they leave behind).
+ *
+ * The file's bytes are decoded ONCE and handed on in a holder (`HeldText`); the bytes themselves
+ * are let go the moment the text exists, because for valid UTF-8 the text encodes back to exactly
+ * those bytes (`Buffer.from(text)` is the original, so nothing needs the buffer to stay). A file
+ * that is not valid UTF-8 keeps its bytes in the holder: the decoding replaced characters, so the
+ * bytes cannot be rebuilt, and the old path uploaded the original bytes when nothing changed.
+ * The order stays image replacement, slim, redact.
+ */
+import { isUtf8 } from "node:buffer";
+import { mightHoldEmbeddedImages } from "@bli-cockpit/telemetry-core";
+import { describeError } from "../health-detail.js";
+/** Decode once and release the bytes, unless the decoding was lossy and the bytes are the only exact copy. */
+export function holdDecodedText(source) {
+    const bytes = source.bytes;
+    const text = bytes.toString("utf8");
+    const lossless = isUtf8(bytes);
+    source.bytes = null;
+    if (!lossless) {
+        console.error("[raw-evidence] transcript is not valid UTF-8; keeping its bytes beside the text so an unchanged file uploads as it is", JSON.stringify({ reason: "invalid_utf8_bytes_kept", byte_size: bytes.byteLength }));
+    }
+    return { text, exactBytes: lossless ? null : bytes };
+}
+/**
+ * BLI-16977: describe each distinct embedded image once and, when the one switch says so, swap the
+ * base64 for a short marker BEFORE masking, so the masker and the content hash see the final bytes.
+ * The text is unchanged (the very same decoding) unless this sync has an image session AND the switch
+ * is `describe_and_drop`; in `describe_and_keep` it always comes back untouched. Never throws: an
+ * image step that breaks hands back the original text and says so.
+ */
+export async function prepareTranscriptBody(collection, file) {
+    const session = collection.budget.images;
+    const agent = file.kind.startsWith("codex") ? "codex" : file.kind.startsWith("claude") ? "claude_code" : null;
+    const untouched = (volatile, pendingImages) => ({ held: holdDecodedText(file.source), volatile, pendingImages });
+    if (!session || !agent)
+        return untouched(false, null);
+    // Drop mode slims every transcript, image-bearing or not; keep mode only describes the ones that can hold an image.
+    if (session.mode !== "describe_and_drop" && !mightHoldEmbeddedImages(file.source.bytes))
+        return untouched(false, null);
+    let held;
+    try {
+        held = holdDecodedText(file.source);
+        const prepared = await session.prepareTranscript({ text: held.text, source: agent, sessionId: file.sessionId });
+        // BLI-3936: "volatile" = the next tick could build different bytes, or do image work, from
+        // this very file: an image still without a stored description (keep mode never replaces, so
+        // any image counts) or a slim step that failed and should be retried.
+        const imagesPending = session.mode === "describe_and_drop"
+            ? prepared.imageCount > prepared.replacedCount
+            : prepared.imageCount > 0;
+        const slimFailed = session.mode === "describe_and_drop" && prepared.slim === null;
+        const volatile = imagesPending || slimFailed;
+        // BLI-22794: volatile ONLY because of images without a stored description (the slim step held
+        // up): the bytes change only when one of these gains one, so the memo may keep the entry.
+        const pendingImages = volatile && !slimFailed ? prepared.pendingImages : null;
+        if (prepared.changed) {
+            // The old path encoded the new text and read it back; a text with no lone surrogate (slim output
+            // re-escapes them) is the same either way, so the new text is the exact decoding of the new bytes.
+            held.text = prepared.text;
+            held.exactBytes = null;
+        }
+        return { held, volatile, pendingImages };
+    }
+    catch (error) {
+        console.error("[raw-evidence] image or slim step failed, uploading the transcript as it was", JSON.stringify({ reason: "image_step_failed", kind: file.kind, ...describeError(error) }));
+        return { held: held ?? holdDecodedText(file.source), volatile: true, pendingImages: null };
+    }
+}

package/dist/adapters/raw-evidence-transcript-file.js

@@ -21,8 +21,8 @@ import path from "node:path";
 import { describeError } from "../health-detail.js";
 import { DELIVERY_BACKOFF_HOLDING_REASON, evidenceSourceKey, } from "../raw-evidence-staging.js";
 import { isSecretLikePath, sha256, shortHash, } from "./raw-evidence-keys.js";
-import { maskSecretsInTranscript, } from "./raw-evidence-sanitize.js";
-import { mightHoldEmbeddedImages } from "@bli-cockpit/telemetry-core";
+import { maskSecretsInHeldTranscript } from "./raw-evidence-sanitize-held.js";
+import { prepareTranscriptBody } from "./raw-evidence-transcript-body.js";
 import { markBudgetCapApplied, markCapApplied, } from "./raw-evidence-completeness.js";
 import { evidenceEntry } from "./raw-evidence-manifest.js";
 import { prepareRecipe, statSourceForMemo, } from "./raw-evidence-prepare-memo.js";
@@ -64,9 +64,9 @@ export async function collectOneEvidenceFile(collection, options) {
         markCapApplied(collection, options.kind, "max_file_bytes");
         return skip("file_too_large");
     }
-    const { sanitized, contentHash, stubClaim, volatile, pendingImages, redactedEntry } = await prepareTranscriptForUpload(collection, options, names, read.bytes);
+    const { sanitized, contentHash, stubClaim, volatile, pendingImages, redactedEntry } = await prepareTranscriptForUpload(collection, options, names, read.source);
     await rememberPreparedTranscript(collection, options, names, {
-        stamp, recipe, readBytes: read.bytes.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, pendingImages, redactedEntry,
+        stamp, recipe, readBytes: read.byteLength, contentHash, byteSize: sanitized.bytes.byteLength, volatile, pendingImages, redactedEntry,
     });
     if (collection.skipContentHashes.has(contentHash) && !stubClaim) {
         collection.reused.push({
@@ -95,17 +95,14 @@ function recordEvidenceSkip(collection, kind, label, reason) {
     collection.skipped.push({ kind, label, reason });
     return false;
 }
-async function prepareTranscriptForUpload(collection, options, names, bytes) {
+async function prepareTranscriptForUpload(collection, options, names, source) {
     // BLI-16977: describe each distinct embedded image once and, when the one switch says so, swap the
     // base64 for a short marker BEFORE masking, so the masker and the content hash see the final bytes.
-    const images = await prepareEmbeddedImages(collection, {
-        bytes,
-        kind: options.kind,
-        sessionId: options.sessionId,
-    });
+    // BLI-23317: the text is decoded once and travels in a holder the mask step clears, so the file's
+    // bytes, the decoded text and the masked text are never all alive together.
+    const body = await prepareTranscriptBody(collection, { source, kind: options.kind, sessionId: options.sessionId });
     const redactedBefore = collection.redacted.length;
-    const sanitized = maskSecretsInTranscript(collection, {
-        raw: images.bytes,
+    const sanitized = maskSecretsInHeldTranscript(collection, body.held, {
         kind: options.kind,
         evidenceLabel: names.evidenceLabel,
         secretLikeFileName: names.secretLikeFileName,
@@ -122,47 +119,7 @@ async function prepareTranscriptForUpload(collection, options, names, bytes) {
         byteSize: sanitized.bytes.byteLength,
         alreadyDurable: collection.skipContentHashes.has(contentHash),
     });
-    return { sanitized, contentHash, stubClaim, volatile: images.volatile, pendingImages: images.pendingImages, redactedEntry };
-}
-/**
- * The transcript bytes after the image step. Unchanged (the very same buffer) unless this sync has
- * an image session AND the switch is `describe_and_drop` (images become markers where a description
- * is stored, then the slim rules run, see src/slim/). In `describe_and_keep` the bytes always come
- * back untouched. Never throws: an image step that breaks hands back the original bytes and says so.
- */
-async function prepareEmbeddedImages(collection, file) {
-    const session = collection.budget.images;
-    const source = file.kind.startsWith("codex") ? "codex" : file.kind.startsWith("claude") ? "claude_code" : null;
-    if (!session || !source)
-        return { bytes: file.bytes, volatile: false, pendingImages: null };
-    // Drop mode slims every transcript, image-bearing or not; keep mode only describes the ones that can hold an image.
-    if (session.mode !== "describe_and_drop" && !mightHoldEmbeddedImages(file.bytes))
-        return { bytes: file.bytes, volatile: false, pendingImages: null };
-    try {
-        const prepared = await session.prepareTranscript({
-            text: file.bytes.toString("utf8"),
-            source,
-            sessionId: file.sessionId,
-        });
-        // BLI-3936: "volatile" = the next tick could build different bytes, or do image work, from
-        // this very file: an image still without a stored description (keep mode never replaces, so
-        // any image counts) or a slim step that failed and should be retried.
-        const imagesPending = session.mode === "describe_and_drop"
-            ? prepared.imageCount > prepared.replacedCount
-            : prepared.imageCount > 0;
-        const slimFailed = session.mode === "describe_and_drop" && prepared.slim === null;
-        const volatile = imagesPending || slimFailed;
-        // BLI-22794: volatile ONLY because of images without a stored description (the slim step held
-        // up): the bytes change only when one of these gains one, so the memo may keep the entry.
-        const pendingImages = volatile && !slimFailed ? prepared.pendingImages : null;
-        if (!prepared.changed)
-            return { bytes: file.bytes, volatile, pendingImages };
-        return { bytes: Buffer.from(prepared.text, "utf8"), volatile, pendingImages };
-    }
-    catch (error) {
-        console.error("[raw-evidence] image or slim step failed, uploading the transcript as it was", JSON.stringify({ reason: "image_step_failed", kind: file.kind, ...describeError(error) }));
-        return { bytes: file.bytes, volatile: true, pendingImages: null };
-    }
+    return { sanitized, contentHash, stubClaim, volatile: body.volatile, pendingImages: body.pendingImages, redactedEntry };
 }
 /**
  * BLI-3936: the answer for a transcript whose file has not changed since a tick that found its
@@ -348,5 +305,5 @@ async function readEvidenceFileWithinCap(filePath, maxFileBytes) {
     if (maxFileBytes && bytes.byteLength > maxFileBytes) {
         return { status: "too_large" };
     }
-    return { status: "ok", bytes };
+    return { status: "ok", source: { bytes }, byteLength: bytes.byteLength };
 }

package/dist/adapters/transcript-line-stream.js

@@ -0,0 +1,47 @@
+/**
+ * A file's lines, one at a time, without ever holding the whole file (BLI-23317).
+ *
+ * `fs.readFile(path, "utf8")` followed by `.split("\n")` keeps the file's bytes, its decoded string and
+ * the array of lines alive together; for a 200 MB transcript that is the biggest allocation the image
+ * reader makes. This yields exactly the lines `split("\n")` would (the last one, possibly empty, is the
+ * text after the final newline), each decoded as UTF-8 with the same replacement of invalid sequences:
+ * a "\n" byte never occurs inside a multi-byte UTF-8 sequence, so decoding line by line equals decoding
+ * the whole file and cutting it. A line is held as bytes until its newline arrives, so the peak is the
+ * longest line, not the file.
+ *
+ * A file that cannot be opened or read yields no lines, as `readFile(...).catch(() => null)` did, and
+ * says why (a read that fails part way keeps the lines already yielded; the log names it).
+ */
+import fs from "node:fs";
+import { describeError } from "../health-detail.js";
+const NEWLINE = 0x0a;
+/** Read size per chunk. A long line spans many chunks; its parts wait as buffers until the line ends. */
+const CHUNK_BYTES = 1024 * 1024;
+export async function* linesOfFile(filePath) {
+    const stream = fs.createReadStream(filePath, { highWaterMark: CHUNK_BYTES });
+    let parts = [];
+    let lines = 0;
+    try {
+        for await (const chunk of stream) {
+            let start = 0;
+            for (let newline = chunk.indexOf(NEWLINE, start); newline !== -1; newline = chunk.indexOf(NEWLINE, start)) {
+                parts.push(chunk.subarray(start, newline));
+                const line = (parts.length === 1 ? parts[0] : Buffer.concat(parts)).toString("utf8");
+                parts = [];
+                start = newline + 1;
+                lines += 1;
+                yield line;
+            }
+            // A line that outlives its chunk must not pin the whole chunk, so its tail is copied.
+            if (start < chunk.length)
+                parts.push(Buffer.from(chunk.subarray(start)));
+        }
+        yield Buffer.concat(parts).toString("utf8");
+    }
+    catch (error) {
+        console.error("[transcript-line-stream] file could not be read; its lines stop here", JSON.stringify({ reason: "file_read_failed", lines_yielded: lines, ...describeError(error) }));
+    }
+    finally {
+        stream.destroy();
+    }
+}

package/dist/commands/public-root.js

@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
   }
 
   if (command === "--version" || command === "-V" || command === "version") {
-    writeLine(io?.stdout ?? process.stdout, "0.2.263");
+    writeLine(io?.stdout ?? process.stdout, "0.2.264");
     return 0;
   }
 

package/dist/image-description-session.js

@@ -5,10 +5,10 @@
  * Failures never fail the transcript. Logs contain metadata, never image bytes
  * or descriptions.
  */
-import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, base64Of, findEmbeddedImages, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
+import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, base64Of, findEmbeddedImages, formatImageMarker, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
 import { postImageToDescribeDoor } from "./image-describe-client.js";
 import { resolveImageHandlingMode } from "./image-mode-state.js";
-import { slimOrKeep, slimTranscript } from "./slim/slim-transcript.js";
+import { slimOrKeep, slimPassOrNull, slimTranscript } from "./slim/slim-transcript.js";
 import { imageStateKey, imageRetryDelayMs, readDescribedImageKeys, readFailedImages, writeImageDescriptionState, } from "./image-description-state.js";
 /** New images sent to the door in one sync (all worktrees together); the count never refills within a tick. */
 export const IMAGE_DESCRIBE_MAX_PER_RUN = 100;
@@ -39,10 +39,16 @@ export async function openImageDescriptions(options) {
         isDescribed: (sha256) => described.has(imageStateKey(sha256)),
         hasDescribeRoom: (sha256s) => describeRoomLeft(run, sha256s),
         async prepareTranscript(input) {
-            const afterImages = await describeAndReplace(run, input);
+            const { swap, ...afterImages } = await describeAndReplace(run, input);
             if (mode !== "describe_and_drop")
                 return { ...afterImages, text: input.text, changed: false, slim: null };
-            const slim = slimOrKeep(afterImages.text, input.source, { sessionId: input.sessionId });
+            // BLI-23317: the image swap and the slim rules are ONE pass over the lines (one joined string, no
+            // intermediate copy). A pass that cannot run falls back to the two-step order, which is the same bytes.
+            const pass = slimPassOrNull(input.text, input.source, { sessionId: input.sessionId }, swap);
+            if (pass)
+                return { ...afterImages, text: pass.text, changed: pass.changed, slim: pass.stats };
+            const replaced = replaceImagesWithMarkers(input.text, swap.images, swap.replace);
+            const slim = slimOrKeep(replaced.text, input.source, { sessionId: input.sessionId });
             return { ...afterImages, text: slim.text, changed: slim.text !== input.text, slim: slim.stats };
         },
     };
@@ -110,8 +116,9 @@ function rememberFailure(run, key, outcome) {
 async function describeAndReplace(run, input) {
     const { mode, described, options, orderedKeys, failedImages } = run;
     const images = findEmbeddedImages(input.text);
+    const swap = { images, replace: (image) => described.has(imageStateKey(image.sha256)) };
     if (images.length === 0)
-        return { text: input.text, pendingImages: [], imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
+        return { swap, pendingImages: [], imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
     const unique = new Map();
     for (const image of images)
         if (!unique.has(image.sha256))
@@ -126,9 +133,7 @@ async function describeAndReplace(run, input) {
     }
     const deferredOnly = counts.over_budget + counts.backing_off > 0
         && counts.described_now + counts.known + counts.failed + counts.too_large + counts.settled_earlier === 0;
-    const replaced = mode === "describe_and_drop"
-        ? replaceImagesWithMarkers(input.text, images, (image) => described.has(imageStateKey(image.sha256)))
-        : { text: input.text, replacedCount: 0, charsSaved: 0 };
+    const replaced = mode === "describe_and_drop" ? countImageSwaps(swap) : { replacedCount: 0, charsSaved: 0 };
     const pendingImages = [...unique.keys()].filter((sha256) => !described.has(imageStateKey(sha256)));
     console.error(TAG + " transcript prepared", JSON.stringify({
         mode,
@@ -143,7 +148,7 @@ async function describeAndReplace(run, input) {
         kept_without_description: mode === "describe_and_drop" ? images.length - replaced.replacedCount : 0,
     }));
     return {
-        text: replaced.text,
+        swap,
         pendingImages,
         imageCount: images.length,
         uniqueCount: unique.size,
@@ -152,6 +157,19 @@ async function describeAndReplace(run, input) {
         deferredOnly,
     };
 }
+/** What `replaceImagesWithMarkers` would report for this swap, without building the text it would make. */
+function countImageSwaps(swap) {
+    let replacedCount = 0;
+    let charsSaved = 0;
+    for (const image of swap.images) {
+        if (!swap.replace(image))
+            continue;
+        const marker = formatImageMarker({ sha256: image.sha256, mediaType: image.mediaType, byteSize: image.byteSize });
+        replacedCount += 1;
+        charsSaved += image.replaceEnd - image.replaceStart - marker.length;
+    }
+    return { replacedCount, charsSaved };
+}
 function selectImagesToDescribe(run, unique, counts) {
     const { described, settledThisRun, failedImages, now } = run;
     const toSend = [];

package/dist/slim/slim-transcript.js

@@ -22,6 +22,7 @@
  * What each rule keeps and drops: claude-rules.ts and codex-rules.ts. The table for people:
  * docs/architecture/image-descriptions.md.
  */
+import { replaceImagesWithMarkers } from "@bli-cockpit/telemetry-core";
 import { newClaudeSlimState, slimClaudeRecord } from "./claude-rules.js";
 import { newCodexSlimState, slimCodexRecord } from "./codex-rules.js";
 import { LineEdits, SeenBlocks, isJsonObject } from "./slim-support.js";
@@ -103,17 +104,89 @@ function slimOneLine(line, applyRules, stats) {
     stats.linesModified += 1;
     return JSON.stringify(record) + (line.endsWith("\r") ? "\r" : "");
 }
-/** The slim form of a whole transcript's text, line for line. */
-export function slimTranscript(text, kind) {
+/**
+ * The slim form of a whole transcript's text, line for line, with the image swap folded into the
+ * same pass (BLI-23317). Output is byte-identical to `replaceImagesWithMarkers` followed by the old
+ * split/map/join: lines are found by `indexOf`, never `split`; a line no rule touched stays part of
+ * a slice of the input and is not copied on its own, so the only new allocation is the one joined
+ * result (and only when some line changed). An image sits inside one line (its characters exclude
+ * "\n"), so it is swapped inside that line before the rules read it, as the two-step order did.
+ */
+export function slimPass(text, kind, swap) {
     const slimmer = createSlimmer(kind);
-    const lines = text.split("\n");
-    const out = new Array(lines.length);
-    for (let index = 0; index < lines.length; index += 1)
-        out[index] = slimmer.slimLine(lines[index]);
+    const pieces = [];
+    let emitted = 0;
+    let nextImage = 0;
+    let replacedCount = 0;
+    let charsSaved = 0;
+    for (let position = 0;;) {
+        const newline = text.indexOf("\n", position);
+        const end = newline === -1 ? text.length : newline;
+        let line = text.slice(position, end);
+        let swapped = false;
+        if (swap) {
+            const local = [];
+            while (nextImage < swap.images.length && swap.images[nextImage].replaceStart < end) {
+                const image = swap.images[nextImage];
+                nextImage += 1;
+                if (image.replaceStart < position || image.replaceEnd > end)
+                    throw new Error("image_spans_lines");
+                local.push({
+                    ...image,
+                    replaceStart: image.replaceStart - position,
+                    replaceEnd: image.replaceEnd - position,
+                    base64Start: image.base64Start - position,
+                    base64End: image.base64End - position,
+                });
+            }
+            if (local.length > 0) {
+                const replaced = replaceImagesWithMarkers(line, local, swap.replace);
+                if (replaced.replacedCount > 0) {
+                    line = replaced.text;
+                    swapped = true;
+                    replacedCount += replaced.replacedCount;
+                    charsSaved += replaced.charsSaved;
+                }
+            }
+        }
+        const out = slimmer.slimLine(line);
+        if (swapped || out !== line) {
+            pieces.push(text.slice(emitted, position), out);
+            emitted = end;
+        }
+        if (newline === -1)
+            break;
+        position = newline + 1;
+    }
     const stats = slimmer.stats();
-    return { text: stats.linesModified === 0 ? text : out.join("\n"), stats };
+    if (pieces.length === 0)
+        return { text, stats, replacedCount, charsSaved, changed: false };
+    pieces.push(text.slice(emitted));
+    return { text: pieces.join(""), stats, replacedCount, charsSaved, changed: true };
+}
+/** The slim form of a whole transcript's text, line for line. */
+export function slimTranscript(text, kind) {
+    const slim = slimPass(text, kind);
+    return { text: slim.text, stats: slim.stats };
 }
 const TAG = "[slim-transcript]";
+function logSlimmed(kind, sessionId, stats, startedAt) {
+    console.error(TAG + (stats.linesModified === 0 ? " nothing to strip" : " transcript slimmed"), JSON.stringify({
+        source: kind,
+        session_id: sessionId.slice(0, 80),
+        lines: stats.lines,
+        lines_modified: stats.linesModified,
+        lines_failed: stats.linesFailed,
+        bytes_before: stats.bytesBefore,
+        bytes_after: stats.bytesAfter,
+        chars_by_rule: stats.charsByRule,
+        lines_by_rule: stats.linesByRule,
+        elapsed_ms: Date.now() - startedAt,
+    }));
+}
+function logSlimFailed(kind, error, bytes) {
+    console.error(TAG + " failed, uploading the transcript as it was", JSON.stringify({ reason: "slim_failed", source: kind, error_name: error instanceof Error ? error.name : typeof error, bytes }));
+}
 /**
  * `slimTranscript` that can never fail a transcript: any exception returns the text it was given and
  * logs the reason. Metadata only: counts, sizes, rule names. Never a line, never a field value.
@@ -122,23 +195,27 @@ export function slimOrKeep(text, kind, label) {
     const startedAt = Date.now();
     try {
         const slim = slimTranscript(text, kind);
-        const stats = slim.stats;
-        console.error(TAG + (stats.linesModified === 0 ? " nothing to strip" : " transcript slimmed"), JSON.stringify({
-            source: kind,
-            session_id: label.sessionId.slice(0, 80),
-            lines: stats.lines,
-            lines_modified: stats.linesModified,
-            lines_failed: stats.linesFailed,
-            bytes_before: stats.bytesBefore,
-            bytes_after: stats.bytesAfter,
-            chars_by_rule: stats.charsByRule,
-            lines_by_rule: stats.linesByRule,
-            elapsed_ms: Date.now() - startedAt,
-        }));
-        return { text: slim.text, stats };
+        logSlimmed(kind, label.sessionId, slim.stats, startedAt);
+        return { text: slim.text, stats: slim.stats };
     }
     catch (error) {
-        console.error(TAG + " failed, uploading the transcript as it was", JSON.stringify({ reason: "slim_failed", source: kind, error_name: error instanceof Error ? error.name : typeof error, bytes: typeof text === "string" ? text.length : 0 }));
+        logSlimFailed(kind, error, typeof text === "string" ? text.length : 0);
         return { text, stats: null };
     }
+}
+/**
+ * `slimPass` that can never fail a transcript: a pass that throws (an image that would span lines,
+ * an exhausted string) logs the reason and returns null, and the caller takes the two-step path.
+ */
+export function slimPassOrNull(text, kind, label, swap) {
+    const startedAt = Date.now();
+    try {
+        const slim = slimPass(text, kind, swap);
+        logSlimmed(kind, label.sessionId, slim.stats, startedAt);
+        return slim;
+    }
+    catch (error) {
+        logSlimFailed(kind, error, text.length);
+        return null;
+    }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.263",
+  "version": "0.2.264",
   "private": false,
   "type": "module",
   "bin": {