@bli-cockpit/cli 0.2.266 → 0.2.268

package/dist/adapters/raw-evidence-transcript-stage.js

@@ -0,0 +1,84 @@
+/**
+ * Put an accepted transcript's bytes on disk and add its manifest entry (split out of
+ * `raw-evidence-transcript-file.ts`, BLI-23317 stage D).
+ *
+ * The mask step's answer comes in two shapes (`MaskedBody`): bytes in memory, or (a clean streamed
+ * transcript) a file already on disk and hashed. Both publish the same object under the same name with
+ * the same manifest entry; the second never holds the bytes.
+ */
+import path from "node:path";
+import { shortHash } from "./raw-evidence-keys.js";
+import { evidenceEntry } from "./raw-evidence-manifest.js";
+import { describeSnapshot, describeStagedSnapshot, supersedeClaimFor, } from "./raw-evidence-snapshot-supersede.js";
+import { stageEvidenceBytes } from "./raw-evidence-collection-state.js";
+import { stageEvidenceFile } from "./raw-evidence-stage-file.js";
+import { discardStagedOutput } from "./staged-output.js";
+export function byteLengthOf(body) {
+    return "staged" in body ? body.staged.byteLength : body.bytes.byteLength;
+}
+/** A staged partial that will not be published (content already durable, budget deferral) is deleted, never left behind. */
+export async function dropStagedBody(body, reason) {
+    if ("staged" in body)
+        await discardStagedOutput(body.staged, reason);
+}
+/** Put an accepted transcript's bytes on disk and add its manifest entry. */
+export async function stageOneTranscript(collection, file) {
+    collection.index.value += 1;
+    const relativePath = path.join("files", `${String(collection.index.value).padStart(3, "0")}-${shortHash(file.filePath)}-${file.names.packedFileName}`);
+    const prior = collection.priorSnapshots.get(file.names.sourceKey);
+    // BLI-11088: remember this snapshot's shape, and claim the last one when these bytes rebuild it. A
+    // streamed clean file (BLI-23317 stage D) is read once from its partial, before it is published.
+    let content;
+    let snapshot;
+    let supersedes;
+    let staged;
+    if ("staged" in file.sanitized) {
+        const partial = file.sanitized.staged;
+        content = { sha256: partial.sha256, byteLength: partial.byteLength };
+        try {
+            const described = await describeStagedSnapshot({
+                filePath: partial.path, byteLength: partial.byteLength, contentHash: file.contentHash, kind: file.kind, prior, claim: file.stubClaim === null,
+            });
+            snapshot = described.snapshot;
+            supersedes = file.stubClaim ?? described.supersedes;
+        }
+        catch (error) {
+            await discardStagedOutput(partial, "snapshot_read_failed");
+            throw error;
+        }
+        staged = await stageEvidenceFile(collection, { staged: partial, fileName: path.basename(relativePath), kind: file.kind });
+    }
+    else {
+        content = file.sanitized.bytes;
+        snapshot = describeSnapshot(file.sanitized.bytes);
+        supersedes = file.stubClaim ?? supersedeClaimFor({ bytes: file.sanitized.bytes, contentHash: file.contentHash, kind: file.kind, prior });
+        staged = await stageEvidenceBytes(collection, {
+            contentHash: file.contentHash,
+            bytes: file.sanitized.bytes,
+            fileName: path.basename(relativePath),
+            kind: file.kind,
+            sourceKey: file.names.sourceKey,
+        });
+    }
+    collection.entries.push(evidenceEntry({
+        context: collection.context,
+        kind: file.kind,
+        packId: collection.packId,
+        localPath: staged.local_path,
+        relativePath,
+        mediaType: file.mediaType,
+        redactedSummary: file.sanitized.status === "redacted"
+            ? `${file.redactedSummary} Secret-like values were deterministically redacted before upload.`
+            : file.redactedSummary,
+        // Both branches carry a record now (BLI-3277), so "was anything
+        // replaced?" is the status, never the presence of `redaction`.
+        redaction: file.sanitized.redaction,
+        bytes: content,
+        codexSessionId: file.sessionId,
+        contentAddress: file.contentAddress(file.contentHash.slice(0, 16)),
+        stagedInPack: staged.staged_in_pack,
+        sourceKey: file.names.sourceKey,
+        snapshot,
+        supersedes,
+    }));
+}

package/dist/adapters/raw-evidence-transcript-stream.js

@@ -35,10 +35,10 @@ export async function streamTranscriptBody(session, agent, file) {
         return null;
     }
     try {
-        const prepared = await session.prepareTranscriptFile({ filePath: file.filePath, source: agent, sessionId: file.sessionId, maxFileBytes: file.maxFileBytes });
+        const prepared = await session.prepareTranscriptFile({ filePath: file.filePath, source: agent, sessionId: file.sessionId, maxFileBytes: file.maxFileBytes, stageDir: file.stageDir });
         const { volatile, pendingImages } = volatilityOf(session.mode, prepared);
-        console.error(TAG + " transcript streamed through image swap and slim", JSON.stringify({ kind: file.kind, byte_size: prepared.readBytes, changed: prepared.changed, images: prepared.imageCount, volatile }));
-        return { held: { text: prepared.text, exactBytes: null }, volatile, pendingImages, readBytes: prepared.readBytes };
+        console.error(TAG + " transcript streamed through image swap and slim", JSON.stringify({ kind: file.kind, byte_size: prepared.readBytes, changed: prepared.changed, images: prepared.imageCount, volatile, staged_bytes: prepared.staged?.byteLength ?? null }));
+        return { held: { text: prepared.text, exactBytes: null, ...(prepared.staged ? { staged: prepared.staged } : {}) }, volatile, pendingImages, readBytes: prepared.readBytes };
     }
     catch (error) {
         if (error instanceof TranscriptReadError || error instanceof TranscriptTooLargeError)

package/dist/adapters/staged-output.js

@@ -0,0 +1,128 @@
+/**
+ * The slimmed transcript, written to staging as it is made (BLI-23317, stage D).
+ *
+ * Before this, the streamed pass wrote every slimmed line into one buffer sized to the INPUT, decoded
+ * it to the string the sanitizer takes, and the clean outcome then encoded that string AGAIN into the
+ * bytes it staged. Now each slimmed line is encoded once, hashed and appended to a `.partial` file in
+ * the pack's `files` directory as it arrives. When the sanitizer decides the file is clean, those
+ * bytes ARE the upload: the partial is renamed into place (or dropped when the content is already
+ * durable or the budget defers it), and nothing re-encodes or re-hashes the file. Any other outcome
+ * (a secret was masked) discards the partial and builds its bytes from the sanitizer's text, as before.
+ *
+ * Bytes written are exactly `Buffer.from(text, "utf8")` of the text the sanitizer reads: slim output
+ * re-escapes lone surrogates, so encoding never changes a character. The sha256 is over the bytes in
+ * file order, and `byteLength` is their count.
+ */
+import { createHash, randomBytes } from "node:crypto";
+import fsp from "node:fs/promises";
+import path from "node:path";
+import { describeError } from "../health-detail.js";
+const TAG = "[staged-output]";
+/** Bytes gathered before a write: big enough to keep writes few, small enough to stay a rounding error. */
+const FLUSH_BYTES = 1024 * 1024;
+export async function openStagedOutput(directory) {
+    const file = path.join(directory, `.stream-${process.pid}-${randomBytes(6).toString("hex")}.partial`);
+    let handle;
+    try {
+        await fsp.mkdir(directory, { recursive: true });
+        handle = await fsp.open(file, "wx", 0o600);
+    }
+    catch (error) {
+        console.error(TAG + " staging file could not be created, nothing was written", JSON.stringify({ reason: "staging_write_failed", stage: "open", ...describeError(error) }));
+        throw error;
+    }
+    const hash = createHash("sha256");
+    let pieces = [];
+    let pending = 0;
+    let byteLength = 0;
+    let open = true;
+    async function flush() {
+        if (pending === 0)
+            return;
+        const bytes = pieces.length === 1 ? pieces[0] : Buffer.concat(pieces, pending);
+        pieces = [];
+        pending = 0;
+        hash.update(bytes);
+        try {
+            for (let done = 0; done < bytes.length;) {
+                const { bytesWritten } = await handle.write(bytes, done, bytes.length - done);
+                done += bytesWritten;
+            }
+        }
+        catch (error) {
+            console.error(TAG + " staging write failed, the partial is dropped", JSON.stringify({ reason: "staging_write_failed", stage: "write", byte_size: byteLength, ...describeError(error) }));
+            await abandon();
+            throw error;
+        }
+    }
+    async function abandon() {
+        if (open) {
+            open = false;
+            await handle.close().catch(() => undefined);
+        }
+        await fsp.rm(file, { force: true }).catch(() => undefined);
+    }
+    return {
+        async write(text) {
+            if (text.length === 0)
+                return;
+            const bytes = Buffer.from(text, "utf8");
+            pieces.push(bytes);
+            pending += bytes.length;
+            byteLength += bytes.length;
+            if (pending >= FLUSH_BYTES)
+                await flush();
+        },
+        async finish() {
+            await flush();
+            open = false;
+            try {
+                await handle.close();
+            }
+            catch (error) {
+                console.error(TAG + " staging file did not close cleanly, the partial is dropped", JSON.stringify({ reason: "staging_write_failed", stage: "close", ...describeError(error) }));
+                await fsp.rm(file, { force: true }).catch(() => undefined);
+                throw error;
+            }
+            return { path: file, sha256: hash.digest("hex"), byteLength };
+        },
+        abandon,
+    };
+}
+/**
+ * The text of a finished staged file, for the sanitizer, which reads the whole text: the file is read
+ * into one buffer and decoded once. Both exist together for that moment (as the byte buffer and the text
+ * did before), and the buffer is detached and freed on the spot instead of waiting for a collection.
+ * `Buffer.toString("utf8")` keeps a BOM as a character, the same decoding the in-memory sink gives.
+ */
+export async function readStagedText(staged) {
+    if (staged.byteLength === 0)
+        return "";
+    const handle = await fsp.open(staged.path, "r");
+    try {
+        // Never the shared pool: the backing store may be detached below.
+        const bytes = Buffer.allocUnsafeSlow(staged.byteLength);
+        for (let done = 0; done < bytes.length;) {
+            const { bytesRead } = await handle.read(bytes, done, bytes.length - done, done);
+            if (bytesRead === 0)
+                throw new Error("staged_file_shorter_than_written");
+            done += bytesRead;
+        }
+        const text = bytes.toString("utf8");
+        bytes.buffer.transfer?.(0);
+        return text;
+    }
+    finally {
+        await handle.close().catch(() => undefined);
+    }
+}
+/** Delete a staged partial that will not be published. A file already gone is fine; anything else is logged. */
+export async function discardStagedOutput(staged, reason) {
+    try {
+        await fsp.rm(staged.path, { force: true });
+        console.error(TAG + " staged partial discarded", JSON.stringify({ reason, byte_size: staged.byteLength }));
+    }
+    catch (error) {
+        console.error(TAG + " staged partial could not be deleted", JSON.stringify({ reason: "staged_partial_delete_failed", why: reason, byte_size: staged.byteLength, ...describeError(error) }));
+    }
+}

package/dist/adapters/transcript-text-stream.js

@@ -1,26 +1,16 @@
 /**
- * A transcript's lines as text, straight from disk, for the pass that never holds the whole file (BLI-23317).
+ * The named endings of reading a transcript as a stream (BLI-23317).
  *
- * Unlike `transcript-line-stream.ts` (the image reader's, which logs a failed read and ends the lines
- * early), this one tells the two endings apart. A read that fails is a `TranscriptReadError`, never a
- * shorter file: the caller maps it to the same `file_read_failed` skip a whole-file read failure gets.
- * Text that is not valid UTF-8 is a `TranscriptNotStreamable`: the streamed path replaces nothing, so
- * the caller falls back to the whole-file path, which keeps the original bytes for the clean branch.
+ * The reader itself is `pinned-transcript.ts` (stage D: one descriptor, a pinned length, byte lines).
+ * What lives here is the vocabulary the callers map to skips and fallbacks:
  *
- * One `TextDecoder` (fatal, BOM kept, `stream: true`) lives across all chunks, so a character split
- * between two chunks is valid input, and `decode()` at EOF rejects a file that ends mid-character.
- * For valid UTF-8 it yields exactly the string `Buffer.toString("utf8")` makes, cut at "\n" like
- * `split("\n")`: the last line is whatever follows the final newline, empty or not.
- *
- * A line is held as pieces until its newline arrives, so the peak is the longest line (a single very
- * long line is still concatenated whole; that bound is not claimed to be small).
+ *  - `TranscriptReadError`: a read that failed is never a shorter file; the caller maps it to the same
+ *    `file_read_failed` skip a whole-file read failure gets;
+ *  - `TranscriptTooLargeError`: the file is over this pass's cap (`file_too_large`);
+ *  - `TranscriptNotStreamable`: the streamed path cannot give the whole-file answer for this file (bytes
+ *    that are not valid UTF-8, a file that changed between the passes, an image step that broke), so the
+ *    caller reads the file whole, which keeps the original bytes for the clean branch.
  */
-import fsp from "node:fs/promises";
-import { TextDecoder } from "node:util";
-import { describeError } from "../health-detail.js";
-/** Bytes read per step, into one reused buffer. */
-export const TRANSCRIPT_CHUNK_BYTES = 64 * 1024;
-const CHUNK_BYTES = TRANSCRIPT_CHUNK_BYTES;
 /** The file could not be read to its end. Not EOF: whoever catches it must not treat the lines so far as the file. */
 export class TranscriptReadError extends Error {
     stage;
@@ -49,76 +39,4 @@ export class TranscriptNotStreamable extends Error {
         this.name = "TranscriptNotStreamable";
         this.reason = reason;
     }
-}
-/**
- * The lines of `filePath`. `byteLimit` reads exactly that many bytes (the second pass reads the first
- * pass's count, so a file that grew between the passes is not half-seen) and fails with
- * `file_changed_during_pass` when fewer arrive. `stopAfter` is a ceiling only (the first pass, bounded
- * by the file cap): fewer is fine.
- */
-export function openTranscriptLines(filePath, options = {}) {
-    let bytes = 0;
-    const { byteLimit } = options;
-    const readTo = byteLimit ?? options.stopAfter;
-    async function* lines() {
-        if (readTo === 0) {
-            yield "";
-            return;
-        }
-        const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
-        let handle;
-        let held = [];
-        try {
-            handle = await fsp.open(filePath, "r");
-            // One buffer for the whole pass: the decoder copies out of it, so nothing waits on the collector to free a chunk.
-            const buffer = Buffer.allocUnsafe(CHUNK_BYTES);
-            for (let position = 0;;) {
-                const want = readTo === undefined ? CHUNK_BYTES : Math.min(CHUNK_BYTES, readTo - position);
-                if (want <= 0)
-                    break;
-                const { bytesRead } = await handle.read(buffer, 0, want, position);
-                if (bytesRead === 0)
-                    break;
-                position += bytesRead;
-                bytes += bytesRead;
-                const text = decodeOrThrow(decoder, buffer.subarray(0, bytesRead));
-                let start = 0;
-                for (let newline = text.indexOf("\n", start); newline !== -1; newline = text.indexOf("\n", start)) {
-                    held.push(text.slice(start, newline));
-                    const line = held.length === 1 ? held[0] : held.join("");
-                    held = [];
-                    start = newline + 1;
-                    yield line;
-                }
-                if (start < text.length)
-                    held.push(text.slice(start));
-            }
-            const tail = decodeOrThrow(decoder, undefined);
-            if (tail)
-                held.push(tail);
-            if (byteLimit !== undefined && bytes < byteLimit)
-                throw new TranscriptNotStreamable("file_changed_during_pass");
-            yield held.join("");
-        }
-        catch (error) {
-            if (error instanceof TranscriptNotStreamable)
-                throw error;
-            console.error("[transcript-text-stream] transcript could not be read to its end", JSON.stringify({ reason: "file_read_failed", stage: "stream", bytes_read: bytes, ...describeError(error) }));
-            throw new TranscriptReadError("stream", error);
-        }
-        finally {
-            await handle?.close().catch((error) => {
-                console.error("[transcript-text-stream] file handle did not close", JSON.stringify({ reason: "file_close_failed", ...describeError(error) }));
-            });
-        }
-    }
-    return { lines: lines(), bytesRead: () => bytes };
-}
-function decodeOrThrow(decoder, chunk) {
-    try {
-        return chunk ? decoder.decode(chunk, { stream: true }) : decoder.decode();
-    }
-    catch {
-        throw new TranscriptNotStreamable("invalid_utf8");
-    }
 }

package/dist/commands/public-root.js

@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
   }
 
   if (command === "--version" || command === "-V" || command === "version") {
-    writeLine(io?.stdout ?? process.stdout, "0.2.266");
+    writeLine(io?.stdout ?? process.stdout, "0.2.268");
     return 0;
   }
 

package/dist/image-byte-scan.js

@@ -0,0 +1,185 @@
+/**
+ * Find the embedded images of one transcript line without ever building the base64 as a JS string
+ * (BLI-23317, stage D).
+ *
+ * `findEmbeddedImages` (telemetry-core/transcript-images.ts) is the RULE and stays the test oracle: it
+ * feeds `evidenceRulesFingerprint`, so its regex and its results do not change. This module is the same
+ * rule run on the line's BYTES, which is what the streamed prepare path calls:
+ *
+ *  - a whole-string payload: a base64 run that starts right after a `"`, starts with one of the four
+ *    image starts (`iVBOR`, `/9j/`, `R0lGO`, `UklGR`), is at least 495 characters past that start
+ *    (so 499 for a JPEG, 500 for the others, exactly as the regex counts) and is closed by `"` or `\"`;
+ *  - a data URL: `data:image/<[a-z0-9.+-]+>;base64,` and then a run of at least 500 characters, no closing
+ *    quote needed. The whole URL is the replaced range.
+ *
+ * A match is accepted only when the decoded bytes sniff as PNG, JPEG, GIF or WebP; either way the scan
+ * resumes after the run, as `matchAll` does. All matched characters are ASCII, so a byte offset here is
+ * the character offset there on ASCII text, and the replaced ranges are the same everywhere else.
+ *
+ * Decoding is incremental: the run is decoded in blocks of 48 Ki characters (a multiple of 4, so each
+ * block decodes on its own) straight into a sha256, with the first block sniffed. At no point does a
+ * string of the run, or a Buffer of the whole decoded image, exist. The one exception is a run whose
+ * `=` characters are not plain trailing padding: Node's decoder rules for those are version specific,
+ * so such a run takes the whole-run decode the oracle takes (never seen in real transcripts).
+ *
+ * `byteSize` is the length Node's `Buffer.from(run, "base64")` gives, because every block, including a
+ * short last one, is decoded by that very function.
+ */
+import { createHash } from "node:crypto";
+import { IMAGE_MIN_BASE64_CHARS, sniffImageMediaType, } from "@bli-cockpit/telemetry-core";
+const QUOTE = 0x22;
+const BACKSLASH = 0x5c;
+const EQUALS = 0x3d;
+/** Characters per decoded block. A multiple of 4: each block is a whole number of base64 quanta. */
+const BLOCK_CHARS = 48 * 1024;
+const BASE64 = new Uint8Array(256);
+for (const range of ["AZ", "az", "09"]) {
+    for (let code = range.charCodeAt(0); code <= range.charCodeAt(1); code += 1)
+        BASE64[code] = 1;
+}
+for (const char of "+/=")
+    BASE64[char.charCodeAt(0)] = 1;
+const DATA_URL_LOWER = new Uint8Array(256);
+for (const range of ["az", "09"]) {
+    for (let code = range.charCodeAt(0); code <= range.charCodeAt(1); code += 1)
+        DATA_URL_LOWER[code] = 1;
+}
+for (const char of ".+-")
+    DATA_URL_LOWER[char.charCodeAt(0)] = 1;
+/** The four image starts of a whole-string payload, and the data URL anchor. */
+const STARTS = ["iVBOR", "/9j/", "R0lGO", "UklGR"].map((text) => Buffer.from(text, "latin1"));
+const DATA_URL = Buffer.from("data:image/", "latin1");
+const BASE64_TAG = Buffer.from(";base64,", "latin1");
+/** Is there any chance this line holds an image? Same anchors as the string check, on bytes. */
+export function bytesMayHoldImage(line) {
+    return line.includes(DATA_URL) || STARTS.some((start) => line.includes(start));
+}
+/** The end (exclusive) of the base64-alphabet run that starts at `from`. */
+function runEnd(line, from) {
+    let at = from;
+    while (at < line.length && BASE64[line[at]] === 1)
+        at += 1;
+    return at;
+}
+/**
+ * sha256, size and type of the image `line[from, to)` encodes, or null when it is not a PNG, JPEG, GIF
+ * or WebP. Blocks of the run are decoded one at a time.
+ */
+export function digestBase64Run(line, from, to) {
+    const length = to - from;
+    const inRun = line.subarray(from, to).indexOf(EQUALS);
+    const firstPad = inRun === -1 ? -1 : from + inRun;
+    const padCount = firstPad === -1 ? 0 : to - firstPad;
+    const plainPadding = padCount === 0
+        || (padCount <= 2 && length % 4 === 0 && onlyPaddingFrom(line, firstPad, to));
+    if (!plainPadding) {
+        const bytes = Buffer.from(line.toString("latin1", from, to), "base64");
+        const mediaType = sniffImageMediaType(bytes);
+        return mediaType ? { sha256: createHash("sha256").update(bytes).digest("hex"), mediaType, byteSize: bytes.byteLength } : null;
+    }
+    const hash = createHash("sha256");
+    let byteSize = 0;
+    let mediaType = null;
+    for (let at = from; at < to; at += BLOCK_CHARS) {
+        const block = Buffer.from(line.toString("latin1", at, Math.min(to, at + BLOCK_CHARS)), "base64");
+        if (at === from) {
+            mediaType = sniffImageMediaType(block);
+            if (!mediaType)
+                return null;
+        }
+        hash.update(block);
+        byteSize += block.byteLength;
+    }
+    return mediaType ? { sha256: hash.digest("hex"), mediaType, byteSize } : null;
+}
+/** True when everything from `firstPad` to `to` is `=` (padding at the very end of the run, nothing after it). */
+function onlyPaddingFrom(line, firstPad, to) {
+    for (let at = firstPad; at < to; at += 1)
+        if (line[at] !== EQUALS)
+            return false;
+    return true;
+}
+/** The sha256 of an image's decoded bytes, or null when `line[from, to)` does not decode to an image (used to re-check a payload before it is sent). */
+export function sha256OfBase64Run(line, from, to) {
+    return digestBase64Run(line, from, to)?.sha256 ?? null;
+}
+/**
+ * The next index at or after `from` where `anchor` sits right after a `"`, or -1. `current` is the answer
+ * of the last call: -2 not looked for yet, -1 none left, else still good while it is at or after `from`.
+ */
+function nextQuotedStart(line, anchor, from, current) {
+    if (current === -1 || current >= from)
+        return current;
+    let at = line.indexOf(anchor, from);
+    while (at !== -1 && !(at > 0 && line[at - 1] === QUOTE))
+        at = line.indexOf(anchor, at + 1);
+    return at;
+}
+/**
+ * Every embedded image in `line` (one physical line, no `\n` inside), in order, with byte offsets inside
+ * the line. Equal to `findEmbeddedImages` on the decoded text in everything but the unit of the offsets.
+ */
+export function scanImagesInLine(line) {
+    const found = [];
+    if (line.length < IMAGE_MIN_BASE64_CHARS - 1)
+        return found;
+    // For each whole-string start, the next quoted occurrence (-2: not looked for yet, -1: none left).
+    const nextStart = STARTS.map(() => -2);
+    let nextData = -2;
+    let position = 0;
+    while (position < line.length) {
+        let candidate = -1;
+        let kind = -1;
+        for (let index = 0; index < STARTS.length; index += 1) {
+            nextStart[index] = nextQuotedStart(line, STARTS[index], position, nextStart[index]);
+            const at = nextStart[index];
+            if (at !== -1 && (candidate === -1 || at < candidate)) {
+                candidate = at;
+                kind = index;
+            }
+        }
+        if (nextData !== -1) {
+            nextData = nextData >= position ? nextData : line.indexOf(DATA_URL, position);
+            if (nextData !== -1 && (candidate === -1 || nextData < candidate)) {
+                candidate = nextData;
+                kind = STARTS.length;
+            }
+        }
+        if (candidate === -1)
+            break;
+        if (kind < STARTS.length) {
+            const startLength = STARTS[kind].length;
+            const end = runEnd(line, candidate);
+            const closed = end < line.length && (line[end] === QUOTE || (line[end] === BACKSLASH && line[end + 1] === QUOTE));
+            if (!closed || end - candidate < startLength + (IMAGE_MIN_BASE64_CHARS - 5)) {
+                position = candidate + 1;
+                continue;
+            }
+            position = end;
+            const digest = digestBase64Run(line, candidate, end);
+            if (digest)
+                found.push({ replaceStart: candidate, replaceEnd: end, base64Start: candidate, base64End: end, ...digest });
+            continue;
+        }
+        // data:image/<type>;base64,<run of at least 500>
+        let cursor = candidate + DATA_URL.length;
+        const typeStart = cursor;
+        while (cursor < line.length && DATA_URL_LOWER[line[cursor]] === 1)
+            cursor += 1;
+        if (cursor === typeStart || line.compare(BASE64_TAG, 0, BASE64_TAG.length, cursor, cursor + BASE64_TAG.length) !== 0) {
+            position = candidate + 1;
+            continue;
+        }
+        const base64Start = cursor + BASE64_TAG.length;
+        const end = runEnd(line, base64Start);
+        if (end - base64Start < IMAGE_MIN_BASE64_CHARS) {
+            position = candidate + 1;
+            continue;
+        }
+        position = end;
+        const digest = digestBase64Run(line, base64Start, end);
+        if (digest)
+            found.push({ replaceStart: candidate, replaceEnd: end, base64Start, base64End: end, ...digest });
+    }
+    return found;
+}

package/dist/image-census-stream.js

@@ -1,36 +1,46 @@
 /**
- * Pass one of the streamed transcript: find every embedded image line by line and keep only what the
- * describe step needs, never the text (BLI-23317).
+ * Pass one of the streamed transcript: find every embedded image line by line, on bytes, and keep only
+ * WHERE it is and what it is, never its payload (BLI-23317, stage D).
  *
- * `findEmbeddedImages` over the whole text and over each line give the same images, because an image's
- * characters never include "\n" and the finder's quote and data-URL anchors sit inside one line
- * (`slimPass` already throws when one does not). So the census is the whole-text finder run per line,
- * and what outlives a line is small: one record per distinct image, a tally per image of how often it
- * occurs and how many characters swapping each occurrence saves, and the base64 of only those images
- * the describe step may send (a copy, so no line or chunk stays alive behind it).
+ * Each occurrence is an `EmbeddedImage` whose offsets are absolute byte offsets in the pinned file, so the
+ * swap pass needs no second scan and no second hash, and the describe step reads the one payload it is
+ * about to send straight from those offsets (`readPayload`) instead of a census holding every candidate's
+ * base64. What outlives a line is small: one record per OCCURRENCE (a few hundred bytes), a tally per
+ * distinct image of how often it occurs and how many characters swapping each occurrence saves, and the
+ * first occurrence of each distinct image.
+ *
+ * The finder is `image-byte-scan.ts`, the byte form of `findEmbeddedImages`; a line holds the same
+ * images either way because an image's characters never include "\n". Every line is also checked to be
+ * valid UTF-8, so a file the text path cannot decode is set aside here, before any image is sent.
  */
-import { findEmbeddedImages, formatImageMarker, } from "@bli-cockpit/telemetry-core";
-/** Could this line hold an embedded image? A superset of what the finder matches: its anchors are `base64` and the four image magics. */
-export function lineMayHoldImage(line) {
-    return line.includes("base64") || line.includes("iVBOR") || line.includes("/9j/") || line.includes("R0lGO") || line.includes("UklGR");
-}
-export async function censusOfLines(lines, keepBase64) {
-    const census = { total: 0, unique: new Map(), tally: new Map(), base64: new Map() };
-    for await (const line of lines) {
-        if (!lineMayHoldImage(line))
+import { isUtf8 } from "node:buffer";
+import { formatImageMarker } from "@bli-cockpit/telemetry-core";
+import { TranscriptNotStreamable } from "./adapters/transcript-text-stream.js";
+import { bytesMayHoldImage, scanImagesInLine } from "./image-byte-scan.js";
+export async function censusOfLines(lines) {
+    const census = { total: 0, occurrences: [], unique: new Map(), tally: new Map() };
+    for await (const { start, bytes } of lines) {
+        if (!isUtf8(bytes))
+            throw new TranscriptNotStreamable("invalid_utf8");
+        if (!bytesMayHoldImage(bytes))
             continue;
-        for (const image of findEmbeddedImages(line)) {
+        for (const found of scanImagesInLine(bytes)) {
+            const image = {
+                ...found,
+                replaceStart: found.replaceStart + start,
+                replaceEnd: found.replaceEnd + start,
+                base64Start: found.base64Start + start,
+                base64End: found.base64End + start,
+            };
             census.total += 1;
+            census.occurrences.push(image);
             const marker = formatImageMarker({ sha256: image.sha256, mediaType: image.mediaType, byteSize: image.byteSize });
             const tally = census.tally.get(image.sha256) ?? { count: 0, savedChars: 0 };
             tally.count += 1;
             tally.savedChars += image.replaceEnd - image.replaceStart - marker.length;
             census.tally.set(image.sha256, tally);
-            if (census.unique.has(image.sha256))
-                continue;
-            census.unique.set(image.sha256, image);
-            if (keepBase64(image))
-                census.base64.set(image.sha256, Buffer.from(line.slice(image.base64Start, image.base64End), "latin1"));
+            if (!census.unique.has(image.sha256))
+                census.unique.set(image.sha256, image);
         }
     }
     return census;