@bli-cockpit/cli 0.2.266 → 0.2.268

package/dist/image-description-send.js

@@ -0,0 +1,80 @@
+import { postImageToDescribeDoor } from "./image-describe-client.js";
+import { imageRetryDelayMs, imageStateKey } from "./image-description-state.js";
+const TAG = "[image-descriptions]";
+const CONCURRENCY = 3;
+function markDescribed(run, sha256) {
+    const { described, orderedKeys } = run;
+    const key = imageStateKey(sha256);
+    if (described.has(key))
+        return;
+    described.add(key);
+    orderedKeys.push(key);
+}
+async function sendOne(run, image, payload, input) {
+    const { options, fetchImpl, settledThisRun, failedImages } = run;
+    // Counted before the first await: the budget check that let this send through ran in the same tick.
+    run.sentThisRun += 1;
+    const base64 = await payload;
+    const outcome = await postImageToDescribeDoor({ fetchImpl, dashboardUrl: options.dashboardUrl, deviceToken: options.deviceToken, sleep: options.sleep }, {
+        schema_version: "image-describe.v1",
+        sha256: image.sha256,
+        media_type: image.mediaType,
+        byte_size: image.byteSize,
+        content_base64: base64,
+        source: input.source,
+        session_id: input.sessionId.slice(0, 200),
+    });
+    const key = imageStateKey(image.sha256);
+    if (outcome.ok) {
+        markDescribed(run, image.sha256);
+        settledThisRun.set(image.sha256, outcome.status);
+        if (failedImages[key]) {
+            delete failedImages[key];
+            run.stateDirty = true;
+        }
+        return;
+    }
+    settledThisRun.set(image.sha256, `failed:${outcome.reason}`);
+    rememberFailure(run, key, outcome);
+}
+/**
+ * Remember image-specific HTTP failures for backoff. Transport failures, 401
+ * and 429 say nothing about the image and must not be remembered.
+ */
+function rememberFailure(run, key, outcome) {
+    const { failedImages, now } = run;
+    if (outcome.httpStatus < 400 || outcome.httpStatus === 401 || outcome.httpStatus === 429)
+        return;
+    const attempts = (failedImages[key]?.attempts ?? 0) + 1;
+    const delay = imageRetryDelayMs(attempts, outcome.permanent);
+    failedImages[key] = { reason: outcome.reason, attempts, retry_after: now() + delay };
+    run.stateDirty = true;
+    console.error(TAG + " image not described, will not be re-sent until later", JSON.stringify({ reason: outcome.reason, http_status: outcome.httpStatus, attempts, retry_in_hours: Math.round(delay / 3_600_000), sha256_prefix: key.slice(0, 16) }));
+}
+export async function sendPendingImages(run, toSend, counts, input, base64For) {
+    const { now, maxPerRun, maxRunMs } = run;
+    const callStartedAt = now();
+    let next = 0;
+    let aborted = false;
+    const worker = async () => {
+        while (next < toSend.length && !aborted) {
+            const image = toSend[next++];
+            const timeUp = run.describingMs + (now() - callStartedAt) > maxRunMs;
+            if (run.sentThisRun >= maxPerRun || timeUp) {
+                counts.over_budget += 1;
+                continue;
+            }
+            try {
+                // The payload is read from the source only now, for this one image (BLI-23317).
+                await sendOne(run, image, base64For(image), input);
+            }
+            catch (error) {
+                // A source that changed under the passes ends the whole prepare: the other workers send nothing more.
+                aborted = true;
+                throw error;
+            }
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(CONCURRENCY, toSend.length) }, worker));
+    run.describingMs += now() - callStartedAt;
+}

package/dist/image-description-session.js

@@ -3,16 +3,20 @@
  * Describe each distinct image once. Keep images without stored descriptions;
  * only describe_and_drop replaces described images and then runs the slim rules.
  * Failures never fail the transcript. Logs contain metadata, never image bytes
- * or descriptions.
+ * or descriptions. The sends themselves (three at a time, budgets, backoff) are in
+ * `image-description-send.ts`.
  */
 import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, base64Of, findEmbeddedImages, formatImageMarker, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
-import { postImageToDescribeDoor } from "./image-describe-client.js";
+import { sendPendingImages } from "./image-description-send.js";
 import { resolveImageHandlingMode } from "./image-mode-state.js";
 import { censusOfLines } from "./image-census-stream.js";
-import { openTranscriptLines, TranscriptNotStreamable, TranscriptTooLargeError } from "./adapters/transcript-text-stream.js";
-import { slimLineStream } from "./slim/slim-stream.js";
+import { sha256OfBase64Run } from "./image-byte-scan.js";
+import { openPinnedTranscript } from "./adapters/pinned-transcript.js";
+import { openStagedOutput, readStagedText } from "./adapters/staged-output.js";
+import { TranscriptNotStreamable } from "./adapters/transcript-text-stream.js";
+import { LineBytesSink, slimLineStream } from "./slim/slim-stream.js";
 import { logSlimmed, slimOrKeep, slimPassOrNull, slimTranscript } from "./slim/slim-transcript.js";
-import { imageStateKey, imageRetryDelayMs, readDescribedImageKeys, readFailedImages, writeImageDescriptionState, } from "./image-description-state.js";
+import { imageStateKey, readDescribedImageKeys, readFailedImages, writeImageDescriptionState, } from "./image-description-state.js";
 /** New images sent to the door in one sync (all worktrees together); the count never refills within a tick. */
 export const IMAGE_DESCRIBE_MAX_PER_RUN = 100;
 /**
@@ -22,7 +26,6 @@ export const IMAGE_DESCRIBE_MAX_PER_RUN = 100;
  * spent the budget on that work, so almost nothing was described (BLI-22789: 1 of 1,874, then 10).
  */
 export const IMAGE_DESCRIBE_MAX_RUN_MS = 120_000;
-const CONCURRENCY = 3;
 const TAG = "[image-descriptions]";
 export async function openImageDescriptions(options) {
     const mode = options.mode ?? IMAGE_HANDLING_MODE;
@@ -76,15 +79,18 @@ function findsOfText(text) {
         swapped: (replace) => countImageSwaps({ images, replace }),
     };
 }
-function findsOfCensus(census) {
+function findsOfCensus(census, source) {
     return {
         total: census.total,
         unique: census.unique,
-        base64For: (image) => {
-            const kept = census.base64.get(image.sha256);
-            if (!kept)
-                throw new Error("image_base64_not_kept");
-            return kept.toString("latin1");
+        // The one payload about to be sent is read from its offsets in the pinned file and re-hashed: bytes that
+        // no longer decode to the image the census saw mean the file changed under us, and a request carrying
+        // them would be refused by the door and remembered as a failure of the IMAGE.
+        base64For: async (image) => {
+            const payload = await source.read(image.base64Start, image.base64End - image.base64Start);
+            if (sha256OfBase64Run(payload, 0, payload.length) !== image.sha256)
+                throw new TranscriptNotStreamable("file_changed_during_pass");
+            return payload.toString("latin1");
         },
         swapped: (replace) => {
             let replacedCount = 0;
@@ -102,33 +108,35 @@ function findsOfCensus(census) {
 }
 async function prepareFileInTwoPasses(run, input) {
     const startedAt = Date.now();
-    // Pass one: discover. Nothing of the text is kept; only the images and, for those that may still be sent, their base64.
-    const room = Math.max(0, run.maxPerRun - run.sentThisRun);
-    let wanted = 0;
-    const first = openTranscriptLines(input.filePath, { stopAfter: input.maxFileBytes ? input.maxFileBytes + 1 : undefined });
-    const census = await censusOfLines(first.lines, (image) => {
-        // The same filters `selectImagesToDescribe` applies, so what is kept is what may be sent.
-        const key = imageStateKey(image.sha256);
-        const failure = run.failedImages[key];
-        if (wanted >= room || run.described.has(key) || run.settledThisRun.has(image.sha256) || image.byteSize > IMAGE_DESCRIBE_MAX_BYTES)
-            return false;
-        if (failure && failure.retry_after > run.now())
-            return false;
-        wanted += 1;
-        return true;
-    });
-    const readBytes = first.bytesRead();
-    if (input.maxFileBytes && readBytes > input.maxFileBytes)
-        throw new TranscriptTooLargeError(input.maxFileBytes);
-    // Describe attempts finish and the swap decisions are settled before a single line is swapped, as with the whole text.
-    const { replace, finds: _finds, ...afterImages } = await describeAndReplace(run, input, findsOfCensus(census));
-    // Pass two: swap and slim over exactly the bytes pass one saw.
-    const second = openTranscriptLines(input.filePath, { byteLimit: readBytes });
-    const slim = await slimLineStream(second.lines, input.source, replace, readBytes);
-    if (slim.replacedCount !== afterImages.replacedCount)
-        throw new TranscriptNotStreamable("images_changed_between_passes");
-    logSlimmed(input.source, input.sessionId, slim.stats, startedAt);
-    return { ...afterImages, text: slim.text, changed: slim.changed, slim: slim.stats, readBytes };
+    // The file is opened once and its length pinned: the census offsets, the describe payload reads and the swap
+    // pass all read these bytes, and a source that stopped being them is set aside (`verifyUnchanged`).
+    const source = await openPinnedTranscript(input.filePath, { maxFileBytes: input.maxFileBytes });
+    let writer;
+    try {
+        // Pass one: discover. Nothing of the text is kept; only where each image is and what it is.
+        const census = await censusOfLines(source.lines());
+        // Describe attempts finish and the swap decisions are settled before a single line is swapped, as with the whole text.
+        const { replace, finds: _finds, ...afterImages } = await describeAndReplace(run, input, findsOfCensus(census, source));
+        // Pass two: swap and slim over exactly the bytes pass one saw, writing the result to staging as it goes.
+        writer = input.stageDir ? await openStagedOutput(input.stageDir) : undefined;
+        const memory = writer ? undefined : new LineBytesSink(source.length);
+        const slim = await slimLineStream(source.lines(), input.source, census.occurrences, replace, writer ?? memory);
+        if (slim.replacedCount !== afterImages.replacedCount)
+            throw new TranscriptNotStreamable("images_changed_between_passes");
+        await source.verifyUnchanged();
+        const staged = await writer?.finish();
+        // The sanitizer reads the whole text: from the staged file when one was written, else from the sink's buffer.
+        const text = staged ? await readStagedText(staged) : memory.text();
+        logSlimmed(input.source, input.sessionId, slim.stats, startedAt);
+        return { ...afterImages, text, changed: slim.changed, slim: slim.stats, readBytes: source.length, ...(staged ? { staged } : {}) };
+    }
+    catch (error) {
+        await writer?.abandon();
+        throw error;
+    }
+    finally {
+        await source.close();
+    }
 }
 function describeRoomLeft(run, sha256s) {
     if (run.sentThisRun >= run.maxPerRun)
@@ -143,53 +151,6 @@ function describeRoomLeft(run, sha256s) {
         return !(failure && failure.retry_after > run.now());
     });
 }
-function markDescribed(run, sha256) {
-    const { described, orderedKeys } = run;
-    const key = imageStateKey(sha256);
-    if (described.has(key))
-        return;
-    described.add(key);
-    orderedKeys.push(key);
-}
-async function sendOne(run, image, base64, input) {
-    const { options, fetchImpl, settledThisRun, failedImages } = run;
-    run.sentThisRun += 1;
-    const outcome = await postImageToDescribeDoor({ fetchImpl, dashboardUrl: options.dashboardUrl, deviceToken: options.deviceToken, sleep: options.sleep }, {
-        schema_version: "image-describe.v1",
-        sha256: image.sha256,
-        media_type: image.mediaType,
-        byte_size: image.byteSize,
-        content_base64: base64,
-        source: input.source,
-        session_id: input.sessionId.slice(0, 200),
-    });
-    const key = imageStateKey(image.sha256);
-    if (outcome.ok) {
-        markDescribed(run, image.sha256);
-        settledThisRun.set(image.sha256, outcome.status);
-        if (failedImages[key]) {
-            delete failedImages[key];
-            run.stateDirty = true;
-        }
-        return;
-    }
-    settledThisRun.set(image.sha256, `failed:${outcome.reason}`);
-    rememberFailure(run, key, outcome);
-}
-/**
- * Remember image-specific HTTP failures for backoff. Transport failures, 401
- * and 429 say nothing about the image and must not be remembered.
- */
-function rememberFailure(run, key, outcome) {
-    const { failedImages, now } = run;
-    if (outcome.httpStatus < 400 || outcome.httpStatus === 401 || outcome.httpStatus === 429)
-        return;
-    const attempts = (failedImages[key]?.attempts ?? 0) + 1;
-    const delay = imageRetryDelayMs(attempts, outcome.permanent);
-    failedImages[key] = { reason: outcome.reason, attempts, retry_after: now() + delay };
-    run.stateDirty = true;
-    console.error(TAG + " image not described, will not be re-sent until later", JSON.stringify({ reason: outcome.reason, http_status: outcome.httpStatus, attempts, retry_in_hours: Math.round(delay / 3_600_000), sha256_prefix: key.slice(0, 16) }));
-}
 async function describeAndReplace(run, input, finds) {
     const { mode, described, options, orderedKeys, failedImages } = run;
     const replace = (image) => described.has(imageStateKey(image.sha256));
@@ -276,24 +237,6 @@ function selectImagesToDescribe(run, unique, counts) {
     }
     return toSend;
 }
-async function sendPendingImages(run, toSend, counts, input, base64For) {
-    const { now, maxPerRun, maxRunMs } = run;
-    const callStartedAt = now();
-    let next = 0;
-    const worker = async () => {
-        while (next < toSend.length) {
-            const image = toSend[next++];
-            const timeUp = run.describingMs + (now() - callStartedAt) > maxRunMs;
-            if (run.sentThisRun >= maxPerRun || timeUp) {
-                counts.over_budget += 1;
-                continue;
-            }
-            await sendOne(run, image, base64For(image), input);
-        }
-    };
-    await Promise.all(Array.from({ length: Math.min(CONCURRENCY, toSend.length) }, worker));
-    run.describingMs += now() - callStartedAt;
-}
 function countImageOutcomes(run, toSend, counts) {
     const { settledThisRun } = run;
     const failedReasons = {};

package/dist/image-description-state.js

@@ -2,87 +2,32 @@
  * Which images this machine has already sent to the describe door (BLI-16977).
  *
  * One small file in the collector state directory. It is how "each distinct image is sent once"
- * survives from one 15-minute sync to the next. A key is the first 32 hex characters of the
- * image's sha256 (128 bits: no two images on one laptop will ever share one); the file holds keys
- * only, never a description, never a path, never a byte. If it is lost or corrupt the worst case
- * is the images are sent again and the server answers "known" without calling a model.
+ * survives from one 15-minute sync to the next. The file holds keys only, never a description, never
+ * a path, never a byte. If it is lost or corrupt the worst case is the images are sent again and the
+ * server answers "known" without calling a model.
+ *
+ * BLI-23328: the BLI Memory daemon writes the same file when it describes an image at turn end, so
+ * the format, the keys, the backoff steps and the locked read-merge-write live in telemetry-core
+ * (`image-description-store.ts`), shared by both writers. This module keeps the collector's names.
  */
-import path from "node:path";
-import { isMissingFileFailure } from "./health-detail.js";
-import { readJsonFile, writeJsonFileAtomic } from "./local-state-files.js";
-export const IMAGE_DESCRIPTION_STATE_FILENAME = "image-descriptions.json";
-/** Oldest keys leave first past this many (about 5 MB on disk). */
-export const IMAGE_DESCRIPTION_STATE_MAX_KEYS = 150_000;
-const KEY_CHARS = 32;
-export function imageStateKey(sha256) {
-    return sha256.slice(0, KEY_CHARS);
-}
-export function imageDescriptionStatePath(stateDir) {
-    return path.join(stateDir, IMAGE_DESCRIPTION_STATE_FILENAME);
-}
+import { IMAGE_DESCRIPTION_STATE_FILENAME, IMAGE_DESCRIPTION_STATE_MAX_KEYS, IMAGE_FAILED_STATE_MAX_KEYS, imageDescriptionStatePath, imageRetryDelayMs, imageStateKey, mergeImageDescriptionStates, readImageDescriptionState, updateImageDescriptionState, } from "@bli-cockpit/telemetry-core";
+export { IMAGE_DESCRIPTION_STATE_FILENAME, IMAGE_DESCRIPTION_STATE_MAX_KEYS, IMAGE_FAILED_STATE_MAX_KEYS, imageDescriptionStatePath, imageRetryDelayMs, imageStateKey, };
 /** The keys already described, oldest first. A missing or unreadable file is an empty list, said out loud when unreadable. */
 export async function readDescribedImageKeys(stateDir) {
-    try {
-        const raw = (await readJsonFile(imageDescriptionStatePath(stateDir)));
-        if (!Array.isArray(raw?.described))
-            return [];
-        return raw.described.filter((key) => typeof key === "string" && key.length === KEY_CHARS);
-    }
-    catch (error) {
-        if (!isMissingFileFailure(error)) {
-            console.error("[image-descriptions] state file unreadable, starting from an empty list", JSON.stringify({ reason: "image_state_unreadable" }));
-        }
-        return [];
-    }
-}
-/** Oldest failures leave first past this many. */
-export const IMAGE_FAILED_STATE_MAX_KEYS = 5_000;
-const HOUR_MS = 3_600_000;
-/** A refusal that repeats for the same bytes (a 4xx): a week, then a month. */
-const PERMANENT_BACKOFF_MS = [7 * 24 * HOUR_MS, 30 * 24 * HOUR_MS];
-/** A failure that may pass (the door or the model was down): 1 h, 6 h, 24 h, then 72 h. */
-const TRANSIENT_BACKOFF_MS = [HOUR_MS, 6 * HOUR_MS, 24 * HOUR_MS, 72 * HOUR_MS];
-/** How long to leave an image alone after its Nth failed attempt. */
-export function imageRetryDelayMs(attempts, permanent) {
-    const steps = permanent ? PERMANENT_BACKOFF_MS : TRANSIENT_BACKOFF_MS;
-    return steps[Math.min(Math.max(attempts, 1), steps.length) - 1];
+    return (await readImageDescriptionState(stateDir)).described;
 }
 /** The failures on record, key to record. A missing or unreadable file is an empty map, said out loud when unreadable. */
 export async function readFailedImages(stateDir) {
-    try {
-        const raw = (await readJsonFile(imageDescriptionStatePath(stateDir)));
-        if (!raw?.failed || typeof raw.failed !== "object" || Array.isArray(raw.failed))
-            return {};
-        const kept = {};
-        for (const [key, value] of Object.entries(raw.failed)) {
-            const record = value;
-            if (key.length !== KEY_CHARS || !record)
-                continue;
-            if (typeof record.reason !== "string" || typeof record.attempts !== "number" || typeof record.retry_after !== "number")
-                continue;
-            kept[key] = { reason: record.reason, attempts: record.attempts, retry_after: record.retry_after };
-        }
-        return kept;
-    }
-    catch (error) {
-        if (!isMissingFileFailure(error)) {
-            console.error("[image-descriptions] failed-image list unreadable, starting from an empty list", JSON.stringify({ reason: "image_failed_state_unreadable" }));
-        }
-        return {};
-    }
+    return (await readImageDescriptionState(stateDir)).failed;
 }
+/**
+ * Write this sync's view, merged with whatever the file holds now: the memory daemon may have
+ * described images since the tick opened its copy, and those entries must survive.
+ */
 export async function writeImageDescriptionState(stateDir, state) {
-    const keys = state.described;
-    const kept = keys.length > IMAGE_DESCRIPTION_STATE_MAX_KEYS ? keys.slice(keys.length - IMAGE_DESCRIPTION_STATE_MAX_KEYS) : keys;
-    const failedEntries = Object.entries(state.failed).sort((a, b) => a[1].retry_after - b[1].retry_after);
-    const failed = Object.fromEntries(failedEntries.slice(Math.max(0, failedEntries.length - IMAGE_FAILED_STATE_MAX_KEYS)));
-    await writeJsonFileAtomic(imageDescriptionStatePath(stateDir), {
-        schema_version: "cockpit-image-descriptions.v1",
-        described: kept,
-        failed,
-    });
+    await updateImageDescriptionState(stateDir, (disk) => mergeImageDescriptionStates({ described: [...state.described], failed: { ...state.failed } }, disk));
 }
 /** Writes the described list and keeps whatever failures are on record. */
 export async function writeDescribedImageKeys(stateDir, keys) {
-    await writeImageDescriptionState(stateDir, { described: keys, failed: await readFailedImages(stateDir) });
+    await writeImageDescriptionState(stateDir, { described: keys, failed: {} });
 }

package/dist/slim/slim-stream.js

@@ -1,29 +1,50 @@
 /**
- * Pass two of the streamed transcript: image swap and slimming over lines as they arrive (BLI-23317).
+ * Pass two of the streamed transcript: image swap and slimming over byte lines as they arrive
+ * (BLI-23317).
  *
  * The same per-line work as `slimPass` (swap the chosen images for markers inside the line, then the
- * slim rules), with the lines coming from a stream instead of a decoded string, so the original text
- * and the slimmed text are never both whole in memory. Output is byte-identical to `slimPass` on the
- * joined input; `raw-evidence-transcript-stream.test.ts` and the frozen goldens hold it to that.
+ * slim rules), with the lines coming from a pinned file instead of a decoded string, so the original
+ * text and the slimmed text are never both whole in memory. Output is byte-identical to `slimPass` on
+ * the joined input; `raw-evidence-transcript-stream.test.ts` and the frozen goldens hold it to that.
  *
- * The result is ONE string, because the sanitizer after it takes one. Slimmed lines are written as
- * UTF-8 into one byte buffer sized to the file (off the JS heap, one byte per ASCII character, no
- * per-line garbage waiting on the collector) and decoded once at the end, so the last step holds the
- * bytes beside the string they make, not a second copy of the string. Text that came from valid
- * UTF-8 survives encode and decode unchanged; the slim rules re-escape lone surrogates, so none is
- * ever written.
+ * Stage D: the images were found and hashed ONCE, by the census, and arrive here as occurrences with
+ * absolute byte offsets. A line with no chosen occurrence is decoded as it is; a line with one is
+ * rebuilt from the bytes around it and the marker's bytes, so a described image's base64 is never
+ * decoded to a string, let alone matched by a regex. A line that keeps an image (undescribed, too large)
+ * is decoded whole, as before.
+ *
+ * Each slimmed line goes to a `LineSink` and nowhere else: the staging file (`staged-output.ts`, hashing
+ * as it writes) when the gate allows one, else `LineBytesSink` below, one byte buffer in memory. No
+ * array of line strings is kept: with a non-Latin-1 character in most lines, the strings are two bytes a
+ * character, so they would cost as much as the text they build. The caller takes the text from the sink
+ * once the pass is over, for the sanitizer, which still reads the whole text: redaction is file-global
+ * (the first 200 ranges, absolute offsets, the residual fallback) and is not changed here.
  */
-import { findEmbeddedImages, replaceImagesWithMarkers } from "@bli-cockpit/telemetry-core";
-import { lineMayHoldImage } from "../image-census-stream.js";
+import { formatImageMarker } from "@bli-cockpit/telemetry-core";
+import { TextDecoder } from "node:util";
+import { TranscriptNotStreamable } from "../adapters/transcript-text-stream.js";
 import { createSlimmer } from "./slim-transcript.js";
 const NEWLINE = 0x0a;
-/** UTF-8 bytes of the lines so far, in a buffer that starts at the input's size and grows only if the output outgrows it. */
-class LineBytes {
+/**
+ * UTF-8 bytes of the lines so far, in a buffer that starts at the input's size (pages never written are
+ * never resident) and grows only if the output outgrows it. The sink when nothing is staged to disk.
+ */
+export class LineBytesSink {
     buffer;
     length = 0;
     constructor(capacity) {
         this.buffer = Buffer.allocUnsafe(Math.max(capacity, 1024));
     }
+    async write(text) {
+        if (text === "\n") {
+            this.reserve(1);
+            this.buffer[this.length++] = NEWLINE;
+            return;
+        }
+        const size = Buffer.byteLength(text);
+        this.reserve(size);
+        this.length += this.buffer.write(text, this.length, size, "utf8");
+    }
     reserve(extra) {
         const needed = this.length + extra;
         if (needed <= this.buffer.length)
@@ -32,46 +53,68 @@ class LineBytes {
         this.buffer.copy(grown, 0, 0, this.length);
         this.buffer = grown;
     }
-    newline() {
-        this.reserve(1);
-        this.buffer[this.length++] = NEWLINE;
-    }
-    line(text) {
-        const size = Buffer.byteLength(text);
-        this.reserve(size);
-        this.length += this.buffer.write(text, this.length, size, "utf8");
-    }
     text() {
         return this.buffer.toString("utf8", 0, this.length);
     }
 }
-export async function slimLineStream(lines, kind, replace, inputBytes) {
+export async function slimLineStream(lines, kind, occurrences, replace, sink) {
     const slimmer = createSlimmer(kind);
-    const output = new LineBytes(inputBytes);
+    // One decoder for the pass: fatal on invalid UTF-8, BOM kept as a character like `Buffer.toString`.
+    const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+    const decode = (bytes) => {
+        try {
+            return decoder.decode(bytes);
+        }
+        catch {
+            throw new TranscriptNotStreamable("invalid_utf8");
+        }
+    };
+    let next = 0;
     let replacedCount = 0;
     let charsSaved = 0;
     let changed = false;
     let first = true;
-    for await (const original of lines) {
-        let line = original;
-        if (lineMayHoldImage(line)) {
-            const found = findEmbeddedImages(line);
-            if (found.length > 0) {
-                const replaced = replaceImagesWithMarkers(line, found, replace);
-                if (replaced.replacedCount > 0) {
-                    line = replaced.text;
-                    replacedCount += replaced.replacedCount;
-                    charsSaved += replaced.charsSaved;
-                }
+    for await (const { start, bytes } of lines) {
+        const end = start + bytes.length;
+        // Occurrences sit inside one line; one that starts before this line was missed by the census's cut.
+        if (next < occurrences.length && occurrences[next].replaceStart < start)
+            throw new TranscriptNotStreamable("images_changed_between_passes");
+        const chosen = [];
+        for (; next < occurrences.length && occurrences[next].replaceStart < end; next += 1) {
+            const image = occurrences[next];
+            if (image.replaceEnd > end)
+                throw new TranscriptNotStreamable("images_changed_between_passes");
+            if (replace(image))
+                chosen.push(image);
+        }
+        let line;
+        if (chosen.length === 0) {
+            line = decode(bytes);
+        }
+        else {
+            const pieces = [];
+            let cursor = 0;
+            for (const image of chosen) {
+                const marker = formatImageMarker({ sha256: image.sha256, mediaType: image.mediaType, byteSize: image.byteSize });
+                pieces.push(bytes.subarray(cursor, image.replaceStart - start), Buffer.from(marker, "utf8"));
+                cursor = image.replaceEnd - start;
+                replacedCount += 1;
+                charsSaved += image.replaceEnd - image.replaceStart - marker.length;
             }
+            pieces.push(bytes.subarray(cursor));
+            line = decode(Buffer.concat(pieces));
+            changed = true;
         }
-        const out = slimmer.slimLine(line);
-        if (line !== original || out !== original)
+        const slimmed = slimmer.slimLine(line);
+        // An unswapped line is its own decoding, so it is changed exactly when the slim rules rewrote it.
+        if (chosen.length === 0 && slimmed !== line)
             changed = true;
         if (!first)
-            output.newline();
+            await sink.write("\n");
         first = false;
-        output.line(out);
+        await sink.write(slimmed);
     }
-    return { text: output.text(), stats: slimmer.stats(), replacedCount, charsSaved, changed };
+    if (next < occurrences.length)
+        throw new TranscriptNotStreamable("images_changed_between_passes");
+    return { stats: slimmer.stats(), replacedCount, charsSaved, changed };
 }

package/dist/staging-gate.js

@@ -172,13 +172,10 @@ export async function openStagingAllowance(stateDir, env, deps = {}) {
     return { snapshot, admitted_bytes: 0, deferred: {} };
 }
 /**
- * May a file of this size be copied into staging now? Returns the reason it may
- * not, or null after charging the allowance. A copy that already exists on disk
- * costs nothing and must be passed as `alreadyStaged` so it is never refused.
+ * The gate's decision for a file of this size, with nothing charged, counted or logged: the one rule
+ * `admitToStaging` applies and `roomToStageUpTo` asks about beforehand.
  */
-export function admitToStaging(allowance, byteLength, options = {}) {
-    if (options.alreadyStaged)
-        return null;
+function decideAdmission(allowance, byteLength) {
     const { snapshot } = allowance;
     const payloadCap = Math.floor(snapshot.cap_bytes * (1 - STAGING_MANIFEST_RESERVE_FRACTION));
     const spent = allowance.admitted_bytes;
@@ -189,16 +186,6 @@ export function admitToStaging(allowance, byteLength, options = {}) {
         snapshot.floor_bytes !== null &&
         freeNow - byteLength < snapshot.floor_bytes;
     const smallForFree = freeNow !== null && freeNow > 0 && byteLength <= freeNow * LOW_DISK_SMALL_FILE_FRACTION;
-    if (belowFloor && smallForFree) {
-        console.error("[staging-gate] low-disk floor waived, file is under 1% of free space", JSON.stringify({
-            event: STAGING_DEFERRED,
-            reason: "low_disk_waived_small_file",
-            kind: options.kind ?? null,
-            file_bytes: byteLength,
-            free_bytes: freeNow,
-            floor_bytes: snapshot.floor_bytes,
-        }));
-    }
     if (belowFloor && !smallForFree) {
         reason = "low_disk";
     }
@@ -209,6 +196,39 @@ export function admitToStaging(allowance, byteLength, options = {}) {
         (stagedNow > 0 && stagedNow + byteLength > payloadCap)) {
         reason = "staging_at_cap";
     }
+    return { reason, waivedSmallFile: belowFloor && smallForFree, freeNow };
+}
+/**
+ * Would the gate admit a file of up to this size right now? A read-only question (BLI-23317 stage D): the
+ * streamed transcript pass writes its output to staging while it is made, BEFORE the real size is known
+ * and the real admission runs, so it asks first with the source file's size as the bound. A "no" means no
+ * byte is written ahead of the gate; the ordinary path then refuses the file exactly as it always did.
+ */
+export function roomToStageUpTo(allowance, byteLength) {
+    if (!allowance)
+        return null;
+    return decideAdmission(allowance, byteLength).reason;
+}
+/**
+ * May a file of this size be copied into staging now? Returns the reason it may
+ * not, or null after charging the allowance. A copy that already exists on disk
+ * costs nothing and must be passed as `alreadyStaged` so it is never refused.
+ */
+export function admitToStaging(allowance, byteLength, options = {}) {
+    if (options.alreadyStaged)
+        return null;
+    const { snapshot } = allowance;
+    const { reason, waivedSmallFile, freeNow } = decideAdmission(allowance, byteLength);
+    if (waivedSmallFile) {
+        console.error("[staging-gate] low-disk floor waived, file is under 1% of free space", JSON.stringify({
+            event: STAGING_DEFERRED,
+            reason: "low_disk_waived_small_file",
+            kind: options.kind ?? null,
+            file_bytes: byteLength,
+            free_bytes: freeNow,
+            floor_bytes: snapshot.floor_bytes,
+        }));
+    }
     if (!reason) {
         allowance.admitted_bytes += byteLength;
         return null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.266",
+  "version": "0.2.268",
   "private": false,
   "type": "module",
   "bin": {
@@ -27,9 +27,9 @@
     "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
   },
   "dependencies": {
-    "@bli-cockpit/memory-mcp": "0.1.78",
-    "@bli-cockpit/mcp": "0.1.162",
-    "@bli-cockpit/telemetry-core": "0.1.85"
+    "@bli-cockpit/memory-mcp": "0.1.79",
+    "@bli-cockpit/mcp": "0.1.163",
+    "@bli-cockpit/telemetry-core": "0.1.86"
   },
   "repository": {
     "type": "git",