@bli-cockpit/cli 0.2.263 → 0.2.265

package/dist/adapters/transcript-line-stream.js

@@ -0,0 +1,47 @@
+/**
+ * A file's lines, one at a time, without ever holding the whole file (BLI-23317).
+ *
+ * `fs.readFile(path, "utf8")` followed by `.split("\n")` keeps the file's bytes, its decoded string and
+ * the array of lines alive together; for a 200 MB transcript that is the biggest allocation the image
+ * reader makes. This yields exactly the lines `split("\n")` would (the last one, possibly empty, is the
+ * text after the final newline), each decoded as UTF-8 with the same replacement of invalid sequences:
+ * a "\n" byte never occurs inside a multi-byte UTF-8 sequence, so decoding line by line equals decoding
+ * the whole file and cutting it. A line is held as bytes until its newline arrives, so the peak is the
+ * longest line, not the file.
+ *
+ * A file that cannot be opened or read yields no lines, as `readFile(...).catch(() => null)` did, and
+ * says why (a read that fails part way keeps the lines already yielded; the log names it).
+ */
+import fs from "node:fs";
+import { describeError } from "../health-detail.js";
+const NEWLINE = 0x0a;
+/** Read size per chunk. A long line spans many chunks; its parts wait as buffers until the line ends. */
+const CHUNK_BYTES = 1024 * 1024;
+export async function* linesOfFile(filePath) {
+    const stream = fs.createReadStream(filePath, { highWaterMark: CHUNK_BYTES });
+    let parts = [];
+    let lines = 0;
+    try {
+        for await (const chunk of stream) {
+            let start = 0;
+            for (let newline = chunk.indexOf(NEWLINE, start); newline !== -1; newline = chunk.indexOf(NEWLINE, start)) {
+                parts.push(chunk.subarray(start, newline));
+                const line = (parts.length === 1 ? parts[0] : Buffer.concat(parts)).toString("utf8");
+                parts = [];
+                start = newline + 1;
+                lines += 1;
+                yield line;
+            }
+            // A line that outlives its chunk must not pin the whole chunk, so its tail is copied.
+            if (start < chunk.length)
+                parts.push(Buffer.from(chunk.subarray(start)));
+        }
+        yield Buffer.concat(parts).toString("utf8");
+    }
+    catch (error) {
+        console.error("[transcript-line-stream] file could not be read; its lines stop here", JSON.stringify({ reason: "file_read_failed", lines_yielded: lines, ...describeError(error) }));
+    }
+    finally {
+        stream.destroy();
+    }
+}

package/dist/adapters/transcript-text-stream.js

@@ -0,0 +1,124 @@
+/**
+ * A transcript's lines as text, straight from disk, for the pass that never holds the whole file (BLI-23317).
+ *
+ * Unlike `transcript-line-stream.ts` (the image reader's, which logs a failed read and ends the lines
+ * early), this one tells the two endings apart. A read that fails is a `TranscriptReadError`, never a
+ * shorter file: the caller maps it to the same `file_read_failed` skip a whole-file read failure gets.
+ * Text that is not valid UTF-8 is a `TranscriptNotStreamable`: the streamed path replaces nothing, so
+ * the caller falls back to the whole-file path, which keeps the original bytes for the clean branch.
+ *
+ * One `TextDecoder` (fatal, BOM kept, `stream: true`) lives across all chunks, so a character split
+ * between two chunks is valid input, and `decode()` at EOF rejects a file that ends mid-character.
+ * For valid UTF-8 it yields exactly the string `Buffer.toString("utf8")` makes, cut at "\n" like
+ * `split("\n")`: the last line is whatever follows the final newline, empty or not.
+ *
+ * A line is held as pieces until its newline arrives, so the peak is the longest line (a single very
+ * long line is still concatenated whole; that bound is not claimed to be small).
+ */
+import fsp from "node:fs/promises";
+import { TextDecoder } from "node:util";
+import { describeError } from "../health-detail.js";
+/** Bytes read per step, into one reused buffer. */
+export const TRANSCRIPT_CHUNK_BYTES = 64 * 1024;
+const CHUNK_BYTES = TRANSCRIPT_CHUNK_BYTES;
+/** The file could not be read to its end. Not EOF: whoever catches it must not treat the lines so far as the file. */
+export class TranscriptReadError extends Error {
+    stage;
+    underlying;
+    constructor(stage, underlying) {
+        super(`transcript read failed at ${stage}`);
+        this.name = "TranscriptReadError";
+        this.stage = stage;
+        this.underlying = underlying;
+    }
+}
+/** The file is bigger than this pass's cap. */
+export class TranscriptTooLargeError extends Error {
+    maxFileBytes;
+    constructor(maxFileBytes) {
+        super("transcript over the file cap");
+        this.name = "TranscriptTooLargeError";
+        this.maxFileBytes = maxFileBytes;
+    }
+}
+/** The streamed path cannot give the whole-file answer for this file; read it whole instead. `reason` is a label. */
+export class TranscriptNotStreamable extends Error {
+    reason;
+    constructor(reason) {
+        super(`transcript not streamable: ${reason}`);
+        this.name = "TranscriptNotStreamable";
+        this.reason = reason;
+    }
+}
+/**
+ * The lines of `filePath`. `byteLimit` reads exactly that many bytes (the second pass reads the first
+ * pass's count, so a file that grew between the passes is not half-seen) and fails with
+ * `file_changed_during_pass` when fewer arrive. `stopAfter` is a ceiling only (the first pass, bounded
+ * by the file cap): fewer is fine.
+ */
+export function openTranscriptLines(filePath, options = {}) {
+    let bytes = 0;
+    const { byteLimit } = options;
+    const readTo = byteLimit ?? options.stopAfter;
+    async function* lines() {
+        if (readTo === 0) {
+            yield "";
+            return;
+        }
+        const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+        let handle;
+        let held = [];
+        try {
+            handle = await fsp.open(filePath, "r");
+            // One buffer for the whole pass: the decoder copies out of it, so nothing waits on the collector to free a chunk.
+            const buffer = Buffer.allocUnsafe(CHUNK_BYTES);
+            for (let position = 0;;) {
+                const want = readTo === undefined ? CHUNK_BYTES : Math.min(CHUNK_BYTES, readTo - position);
+                if (want <= 0)
+                    break;
+                const { bytesRead } = await handle.read(buffer, 0, want, position);
+                if (bytesRead === 0)
+                    break;
+                position += bytesRead;
+                bytes += bytesRead;
+                const text = decodeOrThrow(decoder, buffer.subarray(0, bytesRead));
+                let start = 0;
+                for (let newline = text.indexOf("\n", start); newline !== -1; newline = text.indexOf("\n", start)) {
+                    held.push(text.slice(start, newline));
+                    const line = held.length === 1 ? held[0] : held.join("");
+                    held = [];
+                    start = newline + 1;
+                    yield line;
+                }
+                if (start < text.length)
+                    held.push(text.slice(start));
+            }
+            const tail = decodeOrThrow(decoder, undefined);
+            if (tail)
+                held.push(tail);
+            if (byteLimit !== undefined && bytes < byteLimit)
+                throw new TranscriptNotStreamable("file_changed_during_pass");
+            yield held.join("");
+        }
+        catch (error) {
+            if (error instanceof TranscriptNotStreamable)
+                throw error;
+            console.error("[transcript-text-stream] transcript could not be read to its end", JSON.stringify({ reason: "file_read_failed", stage: "stream", bytes_read: bytes, ...describeError(error) }));
+            throw new TranscriptReadError("stream", error);
+        }
+        finally {
+            await handle?.close().catch((error) => {
+                console.error("[transcript-text-stream] file handle did not close", JSON.stringify({ reason: "file_close_failed", ...describeError(error) }));
+            });
+        }
+    }
+    return { lines: lines(), bytesRead: () => bytes };
+}
+function decodeOrThrow(decoder, chunk) {
+    try {
+        return chunk ? decoder.decode(chunk, { stream: true }) : decoder.decode();
+    }
+    catch {
+        throw new TranscriptNotStreamable("invalid_utf8");
+    }
+}

package/dist/commands/public-root.js

@@ -21,7 +21,7 @@ export async function runCockpitCli(argv, io) {
   }
 
   if (command === "--version" || command === "-V" || command === "version") {
-    writeLine(io?.stdout ?? process.stdout, "0.2.263");
+    writeLine(io?.stdout ?? process.stdout, "0.2.265");
     return 0;
   }
 

package/dist/image-census-stream.js

@@ -0,0 +1,37 @@
+/**
+ * Pass one of the streamed transcript: find every embedded image line by line and keep only what the
+ * describe step needs, never the text (BLI-23317).
+ *
+ * `findEmbeddedImages` over the whole text and over each line give the same images, because an image's
+ * characters never include "\n" and the finder's quote and data-URL anchors sit inside one line
+ * (`slimPass` already throws when one does not). So the census is the whole-text finder run per line,
+ * and what outlives a line is small: one record per distinct image, a tally per image of how often it
+ * occurs and how many characters swapping each occurrence saves, and the base64 of only those images
+ * the describe step may send (a copy, so no line or chunk stays alive behind it).
+ */
+import { findEmbeddedImages, formatImageMarker, } from "@bli-cockpit/telemetry-core";
+/** Could this line hold an embedded image? A superset of what the finder matches: its anchors are `base64` and the four image magics. */
+export function lineMayHoldImage(line) {
+    return line.includes("base64") || line.includes("iVBOR") || line.includes("/9j/") || line.includes("R0lGO") || line.includes("UklGR");
+}
+export async function censusOfLines(lines, keepBase64) {
+    const census = { total: 0, unique: new Map(), tally: new Map(), base64: new Map() };
+    for await (const line of lines) {
+        if (!lineMayHoldImage(line))
+            continue;
+        for (const image of findEmbeddedImages(line)) {
+            census.total += 1;
+            const marker = formatImageMarker({ sha256: image.sha256, mediaType: image.mediaType, byteSize: image.byteSize });
+            const tally = census.tally.get(image.sha256) ?? { count: 0, savedChars: 0 };
+            tally.count += 1;
+            tally.savedChars += image.replaceEnd - image.replaceStart - marker.length;
+            census.tally.set(image.sha256, tally);
+            if (census.unique.has(image.sha256))
+                continue;
+            census.unique.set(image.sha256, image);
+            if (keepBase64(image))
+                census.base64.set(image.sha256, Buffer.from(line.slice(image.base64Start, image.base64End), "latin1"));
+        }
+    }
+    return census;
+}

package/dist/image-description-session.js

@@ -5,10 +5,13 @@
  * Failures never fail the transcript. Logs contain metadata, never image bytes
  * or descriptions.
  */
-import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, base64Of, findEmbeddedImages, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
+import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, base64Of, findEmbeddedImages, formatImageMarker, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
 import { postImageToDescribeDoor } from "./image-describe-client.js";
 import { resolveImageHandlingMode } from "./image-mode-state.js";
-import { slimOrKeep, slimTranscript } from "./slim/slim-transcript.js";
+import { censusOfLines } from "./image-census-stream.js";
+import { openTranscriptLines, TranscriptNotStreamable, TranscriptTooLargeError } from "./adapters/transcript-text-stream.js";
+import { slimLineStream } from "./slim/slim-stream.js";
+import { logSlimmed, slimOrKeep, slimPassOrNull, slimTranscript } from "./slim/slim-transcript.js";
 import { imageStateKey, imageRetryDelayMs, readDescribedImageKeys, readFailedImages, writeImageDescriptionState, } from "./image-description-state.js";
 /** New images sent to the door in one sync (all worktrees together); the count never refills within a tick. */
 export const IMAGE_DESCRIBE_MAX_PER_RUN = 100;
@@ -34,19 +37,99 @@ export async function openImageDescriptions(options) {
     const maxRunMs = options.maxRunMs ?? IMAGE_DESCRIBE_MAX_RUN_MS;
     const sentThisRun = 0;
     const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now, describingMs: 0, maxPerRun, maxRunMs, sentThisRun };
+    const streamMinBytes = options.streamMinBytes ?? STREAM_MIN_BYTES;
     return {
         mode,
+        streamMinBytes,
         isDescribed: (sha256) => described.has(imageStateKey(sha256)),
         hasDescribeRoom: (sha256s) => describeRoomLeft(run, sha256s),
         async prepareTranscript(input) {
-            const afterImages = await describeAndReplace(run, input);
+            const { replace, finds, ...afterImages } = await describeAndReplace(run, input, findsOfText(input.text));
             if (mode !== "describe_and_drop")
                 return { ...afterImages, text: input.text, changed: false, slim: null };
-            const slim = slimOrKeep(afterImages.text, input.source, { sessionId: input.sessionId });
+            // BLI-23317: the image swap and the slim rules are ONE pass over the lines (one joined string, no
+            // intermediate copy). A pass that cannot run falls back to the two-step order, which is the same bytes.
+            const swap = { images: finds.images, replace };
+            const pass = slimPassOrNull(input.text, input.source, { sessionId: input.sessionId }, swap);
+            if (pass)
+                return { ...afterImages, text: pass.text, changed: pass.changed, slim: pass.stats };
+            const replaced = replaceImagesWithMarkers(input.text, swap.images, swap.replace);
+            const slim = slimOrKeep(replaced.text, input.source, { sessionId: input.sessionId });
             return { ...afterImages, text: slim.text, changed: slim.text !== input.text, slim: slim.stats };
         },
+        prepareTranscriptFile: mode === "describe_and_drop" ? (input) => prepareFileInTwoPasses(run, input) : undefined,
     };
 }
+/** Files smaller than this are cheaper to read whole; the streamed path pays a second read. */
+const STREAM_MIN_BYTES = 8 * 1024 * 1024;
+function findsOfText(text) {
+    const images = findEmbeddedImages(text);
+    const unique = new Map();
+    for (const image of images)
+        if (!unique.has(image.sha256))
+            unique.set(image.sha256, image);
+    return {
+        total: images.length,
+        unique,
+        images,
+        base64For: (image) => base64Of(text, image),
+        swapped: (replace) => countImageSwaps({ images, replace }),
+    };
+}
+function findsOfCensus(census) {
+    return {
+        total: census.total,
+        unique: census.unique,
+        base64For: (image) => {
+            const kept = census.base64.get(image.sha256);
+            if (!kept)
+                throw new Error("image_base64_not_kept");
+            return kept.toString("latin1");
+        },
+        swapped: (replace) => {
+            let replacedCount = 0;
+            let charsSaved = 0;
+            for (const [sha256, image] of census.unique) {
+                if (!replace(image))
+                    continue;
+                const tally = census.tally.get(sha256);
+                replacedCount += tally.count;
+                charsSaved += tally.savedChars;
+            }
+            return { replacedCount, charsSaved };
+        },
+    };
+}
+async function prepareFileInTwoPasses(run, input) {
+    const startedAt = Date.now();
+    // Pass one: discover. Nothing of the text is kept; only the images and, for those that may still be sent, their base64.
+    const room = Math.max(0, run.maxPerRun - run.sentThisRun);
+    let wanted = 0;
+    const first = openTranscriptLines(input.filePath, { stopAfter: input.maxFileBytes ? input.maxFileBytes + 1 : undefined });
+    const census = await censusOfLines(first.lines, (image) => {
+        // The same filters `selectImagesToDescribe` applies, so what is kept is what may be sent.
+        const key = imageStateKey(image.sha256);
+        const failure = run.failedImages[key];
+        if (wanted >= room || run.described.has(key) || run.settledThisRun.has(image.sha256) || image.byteSize > IMAGE_DESCRIBE_MAX_BYTES)
+            return false;
+        if (failure && failure.retry_after > run.now())
+            return false;
+        wanted += 1;
+        return true;
+    });
+    const readBytes = first.bytesRead();
+    if (input.maxFileBytes && readBytes > input.maxFileBytes)
+        throw new TranscriptTooLargeError(input.maxFileBytes);
+    // Describe attempts finish and the swap decisions are settled before a single line is swapped, as with the whole text.
+    const { replace, finds: _finds, ...afterImages } = await describeAndReplace(run, input, findsOfCensus(census));
+    // Pass two: swap and slim over exactly the bytes pass one saw.
+    const second = openTranscriptLines(input.filePath, { byteLimit: readBytes });
+    const slim = await slimLineStream(second.lines, input.source, replace, readBytes);
+    if (slim.replacedCount !== afterImages.replacedCount)
+        throw new TranscriptNotStreamable("images_changed_between_passes");
+    logSlimmed(input.source, input.sessionId, slim.stats, startedAt);
+    return { ...afterImages, text: slim.text, changed: slim.changed, slim: slim.stats, readBytes };
+}
 function describeRoomLeft(run, sha256s) {
     if (run.sentThisRun >= run.maxPerRun)
         return false;
@@ -68,7 +151,7 @@ function markDescribed(run, sha256) {
     described.add(key);
     orderedKeys.push(key);
 }
-async function sendOne(run, image, text, input) {
+async function sendOne(run, image, base64, input) {
     const { options, fetchImpl, settledThisRun, failedImages } = run;
     run.sentThisRun += 1;
     const outcome = await postImageToDescribeDoor({ fetchImpl, dashboardUrl: options.dashboardUrl, deviceToken: options.deviceToken, sleep: options.sleep }, {
@@ -76,7 +159,7 @@ async function sendOne(run, image, text, input) {
         sha256: image.sha256,
         media_type: image.mediaType,
         byte_size: image.byteSize,
-        content_base64: base64Of(text, image),
+        content_base64: base64,
         source: input.source,
         session_id: input.sessionId.slice(0, 200),
     });
@@ -107,18 +190,15 @@ function rememberFailure(run, key, outcome) {
     run.stateDirty = true;
     console.error(TAG + " image not described, will not be re-sent until later", JSON.stringify({ reason: outcome.reason, http_status: outcome.httpStatus, attempts, retry_in_hours: Math.round(delay / 3_600_000), sha256_prefix: key.slice(0, 16) }));
 }
-async function describeAndReplace(run, input) {
+async function describeAndReplace(run, input, finds) {
     const { mode, described, options, orderedKeys, failedImages } = run;
-    const images = findEmbeddedImages(input.text);
-    if (images.length === 0)
-        return { text: input.text, pendingImages: [], imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
-    const unique = new Map();
-    for (const image of images)
-        if (!unique.has(image.sha256))
-            unique.set(image.sha256, image);
+    const replace = (image) => described.has(imageStateKey(image.sha256));
+    const { unique } = finds;
+    if (finds.total === 0)
+        return { replace, finds, pendingImages: [], imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
     const counts = { settled_earlier: 0, already: 0, described_now: 0, known: 0, too_large: 0, over_budget: 0, failed: 0, backing_off: 0 };
     const toSend = selectImagesToDescribe(run, unique, counts);
-    await sendPendingImages(run, toSend, counts, input);
+    await sendPendingImages(run, toSend, counts, input, finds.base64For);
     const failedReasons = countImageOutcomes(run, toSend, counts);
     if (counts.described_now + counts.known > 0 || run.stateDirty) {
         await writeImageDescriptionState(options.stateDir, { described: orderedKeys, failed: failedImages });
@@ -126,32 +206,44 @@ async function describeAndReplace(run, input) {
     }
     const deferredOnly = counts.over_budget + counts.backing_off > 0
         && counts.described_now + counts.known + counts.failed + counts.too_large + counts.settled_earlier === 0;
-    const replaced = mode === "describe_and_drop"
-        ? replaceImagesWithMarkers(input.text, images, (image) => described.has(imageStateKey(image.sha256)))
-        : { text: input.text, replacedCount: 0, charsSaved: 0 };
+    const replaced = mode === "describe_and_drop" ? finds.swapped(replace) : { replacedCount: 0, charsSaved: 0 };
     const pendingImages = [...unique.keys()].filter((sha256) => !described.has(imageStateKey(sha256)));
     console.error(TAG + " transcript prepared", JSON.stringify({
         mode,
         source: input.source,
-        images: images.length,
+        images: finds.total,
         unique: unique.size,
         ...counts,
         deferred_only: deferredOnly,
         failed_reasons: failedReasons,
         replaced: replaced.replacedCount,
         chars_saved: replaced.charsSaved,
-        kept_without_description: mode === "describe_and_drop" ? images.length - replaced.replacedCount : 0,
+        kept_without_description: mode === "describe_and_drop" ? finds.total - replaced.replacedCount : 0,
     }));
     return {
-        text: replaced.text,
+        replace,
+        finds,
         pendingImages,
-        imageCount: images.length,
+        imageCount: finds.total,
         uniqueCount: unique.size,
         replacedCount: replaced.replacedCount,
         charsSaved: replaced.charsSaved,
         deferredOnly,
     };
 }
+/** What `replaceImagesWithMarkers` would report for this swap, without building the text it would make. */
+function countImageSwaps(swap) {
+    let replacedCount = 0;
+    let charsSaved = 0;
+    for (const image of swap.images) {
+        if (!swap.replace(image))
+            continue;
+        const marker = formatImageMarker({ sha256: image.sha256, mediaType: image.mediaType, byteSize: image.byteSize });
+        replacedCount += 1;
+        charsSaved += image.replaceEnd - image.replaceStart - marker.length;
+    }
+    return { replacedCount, charsSaved };
+}
 function selectImagesToDescribe(run, unique, counts) {
     const { described, settledThisRun, failedImages, now } = run;
     const toSend = [];
@@ -179,7 +271,7 @@ function selectImagesToDescribe(run, unique, counts) {
     }
     return toSend;
 }
-async function sendPendingImages(run, toSend, counts, input) {
+async function sendPendingImages(run, toSend, counts, input, base64For) {
     const { now, maxPerRun, maxRunMs } = run;
     const callStartedAt = now();
     let next = 0;
@@ -191,7 +283,7 @@ async function sendPendingImages(run, toSend, counts, input) {
                 counts.over_budget += 1;
                 continue;
             }
-            await sendOne(run, image, input.text, input);
+            await sendOne(run, image, base64For(image), input);
         }
     };
     await Promise.all(Array.from({ length: Math.min(CONCURRENCY, toSend.length) }, worker));

package/dist/slim/slim-stream.js

@@ -0,0 +1,77 @@
+/**
+ * Pass two of the streamed transcript: image swap and slimming over lines as they arrive (BLI-23317).
+ *
+ * The same per-line work as `slimPass` (swap the chosen images for markers inside the line, then the
+ * slim rules), with the lines coming from a stream instead of a decoded string, so the original text
+ * and the slimmed text are never both whole in memory. Output is byte-identical to `slimPass` on the
+ * joined input; `raw-evidence-transcript-stream.test.ts` and the frozen goldens hold it to that.
+ *
+ * The result is ONE string, because the sanitizer after it takes one. Slimmed lines are written as
+ * UTF-8 into one byte buffer sized to the file (off the JS heap, one byte per ASCII character, no
+ * per-line garbage waiting on the collector) and decoded once at the end, so the last step holds the
+ * bytes beside the string they make, not a second copy of the string. Text that came from valid
+ * UTF-8 survives encode and decode unchanged; the slim rules re-escape lone surrogates, so none is
+ * ever written.
+ */
+import { findEmbeddedImages, replaceImagesWithMarkers } from "@bli-cockpit/telemetry-core";
+import { lineMayHoldImage } from "../image-census-stream.js";
+import { createSlimmer } from "./slim-transcript.js";
+const NEWLINE = 0x0a;
+/** UTF-8 bytes of the lines so far, in a buffer that starts at the input's size and grows only if the output outgrows it. */
+class LineBytes {
+    buffer;
+    length = 0;
+    constructor(capacity) {
+        this.buffer = Buffer.allocUnsafe(Math.max(capacity, 1024));
+    }
+    reserve(extra) {
+        const needed = this.length + extra;
+        if (needed <= this.buffer.length)
+            return;
+        const grown = Buffer.allocUnsafe(Math.max(needed, Math.ceil(this.buffer.length * 1.25)));
+        this.buffer.copy(grown, 0, 0, this.length);
+        this.buffer = grown;
+    }
+    newline() {
+        this.reserve(1);
+        this.buffer[this.length++] = NEWLINE;
+    }
+    line(text) {
+        const size = Buffer.byteLength(text);
+        this.reserve(size);
+        this.length += this.buffer.write(text, this.length, size, "utf8");
+    }
+    text() {
+        return this.buffer.toString("utf8", 0, this.length);
+    }
+}
+export async function slimLineStream(lines, kind, replace, inputBytes) {
+    const slimmer = createSlimmer(kind);
+    const output = new LineBytes(inputBytes);
+    let replacedCount = 0;
+    let charsSaved = 0;
+    let changed = false;
+    let first = true;
+    for await (const original of lines) {
+        let line = original;
+        if (lineMayHoldImage(line)) {
+            const found = findEmbeddedImages(line);
+            if (found.length > 0) {
+                const replaced = replaceImagesWithMarkers(line, found, replace);
+                if (replaced.replacedCount > 0) {
+                    line = replaced.text;
+                    replacedCount += replaced.replacedCount;
+                    charsSaved += replaced.charsSaved;
+                }
+            }
+        }
+        const out = slimmer.slimLine(line);
+        if (line !== original || out !== original)
+            changed = true;
+        if (!first)
+            output.newline();
+        first = false;
+        output.line(out);
+    }
+    return { text: output.text(), stats: slimmer.stats(), replacedCount, charsSaved, changed };
+}