@bli-cockpit/cli 0.2.277 → 0.2.279

package/dist/image-store-send.js

@@ -0,0 +1,326 @@
+/**
+ * The store step of `store_and_drop` (BLI-23329): put each distinct image of a transcript on Tower once, and say which
+ * ones Tower confirmed THIS tick, so only those are replaced by their marker.
+ *
+ * "Stored" means one thing here: Tower answered `already_committed` (begin) or `committed` / `already_committed`
+ * (commit) for this image AND this session, and the server records the session as a user of the image in the same step
+ * that answers (`apps/dashboard/src/lib/ambient/image-store/image-store-door.ts`). Nothing local is trusted: no list of
+ * "stored" keys survives the tick, because a forget on Tower can make it a lie. Everything else (offline, refused,
+ * over budget, a 5xx, a changed source) leaves the image inline, which is today's behaviour; an image is never lost.
+ *
+ * Order per transcript: a cheap begin probe for every candidate (25 per request); the ones already on Tower are done;
+ * the rest are decoded to a temp file block by block (`image-store-stage.ts`) and sent through the ordinary chunked
+ * upload client. Logs carry counts, reason labels and the first 16 characters of a hash; never bytes, paths or ids.
+ */
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { IMAGE_STORE_MAX_BYTES, IMAGE_STORE_MAX_PER_RUN, IMAGE_USE_SESSION_ID, IMAGE_STORE_MAX_RUN_MS, RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceUploadBeginResponseSchema, imageObjectKey, } from "@bli-cockpit/telemetry-core";
+import { uploadRawEvidenceFilesChunked } from "./evidence-upload-client.js";
+import { requestJson, safeFailureDetail } from "./evidence-upload-transport.js";
+import { stageDecodedImage } from "./image-store-stage.js";
+import { isBackingOff, readStoreFailures, rememberStoreFailure, storeFailureKey, writeStoreFailures } from "./image-store-state.js";
+const TAG = "[image-store]";
+/** Distinct (image, session) pairs asked about per sync. Asking is cheap (one begin per 25); uploading is what the other budget bounds. */
+export const IMAGE_STORE_MAX_CONFIRMS_PER_RUN = 2_000;
+/** Images sent to the upload client together; the time budget is checked between groups. */
+const UPLOAD_GROUP = 10;
+const BEGIN_PATH = "/api/ambient/evidence/upload/begin";
+/** What the server says when it, not the image, is the problem: retried next tick, never remembered against the image. */
+const SERVER_SIDE_TRANSIENT = /image_store_unavailable|image_ledger_not_committed|image_bytes_missing/;
+/** Temp directories of images being sent are named with this; a run that died leaves one behind. */
+export const IMAGE_STORE_TEMP_PREFIX = "tower-image-store-";
+const STALE_TEMP_MS = 6 * 3_600_000;
+/** Remove staging directories older than six hours (a run that was killed mid-upload). Best effort, said out loud. */
+export async function pruneStaleImageStoreTemp(now, tempRoot = os.tmpdir()) {
+    let removed = 0;
+    try {
+        for (const name of await fs.readdir(tempRoot)) {
+            if (!name.startsWith(IMAGE_STORE_TEMP_PREFIX))
+                continue;
+            const dir = path.join(tempRoot, name);
+            const info = await fs.stat(dir).catch(() => null);
+            if (!info?.isDirectory() || now - info.mtimeMs < STALE_TEMP_MS)
+                continue;
+            await fs.rm(dir, { recursive: true, force: true });
+            removed += 1;
+        }
+    }
+    catch (error) {
+        console.error(`${TAG} could not prune old staging directories`, JSON.stringify({ reason: "image_store_temp_prune_failed", error_name: error instanceof Error ? error.name : typeof error }));
+        return removed;
+    }
+    if (removed > 0)
+        console.error(`${TAG} removed staging directories a killed run left behind`, JSON.stringify({ reason: "image_store_temp_pruned", removed }));
+    return removed;
+}
+export async function openImageStoreRun(stateDir, overrides = {}) {
+    await pruneStaleImageStoreTemp(Date.now());
+    return {
+        stored: new Set(),
+        failures: await readStoreFailures(stateDir),
+        failuresDirty: false,
+        uploadedThisRun: 0,
+        confirmedThisRun: 0,
+        storingMs: 0,
+        unavailable: null,
+        maxUploads: overrides.maxUploads ?? IMAGE_STORE_MAX_PER_RUN,
+        maxConfirms: overrides.maxConfirms ?? IMAGE_STORE_MAX_CONFIRMS_PER_RUN,
+        maxRunMs: overrides.maxRunMs ?? IMAGE_STORE_MAX_RUN_MS,
+    };
+}
+export const storedKey = (sha256, sessionId) => `${sha256}|${sessionId}`;
+export function isStoredThisRun(store, sha256, sessionId) {
+    return store.stored.has(storedKey(sha256, sessionId));
+}
+/**
+ * Could a prepare pass still store at least one of these images right now? False when the sync's upload or time budget
+ * is spent, Tower was unreachable this sync, or every one is backing off after a refusal. While true, a remembered
+ * transcript whose store failed is rebuilt, so a closed session whose store failed is stored on the next tick with room.
+ */
+export function hasStoreRoom(store, sha256s, now) {
+    if (store.unavailable)
+        return false;
+    if (store.uploadedThisRun >= store.maxUploads || store.storingMs > store.maxRunMs)
+        return false;
+    return sha256s.some((sha256) => !isBackingOff(store.failures, storeFailureKey(sha256, "", false), now));
+}
+function pointerFor(operatorId, image) {
+    const objectKey = imageObjectKey(operatorId, image.sha256, image.mediaType);
+    return {
+        raw_evidence_pointer_id: objectKey,
+        privacy_classification: "remote_durable_raw_evidence",
+        retention_policy: { mode: "remote_durable", privacy_classification: "remote_durable_raw_evidence" },
+        storage_scope: "remote_object",
+        storage_bucket: "ambient-raw-evidence",
+        object_key: objectKey,
+        content_hash_sha256: image.sha256,
+        byte_size: image.byteSize,
+        media_type: image.mediaType,
+    };
+}
+function* groupsOf(items, size) {
+    for (let at = 0; at < items.length; at += size)
+        yield items.slice(at, at + size);
+}
+export async function storeImages(store, input) {
+    const counts = { settled_earlier: 0, too_large_to_store: 0, backing_off: 0, over_budget: 0, unavailable: 0, stored_existing: 0, stored_new: 0, failed: 0 };
+    if (input.images.length === 0)
+        return counts;
+    if (!input.provenance) {
+        console.error(`${TAG} nothing stored, images stay inline`, JSON.stringify({ reason: "store_provenance_missing", images: input.images.length }));
+        counts.unavailable = input.images.length;
+        return counts;
+    }
+    // The server records the session by this id (the id forget keys on); one it would refuse is not worth a request.
+    if (!IMAGE_USE_SESSION_ID.test(input.sessionId)) {
+        console.error(`${TAG} nothing stored, images stay inline`, JSON.stringify({ reason: "store_session_id_unusable", images: input.images.length }));
+        counts.unavailable = input.images.length;
+        return counts;
+    }
+    const provenance = input.provenance;
+    const startedAt = input.now();
+    const options = {
+        fetchImpl: input.fetchImpl, dashboardUrl: input.dashboardUrl, deviceToken: input.deviceToken, provenance,
+        generatedAt: new Date(startedAt).toISOString(), files: [], ...(input.sleep ? { sleep: input.sleep } : {}),
+    };
+    const use = { session_id: input.sessionId, source: input.source };
+    const candidates = selectCandidates(store, input, counts);
+    const needUpload = await probeAll(store, input, options, use, candidates, counts);
+    await uploadAll(store, input, options, use, needUpload, counts, startedAt);
+    store.storingMs += input.now() - startedAt;
+    if (store.failuresDirty) {
+        try {
+            await writeStoreFailures(input.stateDir, store.failures);
+            store.failuresDirty = false;
+        }
+        catch (error) {
+            console.error(`${TAG} could not record store refusals; they are forgotten at the end of this sync`, JSON.stringify({ reason: "image_store_failures_write_failed", error_name: error instanceof Error ? error.name : typeof error }));
+        }
+    }
+    console.error(`${TAG} transcript stored`, JSON.stringify({ source: input.source, images: input.images.length, ...counts, storing_ms: input.now() - startedAt }));
+    return counts;
+}
+function selectCandidates(store, input, counts) {
+    const candidates = [];
+    for (const image of input.images) {
+        if (isStoredThisRun(store, image.sha256, input.sessionId)) {
+            counts.settled_earlier += 1;
+            continue;
+        }
+        if (image.byteSize > IMAGE_STORE_MAX_BYTES) {
+            counts.too_large_to_store += 1;
+            continue;
+        }
+        if (store.unavailable) {
+            counts.unavailable += 1;
+            continue;
+        }
+        const imageKey = storeFailureKey(image.sha256, input.sessionId, false);
+        const sessionKey = storeFailureKey(image.sha256, input.sessionId, true);
+        if (isBackingOff(store.failures, imageKey, input.now()) || isBackingOff(store.failures, sessionKey, input.now())) {
+            counts.backing_off += 1;
+            continue;
+        }
+        if (store.confirmedThisRun >= store.maxConfirms || store.storingMs > store.maxRunMs) {
+            counts.over_budget += 1;
+            continue;
+        }
+        store.confirmedThisRun += 1;
+        candidates.push(image);
+    }
+    return candidates;
+}
+/** Ask Tower about every candidate; the ones it already holds are confirmed here, the rest come back to be uploaded. */
+async function probeAll(store, input, options, use, candidates, counts) {
+    const needUpload = [];
+    const operatorId = options.provenance.operator_id;
+    for (const batch of groupsOf(candidates, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN)) {
+        if (store.unavailable) {
+            counts.unavailable += batch.length;
+            continue;
+        }
+        const response = await requestJson(options, BEGIN_PATH, {
+            schema_version: "ambient-raw-evidence-upload-begin.v1",
+            generated_at: options.generatedAt,
+            provenance: options.provenance,
+            objects: batch.map((image) => ({
+                pointer: pointerFor(operatorId, image),
+                chunk_size_bytes: RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES,
+                chunk_count: Math.max(1, Math.ceil(image.byteSize / RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES)),
+                image_use: use,
+            })),
+        });
+        const parsed = response.ok ? RawEvidenceUploadBeginResponseSchema.safeParse(response.body) : null;
+        if (!parsed?.success) {
+            settleUnreachable(store, response.status, safeFailureDetail(response.body), response.ok ? "begin_invalid_response" : null, batch, counts);
+            continue;
+        }
+        const byKey = new Map(parsed.data.objects.map((object) => [object.object_key, object]));
+        for (const image of batch) {
+            const answer = byKey.get(imageObjectKey(operatorId, image.sha256, image.mediaType));
+            if (!answer) {
+                fail(store, input, image, "begin_missing_disposition", counts);
+                continue;
+            }
+            if (answer.disposition === "already_committed") {
+                store.stored.add(storedKey(image.sha256, input.sessionId));
+                counts.stored_existing += 1;
+            }
+            else if (answer.disposition === "conflict") {
+                fail(store, input, image, answer.reason ?? "upload_conflict", counts);
+            }
+            else {
+                needUpload.push(image);
+            }
+        }
+    }
+    return needUpload;
+}
+/**
+ * A whole `begin` request that did not come back as an answer (unreachable, a 5xx, a 4xx, a reply this CLI cannot read)
+ * says nothing about any one picture: a 400 is a request one end could not read (a server that does not know this CLI's
+ * fields, say), and remembering it against every picture for a week would punish the pictures for the wiring. Leave the
+ * rest of this sync's images inline, once, saying why; the next tick asks again.
+ */
+function settleUnreachable(store, status, detail, label, batch, counts) {
+    const reason = label ?? `begin_failed_http_${status}${detail ? `_${detail}` : ""}`;
+    if (!store.unavailable) {
+        store.unavailable = reason;
+        console.error(`${TAG} Tower cannot take images right now; every image stays inline for the rest of this sync`, JSON.stringify({ reason, http_status: status, server_reason: detail ?? "none" }));
+    }
+    counts.unavailable += batch.length;
+}
+/** What a refusal is about decides whether it is remembered, for how long, and against what. */
+function fail(store, input, image, reason, counts) {
+    counts.failed += 1;
+    const verdict = classifyStoreFailure(reason);
+    if (verdict.remember) {
+        const key = storeFailureKey(image.sha256, input.sessionId, verdict.aboutSession);
+        const record = rememberStoreFailure(store.failures, key, `store:${reason}`.slice(0, 90), verdict.permanent, input.now());
+        store.failuresDirty = true;
+        console.error(`${TAG} image not stored, will not be re-sent until later`, JSON.stringify({ reason, attempts: record.attempts, retry_in_hours: Math.round((record.retry_after - input.now()) / 3_600_000), sha256_prefix: image.sha256.slice(0, 16) }));
+    }
+    else {
+        console.error(`${TAG} image not stored this tick, retried on the next`, JSON.stringify({ reason, sha256_prefix: image.sha256.slice(0, 16) }));
+    }
+}
+export function classifyStoreFailure(reason) {
+    if (/image_session_forgotten/.test(reason))
+        return { remember: true, permanent: true, aboutSession: true };
+    if (SERVER_SIDE_TRANSIENT.test(reason))
+        return { remember: false, permanent: false, aboutSession: false };
+    const status = Number(/http_(\d{3})/.exec(reason)?.[1] ?? Number.NaN);
+    if (Number.isNaN(status)) {
+        // A conflict named by the server (no HTTP status in the label) is about these bytes: leave it alone for a while.
+        // A local fault (a file that could not be read or staged) says nothing about the image and is not remembered.
+        const local = /^(file_|staged_|image_source_changed|image_stage_|empty_file|upload_request_|legacy_)/.test(reason);
+        return { remember: !local, permanent: !local, aboutSession: false };
+    }
+    if (status === 0 || status === 401 || status === 429)
+        return { remember: false, permanent: false, aboutSession: false };
+    if (status >= 500)
+        return { remember: true, permanent: false, aboutSession: false };
+    return { remember: true, permanent: true, aboutSession: false };
+}
+async function uploadAll(store, input, options, use, needUpload, counts, startedAt) {
+    if (needUpload.length === 0)
+        return;
+    const dir = await fs.mkdtemp(path.join(os.tmpdir(), IMAGE_STORE_TEMP_PREFIX));
+    const operatorId = options.provenance.operator_id;
+    try {
+        for (const group of groupsOf(needUpload, UPLOAD_GROUP)) {
+            if (store.unavailable) {
+                counts.unavailable += group.length;
+                continue;
+            }
+            const room = store.maxUploads - store.uploadedThisRun;
+            const timeUp = store.storingMs + (input.now() - startedAt) > store.maxRunMs;
+            const take = timeUp ? [] : group.slice(0, Math.max(0, room));
+            counts.over_budget += group.length - take.length;
+            if (take.length === 0)
+                continue;
+            store.uploadedThisRun += take.length;
+            const files = [];
+            const imageOf = new Map();
+            for (const image of take) {
+                const staged = await stageDecodedImage(dir, image, input.blocksOf(image));
+                if (!staged.ok) {
+                    fail(store, input, image, staged.reason, counts);
+                    continue;
+                }
+                files.push({ pointer: pointerFor(operatorId, image), local_path: staged.localPath, kind: "image_store_object", codex_session_id: input.sessionId, image_use: use });
+                imageOf.set(image.sha256, image);
+            }
+            if (files.length === 0)
+                continue;
+            const result = await uploadRawEvidenceFilesChunked({ ...options, files });
+            if (result.used_legacy_fallback) {
+                for (const image of imageOf.values())
+                    fail(store, input, image, "legacy_route_cannot_store_images", counts);
+                continue;
+            }
+            for (const outcome of result.outcomes)
+                settleOutcome(store, input, outcome, imageOf, counts);
+            for (const file of files)
+                await fs.rm(file.local_path, { force: true }).catch(() => undefined);
+        }
+    }
+    finally {
+        await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined);
+    }
+}
+function settleOutcome(store, input, outcome, imageOf, counts) {
+    const image = imageOf.get(outcome.pointer.content_hash_sha256 ?? "");
+    if (!image)
+        return;
+    // Only the server's own "committed" / "already_committed" counts; both are sent after the session is recorded.
+    // `reused_existing` for any other reason (a 409 that says someone else committed) is NOT a confirmation for us.
+    const confirmed = outcome.upload_state === "uploaded" || (outcome.upload_state === "reused_existing" && outcome.reason === "already_committed");
+    if (confirmed) {
+        store.stored.add(storedKey(image.sha256, input.sessionId));
+        counts.stored_new += 1;
+        return;
+    }
+    fail(store, input, image, outcome.reason ?? "upload_failed", counts);
+}

package/dist/image-store-stage.js

@@ -0,0 +1,54 @@
+/**
+ * Decode one embedded image from its base64 text into a private temp file, one block at a time (BLI-23329).
+ *
+ * The upload client sends files by offset, so the image has to exist as bytes on disk. It is decoded in blocks of
+ * 48 Ki characters (a multiple of 4: each block is whole base64 quanta), straight into a sha256 and the file, so
+ * neither the base64 text of a large image nor its decoded bytes are ever held whole. The result is checked
+ * against what the census saw: a source that no longer decodes to the same sha256 and size (the transcript changed
+ * between the passes) is refused by name and the file removed. The file is mode 0600, in a directory this run made,
+ * and the caller removes the directory when the run is done.
+ */
+import { createHash } from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+/** Characters per block. A multiple of 4. */
+export const IMAGE_STORE_BLOCK_CHARS = 48 * 1024;
+/** The base64 blocks of an image held in a string: slices of the run, each a whole number of quanta. */
+export function* blocksOfText(text, image) {
+    for (let at = image.base64Start; at < image.base64End; at += IMAGE_STORE_BLOCK_CHARS) {
+        yield text.slice(at, Math.min(image.base64End, at + IMAGE_STORE_BLOCK_CHARS));
+    }
+}
+export async function stageDecodedImage(dir, image, blocks) {
+    const localPath = path.join(dir, `${image.sha256}.bin`);
+    const hash = createHash("sha256");
+    let byteSize = 0;
+    let handle;
+    try {
+        handle = await fs.open(localPath, "wx", 0o600);
+        for await (const block of blocks) {
+            const bytes = Buffer.from(block, "base64");
+            hash.update(bytes);
+            byteSize += bytes.byteLength;
+            await handle.write(bytes);
+        }
+        await handle.close();
+        handle = undefined;
+    }
+    catch (error) {
+        await handle?.close().catch(() => undefined);
+        await fs.rm(localPath, { force: true }).catch(() => undefined);
+        // A source that cannot be read to the end (the transcript changed or vanished) is a changed source; a disk that
+        // cannot be written is the other label. Neither says anything about the image itself.
+        const code = error?.code;
+        const writeFault = code === "ENOSPC" || code === "EACCES" || code === "EPERM" || code === "EEXIST" || code === "EROFS";
+        console.error("[image-store] staging an image failed", JSON.stringify({ reason: writeFault ? "image_stage_write_failed" : "image_source_changed", code: code ?? null }));
+        return { ok: false, reason: writeFault ? "image_stage_write_failed" : "image_source_changed" };
+    }
+    if (hash.digest("hex") !== image.sha256 || byteSize !== image.byteSize) {
+        await fs.rm(localPath, { force: true }).catch(() => undefined);
+        console.error("[image-store] decoded image is not the one the census saw", JSON.stringify({ reason: "image_source_changed", expected_bytes: image.byteSize, decoded_bytes: byteSize }));
+        return { ok: false, reason: "image_source_changed" };
+    }
+    return { ok: true, localPath, byteSize };
+}

package/dist/image-store-state.js

@@ -0,0 +1,78 @@
+/**
+ * Which images Tower refused to store, and until when this machine leaves them alone (BLI-23329).
+ *
+ * Kept in its OWN file, apart from the described/failed record of the describe door
+ * (`image-descriptions.json`, shared with the BLI Memory daemon). Two reasons, both found in plan review:
+ *  - the describe record's merge deletes a failure for every image that is described by now, whatever the failure
+ *    was about (`mergeImageDescriptionStates`), so a refused store of an image the describer knows would be
+ *    forgotten at the next write and retried on every tick;
+ *  - the daemon rewrites that file from its own copy; a field it does not know can be dropped.
+ * This file is written by the sync tick alone (one writer under the sync lock), so neither can touch it.
+ *
+ * Holds keys, reason labels, attempt counts and times only. Never a path, a session id or a byte.
+ */
+import { createHash } from "node:crypto";
+import path from "node:path";
+import { imageRetryDelayMs, imageStateKey } from "./image-description-state.js";
+import { isMissingFileError, readJsonFile, writeJsonFileAtomic } from "./local-state-files.js";
+export const IMAGE_STORE_FAILURES_FILENAME = "image-store-failures.json";
+export const IMAGE_STORE_FAILURES_SCHEMA = "cockpit-image-store-failures.v1";
+/** Oldest records leave first past this many. */
+export const IMAGE_STORE_FAILURES_MAX = 5_000;
+const KEY_CHARS = 32;
+const TAG = "[image-store]";
+export function imageStoreFailuresPath(stateDir) {
+    return path.join(stateDir, IMAGE_STORE_FAILURES_FILENAME);
+}
+/**
+ * The key a refusal is remembered under. Most refusals are about the IMAGE (its bytes, its type): the image's key.
+ * A refusal that is about the SESSION (a founder forgot it) must not stop another session storing the same picture,
+ * so it is remembered under a key that mixes both in.
+ */
+export function storeFailureKey(sha256, sessionId, aboutSession) {
+    if (!aboutSession)
+        return imageStateKey(sha256);
+    return createHash("sha256").update(`${sessionId}|${sha256}`).digest("hex").slice(0, KEY_CHARS);
+}
+/** The failures on record. A missing file is empty; an unreadable or damaged one is empty and said out loud. */
+export async function readStoreFailures(stateDir) {
+    let raw;
+    try {
+        raw = await readJsonFile(imageStoreFailuresPath(stateDir));
+    }
+    catch (error) {
+        if (!isMissingFileError(error)) {
+            console.error(TAG + " store failure record unreadable, starting from an empty one", JSON.stringify({ reason: "image_store_failures_unreadable" }));
+        }
+        return {};
+    }
+    const record = (raw && typeof raw === "object" ? raw : {});
+    const failed = {};
+    if (record.failed && typeof record.failed === "object" && !Array.isArray(record.failed)) {
+        for (const [key, value] of Object.entries(record.failed)) {
+            const entry = value;
+            if (key.length !== KEY_CHARS || !entry)
+                continue;
+            if (typeof entry.reason !== "string" || typeof entry.attempts !== "number" || typeof entry.retry_after !== "number")
+                continue;
+            failed[key] = { reason: entry.reason, attempts: entry.attempts, retry_after: entry.retry_after };
+        }
+    }
+    return failed;
+}
+export async function writeStoreFailures(stateDir, failures) {
+    const entries = Object.entries(failures).sort((a, b) => a[1].retry_after - b[1].retry_after);
+    const kept = Object.fromEntries(entries.slice(Math.max(0, entries.length - IMAGE_STORE_FAILURES_MAX)));
+    await writeJsonFileAtomic(imageStoreFailuresPath(stateDir), { schema_version: IMAGE_STORE_FAILURES_SCHEMA, failed: kept });
+}
+/** Remember one more refusal of this image and push its next try out along the same ladder the describe door uses. */
+export function rememberStoreFailure(failures, key, reason, permanent, now) {
+    const attempts = (failures[key]?.attempts ?? 0) + 1;
+    const record = { reason, attempts, retry_after: now + imageRetryDelayMs(attempts, permanent) };
+    failures[key] = record;
+    return record;
+}
+export function isBackingOff(failures, key, now) {
+    const record = failures[key];
+    return Boolean(record && record.retry_after > now);
+}

package/dist/slim/keep-line-copy.js

@@ -0,0 +1,34 @@
+/**
+ * Pass two of the streamed transcript when the machine keeps its images (`describe_and_keep`, BLI-23524).
+ *
+ * Keep mode never swaps an image and never slims, so the upload is the file itself. The drop modes' pass two
+ * (`slim-stream.ts`) rebuilds each line; this one copies each line, in order, to the same `LineSink`, so the file
+ * never sits in memory beside its decoded text and its re-encoded output (a 500 MB cap meant 2 to 3 copies).
+ *
+ * Byte identity with the whole-file read is by construction: the lines are cut at 0x0a (the same cut as splitting
+ * the decoded text at "\n" for valid UTF-8), joined again with "\n", and decoded with the same rules as
+ * `Buffer.toString("utf8")` (a BOM stays a character). A file that is not valid UTF-8 is not copied at all: the
+ * whole-file path uploads such a file's original bytes, and only a whole read can keep them, so the caller is told
+ * `invalid_utf8` and reads it whole. (The census pass has already refused such a file before any image was sent;
+ * this check is the same one at the copy, so a file rewritten between the passes cannot slip through.)
+ */
+import { TextDecoder } from "node:util";
+import { TranscriptNotStreamable } from "../adapters/transcript-text-stream.js";
+export async function copyLinesStream(lines, sink) {
+    const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
+    let count = 0;
+    for await (const { bytes } of lines) {
+        let line;
+        try {
+            line = decoder.decode(bytes);
+        }
+        catch {
+            throw new TranscriptNotStreamable("invalid_utf8");
+        }
+        if (count > 0)
+            await sink.write("\n");
+        count += 1;
+        await sink.write(line);
+    }
+    return { stats: null, replacedCount: 0, charsSaved: 0, changed: false, lines: count };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.277",
+  "version": "0.2.279",
   "private": false,
   "type": "module",
   "bin": {
@@ -27,9 +27,9 @@
     "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-surface-doors.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-exit-contract.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-cli-doctor-json.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
   },
   "dependencies": {
-    "@bli-cockpit/memory-mcp": "0.1.84",
-    "@bli-cockpit/mcp": "0.1.168",
-    "@bli-cockpit/telemetry-core": "0.1.91",
+    "@bli-cockpit/memory-mcp": "0.1.86",
+    "@bli-cockpit/mcp": "0.1.170",
+    "@bli-cockpit/telemetry-core": "0.1.93",
     "unpdf": "1.8.1"
   },
   "repository": {