@bli-cockpit/cli 0.2.277 → 0.2.279

package/dist/evidence-upload-object.js

@@ -1,10 +1,46 @@
 import { putSignedChunk } from "./evidence-upload-direct.js";
 import { compressEvidenceBytes } from "@bli-cockpit/telemetry-core/evidence-storage-encoding";
 import { chooseUploadEncoding } from "./evidence-upload-encoding.js";
-import { COMMIT_CRASHED_PLATFORM, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
+import { COMMIT_CRASHED_PLATFORM, EXTEND_BASE_UNAVAILABLE, RawEvidenceUploadCommitResponseSchema, isPermanentUploadFailure, } from "@bli-cockpit/telemetry-core";
 import { StagedFileError, readVerifiedChunk } from "./evidence-upload-staged.js";
 import { beginOneObject } from "./evidence-upload-terminal.js";
 import { isNonJsonResponseBody, requestJson, safeFailureDetail, } from "./evidence-upload-transport.js";
+/**
+ * The chunk plan this `begin` answer commits us to (BLI-23345): the tail plan when the server accepted
+ * exactly the one we offered, the whole file otherwise. A tail plan the server states differently from
+ * ours is not guessed at; the caller re-offers the whole object, which resets the row.
+ */
+function planForDisposition(entry, disposition) {
+    const accepted = disposition.extend;
+    const tail = entry.tail;
+    if (!accepted) {
+        if (tail) {
+            console.error("[evidence-upload] tail upload not used; sending the whole object", JSON.stringify({
+                reason: disposition.extend_refused ? `extend_refused:${disposition.extend_refused}` : "extend_not_answered",
+                upload_id: disposition.upload_id, byte_size: entry.byteSize,
+            }));
+        }
+        return { plan: "whole", view: entry };
+    }
+    if (!tail || accepted.base_object_key !== tail.base.object_key || accepted.base_content_hash_sha256 !== tail.base.content_hash_sha256
+        || accepted.base_byte_size !== tail.base.byte_size || accepted.tail_offset !== tail.offset
+        || accepted.tail_size !== tail.byteSize || accepted.chunk_count !== tail.chunkCount) {
+        return { plan: "mismatch" };
+    }
+    return { plan: "tail", view: { ...entry, byteSize: tail.byteSize, chunkHashes: tail.chunkHashes, chunkCount: tail.chunkCount, readOffset: tail.offset } };
+}
+/**
+ * Re-offer the whole object from the same staged file: begin without the tail plan resets the server's
+ * row to a whole plan, so nothing is lost and nothing is double-counted.
+ */
+async function reofferWhole(options, entry, chunkSizeBytes, restageAttempted, reason, uploadId) {
+    console.error("[evidence-upload] re-offering the whole object", JSON.stringify({ reason, upload_id: uploadId, byte_size: entry.byteSize }));
+    const wholeEntry = { ...entry, tail: undefined, readOffset: undefined, duplicates: [] };
+    const restarted = await beginOneObject(options, wholeEntry, chunkSizeBytes);
+    if (!restarted)
+        return failedOutcome(entry.file, `${reason}:begin_failed`);
+    return uploadOneObject(options, wholeEntry, restarted, chunkSizeBytes, restageAttempted);
+}
 export async function uploadOneObject(options, entry, disposition, chunkSizeBytes, restageAttempted = false) {
     const objectKey = entry.file.pointer.object_key ?? "";
     if (disposition.disposition === "already_committed") {
@@ -15,11 +51,19 @@ export async function uploadOneObject(options, entry, disposition, chunkSizeByte
     if (disposition.disposition === "conflict" || !disposition.upload_id) {
         return failedOutcome(entry.file, disposition.reason ?? "upload_conflict");
     }
+    const picked = planForDisposition(entry, disposition);
+    if (picked.plan === "mismatch")
+        return reofferWhole(options, entry, chunkSizeBytes, restageAttempted, "extend_plan_mismatch", disposition.upload_id);
+    const view = picked.view;
     const directChunks = [];
-    const chunked = await uploadObjectChunks(options, entry, disposition, chunkSizeBytes, objectKey, directChunks);
+    const chunked = await uploadObjectChunks(options, view, disposition, chunkSizeBytes, objectKey, directChunks, picked.plan, entry.byteSize);
     if (!chunked.ok)
         return chunked.outcome;
-    const committed = await commitUploadedObject(options, entry, disposition, objectKey, chunked.uploadedChunks, directChunks);
+    const committed = await commitUploadedObject(options, view, disposition, objectKey, chunked.uploadedChunks, directChunks);
+    if (picked.plan === "tail" && committed.reason === `commit_failed_http_409_${EXTEND_BASE_UNAVAILABLE}`) {
+        // The base was deleted (supersede) between begin and commit. The staged whole file is intact.
+        return reofferWhole(options, entry, chunkSizeBytes, restageAttempted, EXTEND_BASE_UNAVAILABLE, disposition.upload_id);
+    }
     if (committed.reason !== "commit_failed_http_409_staging_missing_between_passes")
         return committed;
     if (restageAttempted)
@@ -36,9 +80,10 @@ export async function uploadOneObject(options, entry, disposition, chunkSizeByte
     }
     return retried;
 }
-async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, objectKey, directChunks) {
+async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, objectKey, directChunks, plan, wholeByteSize) {
     const received = new Set(disposition.received_chunk_indexes);
     let uploadedChunks = 0;
+    const meter = { wireBytes: 0 };
     if (disposition.disposition === "already_committed" || disposition.commit_ready) {
         return { ok: true, uploadedChunks };
     }
@@ -55,7 +100,7 @@ async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, o
         try {
             // Re-read from disk by offset and check against the plan-time hash; the
             // buffer lives only for this iteration (BLI-23317).
-            chunk = await readVerifiedChunk(entry.file.local_path, index * chunkSizeBytes, Math.min(chunkSizeBytes, entry.byteSize - index * chunkSizeBytes), entry.chunkHashes[index]);
+            chunk = await readVerifiedChunk(entry.file.local_path, (entry.readOffset ?? 0) + index * chunkSizeBytes, Math.min(chunkSizeBytes, entry.byteSize - index * chunkSizeBytes), entry.chunkHashes[index]);
         }
         catch (error) {
             const reason = error instanceof StagedFileError ? error.reason : "file_read_failed";
@@ -67,7 +112,7 @@ async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, o
         }
         const offer = disposition.signed_uploads?.find(upload => upload.chunk_index === index);
         const direct = offer
-            ? await putSignedChunk(options, offer.signed_url, chunk, disposition.upload_id, index, encoding.codec)
+            ? await putSignedChunk(options, offer.signed_url, chunk, disposition.upload_id, index, encoding.codec, meter)
             : "refused";
         if (direct === "deadline") {
             // BLI-22801: the chunk route would push the same bytes over the same slow link.
@@ -82,19 +127,27 @@ async function uploadObjectChunks(options, entry, disposition, chunkSizeBytes, o
             uploadedChunks += 1;
             continue;
         }
-        const failure = await uploadChunkThroughEndpoint(options, entry, disposition, objectKey, index, chunk, uploadedChunks, Boolean(offer), encoding.codec);
+        const failure = await uploadChunkThroughEndpoint(options, entry, disposition, objectKey, index, chunk, uploadedChunks, Boolean(offer), encoding.codec, meter);
         if (failure)
             return failure;
         uploadedChunks += 1;
     }
+    // The number the ticket is measured by: compressed bytes that actually crossed the wire for this object,
+    // against the whole object it replaced (sizes and labels only).
+    console.error("[evidence-upload] object bytes sent", JSON.stringify({
+        reason: "object_bytes_sent", upload_id: disposition.upload_id, plan,
+        whole_byte_size: wholeByteSize, planned_byte_size: entry.byteSize, chunks_sent: uploadedChunks,
+        wire_bytes_sent: meter.wireBytes, chunk_encoding: encoding.codec,
+    }));
     return { ok: true, uploadedChunks };
 }
-async function uploadChunkThroughEndpoint(options, entry, disposition, objectKey, index, chunk, uploadedChunks, hadSignedOffer, codec) {
+async function uploadChunkThroughEndpoint(options, entry, disposition, objectKey, index, chunk, uploadedChunks, hadSignedOffer, codec, meter) {
     console.error("[evidence-upload] using chunk route", JSON.stringify({
         reason: hadSignedOffer ? "signed_upload_refused_fallback" : "signed_upload_not_offered",
         upload_id: disposition.upload_id, chunk_index: index,
     }));
     const encodedChunk = compressEvidenceBytes(chunk, codec);
+    meter.wireBytes += encodedChunk.length;
     console.error("[evidence-upload] chunk encoded", JSON.stringify({
         upload_id: disposition.upload_id, chunk_index: index, chunk_encoding: codec,
         bytes_before: chunk.length, bytes_after: encodedChunk.length,
@@ -152,6 +205,8 @@ async function commitUploadedObject(options, entry, disposition, objectKey, uplo
         upload_id: disposition.upload_id,
         object_key: objectKey,
         ...(disposition.signed_uploads ? { direct_chunks: directChunks } : {}),
+        // BLI-23329: the commit of an image object records this session as a user of the image.
+        ...(entry.file.image_use ? { image_use: entry.file.image_use } : {}),
     });
     if (!commit.ok)
         return rejectCommit(commit, entry, disposition, uploadedChunks);

package/dist/evidence-upload-plan.js

@@ -4,12 +4,13 @@ import { requestJson, safeFailureDetail } from "./evidence-upload-transport.js";
 import { failedOutcome, uploadOneObject } from "./evidence-upload-object.js";
 import { duplicateOutcome, rekeyConflictedEntry, reportAbandonedUpload, } from "./evidence-upload-terminal.js";
 import { describeError } from "./health-detail.js";
-import { StagedFileError, fingerprintStagedFile } from "./evidence-upload-staged.js";
+import { StagedFileError, fingerprintStagedFile, fingerprintStagedTail, } from "./evidence-upload-staged.js";
 export async function loadPlannedEntries(options, chunkSizeBytes, outcomes) {
     const loaded = [];
     const loadedByObjectKey = new Map();
     for (const file of options.files) {
-        const measured = await measureStagedUpload(file, chunkSizeBytes, outcomes);
+        const base = tailBaseFor(file, chunkSizeBytes);
+        const measured = await measureStagedUpload(file, chunkSizeBytes, outcomes, base?.byte_size);
         if (!measured)
             continue;
         // Content-addressed keys collide for byte-identical files (e.g. a resumed
@@ -28,6 +29,7 @@ export async function loadPlannedEntries(options, chunkSizeBytes, outcomes) {
             objectSha256: measured.objectSha256,
             chunkHashes: measured.chunkHashes,
             chunkCount: measured.chunkHashes.length,
+            ...(base ? await planTail(file, base, measured, chunkSizeBytes) : {}),
             duplicates: [],
         };
         loaded.push(entry);
@@ -37,7 +39,62 @@ export async function loadPlannedEntries(options, chunkSizeBytes, outcomes) {
     }
     return loaded;
 }
-async function measureStagedUpload(file, chunkSizeBytes, outcomes) {
+/** A transcript that extends a committed snapshot by at least one chunk's worth is worth a tail plan. */
+function tailBaseFor(file, chunkSizeBytes) {
+    const base = file.extend_base;
+    if (!base || (file.pointer.byte_size ?? 0) <= base.byte_size)
+        return null;
+    if (base.byte_size < chunkSizeBytes)
+        return null;
+    return base;
+}
+/** The tail plan, or nothing: the base must still be exactly the staged file's first bytes. */
+async function planTail(file, base, measured, chunkSizeBytes) {
+    if (measured.prefixSha256 !== base.content_hash_sha256) {
+        // The prepare step proved this prefix a moment ago; the file moved since. Whole upload, said out loud.
+        console.error("[evidence-upload] tail upload not planned; the staged file no longer starts with its base", JSON.stringify({
+            reason: "extend_prefix_changed", kind: file.kind ?? "unknown", byte_size: measured.byteSize, base_byte_size: base.byte_size,
+        }));
+        return {};
+    }
+    try {
+        const chunkHashes = await fingerprintStagedTail(file.local_path, measured.byteSize, base.byte_size, chunkSizeBytes);
+        console.error("[evidence-upload] tail planned", JSON.stringify({
+            reason: "extend_planned", kind: file.kind ?? "unknown", byte_size: measured.byteSize,
+            base_byte_size: base.byte_size, tail_byte_size: measured.byteSize - base.byte_size, tail_chunk_count: chunkHashes.length,
+        }));
+        return { tail: { base, offset: base.byte_size, byteSize: measured.byteSize - base.byte_size, chunkHashes, chunkCount: chunkHashes.length } };
+    }
+    catch (error) {
+        console.error("[evidence-upload] tail upload not planned; the staged file could not be read", JSON.stringify({
+            reason: "extend_tail_unreadable", byte_size: measured.byteSize,
+            detail: error instanceof StagedFileError ? error.reason : "read_error",
+        }));
+        return {};
+    }
+}
+/** One `begin` object. The tail plan rides only where `tailAllowed` (an old server answers 400 to the extra field). */
+export function beginObjectFor(entry, chunkSizeBytes, tailAllowed) {
+    const tail = tailAllowed ? entry.tail : undefined;
+    return {
+        pointer: entry.file.pointer,
+        chunk_size_bytes: chunkSizeBytes,
+        chunk_count: tail ? tail.chunkCount : entry.chunkCount,
+        ...(tail ? { extend: {
+                base_object_key: tail.base.object_key,
+                base_content_hash_sha256: tail.base.content_hash_sha256,
+                base_byte_size: tail.base.byte_size,
+            } } : {}),
+        // BLI-23329: an image-store object names the session it is staged for.
+        ...(entry.file.image_use ? { image_use: entry.file.image_use } : {}),
+    };
+}
+/** Runs that learned the server does not know tail plans; the answer holds for the rest of the run. */
+const serverWithoutTailPlans = new WeakSet();
+export function tailPlansAllowed(options) {
+    return !serverWithoutTailPlans.has(options);
+}
+async function measureStagedUpload(file, chunkSizeBytes, outcomes, prefixAt) {
     let stat;
     try {
         stat = await fs.stat(file.local_path);
@@ -64,7 +121,7 @@ async function measureStagedUpload(file, chunkSizeBytes, outcomes) {
         return null;
     }
     try {
-        const fingerprint = await fingerprintStagedFile(file.local_path, stat.size, chunkSizeBytes);
+        const fingerprint = await fingerprintStagedFile(file.local_path, stat.size, chunkSizeBytes, undefined, prefixAt);
         console.error("[evidence-upload] staged object measured", JSON.stringify({
             byte_size: fingerprint.byteSize,
             chunk_count: fingerprint.chunkHashes.length,
@@ -108,16 +165,23 @@ export function batches(items, size) {
  * so each branch stays a guard clause rather than nested conditionals.
  */
 export async function resolveBeginBatch(options, totalLoadedCount, batch, chunkSizeBytes, outcomes, resolvedEntries) {
+    const offersTails = tailPlansAllowed(options) && batch.some((entry) => entry.tail);
     const begin = await requestJson(options, "/api/ambient/evidence/upload/begin", {
         schema_version: "ambient-raw-evidence-upload-begin.v1",
         generated_at: options.generatedAt,
         provenance: options.provenance,
-        objects: batch.map((entry) => ({
-            pointer: entry.file.pointer,
-            chunk_size_bytes: chunkSizeBytes,
-            chunk_count: entry.chunkCount,
-        })),
+        objects: batch.map((entry) => beginObjectFor(entry, chunkSizeBytes, tailPlansAllowed(options))),
     });
+    if (offersTails && begin.status === 400) {
+        // A server from before BLI-23345 rejects the extra field. Ask again as every collector always has,
+        // and remember for the run; server deploys first, so this is the rollout window only.
+        serverWithoutTailPlans.add(options);
+        console.error("[evidence-upload] server does not know tail plans; offering whole objects", JSON.stringify({
+            reason: "extend_not_supported", http_status: begin.status, server_reason: safeFailureDetail(begin.body) ?? "none",
+            objects_in_batch: batch.length,
+        }));
+        return resolveBeginBatch(options, totalLoadedCount, batch, chunkSizeBytes, outcomes, resolvedEntries);
+    }
     // 404 means an old dashboard without the chunk routes; a persistent 5xx
     // (after retries) covers a new dashboard whose ledger migration has not
     // been applied yet. Both still serve the legacy v1 route.
@@ -241,6 +305,8 @@ export function readBeginDispositions(body) {
             received_chunk_indexes: entry.received_chunk_indexes,
             commit_ready: entry.commit_ready === true,
             signed_uploads: entry.signed_uploads,
+            ...(entry.extend ? { extend: entry.extend } : {}),
+            ...(entry.extend_refused ? { extend_refused: entry.extend_refused } : {}),
             reason: entry.reason ?? null,
             raw_evidence_pointer_id: entry.raw_evidence_pointer_id,
             ...(entry.accepted_encodings ? { accepted_encodings: entry.accepted_encodings } : {}),

package/dist/evidence-upload-staged.js

@@ -11,9 +11,14 @@ export class StagedFileError extends Error {
         this.detail = detail;
     }
 }
-/** Read exactly `length` bytes at `position`, looping over short reads. */
-export async function readExactly(handle, length, position) {
-    const buffer = Buffer.allocUnsafe(length);
+/**
+ * Read exactly `length` bytes at `position`, looping over short reads. With `into` (at least `length` bytes) the
+ * read lands in that buffer and the answer is a view of it, so a loop over a big file allocates once instead of
+ * once per chunk (BLI-23524: per-chunk buffers are garbage the collector reclaims lazily, and a 100 MB file
+ * left tens of MB of it resident).
+ */
+export async function readExactly(handle, length, position, into) {
+    const buffer = into ? into.subarray(0, length) : Buffer.allocUnsafe(length);
     let filled = 0;
     while (filled < length) {
         const { bytesRead } = await handle.read(buffer, filled, length - filled, position + filled);
@@ -28,25 +33,87 @@ export async function readExactly(handle, length, position) {
  * one chunk. Throws StagedFileError when the file vanishes or its length
  * moves while it is being read.
  */
-export async function fingerprintStagedFile(localPath, expectedSize, chunkSizeBytes, openFile = fs.open) {
+export async function fingerprintStagedFile(localPath, expectedSize, chunkSizeBytes, openFile = fs.open, prefixAt) {
     const handle = await openStaged(localPath, openFile);
     try {
         const whole = crypto.createHash("sha256");
         const chunkHashes = [];
+        // One buffer for every chunk: each is hashed before the next read, and none is kept.
+        const reused = Buffer.allocUnsafeSlow(Math.min(chunkSizeBytes, Math.max(expectedSize, 1)));
+        let prefixSha256;
         for (let offset = 0; offset < expectedSize; offset += chunkSizeBytes) {
             const want = Math.min(chunkSizeBytes, expectedSize - offset);
-            const chunk = await readExactly(handle, want, offset);
+            const chunk = await readExactly(handle, want, offset, reused);
             if (chunk.length !== want) {
                 throw new StagedFileError("staged_file_changed", "file shrank while planning");
             }
-            whole.update(chunk);
+            if (prefixAt !== undefined && prefixAt > offset && prefixAt <= offset + want) {
+                const cut = prefixAt - offset;
+                whole.update(chunk.subarray(0, cut));
+                prefixSha256 = whole.copy().digest("hex");
+                whole.update(chunk.subarray(cut));
+            }
+            else {
+                whole.update(chunk);
+            }
             chunkHashes.push(crypto.createHash("sha256").update(chunk).digest("hex"));
         }
         const probe = await readExactly(handle, 1, expectedSize);
         if (probe.length !== 0) {
             throw new StagedFileError("staged_file_changed", "file grew while planning");
         }
-        return { byteSize: expectedSize, objectSha256: whole.digest("hex"), chunkHashes };
+        return {
+            byteSize: expectedSize,
+            objectSha256: whole.digest("hex"),
+            chunkHashes,
+            ...(prefixSha256 !== undefined ? { prefixSha256 } : {}),
+        };
+    }
+    finally {
+        await handle.close();
+    }
+}
+/**
+ * The chunk hashes of the TAIL of a staged file, `[start, expectedSize)` cut into chunks that begin at
+ * `start` (BLI-23345). One pass over the tail only; memory is one chunk.
+ */
+export async function fingerprintStagedTail(localPath, expectedSize, start, chunkSizeBytes, openFile = fs.open) {
+    const handle = await openStaged(localPath, openFile);
+    try {
+        const chunkHashes = [];
+        const reused = Buffer.allocUnsafeSlow(Math.min(chunkSizeBytes, Math.max(expectedSize - start, 1)));
+        for (let offset = start; offset < expectedSize; offset += chunkSizeBytes) {
+            const want = Math.min(chunkSizeBytes, expectedSize - offset);
+            const chunk = await readExactly(handle, want, offset, reused);
+            if (chunk.length !== want) {
+                throw new StagedFileError("staged_file_changed", "file shrank while planning the tail");
+            }
+            chunkHashes.push(crypto.createHash("sha256").update(chunk).digest("hex"));
+        }
+        return chunkHashes;
+    }
+    finally {
+        await handle.close();
+    }
+}
+/**
+ * The file's bytes in order, `chunkBytes` at a time, every piece a view of the SAME buffer: valid until the next
+ * piece is asked for, so a consumer must use it (hash it, count it) before looping on. Memory is one chunk
+ * whatever the file's size; the descriptor closes when the consumer stops, however it stops. A failed read throws,
+ * never ends the stream early (BLI-23524).
+ */
+export async function* chunksOfFileReusingOneBuffer(localPath, chunkBytes) {
+    const handle = await openStaged(localPath, fs.open);
+    try {
+        const buffer = Buffer.allocUnsafeSlow(chunkBytes);
+        for (let position = 0;; position += chunkBytes) {
+            const { bytesRead } = await handle.read(buffer, 0, chunkBytes, position);
+            if (bytesRead === 0)
+                return;
+            yield buffer.subarray(0, bytesRead);
+            if (bytesRead < chunkBytes)
+                return;
+        }
     }
     finally {
         await handle.close();

package/dist/evidence-upload-terminal.js

@@ -1,5 +1,5 @@
-import { RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, isRekeyableUploadConflict, } from "@bli-cockpit/telemetry-core";
-import { readBeginDispositions } from "./evidence-upload-plan.js";
+import { RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, isImageObjectKey, isRekeyableUploadConflict, } from "@bli-cockpit/telemetry-core";
+import { beginObjectFor, readBeginDispositions, tailPlansAllowed } from "./evidence-upload-plan.js";
 import { rekeyedEvidencePointer } from "./evidence-upload-rekey.js";
 import { requestJson, safeFailureDetail } from "./evidence-upload-transport.js";
 /**
@@ -19,6 +19,9 @@ export async function rekeyConflictedEntry(options, entry, disposition, chunkSiz
         !isRekeyableUploadConflict(disposition.reason)) {
         return { entry, disposition };
     }
+    // BLI-23329: an image object's name IS its sha256; a different name would not be an image object any more.
+    if (isImageObjectKey(entry.file.pointer.object_key ?? ""))
+        return { entry, disposition };
     const contentHashPrefix = (entry.file.pointer.content_hash_sha256 ?? "none").slice(0, 12);
     const rekeyedPointer = rekeyedEvidencePointer(entry.file.pointer);
     if (!rekeyedPointer) {
@@ -68,13 +71,7 @@ export async function beginOneObject(options, entry, chunkSizeBytes) {
         schema_version: "ambient-raw-evidence-upload-begin.v1",
         generated_at: options.generatedAt,
         provenance: options.provenance,
-        objects: [
-            {
-                pointer: entry.file.pointer,
-                chunk_size_bytes: chunkSizeBytes,
-                chunk_count: entry.chunkCount,
-            },
-        ],
+        objects: [beginObjectFor(entry, chunkSizeBytes, tailPlansAllowed(options))],
     });
     if (!response.ok) {
         console.error("[evidence-rekey] begin refused the re-keyed object; keeping the original conflict", JSON.stringify({

package/dist/image-description-session.js

@@ -6,8 +6,10 @@
  * or descriptions. The sends themselves (three at a time, budgets, backoff) are in
  * `image-description-send.ts`.
  */
-import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, base64Of, findEmbeddedImages, formatImageMarker, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
+import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, IMAGE_STORE_MAX_BYTES, base64Of, findEmbeddedImages, formatImageMarker, modeDropsImages, modeStoresImages, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
 import { sendPendingImages } from "./image-description-send.js";
+import { hasStoreRoom, isStoredThisRun, openImageStoreRun, storeImages } from "./image-store-send.js";
+import { IMAGE_STORE_BLOCK_CHARS, blocksOfText } from "./image-store-stage.js";
 import { resolveImageHandlingMode } from "./image-mode-state.js";
 import { censusOfLines } from "./image-census-stream.js";
 import { sha256OfBase64Run } from "./image-byte-scan.js";
@@ -15,6 +17,7 @@ import { addPdfsToFinds, mergeOccurrences, pdfsOfText } from "./pdf-transcript-s
 import { openPinnedTranscript } from "./adapters/pinned-transcript.js";
 import { openStagedOutput, readStagedText } from "./adapters/staged-output.js";
 import { TranscriptNotStreamable } from "./adapters/transcript-text-stream.js";
+import { copyLinesStream } from "./slim/keep-line-copy.js";
 import { LineBytesSink, slimLineStream } from "./slim/slim-stream.js";
 import { logSlimmed, slimOrKeep, slimPassOrNull, slimTranscript } from "./slim/slim-transcript.js";
 import { imageStateKey, readDescribedImageKeys, readFailedImages, writeImageDescriptionState, } from "./image-description-state.js";
@@ -40,21 +43,25 @@ export async function openImageDescriptions(options) {
     const maxPerRun = options.maxPerRun ?? IMAGE_DESCRIBE_MAX_PER_RUN;
     const maxRunMs = options.maxRunMs ?? IMAGE_DESCRIBE_MAX_RUN_MS;
     const sentThisRun = 0;
-    const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now, describingMs: 0, maxPerRun, maxRunMs, sentThisRun };
+    const store = modeStoresImages(mode)
+        ? await openImageStoreRun(options.stateDir, { ...(options.storeMaxUploads === undefined ? {} : { maxUploads: options.storeMaxUploads }), ...(options.storeMaxConfirms === undefined ? {} : { maxConfirms: options.storeMaxConfirms }), ...(options.storeMaxRunMs === undefined ? {} : { maxRunMs: options.storeMaxRunMs }) })
+        : undefined;
+    const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now, describingMs: 0, maxPerRun, maxRunMs, sentThisRun, ...(store ? { store } : {}) };
     const streamMinBytes = options.streamMinBytes ?? STREAM_MIN_BYTES;
     return {
         mode,
         streamMinBytes,
         isDescribed: (sha256) => described.has(imageStateKey(sha256)),
         hasDescribeRoom: (sha256s) => describeRoomLeft(run, sha256s),
+        hasStoreRoom: (sha256s) => (run.store ? hasStoreRoom(run.store, sha256s, now()) : false),
         async prepareTranscript(input) {
-            // BLI-23347: in drop mode each base64 PDF becomes a marker block (its text, or its described page images).
+            // BLI-23347: in a drop mode (describe_and_drop, store_and_drop) each base64 PDF becomes a marker block (its text, or its described page images).
             const imageFinds = findsOfText(input.text);
-            const finds0 = mode === "describe_and_drop"
+            const finds0 = modeDropsImages(mode)
                 ? await addPdfsToFinds(imageFinds, pdfsOfText(input.text), pdfContext(run, { stateDir: options.stateDir, read: async (pdf) => input.text.slice(pdf.base64Start, pdf.base64End), changed: () => new Error("pdf_changed_in_text") }))
                 : imageFinds;
             const { replace, finds, ...afterImages } = await describeAndReplace(run, input, finds0);
-            if (mode !== "describe_and_drop")
+            if (!modeDropsImages(mode))
                 return { ...afterImages, text: input.text, changed: false, slim: null };
             // BLI-23317: the image swap and the slim rules are ONE pass over the lines (one joined string, no
             // intermediate copy). A pass that cannot run falls back to the two-step order, which is the same bytes.
@@ -66,7 +73,7 @@ export async function openImageDescriptions(options) {
             const slim = slimOrKeep(replaced.text, input.source, { sessionId: input.sessionId });
             return { ...afterImages, text: slim.text, changed: slim.text !== input.text, slim: slim.stats };
         },
-        prepareTranscriptFile: mode === "describe_and_drop" ? (input) => prepareFileInTwoPasses(run, input) : undefined,
+        prepareTranscriptFile: (input) => prepareFileInTwoPasses(run, input),
     };
 }
 /** Files smaller than this are cheaper to read whole; the streamed path pays a second read. */
@@ -82,9 +89,17 @@ function findsOfText(text) {
         unique,
         images,
         base64For: (image) => base64Of(text, image),
+        blocksOf: (image) => blocksOfText(text, image),
         swapped: (replace) => countImageSwaps({ images, replace }),
     };
 }
+/** The base64 of an image from the pinned file, in blocks of `IMAGE_STORE_BLOCK_CHARS`; each read is bounded, never the whole run. */
+async function* blocksOfPinned(source, image) {
+    for (let at = image.base64Start; at < image.base64End; at += IMAGE_STORE_BLOCK_CHARS) {
+        const block = await source.read(at, Math.min(image.base64End, at + IMAGE_STORE_BLOCK_CHARS) - at);
+        yield block.toString("latin1");
+    }
+}
 function findsOfCensus(census, source) {
     return {
         total: census.total,
@@ -98,6 +113,8 @@ function findsOfCensus(census, source) {
                 throw new TranscriptNotStreamable("file_changed_during_pass");
             return payload.toString("latin1");
         },
+        // The same pinned bytes, in 48 Ki-character blocks, so a large image is decoded to a file without being held whole.
+        blocksOf: (image) => blocksOfPinned(source, image),
         swapped: (replace) => {
             let replacedCount = 0;
             let charsSaved = 0;
@@ -123,7 +140,9 @@ async function prepareFileInTwoPasses(run, input) {
         const census = await censusOfLines(source.lines());
         // Describe attempts finish and the swap decisions are settled before a single line is swapped, as with the whole text.
         // BLI-23347: the census also found the PDFs; they are converted here and swapped in the same pass as the images.
-        const pdfFinds = await addPdfsToFinds(findsOfCensus(census, source), census.pdfs, pdfContext(run, {
+        const keeps = !modeDropsImages(run.mode); // BLI-23524: nothing is swapped, so no PDFs and pass two is a copy
+        const imageFinds = findsOfCensus(census, source);
+        const pdfFinds = keeps ? imageFinds : await addPdfsToFinds(imageFinds, census.pdfs, pdfContext(run, {
             stateDir: run.options.stateDir,
             read: (pdf) => source.read(pdf.base64Start, pdf.base64End - pdf.base64Start),
             changed: () => new TranscriptNotStreamable("file_changed_during_pass"),
@@ -132,14 +151,15 @@ async function prepareFileInTwoPasses(run, input) {
         // Pass two: swap and slim over exactly the bytes pass one saw, writing the result to staging as it goes.
         writer = input.stageDir ? await openStagedOutput(input.stageDir) : undefined;
         const memory = writer ? undefined : new LineBytesSink(source.length);
-        const slim = await slimLineStream(source.lines(), input.source, mergeOccurrences(census.occurrences, finds.extraOccurrences), replace, writer ?? memory);
+        const slim = keeps ? await copyLinesStream(source.lines(), writer ?? memory) : await slimLineStream(source.lines(), input.source, mergeOccurrences(census.occurrences, finds.extraOccurrences), replace, writer ?? memory);
         if (slim.replacedCount !== afterImages.replacedCount)
             throw new TranscriptNotStreamable("images_changed_between_passes");
         await source.verifyUnchanged();
         const staged = await writer?.finish();
         // The sanitizer reads the whole text: from the staged file when one was written, else from the sink's buffer.
         const text = staged ? (input.omitText ? "" : await readStagedText(staged)) : memory.text();
-        logSlimmed(input.source, input.sessionId, slim.stats, startedAt);
+        if (slim.stats)
+            logSlimmed(input.source, input.sessionId, slim.stats, startedAt);
         return { ...afterImages, text, changed: slim.changed, slim: slim.stats, readBytes: source.length, ...(staged ? { staged } : {}) };
     }
     catch (error) {
@@ -164,8 +184,9 @@ function describeRoomLeft(run, sha256s) {
     });
 }
 async function describeAndReplace(run, input, finds) {
-    const { mode, described, options, orderedKeys, failedImages } = run;
-    const replace = (image) => described.has(imageStateKey(image.sha256));
+    const { mode, described, options, orderedKeys, failedImages, store } = run;
+    // describe_and_drop replaces what has a description; store_and_drop replaces what Tower confirmed for THIS session this tick.
+    const replace = (image) => store ? isStoredThisRun(store, image.sha256, input.sessionId) : described.has(imageStateKey(image.sha256));
     const { unique } = finds;
     if (finds.total === 0)
         return { replace, finds, pendingImages: [], imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
@@ -177,14 +198,26 @@ async function describeAndReplace(run, input, finds) {
         await writeImageDescriptionState(options.stateDir, { described: orderedKeys, failed: failedImages });
         run.stateDirty = false;
     }
+    // store_and_drop: after describing, before the swap. Whatever Tower does not confirm stays inline (BLI-23329).
+    const stored = store
+        ? await storeImages(store, {
+            fetchImpl: run.fetchImpl, dashboardUrl: options.dashboardUrl, deviceToken: options.deviceToken, stateDir: options.stateDir,
+            ...(options.sleep ? { sleep: options.sleep } : {}), now: run.now, source: input.source, sessionId: input.sessionId,
+            provenance: input.provenance, images: [...unique.values()], blocksOf: finds.blocksOf,
+        })
+        : null;
     const deferredOnly = counts.over_budget + counts.backing_off > 0
         && counts.described_now + counts.known + counts.failed + counts.too_large + counts.settled_earlier === 0;
-    const replaced = mode === "describe_and_drop" ? finds.swapped(replace) : { replacedCount: 0, charsSaved: 0 };
+    const replaced = modeDropsImages(mode) ? finds.swapped(replace) : { replacedCount: 0, charsSaved: 0 };
     // BLI-23317: an image over the describe limit is never sent, so it can never gain a description and
     // never changes the output. Listing it made the describer look like it still had room, and the
     // transcript was rebuilt on every tick (rebuilt_describer_has_room). Counted in `too_large` above.
+    // store_and_drop: what is still pending is every image Tower did not confirm this tick (described or not), unless it is
+    // over the store's size limit and so can never be stored; the memo reuses the entry only while there is no room to store.
     const pendingImages = [...unique.values()]
-        .filter((image) => !described.has(imageStateKey(image.sha256)) && image.byteSize <= IMAGE_DESCRIBE_MAX_BYTES)
+        .filter((image) => store
+        ? !isStoredThisRun(store, image.sha256, input.sessionId) && image.byteSize <= IMAGE_STORE_MAX_BYTES
+        : !described.has(imageStateKey(image.sha256)) && image.byteSize <= IMAGE_DESCRIBE_MAX_BYTES)
         .map((image) => image.sha256);
     console.error(TAG + " transcript prepared", JSON.stringify({
         mode,
@@ -196,7 +229,8 @@ async function describeAndReplace(run, input, finds) {
         failed_reasons: failedReasons,
         replaced: replaced.replacedCount,
         chars_saved: replaced.charsSaved,
-        kept_without_description: mode === "describe_and_drop" ? finds.total - replaced.replacedCount : 0,
+        kept_without_description: modeDropsImages(mode) ? finds.total - replaced.replacedCount : 0,
+        ...(stored ? { store: stored } : {}),
     }));
     return {
         replace,
@@ -281,7 +315,7 @@ function countImageOutcomes(run, toSend, counts) {
  * Drop mode replaces known images, then slims; keep mode preserves all text.
  */
 export function uploadFormOfTranscript(text, source, mode, describedKeys) {
-    if (mode !== "describe_and_drop")
+    if (!modeDropsImages(mode))
         return text;
     const images = findEmbeddedImages(text);
     const replaced = replaceImagesWithMarkers(text, images, (image) => describedKeys.has(imageStateKey(image.sha256)));

package/dist/image-mode-state.js

@@ -13,20 +13,20 @@
  *                                                 transcript as a new object)
  *   else `IMAGE_HANDLING_MODE`
  *
- * The file also remembers, sticky, that this machine EVER ran slim (`drop_seen_at`). A machine put
+ * The file also remembers, sticky, that this machine EVER ran slim (`drop_seen_at`; `store_and_drop` is slim too, BLI-23329). A machine put
  * back on keep still holds slim server copies of the files it uploaded while slim, and
  * `cockpit sources check` must go on treating those as "keep your local file".
  *
  * Holds the mode and two timestamps. Never an id, a path or a byte.
  */
 import path from "node:path";
-import { IMAGE_HANDLING_MODE } from "@bli-cockpit/telemetry-core";
+import { IMAGE_HANDLING_MODE, modeDropsImages } from "@bli-cockpit/telemetry-core";
 import { describeError, isMissingFileFailure } from "./health-detail.js";
 import { readJsonFile, writeJsonFileAtomic } from "./local-state-files.js";
 export const IMAGE_MODE_STATE_FILENAME = "image-handling-mode.json";
 const TAG = "[image-mode]";
 export function isImageHandlingMode(value) {
-    return value === "describe_and_keep" || value === "describe_and_drop";
+    return value === "describe_and_keep" || value === "describe_and_drop" || value === "store_and_drop";
 }
 export function imageModeStatePath(stateDir) {
     return path.join(stateDir, IMAGE_MODE_STATE_FILENAME);
@@ -87,7 +87,7 @@ export async function resolveImageHandlingMode(stateDir, read) {
         mode,
         source: state ? "cache" : damaged ? "damaged_cache" : "default",
         // A file that is there but unreadable is "ever slim, unknown": fail closed (BLI-17273).
-        slimAware: mode === "describe_and_drop" || state?.drop_seen_at != null || damaged,
+        slimAware: modeDropsImages(mode) || state?.drop_seen_at != null || damaged,
         state,
     };
 }
@@ -109,7 +109,7 @@ export async function recordObservedImageMode(stateDir, observed, now = () => ne
         mode: observed,
         observed_at: stamp,
         // Never rebuild "never ran slim" over a file we could not read: keep what parsed, else the unknown marker.
-        drop_seen_at: previous?.drop_seen_at ?? (damaged ? (recoveredDropSeenAt ?? DROP_SEEN_UNKNOWN) : observed === "describe_and_drop" ? stamp : null),
+        drop_seen_at: previous?.drop_seen_at ?? (damaged ? (recoveredDropSeenAt ?? DROP_SEEN_UNKNOWN) : modeDropsImages(observed) ? stamp : null),
     };
     if (damaged) {
         console.error(TAG + " rewriting a damaged mode file; ever-slim history kept", JSON.stringify({ reason: "image_mode_cache_damaged_rewritten", drop_seen_recovered: recoveredDropSeenAt !== null }));