@bli-cockpit/cli 0.2.276 → 0.2.278

package/dist/commands/sources-verdict.js

@@ -38,7 +38,7 @@ import path from "node:path";
 import { RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES, REDACTION_STUB_SHA256, } from "@bli-cockpit/telemetry-core";
 import { isWholeFileMask, sanitizeTextEvidenceForUpload } from "../adapters/raw-evidence-sanitize.js";
 import { isSecretLikePath } from "../adapters/raw-evidence-keys.js";
-import { IMAGE_HANDLING_MODE } from "@bli-cockpit/telemetry-core";
+import { IMAGE_HANDLING_MODE, modeDropsImages } from "@bli-cockpit/telemetry-core";
 import { uploadFormOfTranscript } from "../image-description-session.js";
 export async function judgeSource(input) {
     const { file, remotes, cursorEntry } = input;
@@ -161,7 +161,7 @@ export function fileFacts(file, maxBytes = RAW_EVIDENCE_UPLOAD_MAX_FILE_BYTES,
 upload = {}) {
     const mode = upload.mode ?? IMAGE_HANDLING_MODE;
     const describedKeys = upload.describedKeys ?? new Set();
-    const slimAware = upload.slimAware ?? mode === "describe_and_drop";
+    const slimAware = upload.slimAware ?? modeDropsImages(mode);
     const redactForUpload = (text, original) => redactTranscriptForm(file, text, original);
     let split = null;
     let whole = null;

package/dist/evidence-upload-object.js

@@ -152,6 +152,8 @@ async function commitUploadedObject(options, entry, disposition, objectKey, uplo
         upload_id: disposition.upload_id,
         object_key: objectKey,
         ...(disposition.signed_uploads ? { direct_chunks: directChunks } : {}),
+        // BLI-23329: the commit of an image object records this session as a user of the image.
+        ...(entry.file.image_use ? { image_use: entry.file.image_use } : {}),
     });
     if (!commit.ok)
         return rejectCommit(commit, entry, disposition, uploadedChunks);

package/dist/evidence-upload-plan.js

@@ -116,6 +116,8 @@ export async function resolveBeginBatch(options, totalLoadedCount, batch, chunkS
             pointer: entry.file.pointer,
             chunk_size_bytes: chunkSizeBytes,
             chunk_count: entry.chunkCount,
+            // BLI-23329: an image-store object names the session it is staged for.
+            ...(entry.file.image_use ? { image_use: entry.file.image_use } : {}),
         })),
     });
     // 404 means an old dashboard without the chunk routes; a persistent 5xx

package/dist/evidence-upload-terminal.js

@@ -1,4 +1,4 @@
-import { RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, isRekeyableUploadConflict, } from "@bli-cockpit/telemetry-core";
+import { RawEvidenceRedactionMetadataSchema, RawEvidenceUploadBeginResponseSchema, isImageObjectKey, isRekeyableUploadConflict, } from "@bli-cockpit/telemetry-core";
 import { readBeginDispositions } from "./evidence-upload-plan.js";
 import { rekeyedEvidencePointer } from "./evidence-upload-rekey.js";
 import { requestJson, safeFailureDetail } from "./evidence-upload-transport.js";
@@ -19,6 +19,9 @@ export async function rekeyConflictedEntry(options, entry, disposition, chunkSiz
         !isRekeyableUploadConflict(disposition.reason)) {
         return { entry, disposition };
     }
+    // BLI-23329: an image object's name IS its sha256; a different name would not be an image object any more.
+    if (isImageObjectKey(entry.file.pointer.object_key ?? ""))
+        return { entry, disposition };
     const contentHashPrefix = (entry.file.pointer.content_hash_sha256 ?? "none").slice(0, 12);
     const rekeyedPointer = rekeyedEvidencePointer(entry.file.pointer);
     if (!rekeyedPointer) {
@@ -73,6 +76,7 @@ export async function beginOneObject(options, entry, chunkSizeBytes) {
                 pointer: entry.file.pointer,
                 chunk_size_bytes: chunkSizeBytes,
                 chunk_count: entry.chunkCount,
+                ...(entry.file.image_use ? { image_use: entry.file.image_use } : {}),
             },
         ],
     });

package/dist/image-description-session.js

@@ -6,8 +6,10 @@
  * or descriptions. The sends themselves (three at a time, budgets, backoff) are in
  * `image-description-send.ts`.
  */
-import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, base64Of, findEmbeddedImages, formatImageMarker, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
+import { IMAGE_DESCRIBE_MAX_BYTES, IMAGE_HANDLING_MODE, IMAGE_STORE_MAX_BYTES, base64Of, findEmbeddedImages, formatImageMarker, modeDropsImages, modeStoresImages, replaceImagesWithMarkers, } from "@bli-cockpit/telemetry-core";
 import { sendPendingImages } from "./image-description-send.js";
+import { hasStoreRoom, isStoredThisRun, openImageStoreRun, storeImages } from "./image-store-send.js";
+import { IMAGE_STORE_BLOCK_CHARS, blocksOfText } from "./image-store-stage.js";
 import { resolveImageHandlingMode } from "./image-mode-state.js";
 import { censusOfLines } from "./image-census-stream.js";
 import { sha256OfBase64Run } from "./image-byte-scan.js";
@@ -40,21 +42,25 @@ export async function openImageDescriptions(options) {
     const maxPerRun = options.maxPerRun ?? IMAGE_DESCRIBE_MAX_PER_RUN;
     const maxRunMs = options.maxRunMs ?? IMAGE_DESCRIBE_MAX_RUN_MS;
     const sentThisRun = 0;
-    const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now, describingMs: 0, maxPerRun, maxRunMs, sentThisRun };
+    const store = modeStoresImages(mode)
+        ? await openImageStoreRun(options.stateDir, { ...(options.storeMaxUploads === undefined ? {} : { maxUploads: options.storeMaxUploads }), ...(options.storeMaxConfirms === undefined ? {} : { maxConfirms: options.storeMaxConfirms }), ...(options.storeMaxRunMs === undefined ? {} : { maxRunMs: options.storeMaxRunMs }) })
+        : undefined;
+    const run = { options, mode, fetchImpl, described, orderedKeys, failedImages, stateDirty, settledThisRun, now, describingMs: 0, maxPerRun, maxRunMs, sentThisRun, ...(store ? { store } : {}) };
     const streamMinBytes = options.streamMinBytes ?? STREAM_MIN_BYTES;
     return {
         mode,
         streamMinBytes,
         isDescribed: (sha256) => described.has(imageStateKey(sha256)),
         hasDescribeRoom: (sha256s) => describeRoomLeft(run, sha256s),
+        hasStoreRoom: (sha256s) => (run.store ? hasStoreRoom(run.store, sha256s, now()) : false),
         async prepareTranscript(input) {
-            // BLI-23347: in drop mode each base64 PDF becomes a marker block (its text, or its described page images).
+            // BLI-23347: in a drop mode (describe_and_drop, store_and_drop) each base64 PDF becomes a marker block (its text, or its described page images).
             const imageFinds = findsOfText(input.text);
-            const finds0 = mode === "describe_and_drop"
+            const finds0 = modeDropsImages(mode)
                 ? await addPdfsToFinds(imageFinds, pdfsOfText(input.text), pdfContext(run, { stateDir: options.stateDir, read: async (pdf) => input.text.slice(pdf.base64Start, pdf.base64End), changed: () => new Error("pdf_changed_in_text") }))
                 : imageFinds;
             const { replace, finds, ...afterImages } = await describeAndReplace(run, input, finds0);
-            if (mode !== "describe_and_drop")
+            if (!modeDropsImages(mode))
                 return { ...afterImages, text: input.text, changed: false, slim: null };
             // BLI-23317: the image swap and the slim rules are ONE pass over the lines (one joined string, no
             // intermediate copy). A pass that cannot run falls back to the two-step order, which is the same bytes.
@@ -66,7 +72,7 @@ export async function openImageDescriptions(options) {
             const slim = slimOrKeep(replaced.text, input.source, { sessionId: input.sessionId });
             return { ...afterImages, text: slim.text, changed: slim.text !== input.text, slim: slim.stats };
         },
-        prepareTranscriptFile: mode === "describe_and_drop" ? (input) => prepareFileInTwoPasses(run, input) : undefined,
+        prepareTranscriptFile: modeDropsImages(mode) ? (input) => prepareFileInTwoPasses(run, input) : undefined,
     };
 }
 /** Files smaller than this are cheaper to read whole; the streamed path pays a second read. */
@@ -82,9 +88,17 @@ function findsOfText(text) {
         unique,
         images,
         base64For: (image) => base64Of(text, image),
+        blocksOf: (image) => blocksOfText(text, image),
         swapped: (replace) => countImageSwaps({ images, replace }),
     };
 }
+/** The base64 of an image from the pinned file, in blocks of `IMAGE_STORE_BLOCK_CHARS`; each read is bounded, never the whole run. */
+async function* blocksOfPinned(source, image) {
+    for (let at = image.base64Start; at < image.base64End; at += IMAGE_STORE_BLOCK_CHARS) {
+        const block = await source.read(at, Math.min(image.base64End, at + IMAGE_STORE_BLOCK_CHARS) - at);
+        yield block.toString("latin1");
+    }
+}
 function findsOfCensus(census, source) {
     return {
         total: census.total,
@@ -98,6 +112,8 @@ function findsOfCensus(census, source) {
                 throw new TranscriptNotStreamable("file_changed_during_pass");
             return payload.toString("latin1");
         },
+        // The same pinned bytes, in 48 Ki-character blocks, so a large image is decoded to a file without being held whole.
+        blocksOf: (image) => blocksOfPinned(source, image),
         swapped: (replace) => {
             let replacedCount = 0;
             let charsSaved = 0;
@@ -164,8 +180,9 @@ function describeRoomLeft(run, sha256s) {
     });
 }
 async function describeAndReplace(run, input, finds) {
-    const { mode, described, options, orderedKeys, failedImages } = run;
-    const replace = (image) => described.has(imageStateKey(image.sha256));
+    const { mode, described, options, orderedKeys, failedImages, store } = run;
+    // describe_and_drop replaces what has a description; store_and_drop replaces what Tower confirmed for THIS session this tick.
+    const replace = (image) => store ? isStoredThisRun(store, image.sha256, input.sessionId) : described.has(imageStateKey(image.sha256));
     const { unique } = finds;
     if (finds.total === 0)
         return { replace, finds, pendingImages: [], imageCount: 0, uniqueCount: 0, replacedCount: 0, charsSaved: 0, deferredOnly: false };
@@ -177,14 +194,26 @@ async function describeAndReplace(run, input, finds) {
         await writeImageDescriptionState(options.stateDir, { described: orderedKeys, failed: failedImages });
         run.stateDirty = false;
     }
+    // store_and_drop: after describing, before the swap. Whatever Tower does not confirm stays inline (BLI-23329).
+    const stored = store
+        ? await storeImages(store, {
+            fetchImpl: run.fetchImpl, dashboardUrl: options.dashboardUrl, deviceToken: options.deviceToken, stateDir: options.stateDir,
+            ...(options.sleep ? { sleep: options.sleep } : {}), now: run.now, source: input.source, sessionId: input.sessionId,
+            provenance: input.provenance, images: [...unique.values()], blocksOf: finds.blocksOf,
+        })
+        : null;
     const deferredOnly = counts.over_budget + counts.backing_off > 0
         && counts.described_now + counts.known + counts.failed + counts.too_large + counts.settled_earlier === 0;
-    const replaced = mode === "describe_and_drop" ? finds.swapped(replace) : { replacedCount: 0, charsSaved: 0 };
+    const replaced = modeDropsImages(mode) ? finds.swapped(replace) : { replacedCount: 0, charsSaved: 0 };
     // BLI-23317: an image over the describe limit is never sent, so it can never gain a description and
     // never changes the output. Listing it made the describer look like it still had room, and the
     // transcript was rebuilt on every tick (rebuilt_describer_has_room). Counted in `too_large` above.
+    // store_and_drop: what is still pending is every image Tower did not confirm this tick (described or not), unless it is
+    // over the store's size limit and so can never be stored; the memo reuses the entry only while there is no room to store.
     const pendingImages = [...unique.values()]
-        .filter((image) => !described.has(imageStateKey(image.sha256)) && image.byteSize <= IMAGE_DESCRIBE_MAX_BYTES)
+        .filter((image) => store
+        ? !isStoredThisRun(store, image.sha256, input.sessionId) && image.byteSize <= IMAGE_STORE_MAX_BYTES
+        : !described.has(imageStateKey(image.sha256)) && image.byteSize <= IMAGE_DESCRIBE_MAX_BYTES)
         .map((image) => image.sha256);
     console.error(TAG + " transcript prepared", JSON.stringify({
         mode,
@@ -196,7 +225,8 @@ async function describeAndReplace(run, input, finds) {
         failed_reasons: failedReasons,
         replaced: replaced.replacedCount,
         chars_saved: replaced.charsSaved,
-        kept_without_description: mode === "describe_and_drop" ? finds.total - replaced.replacedCount : 0,
+        kept_without_description: modeDropsImages(mode) ? finds.total - replaced.replacedCount : 0,
+        ...(stored ? { store: stored } : {}),
     }));
     return {
         replace,
@@ -281,7 +311,7 @@ function countImageOutcomes(run, toSend, counts) {
  * Drop mode replaces known images, then slims; keep mode preserves all text.
  */
 export function uploadFormOfTranscript(text, source, mode, describedKeys) {
-    if (mode !== "describe_and_drop")
+    if (!modeDropsImages(mode))
         return text;
     const images = findEmbeddedImages(text);
     const replaced = replaceImagesWithMarkers(text, images, (image) => describedKeys.has(imageStateKey(image.sha256)));

package/dist/image-mode-state.js

@@ -13,20 +13,20 @@
  *                                                 transcript as a new object)
  *   else `IMAGE_HANDLING_MODE`
  *
- * The file also remembers, sticky, that this machine EVER ran slim (`drop_seen_at`). A machine put
+ * The file also remembers, sticky, that this machine EVER ran slim (`drop_seen_at`; `store_and_drop` is slim too, BLI-23329). A machine put
  * back on keep still holds slim server copies of the files it uploaded while slim, and
  * `cockpit sources check` must go on treating those as "keep your local file".
  *
  * Holds the mode and two timestamps. Never an id, a path or a byte.
  */
 import path from "node:path";
-import { IMAGE_HANDLING_MODE } from "@bli-cockpit/telemetry-core";
+import { IMAGE_HANDLING_MODE, modeDropsImages } from "@bli-cockpit/telemetry-core";
 import { describeError, isMissingFileFailure } from "./health-detail.js";
 import { readJsonFile, writeJsonFileAtomic } from "./local-state-files.js";
 export const IMAGE_MODE_STATE_FILENAME = "image-handling-mode.json";
 const TAG = "[image-mode]";
 export function isImageHandlingMode(value) {
-    return value === "describe_and_keep" || value === "describe_and_drop";
+    return value === "describe_and_keep" || value === "describe_and_drop" || value === "store_and_drop";
 }
 export function imageModeStatePath(stateDir) {
     return path.join(stateDir, IMAGE_MODE_STATE_FILENAME);
@@ -87,7 +87,7 @@ export async function resolveImageHandlingMode(stateDir, read) {
         mode,
         source: state ? "cache" : damaged ? "damaged_cache" : "default",
         // A file that is there but unreadable is "ever slim, unknown": fail closed (BLI-17273).
-        slimAware: mode === "describe_and_drop" || state?.drop_seen_at != null || damaged,
+        slimAware: modeDropsImages(mode) || state?.drop_seen_at != null || damaged,
         state,
     };
 }
@@ -109,7 +109,7 @@ export async function recordObservedImageMode(stateDir, observed, now = () => ne
         mode: observed,
         observed_at: stamp,
         // Never rebuild "never ran slim" over a file we could not read: keep what parsed, else the unknown marker.
-        drop_seen_at: previous?.drop_seen_at ?? (damaged ? (recoveredDropSeenAt ?? DROP_SEEN_UNKNOWN) : observed === "describe_and_drop" ? stamp : null),
+        drop_seen_at: previous?.drop_seen_at ?? (damaged ? (recoveredDropSeenAt ?? DROP_SEEN_UNKNOWN) : modeDropsImages(observed) ? stamp : null),
     };
     if (damaged) {
         console.error(TAG + " rewriting a damaged mode file; ever-slim history kept", JSON.stringify({ reason: "image_mode_cache_damaged_rewritten", drop_seen_recovered: recoveredDropSeenAt !== null }));

package/dist/image-store-send.js

@@ -0,0 +1,326 @@
+/**
+ * The store step of `store_and_drop` (BLI-23329): put each distinct image of a transcript on Tower once, and say which
+ * ones Tower confirmed THIS tick, so only those are replaced by their marker.
+ *
+ * "Stored" means one thing here: Tower answered `already_committed` (begin) or `committed` / `already_committed`
+ * (commit) for this image AND this session, and the server records the session as a user of the image in the same step
+ * that answers (`apps/dashboard/src/lib/ambient/image-store/image-store-door.ts`). Nothing local is trusted: no list of
+ * "stored" keys survives the tick, because a forget on Tower can make it a lie. Everything else (offline, refused,
+ * over budget, a 5xx, a changed source) leaves the image inline, which is today's behaviour; an image is never lost.
+ *
+ * Order per transcript: a cheap begin probe for every candidate (25 per request); the ones already on Tower are done;
+ * the rest are decoded to a temp file block by block (`image-store-stage.ts`) and sent through the ordinary chunked
+ * upload client. Logs carry counts, reason labels and the first 16 characters of a hash; never bytes, paths or ids.
+ */
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { IMAGE_STORE_MAX_BYTES, IMAGE_STORE_MAX_PER_RUN, IMAGE_USE_SESSION_ID, IMAGE_STORE_MAX_RUN_MS, RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN, RawEvidenceUploadBeginResponseSchema, imageObjectKey, } from "@bli-cockpit/telemetry-core";
+import { uploadRawEvidenceFilesChunked } from "./evidence-upload-client.js";
+import { requestJson, safeFailureDetail } from "./evidence-upload-transport.js";
+import { stageDecodedImage } from "./image-store-stage.js";
+import { isBackingOff, readStoreFailures, rememberStoreFailure, storeFailureKey, writeStoreFailures } from "./image-store-state.js";
+const TAG = "[image-store]";
+/** Distinct (image, session) pairs asked about per sync. Asking is cheap (one begin per 25); uploading is what the other budget bounds. */
+export const IMAGE_STORE_MAX_CONFIRMS_PER_RUN = 2_000;
+/** Images sent to the upload client together; the time budget is checked between groups. */
+const UPLOAD_GROUP = 10;
+const BEGIN_PATH = "/api/ambient/evidence/upload/begin";
+/** What the server says when it, not the image, is the problem: retried next tick, never remembered against the image. */
+const SERVER_SIDE_TRANSIENT = /image_store_unavailable|image_ledger_not_committed|image_bytes_missing/;
+/** Temp directories of images being sent are named with this; a run that died leaves one behind. */
+export const IMAGE_STORE_TEMP_PREFIX = "tower-image-store-";
+const STALE_TEMP_MS = 6 * 3_600_000;
+/** Remove staging directories older than six hours (a run that was killed mid-upload). Best effort, said out loud. */
+export async function pruneStaleImageStoreTemp(now, tempRoot = os.tmpdir()) {
+    let removed = 0;
+    try {
+        for (const name of await fs.readdir(tempRoot)) {
+            if (!name.startsWith(IMAGE_STORE_TEMP_PREFIX))
+                continue;
+            const dir = path.join(tempRoot, name);
+            const info = await fs.stat(dir).catch(() => null);
+            if (!info?.isDirectory() || now - info.mtimeMs < STALE_TEMP_MS)
+                continue;
+            await fs.rm(dir, { recursive: true, force: true });
+            removed += 1;
+        }
+    }
+    catch (error) {
+        console.error(`${TAG} could not prune old staging directories`, JSON.stringify({ reason: "image_store_temp_prune_failed", error_name: error instanceof Error ? error.name : typeof error }));
+        return removed;
+    }
+    if (removed > 0)
+        console.error(`${TAG} removed staging directories a killed run left behind`, JSON.stringify({ reason: "image_store_temp_pruned", removed }));
+    return removed;
+}
+export async function openImageStoreRun(stateDir, overrides = {}) {
+    await pruneStaleImageStoreTemp(Date.now());
+    return {
+        stored: new Set(),
+        failures: await readStoreFailures(stateDir),
+        failuresDirty: false,
+        uploadedThisRun: 0,
+        confirmedThisRun: 0,
+        storingMs: 0,
+        unavailable: null,
+        maxUploads: overrides.maxUploads ?? IMAGE_STORE_MAX_PER_RUN,
+        maxConfirms: overrides.maxConfirms ?? IMAGE_STORE_MAX_CONFIRMS_PER_RUN,
+        maxRunMs: overrides.maxRunMs ?? IMAGE_STORE_MAX_RUN_MS,
+    };
+}
+export const storedKey = (sha256, sessionId) => `${sha256}|${sessionId}`;
+export function isStoredThisRun(store, sha256, sessionId) {
+    return store.stored.has(storedKey(sha256, sessionId));
+}
+/**
+ * Could a prepare pass still store at least one of these images right now? False when the sync's upload or time budget
+ * is spent, Tower was unreachable this sync, or every one is backing off after a refusal. While true, a remembered
+ * transcript whose store failed is rebuilt, so a closed session whose store failed is stored on the next tick with room.
+ */
+export function hasStoreRoom(store, sha256s, now) {
+    if (store.unavailable)
+        return false;
+    if (store.uploadedThisRun >= store.maxUploads || store.storingMs > store.maxRunMs)
+        return false;
+    return sha256s.some((sha256) => !isBackingOff(store.failures, storeFailureKey(sha256, "", false), now));
+}
+function pointerFor(operatorId, image) {
+    const objectKey = imageObjectKey(operatorId, image.sha256, image.mediaType);
+    return {
+        raw_evidence_pointer_id: objectKey,
+        privacy_classification: "remote_durable_raw_evidence",
+        retention_policy: { mode: "remote_durable", privacy_classification: "remote_durable_raw_evidence" },
+        storage_scope: "remote_object",
+        storage_bucket: "ambient-raw-evidence",
+        object_key: objectKey,
+        content_hash_sha256: image.sha256,
+        byte_size: image.byteSize,
+        media_type: image.mediaType,
+    };
+}
+function* groupsOf(items, size) {
+    for (let at = 0; at < items.length; at += size)
+        yield items.slice(at, at + size);
+}
+export async function storeImages(store, input) {
+    const counts = { settled_earlier: 0, too_large_to_store: 0, backing_off: 0, over_budget: 0, unavailable: 0, stored_existing: 0, stored_new: 0, failed: 0 };
+    if (input.images.length === 0)
+        return counts;
+    if (!input.provenance) {
+        console.error(`${TAG} nothing stored, images stay inline`, JSON.stringify({ reason: "store_provenance_missing", images: input.images.length }));
+        counts.unavailable = input.images.length;
+        return counts;
+    }
+    // The server records the session by this id (the id forget keys on); one it would refuse is not worth a request.
+    if (!IMAGE_USE_SESSION_ID.test(input.sessionId)) {
+        console.error(`${TAG} nothing stored, images stay inline`, JSON.stringify({ reason: "store_session_id_unusable", images: input.images.length }));
+        counts.unavailable = input.images.length;
+        return counts;
+    }
+    const provenance = input.provenance;
+    const startedAt = input.now();
+    const options = {
+        fetchImpl: input.fetchImpl, dashboardUrl: input.dashboardUrl, deviceToken: input.deviceToken, provenance,
+        generatedAt: new Date(startedAt).toISOString(), files: [], ...(input.sleep ? { sleep: input.sleep } : {}),
+    };
+    const use = { session_id: input.sessionId, source: input.source };
+    const candidates = selectCandidates(store, input, counts);
+    const needUpload = await probeAll(store, input, options, use, candidates, counts);
+    await uploadAll(store, input, options, use, needUpload, counts, startedAt);
+    store.storingMs += input.now() - startedAt;
+    if (store.failuresDirty) {
+        try {
+            await writeStoreFailures(input.stateDir, store.failures);
+            store.failuresDirty = false;
+        }
+        catch (error) {
+            console.error(`${TAG} could not record store refusals; they are forgotten at the end of this sync`, JSON.stringify({ reason: "image_store_failures_write_failed", error_name: error instanceof Error ? error.name : typeof error }));
+        }
+    }
+    console.error(`${TAG} transcript stored`, JSON.stringify({ source: input.source, images: input.images.length, ...counts, storing_ms: input.now() - startedAt }));
+    return counts;
+}
+function selectCandidates(store, input, counts) {
+    const candidates = [];
+    for (const image of input.images) {
+        if (isStoredThisRun(store, image.sha256, input.sessionId)) {
+            counts.settled_earlier += 1;
+            continue;
+        }
+        if (image.byteSize > IMAGE_STORE_MAX_BYTES) {
+            counts.too_large_to_store += 1;
+            continue;
+        }
+        if (store.unavailable) {
+            counts.unavailable += 1;
+            continue;
+        }
+        const imageKey = storeFailureKey(image.sha256, input.sessionId, false);
+        const sessionKey = storeFailureKey(image.sha256, input.sessionId, true);
+        if (isBackingOff(store.failures, imageKey, input.now()) || isBackingOff(store.failures, sessionKey, input.now())) {
+            counts.backing_off += 1;
+            continue;
+        }
+        if (store.confirmedThisRun >= store.maxConfirms || store.storingMs > store.maxRunMs) {
+            counts.over_budget += 1;
+            continue;
+        }
+        store.confirmedThisRun += 1;
+        candidates.push(image);
+    }
+    return candidates;
+}
+/** Ask Tower about every candidate; the ones it already holds are confirmed here, the rest come back to be uploaded. */
+async function probeAll(store, input, options, use, candidates, counts) {
+    const needUpload = [];
+    const operatorId = options.provenance.operator_id;
+    for (const batch of groupsOf(candidates, RAW_EVIDENCE_UPLOAD_MAX_OBJECTS_PER_BEGIN)) {
+        if (store.unavailable) {
+            counts.unavailable += batch.length;
+            continue;
+        }
+        const response = await requestJson(options, BEGIN_PATH, {
+            schema_version: "ambient-raw-evidence-upload-begin.v1",
+            generated_at: options.generatedAt,
+            provenance: options.provenance,
+            objects: batch.map((image) => ({
+                pointer: pointerFor(operatorId, image),
+                chunk_size_bytes: RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES,
+                chunk_count: Math.max(1, Math.ceil(image.byteSize / RAW_EVIDENCE_UPLOAD_DEFAULT_CHUNK_BYTES)),
+                image_use: use,
+            })),
+        });
+        const parsed = response.ok ? RawEvidenceUploadBeginResponseSchema.safeParse(response.body) : null;
+        if (!parsed?.success) {
+            settleUnreachable(store, response.status, safeFailureDetail(response.body), response.ok ? "begin_invalid_response" : null, batch, counts);
+            continue;
+        }
+        const byKey = new Map(parsed.data.objects.map((object) => [object.object_key, object]));
+        for (const image of batch) {
+            const answer = byKey.get(imageObjectKey(operatorId, image.sha256, image.mediaType));
+            if (!answer) {
+                fail(store, input, image, "begin_missing_disposition", counts);
+                continue;
+            }
+            if (answer.disposition === "already_committed") {
+                store.stored.add(storedKey(image.sha256, input.sessionId));
+                counts.stored_existing += 1;
+            }
+            else if (answer.disposition === "conflict") {
+                fail(store, input, image, answer.reason ?? "upload_conflict", counts);
+            }
+            else {
+                needUpload.push(image);
+            }
+        }
+    }
+    return needUpload;
+}
+/**
+ * A whole `begin` request that did not come back as an answer (unreachable, a 5xx, a 4xx, a reply this CLI cannot read)
+ * says nothing about any one picture: a 400 is a request one end could not read (a server that does not know this CLI's
+ * fields, say), and remembering it against every picture for a week would punish the pictures for the wiring. Leave the
+ * rest of this sync's images inline, once, saying why; the next tick asks again.
+ */
+function settleUnreachable(store, status, detail, label, batch, counts) {
+    const reason = label ?? `begin_failed_http_${status}${detail ? `_${detail}` : ""}`;
+    if (!store.unavailable) {
+        store.unavailable = reason;
+        console.error(`${TAG} Tower cannot take images right now; every image stays inline for the rest of this sync`, JSON.stringify({ reason, http_status: status, server_reason: detail ?? "none" }));
+    }
+    counts.unavailable += batch.length;
+}
+/** What a refusal is about decides whether it is remembered, for how long, and against what. */
+function fail(store, input, image, reason, counts) {
+    counts.failed += 1;
+    const verdict = classifyStoreFailure(reason);
+    if (verdict.remember) {
+        const key = storeFailureKey(image.sha256, input.sessionId, verdict.aboutSession);
+        const record = rememberStoreFailure(store.failures, key, `store:${reason}`.slice(0, 90), verdict.permanent, input.now());
+        store.failuresDirty = true;
+        console.error(`${TAG} image not stored, will not be re-sent until later`, JSON.stringify({ reason, attempts: record.attempts, retry_in_hours: Math.round((record.retry_after - input.now()) / 3_600_000), sha256_prefix: image.sha256.slice(0, 16) }));
+    }
+    else {
+        console.error(`${TAG} image not stored this tick, retried on the next`, JSON.stringify({ reason, sha256_prefix: image.sha256.slice(0, 16) }));
+    }
+}
+export function classifyStoreFailure(reason) {
+    if (/image_session_forgotten/.test(reason))
+        return { remember: true, permanent: true, aboutSession: true };
+    if (SERVER_SIDE_TRANSIENT.test(reason))
+        return { remember: false, permanent: false, aboutSession: false };
+    const status = Number(/http_(\d{3})/.exec(reason)?.[1] ?? Number.NaN);
+    if (Number.isNaN(status)) {
+        // A conflict named by the server (no HTTP status in the label) is about these bytes: leave it alone for a while.
+        // A local fault (a file that could not be read or staged) says nothing about the image and is not remembered.
+        const local = /^(file_|staged_|image_source_changed|image_stage_|empty_file|upload_request_|legacy_)/.test(reason);
+        return { remember: !local, permanent: !local, aboutSession: false };
+    }
+    if (status === 0 || status === 401 || status === 429)
+        return { remember: false, permanent: false, aboutSession: false };
+    if (status >= 500)
+        return { remember: true, permanent: false, aboutSession: false };
+    return { remember: true, permanent: true, aboutSession: false };
+}
+async function uploadAll(store, input, options, use, needUpload, counts, startedAt) {
+    if (needUpload.length === 0)
+        return;
+    const dir = await fs.mkdtemp(path.join(os.tmpdir(), IMAGE_STORE_TEMP_PREFIX));
+    const operatorId = options.provenance.operator_id;
+    try {
+        for (const group of groupsOf(needUpload, UPLOAD_GROUP)) {
+            if (store.unavailable) {
+                counts.unavailable += group.length;
+                continue;
+            }
+            const room = store.maxUploads - store.uploadedThisRun;
+            const timeUp = store.storingMs + (input.now() - startedAt) > store.maxRunMs;
+            const take = timeUp ? [] : group.slice(0, Math.max(0, room));
+            counts.over_budget += group.length - take.length;
+            if (take.length === 0)
+                continue;
+            store.uploadedThisRun += take.length;
+            const files = [];
+            const imageOf = new Map();
+            for (const image of take) {
+                const staged = await stageDecodedImage(dir, image, input.blocksOf(image));
+                if (!staged.ok) {
+                    fail(store, input, image, staged.reason, counts);
+                    continue;
+                }
+                files.push({ pointer: pointerFor(operatorId, image), local_path: staged.localPath, kind: "image_store_object", codex_session_id: input.sessionId, image_use: use });
+                imageOf.set(image.sha256, image);
+            }
+            if (files.length === 0)
+                continue;
+            const result = await uploadRawEvidenceFilesChunked({ ...options, files });
+            if (result.used_legacy_fallback) {
+                for (const image of imageOf.values())
+                    fail(store, input, image, "legacy_route_cannot_store_images", counts);
+                continue;
+            }
+            for (const outcome of result.outcomes)
+                settleOutcome(store, input, outcome, imageOf, counts);
+            for (const file of files)
+                await fs.rm(file.local_path, { force: true }).catch(() => undefined);
+        }
+    }
+    finally {
+        await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined);
+    }
+}
+function settleOutcome(store, input, outcome, imageOf, counts) {
+    const image = imageOf.get(outcome.pointer.content_hash_sha256 ?? "");
+    if (!image)
+        return;
+    // Only the server's own "committed" / "already_committed" counts; both are sent after the session is recorded.
+    // `reused_existing` for any other reason (a 409 that says someone else committed) is NOT a confirmation for us.
+    const confirmed = outcome.upload_state === "uploaded" || (outcome.upload_state === "reused_existing" && outcome.reason === "already_committed");
+    if (confirmed) {
+        store.stored.add(storedKey(image.sha256, input.sessionId));
+        counts.stored_new += 1;
+        return;
+    }
+    fail(store, input, image, outcome.reason ?? "upload_failed", counts);
+}

package/dist/image-store-stage.js

@@ -0,0 +1,54 @@
+/**
+ * Decode one embedded image from its base64 text into a private temp file, one block at a time (BLI-23329).
+ *
+ * The upload client sends files by offset, so the image has to exist as bytes on disk. It is decoded in blocks of
+ * 48 Ki characters (a multiple of 4: each block is whole base64 quanta), straight into a sha256 and the file, so
+ * neither the base64 text of a large image nor its decoded bytes are ever held whole. The result is checked
+ * against what the census saw: a source that no longer decodes to the same sha256 and size (the transcript changed
+ * between the passes) is refused by name and the file removed. The file is mode 0600, in a directory this run made,
+ * and the caller removes the directory when the run is done.
+ */
+import { createHash } from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+/** Characters per block. A multiple of 4. */
+export const IMAGE_STORE_BLOCK_CHARS = 48 * 1024;
+/** The base64 blocks of an image held in a string: slices of the run, each a whole number of quanta. */
+export function* blocksOfText(text, image) {
+    for (let at = image.base64Start; at < image.base64End; at += IMAGE_STORE_BLOCK_CHARS) {
+        yield text.slice(at, Math.min(image.base64End, at + IMAGE_STORE_BLOCK_CHARS));
+    }
+}
+export async function stageDecodedImage(dir, image, blocks) {
+    const localPath = path.join(dir, `${image.sha256}.bin`);
+    const hash = createHash("sha256");
+    let byteSize = 0;
+    let handle;
+    try {
+        handle = await fs.open(localPath, "wx", 0o600);
+        for await (const block of blocks) {
+            const bytes = Buffer.from(block, "base64");
+            hash.update(bytes);
+            byteSize += bytes.byteLength;
+            await handle.write(bytes);
+        }
+        await handle.close();
+        handle = undefined;
+    }
+    catch (error) {
+        await handle?.close().catch(() => undefined);
+        await fs.rm(localPath, { force: true }).catch(() => undefined);
+        // A source that cannot be read to the end (the transcript changed or vanished) is a changed source; a disk that
+        // cannot be written is the other label. Neither says anything about the image itself.
+        const code = error?.code;
+        const writeFault = code === "ENOSPC" || code === "EACCES" || code === "EPERM" || code === "EEXIST" || code === "EROFS";
+        console.error("[image-store] staging an image failed", JSON.stringify({ reason: writeFault ? "image_stage_write_failed" : "image_source_changed", code: code ?? null }));
+        return { ok: false, reason: writeFault ? "image_stage_write_failed" : "image_source_changed" };
+    }
+    if (hash.digest("hex") !== image.sha256 || byteSize !== image.byteSize) {
+        await fs.rm(localPath, { force: true }).catch(() => undefined);
+        console.error("[image-store] decoded image is not the one the census saw", JSON.stringify({ reason: "image_source_changed", expected_bytes: image.byteSize, decoded_bytes: byteSize }));
+        return { ok: false, reason: "image_source_changed" };
+    }
+    return { ok: true, localPath, byteSize };
+}