@bli-cockpit/cli 0.2.26 → 0.2.28

package/dist/commands/local.js

@@ -20,7 +20,7 @@ import { collectionRootPathAliases, discoverGitWorktreesInRootsWithStatus, } fro
 import { resolveDiscoveryLimits, saveDiscoveryLimits, } from "../discovery-limits.js";
 import { runAttributedWorktreeSync, matchesLiveSyncWorktree, } from "./session-sync.js";
 import { COLLECTION_ROOT_REQUIRED, missingCollectionRootMessage, normalizeRootsDetailed, resolveOnboardingRoots, rootRejectionExplanation, } from "../onboarding-roots.js";
-import { rawEvidenceGcSummary, runRawEvidenceLocalGc, } from "../raw-evidence-gc.js";
+import { rawEvidenceDedupSummary, rawEvidenceGcSummary, runRawEvidenceLocalGc, sweepDuplicateStagedRawEvidence, } from "../raw-evidence-gc.js";
 import { envWithNodeRuntimeOnPath, runScheduledSelfUpdate, } from "../scheduled-self-update.js";
 import { runAutostartSelfHeal, } from "../autostart-self-heal.js";
 import { enqueueInstallEventEntry, readPendingInstallEventEntries, recordInstallEventAttemptFailure, removeInstallEventEntry, } from "../spool/install-event-outbox.js";
@@ -1808,7 +1808,16 @@ function rawEvidenceSyncLine(sync) {
     const retries = sync.raw_evidence_retry_reasons.length > 0
         ? ` retry_required: ${sync.raw_evidence_retry_reasons.join(",")}`
         : "";
-    return `Raw evidence: uploaded ${sync.raw_evidence_uploaded_object_count} object(s) in ${sync.raw_evidence_uploaded_chunk_count} chunk(s), reused ${sync.raw_evidence_reused_count}, failed ${sync.raw_evidence_failed_count}${failures}${retries}`;
+    // A held object and a nine-day-old first failure both belong on this line.
+    // Neither used to appear anywhere, which is how a 1,030-attempt loop stayed
+    // invisible (BLI-3066).
+    const held = sync.raw_evidence_delivery_held_count > 0
+        ? ` held: ${sync.raw_evidence_delivery_held_count}`
+        : "";
+    const stuck = sync.raw_evidence_stuck_object_count > 0
+        ? ` stuck: ${sync.raw_evidence_stuck_object_count} (worst ${sync.raw_evidence_max_delivery_attempts} attempt(s) since ${sync.raw_evidence_oldest_delivery_failure_at ?? "unknown"})`
+        : "";
+    return `Raw evidence: uploaded ${sync.raw_evidence_uploaded_object_count} object(s) in ${sync.raw_evidence_uploaded_chunk_count} chunk(s), reused ${sync.raw_evidence_reused_count}, failed ${sync.raw_evidence_failed_count}${held}${stuck}${failures}${retries}`;
 }
 function cursorStatusLine(sync) {
     return `Cursor: ${sync.cursor_tracked_object_count} durable object(s) tracked`;
@@ -2488,6 +2497,15 @@ function syncResult(run) {
 }
 async function runSyncLocked(command, io) {
     const collectionRoots = await resolveSyncCollectionRoots(command);
+    // Before anything is collected: collapse byte-identical staged packs. It runs
+    // first, unconditionally and unthrottled, because a machine that already
+    // holds 559 copies of one rollout needs the disk back before it stages
+    // anything else (BLI-3066). Safe by construction — a duplicate is identical
+    // by content hash to the survivor.
+    const dedup = await sweepDuplicateStagedRawEvidence(getCollectorRuntimePaths(command.homeDir), io.env);
+    if (!dedup.skipped && dedup.removed_dirs > 0) {
+        writeLine(io.stdout, rawEvidenceDedupSummary(dedup));
+    }
     const worktrees = await discoverCommandWorktrees(collectionRoots, {
         maxDepth: command.maxDepth,
         maxRepos: command.maxRepos,
@@ -2515,6 +2533,7 @@ async function runSyncLocked(command, io) {
                 repos: rows,
                 codex_sessions: run.summary,
                 raw_evidence_gc: gc,
+                raw_evidence_dedup: dedup,
             }, null, 2));
             return syncResult(run);
         }
@@ -2546,6 +2565,7 @@ async function runSyncLocked(command, io) {
                 collection_complete: run.ok,
                 codex_sessions: run.summary,
                 raw_evidence_gc: gc,
+                raw_evidence_dedup: dedup,
             }, null, 2));
             return syncResult(run);
         }
@@ -2568,6 +2588,7 @@ async function runSyncLocked(command, io) {
             collection_complete: run.ok,
             codex_sessions: run.summary,
             raw_evidence_gc: gc,
+            raw_evidence_dedup: dedup,
         }, null, 2));
         return syncResult(run);
     }
@@ -2801,10 +2822,20 @@ async function runStatus(command, io) {
     writeLine(io.stdout, `pending_health_receipts: ${status.pending_health_receipt_count}`);
     writeLine(io.stdout, `last_health_receipt_failure: ${status.last_health_receipt_failure_reason ?? "none"}`);
     writeLine(io.stdout, `backfill: ${backfillCursorLine(backfillCursor)}`);
+    writeLine(io.stdout, `stuck_evidence: ${stuckEvidenceLine(status)}`);
     for (const detail of status.details)
         writeLine(io.stdout, `- ${detail}`);
     return 0;
 }
+/**
+ * One line that cannot say "fine" while an object has never been accepted.
+ * Named reasons, worst attempt count, and the date it started.
+ */
+function stuckEvidenceLine(status) {
+    if (status.stuck_evidence_object_count === 0)
+        return "none";
+    return `${status.stuck_evidence_object_count} object(s), ${status.stuck_evidence_held_count} held, worst ${status.stuck_evidence_max_attempts} attempt(s) since ${status.stuck_evidence_oldest_failure_at ?? "unknown"} (${status.stuck_evidence_reasons.join(",") || "unknown"})`;
+}
 function displayTicketId(ticketId) {
     return ticketId ?? "general ambient";
 }

package/dist/commands/public-root.js

@@ -15,7 +15,7 @@ export async function runCockpitCli(argv, io) {
   }
 
   if (command === "--version" || command === "-V" || command === "version") {
-    writeLine(io?.stdout ?? process.stdout, "0.2.26");
+    writeLine(io?.stdout ?? process.stdout, "0.2.28");
     return 0;
   }
 

package/dist/local-state.js

@@ -8,6 +8,7 @@ import { normalizeGitOrigin, repoFingerprintFromLocalRoot, repoFingerprintFromOr
 import { isSamePath, normalizeCollectionRoots, } from "./root-normalization.js";
 import { summarizeLocalUploadSpool } from "./spool/local-spool.js";
 import { summarizeInstallEventOutbox } from "./spool/install-event-outbox.js";
+import { readRawEvidenceStagingState, summarizeStuckEvidence, } from "./raw-evidence-staging.js";
 const localCollectorPackage = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
 export const LOCAL_COLLECTOR_VERSION = typeof localCollectorPackage.version === "string"
     ? localCollectorPackage.version
@@ -340,10 +341,12 @@ export async function inspectLocalCollectorStatus(options = {}) {
     const identity = await resolveIdentityOrFallback(repoRoot, options.branch);
     const context = await readLocalWorkContextForRepo(paths, repoRoot).catch(() => null);
     const branch = options.branch ?? identity.branch;
-    const [uploadSpool, healthOutbox] = await Promise.all([
+    const [uploadSpool, healthOutbox, stagingState] = await Promise.all([
         summarizeLocalUploadSpool(paths),
         summarizeInstallEventOutbox(paths),
+        readRawEvidenceStagingState(paths.state_dir),
     ]);
+    const stuckEvidence = summarizeStuckEvidence(stagingState, now);
     const freshness = classifyCollectorFreshness(context, uploadSpool.last_upload_success_at, now);
     const uploadState = !config
         ? "not_installed"
@@ -377,6 +380,9 @@ export async function inspectLocalCollectorStatus(options = {}) {
     if (healthOutbox.pending_count > 0) {
         details.push(`Collector health retry pending: ${healthOutbox.pending_count} sanitized receipt(s) queued since ${healthOutbox.oldest_created_at ?? "unknown"}.`);
     }
+    if (stuckEvidence.stuck_object_count > 0) {
+        details.push(`Raw evidence stuck: ${stuckEvidence.stuck_object_count} object(s) have never been accepted (${stuckEvidence.held_object_count} waiting on backoff, worst ${stuckEvidence.max_attempts} attempt(s) since ${stuckEvidence.oldest_first_failed_at ?? "unknown"}) — reasons: ${stuckEvidence.reasons.join(", ") || "unknown"}.`);
+    }
     return {
         installed: Boolean(config),
         config_file: paths.config_file,
@@ -405,6 +411,11 @@ export async function inspectLocalCollectorStatus(options = {}) {
         pending_health_receipt_count: healthOutbox.pending_count,
         oldest_pending_health_receipt_at: healthOutbox.oldest_created_at,
         last_health_receipt_failure_reason: healthOutbox.last_failure_reason,
+        stuck_evidence_object_count: stuckEvidence.stuck_object_count,
+        stuck_evidence_held_count: stuckEvidence.held_object_count,
+        stuck_evidence_max_attempts: stuckEvidence.max_attempts,
+        stuck_evidence_oldest_failure_at: stuckEvidence.oldest_first_failed_at,
+        stuck_evidence_reasons: stuckEvidence.reasons,
         details,
     };
 }

package/dist/raw-evidence-gc.js

@@ -2,8 +2,15 @@ import crypto from "node:crypto";
 import fs from "node:fs/promises";
 import path from "node:path";
 import { readRawEvidenceCursor } from "./cursors/raw-evidence-cursor.js";
+import { readRawEvidenceStagingState } from "./raw-evidence-staging.js";
 const RAW_EVIDENCE_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
 const SYNC_LOG_MAX_BYTES = 50 * 1024 * 1024;
+/**
+ * A staging directory belongs to one in-flight collection pass. Anything this
+ * old is the remains of a crash or a kill, never live work — the longest sync
+ * observed on the fleet is minutes, not hours.
+ */
+const STAGING_ORPHAN_MS = 6 * 60 * 60 * 1000;
 // GC hashes every file in every old-but-kept dir to confirm uploads. Running
 // that on each 15-min sync would re-read the same gigabytes ~96x/day on
 // machines with unconfirmed evidence, so GC is throttled to once per day.
@@ -63,6 +70,177 @@ export async function runRawEvidenceLocalGc(paths, env = process.env, now = new
 export function rawEvidenceGcSummary(result) {
     return `raw-evidence GC: removed ${result.removed_dirs} dirs, freed ~${formatMb(result.freed_bytes)} MB`;
 }
+/**
+ * Collapse byte-identical staged packs down to one survivor.
+ *
+ * Runs at the START of every sync, unthrottled and with no age requirement,
+ * because it is the only thing that drains a machine that already holds 559
+ * copies of the same rollout (BLI-3066). The GC proper cannot: it only deletes
+ * packs whose every file is already committed, and these were never committed —
+ * that is precisely why they piled up.
+ *
+ * Deleting an uncommitted copy is safe here and only here: the survivor holds
+ * the identical bytes, by content hash, so no evidence is lost. Identity comes
+ * from the manifest's recorded file hashes — cheap (one small JSON per pack)
+ * and exact. A pack with no readable manifest is counted and left alone rather
+ * than guessed at.
+ *
+ * The survivor is the newest directory: it is the one the current content-keyed
+ * pack id resolves to, so a fleet machine converges in one extra sync instead
+ * of oscillating between an old name and a new one.
+ */
+export async function sweepDuplicateStagedRawEvidence(paths, env = process.env, now = new Date()) {
+    const empty = {
+        skipped: false,
+        duplicate_groups: 0,
+        removed_dirs: 0,
+        freed_bytes: 0,
+        removed_staging_dirs: 0,
+        unfingerprintable_dirs: 0,
+    };
+    if (env["COCKPIT_DISABLE_GC"] === "1") {
+        return { ...empty, skipped: true };
+    }
+    const rawEvidenceRoot = path.join(paths.state_dir, "raw-evidence");
+    const dirEntries = await fs
+        .readdir(rawEvidenceRoot, { withFileTypes: true })
+        .catch(() => []);
+    // Packs the staged-object index points into are preferred survivors. Keeping
+    // one of those means the next collection reuses it instead of writing a fresh
+    // copy and re-deleting the old one on the sync after — a two-step oscillation
+    // that would look like the sweep working while the bytes moved every cycle.
+    const staging = await readRawEvidenceStagingState(paths.state_dir);
+    const referencedPackIds = new Set(Object.values(staging.staged).map((entry) => entry.pack_id));
+    let removedStagingDirs = 0;
+    const packs = [];
+    for (const entry of dirEntries) {
+        if (!entry.isDirectory())
+            continue;
+        const dir = path.join(rawEvidenceRoot, entry.name);
+        if (entry.name.startsWith(".staging-")) {
+            const info = await fs.stat(dir).catch(() => null);
+            if (!info)
+                continue;
+            if (now.getTime() - info.mtimeMs < STAGING_ORPHAN_MS)
+                continue;
+            await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined);
+            if (!(await exists(dir)))
+                removedStagingDirs += 1;
+            continue;
+        }
+        if (!entry.name.startsWith("work-"))
+            continue;
+        const info = await fs.stat(dir).catch(() => null);
+        if (!info?.isDirectory())
+            continue;
+        packs.push({
+            dir,
+            mtimeMs: info.mtimeMs,
+            preferred: referencedPackIds.has(entry.name),
+        });
+    }
+    const groups = new Map();
+    let unfingerprintable = 0;
+    for (const pack of packs) {
+        const fingerprint = await packContentFingerprint(pack.dir);
+        if (!fingerprint) {
+            unfingerprintable += 1;
+            continue;
+        }
+        const group = groups.get(fingerprint);
+        if (group) {
+            group.push(pack);
+        }
+        else {
+            groups.set(fingerprint, [pack]);
+        }
+    }
+    let duplicateGroups = 0;
+    let removedDirs = 0;
+    let freedBytes = 0;
+    for (const group of groups.values()) {
+        if (group.length < 2)
+            continue;
+        duplicateGroups += 1;
+        group.sort((a, b) => Number(b.preferred) - Number(a.preferred) || b.mtimeMs - a.mtimeMs);
+        for (const duplicate of group.slice(1)) {
+            const byteSize = await dirByteSize(duplicate.dir);
+            await fs
+                .rm(duplicate.dir, { recursive: true, force: true })
+                .catch(() => undefined);
+            if (await exists(duplicate.dir))
+                continue;
+            removedDirs += 1;
+            freedBytes += byteSize;
+        }
+    }
+    const result = {
+        skipped: false,
+        duplicate_groups: duplicateGroups,
+        removed_dirs: removedDirs,
+        freed_bytes: freedBytes,
+        removed_staging_dirs: removedStagingDirs,
+        unfingerprintable_dirs: unfingerprintable,
+    };
+    // stderr, which launchd captures to `sync.err.log`. Logged on every sweep,
+    // including the boring one: "0 duplicates today" is the only evidence that
+    // the drain is still running at all.
+    console.error("[raw-evidence] dedup sweep", JSON.stringify({
+        reason: removedDirs > 0 ? "dedup_removed" : "dedup_clean",
+        pack_count: packs.length,
+        duplicate_groups: duplicateGroups,
+        removed_dirs: removedDirs,
+        freed_bytes: freedBytes,
+        removed_staging_dirs: removedStagingDirs,
+        unfingerprintable_dirs: unfingerprintable,
+    }));
+    return result;
+}
+export function rawEvidenceDedupSummary(result) {
+    return `raw-evidence dedup: removed ${result.removed_dirs} duplicate pack(s) across ${result.duplicate_groups} group(s), freed ~${formatMb(result.freed_bytes)} MB`;
+}
+/**
+ * Content identity of a pack, read from the manifest it already carries.
+ *
+ * Only the file content hashes are used — not `pack_id`, not `created_at`, not
+ * the branch or ticket labels. Two packs of the same transcript minted 15
+ * minutes apart differ in all of those and in none of these.
+ */
+async function packContentFingerprint(dir) {
+    const manifestBytes = await fs
+        .readFile(path.join(dir, "manifest.json"), "utf8")
+        .catch(() => null);
+    if (!manifestBytes)
+        return null;
+    try {
+        const parsed = JSON.parse(manifestBytes);
+        const hashes = (parsed.files ?? [])
+            .filter((file) => typeof file.content_hash_sha256 === "string" &&
+            file.content_hash_sha256.length === 64)
+            .map((file) => `${String(file.content_hash_sha256)}:${typeof file.byte_size === "number" ? file.byte_size : 0}`)
+            .sort();
+        if (hashes.length === 0)
+            return null;
+        return crypto
+            .createHash("sha256")
+            .update(hashes.join(","), "utf8")
+            .digest("hex");
+    }
+    catch {
+        return null;
+    }
+}
+async function dirByteSize(dir) {
+    const files = await listFiles(dir).catch(() => null);
+    if (!files)
+        return 0;
+    let total = 0;
+    for (const file of files) {
+        const info = await fs.stat(file).catch(() => null);
+        total += info?.isFile() ? info.size : 0;
+    }
+    return total;
+}
 async function inspectRawEvidenceDirForGc(dir, uploadedHashes) {
     const files = await listFiles(dir).catch(() => null);
     if (!files)

package/dist/raw-evidence-staging.js

@@ -0,0 +1,309 @@
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+/**
+ * Durable local state for raw-evidence STAGING and DELIVERY ATTEMPTS.
+ *
+ * Two questions this file answers, and BLI-3066 is what it costs when nothing
+ * does:
+ *
+ * 1. "Have I already staged these exact bytes?" — the pack id used to fold
+ *    `now` into its hash, so every 15-minute sync minted a new directory and
+ *    copied the same transcript into it again. One Codex rollout reached 559
+ *    byte-identical copies, 182 GiB, on one laptop.
+ * 2. "How many times have I already offered this object, and how did that go?"
+ *    — nothing counted, so the same commit failed 1,030 times in nine days at
+ *    full 15-minute cadence, and every attempt looked like the first.
+ *
+ * The cursor (`cursors/raw-evidence.json`) answers the *committed* question and
+ * deliberately stays that way: an entry there means the object is durable
+ * remotely. This file is the pre-commit side — staged but not yet acknowledged —
+ * so a cursor reader can never mistake "on this disk" for "safe in the bucket".
+ *
+ * Metadata only: content hashes, byte sizes, pack ids, pack-relative paths,
+ * reason labels, timestamps. Never content, never an absolute path.
+ */
+export const RAW_EVIDENCE_STAGING_FILENAME = "raw-evidence-staging.json";
+/**
+ * Backoff schedule for an object whose delivery keeps failing.
+ *
+ * 15 min (the scheduler's own cadence, so the first retry is simply the next
+ * sync), doubling to a 6 h ceiling. Nine days of failure is then ~40 attempts
+ * instead of 1,030, and the object is still retried four times a day — a server
+ * fix lands within hours, not on the next reinstall.
+ */
+export const EVIDENCE_DELIVERY_BACKOFF_BASE_MS = 15 * 60 * 1000;
+export const EVIDENCE_DELIVERY_BACKOFF_MAX_MS = 6 * 60 * 60 * 1000;
+/** The reason label written when an object is being held by backoff. */
+export const DELIVERY_BACKOFF_HOLDING_REASON = "delivery_backoff_holding";
+const MAX_TRACKED_STAGED_OBJECTS = 5_000;
+const MAX_TRACKED_DELIVERY_ATTEMPTS = 5_000;
+export function emptyRawEvidenceStagingState() {
+    return {
+        schema_version: "cockpit-raw-evidence-staging.v1",
+        updated_at: null,
+        staged: {},
+        delivery_attempts: {},
+    };
+}
+export function rawEvidenceStagingStatePath(stateDir) {
+    return path.join(stateDir, RAW_EVIDENCE_STAGING_FILENAME);
+}
+export async function readRawEvidenceStagingState(stateDir) {
+    try {
+        const raw = JSON.parse(await fs.readFile(rawEvidenceStagingStatePath(stateDir), "utf8"));
+        return parseStagingState(raw);
+    }
+    catch {
+        return emptyRawEvidenceStagingState();
+    }
+}
+/**
+ * Atomic write, same shape as the cursor: temp sibling, fsync, rename. A
+ * launchd sync and a hand-run sync can race, and a truncated state file would
+ * lose every attempt count at once — which is exactly the thing that must not
+ * be losable.
+ */
+export async function writeRawEvidenceStagingState(stateDir, state) {
+    const filePath = rawEvidenceStagingStatePath(stateDir);
+    await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 });
+    const pruned = pruneStagingState(state);
+    const serialized = `${JSON.stringify(pruned, null, 2)}\n`;
+    const tempPath = `${filePath}.tmp-${process.pid}-${crypto.randomUUID()}`;
+    let handle = null;
+    try {
+        handle = await fs.open(tempPath, "w", 0o600);
+        await handle.writeFile(serialized);
+        await handle.sync();
+    }
+    finally {
+        await handle?.close();
+    }
+    try {
+        await fs.rename(tempPath, filePath);
+    }
+    catch (error) {
+        await fs.rm(tempPath, { force: true }).catch(() => undefined);
+        throw error;
+    }
+    if (process.platform !== "win32") {
+        await fs.chmod(filePath, 0o600).catch(() => undefined);
+    }
+}
+export function recordStagedObject(state, contentHash, entry) {
+    state.staged[contentHash] = entry;
+}
+/**
+ * The absolute path of an already-staged copy of these bytes, or null.
+ *
+ * Verified against the filesystem, not trusted from the index: a dedup sweep,
+ * a GC pass, or an operator with `rm -rf` can all have removed the file since
+ * it was recorded. A stale entry is dropped so the caller re-stages rather than
+ * handing the uploader a path that is not there.
+ */
+export async function resolveStagedObject(rawEvidenceRoot, state, contentHash) {
+    const entry = state.staged[contentHash];
+    if (!entry)
+        return null;
+    const localPath = stagedObjectPath(rawEvidenceRoot, entry);
+    const info = await fs.stat(localPath).catch(() => null);
+    if (!info?.isFile() || info.size !== entry.byte_size) {
+        delete state.staged[contentHash];
+        return null;
+    }
+    return { local_path: localPath, entry };
+}
+export function stagedObjectPath(rawEvidenceRoot, entry) {
+    return path.join(rawEvidenceRoot, entry.pack_id, ...entry.relative_path.split("/").filter(Boolean));
+}
+/**
+ * Delay before the next attempt on an object that has failed `attempts` times.
+ * `attempts` is 1-based: the first failure waits one scheduler cadence.
+ */
+export function evidenceDeliveryBackoffMs(attempts) {
+    const safeAttempts = Math.max(1, Math.floor(attempts));
+    const exponent = Math.min(safeAttempts - 1, 32);
+    const delay = EVIDENCE_DELIVERY_BACKOFF_BASE_MS * 2 ** exponent;
+    return Math.min(delay, EVIDENCE_DELIVERY_BACKOFF_MAX_MS);
+}
+export function recordDeliveryFailure(state, contentHash, options) {
+    const previous = state.delivery_attempts[contentHash];
+    const attempts = (previous?.attempts ?? 0) + 1;
+    const attemptedAtIso = options.attemptedAt.toISOString();
+    const entry = {
+        attempts,
+        first_failed_at: previous?.first_failed_at ?? attemptedAtIso,
+        last_attempt_at: attemptedAtIso,
+        last_reason: options.reason,
+        next_attempt_at: new Date(options.attemptedAt.getTime() + evidenceDeliveryBackoffMs(attempts)).toISOString(),
+        byte_size: options.byteSize ?? previous?.byte_size ?? 0,
+        source_key: options.sourceKey ?? previous?.source_key ?? null,
+    };
+    state.delivery_attempts[contentHash] = entry;
+    return entry;
+}
+/** An object that landed clears its history; the next failure starts at one. */
+export function clearDeliveryAttempt(state, contentHash) {
+    if (!state.delivery_attempts[contentHash])
+        return false;
+    delete state.delivery_attempts[contentHash];
+    return true;
+}
+export function deliveryHold(state, contentHash, now) {
+    if (!contentHash)
+        return null;
+    const entry = state.delivery_attempts[contentHash];
+    if (!entry)
+        return null;
+    return Date.parse(entry.next_attempt_at) > now.getTime() ? entry : null;
+}
+/**
+ * Source keys currently held by backoff.
+ *
+ * Collection consults this BEFORE reading a file, so a held 350 MB transcript
+ * costs nothing at all this cycle — no read, no hash, no copy, no upload. The
+ * hold is still a named, retryable gap, never a silent drop.
+ */
+export function heldSourceKeys(state, now) {
+    const held = new Set();
+    for (const entry of Object.values(state.delivery_attempts)) {
+        if (!entry.source_key)
+            continue;
+        if (Date.parse(entry.next_attempt_at) > now.getTime()) {
+            held.add(entry.source_key);
+        }
+    }
+    return held;
+}
+export function evidenceSourceKey(options) {
+    const source = options.sourcePath
+        ? shortHash(options.sourcePath)
+        : options.label
+            ? shortHash(options.label)
+            : "unknown";
+    return `${options.kind}:${options.sessionId ?? "none"}:${source}`;
+}
+/**
+ * What `cockpit status` and the health receipt need in order to be unable to
+ * read green while an object has been failing for nine days.
+ */
+export function summarizeStuckEvidence(state, now) {
+    const entries = Object.values(state.delivery_attempts);
+    const reasons = new Set();
+    let heldCount = 0;
+    let maxAttempts = 0;
+    let oldest = null;
+    for (const entry of entries) {
+        reasons.add(entry.last_reason);
+        if (Date.parse(entry.next_attempt_at) > now.getTime())
+            heldCount += 1;
+        maxAttempts = Math.max(maxAttempts, entry.attempts);
+        if (!oldest || entry.first_failed_at.localeCompare(oldest) < 0) {
+            oldest = entry.first_failed_at;
+        }
+    }
+    return {
+        stuck_object_count: entries.length,
+        held_object_count: heldCount,
+        max_attempts: maxAttempts,
+        oldest_first_failed_at: oldest,
+        reasons: [...reasons].sort(),
+    };
+}
+/**
+ * Pack id derived from CONTENT, never from the clock.
+ *
+ * The manifest is excluded from the hash on purpose: its own object key
+ * contains the pack id, so folding it in would be circular. Hashes are sorted,
+ * so a change in collection order alone does not mint a new pack.
+ */
+export function contentKeyedRawEvidencePackId(options) {
+    const material = [...options.contentHashes].sort().join(",");
+    return `${options.workContextId}-${shortHash(`${options.workContextId}:${material}`)}`;
+}
+function pruneStagingState(state) {
+    return {
+        ...state,
+        staged: pruneNewest(state.staged, MAX_TRACKED_STAGED_OBJECTS, (entry) => entry.staged_at),
+        delivery_attempts: pruneNewest(state.delivery_attempts, MAX_TRACKED_DELIVERY_ATTEMPTS, (entry) => entry.last_attempt_at),
+    };
+}
+function pruneNewest(record, max, sortKey) {
+    const entries = Object.entries(record);
+    if (entries.length <= max)
+        return record;
+    entries.sort((a, b) => sortKey(b[1]).localeCompare(sortKey(a[1])));
+    return Object.fromEntries(entries.slice(0, max));
+}
+function parseStagingState(value) {
+    if (!value || typeof value !== "object") {
+        return emptyRawEvidenceStagingState();
+    }
+    const record = value;
+    return {
+        schema_version: "cockpit-raw-evidence-staging.v1",
+        updated_at: optionalString(record["updated_at"]),
+        staged: parseRecord(record["staged"], parseStagedEntry),
+        delivery_attempts: parseRecord(record["delivery_attempts"], parseAttemptEntry),
+    };
+}
+function parseRecord(value, parseEntry) {
+    if (!value || typeof value !== "object")
+        return {};
+    const out = {};
+    for (const [key, entry] of Object.entries(value)) {
+        const parsed = parseEntry(entry);
+        if (parsed)
+            out[key] = parsed;
+    }
+    return out;
+}
+function parseStagedEntry(value) {
+    if (!value || typeof value !== "object")
+        return null;
+    const record = value;
+    const packId = optionalString(record["pack_id"]);
+    const relativePath = optionalString(record["relative_path"]);
+    const stagedAt = optionalString(record["staged_at"]);
+    if (!packId || !relativePath || !stagedAt)
+        return null;
+    return {
+        pack_id: packId,
+        relative_path: relativePath,
+        byte_size: optionalNumber(record["byte_size"]),
+        source_key: optionalString(record["source_key"]),
+        staged_at: stagedAt,
+    };
+}
+function parseAttemptEntry(value) {
+    if (!value || typeof value !== "object")
+        return null;
+    const record = value;
+    const lastAttemptAt = optionalString(record["last_attempt_at"]);
+    const nextAttemptAt = optionalString(record["next_attempt_at"]);
+    if (!lastAttemptAt || !nextAttemptAt)
+        return null;
+    const attempts = optionalNumber(record["attempts"]);
+    return {
+        attempts: attempts > 0 ? attempts : 1,
+        first_failed_at: optionalString(record["first_failed_at"]) ?? lastAttemptAt,
+        last_attempt_at: lastAttemptAt,
+        last_reason: optionalString(record["last_reason"]) ?? "unknown",
+        next_attempt_at: nextAttemptAt,
+        byte_size: optionalNumber(record["byte_size"]),
+        source_key: optionalString(record["source_key"]),
+    };
+}
+function optionalString(value) {
+    return typeof value === "string" && value.trim() ? value : null;
+}
+function optionalNumber(value) {
+    return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+function shortHash(value) {
+    return crypto
+        .createHash("sha256")
+        .update(value, "utf8")
+        .digest("hex")
+        .slice(0, 12);
+}