@bli-cockpit/cli 0.2.70 → 0.2.74

package/dist/commands/sync-followups.js

@@ -2,7 +2,8 @@
  * What the sync tick does AFTER collection's own outcome is decided and
  * reported: keep this machine's CLI current on npm `latest` (BLI-2601), put a
  * broken scheduler registration back (BLI-2721), keep BLI Memory registered
- * with both agent hosts (BLI-3580), and stop the laptop filling up (BLI-3619).
+ * with both agent hosts (BLI-3580), re-offer staged evidence that never landed
+ * (BLI-3797), and stop the laptop filling up (BLI-3619).
  *
  * Split out of commands/sync.ts (BLI-3578), moved verbatim. They belong
  * together because they share one rule, and it is the reason both are called
@@ -20,6 +21,7 @@ import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, } from "../local-sta
 import { runAutostartSelfHeal, } from "../autostart-self-heal.js";
 import { runStagingPrune } from "../disk-prune.js";
 import { runEvidenceReconcile, } from "../evidence-reconcile-client.js";
+import { runEvidenceRedelivery, } from "../evidence-redelivery.js";
 import { envWithNodeRuntimeOnPath, runScheduledSelfUpdate, } from "../scheduled-self-update.js";
 /** How many reconcile batches (of up to 500 hashes each) one sync tick may spend. */
 const RECONCILE_BATCHES_PER_TICK = 1;
@@ -215,12 +217,20 @@ function memoryInstallEvent(outcome) {
  * costs one cheap local read and no network call at all, so it never competes
  * with collection for the tick's time. A reconcile failure never blocks the
  * prune that follows it.
+ *
+ * BLI-3797 sits between them: the reconcile has just established which staged
+ * objects the SERVER says it never received, so the redelivery drain re-offers
+ * exactly those, bounded, before the prune runs and can delete whatever landed.
+ * Neither of the two can block the prune, and none of the three can fail a tick.
  */
 export async function runStagingPruneAfterSync(command, io, dashboardUrl, options = {}) {
     const events = [];
     const reconciled = await runReconcileFollowUp(command, io, dashboardUrl, options);
     if (reconciled)
         events.push(reconcileEvent(reconciled));
+    const redelivered = await runRedeliveryFollowUp(command, io, dashboardUrl, options);
+    if (redelivered)
+        events.push(redeliveryEvent(redelivered));
     let result;
     try {
         result = await runStagingPrune(getCollectorRuntimePaths(command.homeDir), {
@@ -269,6 +279,59 @@ async function runReconcileFollowUp(command, io, dashboardUrl, options) {
     });
     return result.reason === "nothing_unknown" ? null : result;
 }
+/**
+ * BLI-3797, between the reconcile above and the prune below, and in that order
+ * for a reason: reconcile turns `unknown` into a server-backed answer, this
+ * re-offers what that answer says never landed, and the prune then deletes
+ * whatever this just made durable. Running the drain first would ask the server
+ * about hashes it is about to be told the truth about; running it after the
+ * prune would leave a tick's worth of freed cap unused.
+ *
+ * Never throws (`runEvidenceRedelivery` already never does); returns `null` for
+ * the boring, common case — nothing on this disk is undelivered — so the steady
+ * state costs no receipt, the same rule the reconcile follow-up above and the
+ * prune's daily throttle below both follow.
+ */
+async function runRedeliveryFollowUp(command, io, dashboardUrl, options) {
+    const result = await runEvidenceRedelivery({
+        homeDir: command.homeDir,
+        dashboardUrl,
+        env: io.env,
+        fetch: io.fetch,
+        ...(options.now ? { now: options.now } : {}),
+    });
+    return result.reason === "nothing_uncommitted" ? null : result;
+}
+/** Counts and byte totals only; no pack id and no hash travels in a receipt. */
+function redeliveryEvent(result) {
+    const detail = [
+        `offered ${result.offered} object(s), ${result.offered_bytes}B, across ${result.packs} pack(s)`,
+        `uploaded ${result.uploaded} (${result.uploaded_bytes}B), reused ${result.reused}, failed ${result.failed}`,
+        `held ${result.held}, deferred ${result.deferred} (${result.deferred_bytes}B)`,
+        result.failure_reasons.length > 0
+            ? `failure_reasons ${result.failure_reasons.join(",")}`
+            : null,
+    ]
+        .filter((part) => Boolean(part))
+        .join("; ");
+    if (result.status === "fail") {
+        return {
+            step: "evidence_redelivery",
+            status: "fail",
+            error_code: result.reason,
+            error_detail: detail,
+        };
+    }
+    if (result.status === "skipped") {
+        return {
+            step: "evidence_redelivery",
+            status: "skipped",
+            error_code: result.reason,
+            error_detail: detail,
+        };
+    }
+    return { step: "evidence_redelivery", status: "ok", error_detail: detail };
+}
 function prunedNothing() {
     return {
         status: "skipped",

package/dist/commands/sync.js

@@ -2,7 +2,7 @@ import { writeLine } from "./cli-io.js";
 import { attributedSyncRunStatus, cursorStatusLine, displayTicketId, rawEvidenceSyncLine, shortSha, worktreeSyncRow, writeAgentSessionSummary, } from "./collection-report.js";
 import { collectionRootConsentAliases } from "./collection-roots.js";
 import { discoverCommandWorktrees } from "./local-discovery.js";
-import { sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
+import { sendCollectorHeartbeatBestEffort, readHeartbeatStagingFacts, } from "./heartbeat.js";
 import { classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
 import { runAttributedWorktreeSync, } from "./session-sync.js";
 import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, runStagingPruneAfterSync, } from "./sync-followups.js";
@@ -98,11 +98,22 @@ export async function runSync(command, io) {
  */
 async function sendSyncHeartbeat(command, io, dashboardUrl, facts) {
     const roots = await resolveSyncCollectionRoots(command).catch(() => []);
+    // BLI-3797: the backlog figure rides the same check-in as the root labels, so
+    // `cockpit ops` learns about undelivered evidence on the SAME tick that proves
+    // the machine is alive. Best-effort: nulls omit the fields rather than
+    // reporting a zero nobody measured.
+    const staging = await readHeartbeatStagingFacts({
+        homeDir: command.homeDir,
+    }).catch(() => ({ bytes: null, reason: null }));
     await sendCollectorHeartbeatBestEffort({
         homeDir: command.homeDir,
         dashboardUrl,
         roots,
-        facts,
+        facts: {
+            ...facts,
+            stagingUncommittedBytes: staging.bytes,
+            stagingUncommittedReason: staging.reason,
+        },
         io,
     }).catch((error) => {
         // The sender already swallows everything it knows about; this is the net

package/dist/disk-retention.js

@@ -131,7 +131,11 @@ function pushDeletion(plan, object, reason) {
 function addToBucket(bucket, object) {
     bucket.count += 1;
     bucket.bytes += object.byte_size;
-    bucket.oldest_age_ms = Math.max(bucket.oldest_age_ms, object.age_ms);
+    // Report the age of the BYTES, not of the last answer about them (BLI-3797).
+    // `age_ms` on a reconciled object is the age of the reconcile reply, which
+    // made `oldest_uncommitted_age_ms` read 0.94 days for a 45-day-old backlog.
+    // Deletion still keys off `age_ms` above — this bucket is report-only.
+    bucket.oldest_age_ms = Math.max(bucket.oldest_age_ms, object.disk_age_ms || object.age_ms);
 }
 function removeFromBucket(bucket, object) {
     bucket.count -= 1;

package/dist/disk-usage.js

@@ -104,6 +104,7 @@ export async function readStagingInventory(paths, now = new Date()) {
         inventory.packs.push(pack);
         accumulatePack(inventory, pack);
     }
+    rankUncommittedReasons(inventory);
     return inventory;
 }
 /**
@@ -150,15 +151,23 @@ async function readPack(dir, packId, options) {
         manifest_present: manifest !== null,
     };
     for (const relative of await listPackFiles(dir)) {
-        const size = await fs
-            .stat(path.join(dir, relative))
-            .then((info) => (info.isFile() ? info.size : 0))
-            .catch(() => 0);
+        // One stat, two facts: how big the staged copy is and WHEN it was written.
+        // The mtime is the only unfalsifiable record of how long undelivered
+        // evidence has been sitting here (BLI-3797) — every other timestamp on the
+        // machine records when something last had an opinion about it.
+        const info = await fs.stat(path.join(dir, relative)).catch(() => null);
+        const size = info?.isFile() ? info.size : 0;
         if (relative === "manifest.json") {
             pack.manifest_bytes += size;
             continue;
         }
-        pack.objects.push(await classifyObject(dir, packId, relative, size, manifest, options));
+        const stagedOnDiskAt = info && Number.isFinite(info.mtimeMs)
+            ? new Date(info.mtimeMs).toISOString()
+            : null;
+        pack.objects.push(await classifyObject(dir, packId, relative, size, manifest, {
+            ...options,
+            stagedOnDiskAt,
+        }));
     }
     return pack;
 }
@@ -167,7 +176,13 @@ async function classifyObject(dir, packId, relative, size, manifest, options) {
         (await hashUnnamedFile(path.join(dir, relative), size, options.budget));
     const stagedAt = options.stagedAt.get(`${packId}/${relative}`) ?? null;
     const committedAt = hash ? options.ledger.committed.get(hash) : undefined;
-    const base = { pack_id: packId, relative_path: relative, byte_size: size };
+    const base = {
+        pack_id: packId,
+        relative_path: relative,
+        byte_size: size,
+        staged_on_disk_at: options.stagedOnDiskAt,
+        disk_age_ms: ageMs(options.stagedOnDiskAt, options.now),
+    };
     if (committedAt) {
         return {
             ...base,
@@ -359,13 +374,49 @@ function accumulatePack(inventory, pack) {
         }
         inventory.uncommitted_count += 1;
         inventory.uncommitted_bytes += object.byte_size;
-        if (object.decided_at &&
+        tallyUncommittedReason(inventory, object);
+        // The age of the BYTES, not of the answer about them (BLI-3797). Falls back
+        // to `decided_at` only when the file has no readable mtime, which is the
+        // one case where the disk itself cannot say.
+        const stagedAt = object.staged_on_disk_at ?? object.decided_at;
+        if (stagedAt &&
             (!inventory.oldest_uncommitted_at ||
-                object.decided_at.localeCompare(inventory.oldest_uncommitted_at) < 0)) {
-            inventory.oldest_uncommitted_at = object.decided_at;
+                stagedAt.localeCompare(inventory.oldest_uncommitted_at) < 0)) {
+            inventory.oldest_uncommitted_at = stagedAt;
         }
     }
 }
+/**
+ * One more object against its reason label. Kept as a mutable array rather than
+ * a Map so the inventory stays a plain serializable value — it rides `--json`
+ * output and a receipt, and a Map would silently become `{}` in both.
+ */
+function tallyUncommittedReason(inventory, object) {
+    const existing = inventory.uncommitted_reasons.find((tally) => tally.reason === object.reason);
+    const tally = existing ?? {
+        reason: object.reason,
+        count: 0,
+        bytes: 0,
+        oldest_disk_age_ms: 0,
+    };
+    tally.count += 1;
+    tally.bytes += object.byte_size;
+    tally.oldest_disk_age_ms = Math.max(tally.oldest_disk_age_ms, object.disk_age_ms || object.age_ms);
+    if (!existing)
+        inventory.uncommitted_reasons.push(tally);
+}
+/**
+ * Rank the reasons by BYTES, then name the biggest one.
+ *
+ * Bytes, not object count, because the question the ranking answers is "what is
+ * holding this laptop over its cap" — 234 objects at 1,836 MB and 17 at 81 MB
+ * are not the same finding, and counting rows would have called them close.
+ */
+function rankUncommittedReasons(inventory) {
+    inventory.uncommitted_reasons.sort((left, right) => right.bytes - left.bytes || right.count - left.count);
+    inventory.dominant_uncommitted_reason =
+        inventory.uncommitted_reasons[0] ?? null;
+}
 function emptyStagingInventory() {
     return {
         pack_count: 0,
@@ -376,6 +427,8 @@ function emptyStagingInventory() {
         uncommitted_count: 0,
         uncommitted_bytes: 0,
         oldest_uncommitted_at: null,
+        uncommitted_reasons: [],
+        dominant_uncommitted_reason: null,
         unknown_count: 0,
         unknown_bytes: 0,
         manifest_bytes: 0,

package/dist/evidence-redelivery-plan.js

@@ -0,0 +1,277 @@
+/**
+ * Rebuilding an upload offer for evidence that is already on this disk.
+ *
+ * BLI-3797. The live sync only ever offers what THIS tick collected: it reads
+ * a transcript, hashes it, stages a copy, uploads that copy, and moves on. If
+ * the upload never happens — the process is killed between staging and
+ * delivery, the envelope is refused, the source file grows so next tick mints a
+ * different hash into a different pack — the staged bytes are orphaned. Nothing
+ * offers them again, ever. `disk-retention.ts` will not delete them (correctly:
+ * the ledger cannot vouch for them), so they sit until the disk fills.
+ *
+ * Measured on the reference Mac 2026-09-06, not reasoned about: 251 objects and
+ * 1,916.9 MB uncommitted against a 2,048 MB cap, 234 of them (1,835.8 MB)
+ * carrying `reconciled_unknown_to_server` — the SERVER was asked and answered
+ * that it has never held those hashes. `raw-evidence-staging.json` held six
+ * delivery-attempt rows in total, so those objects had not failed delivery;
+ * they had never been offered. The oldest pack was written 2026-07-23, 45 days
+ * earlier, and `cockpit doctor`'s fix sentence said "run `cockpit sync` to
+ * deliver them" — a sentence that could not come true.
+ *
+ * The repair is possible because each pack's own `manifest.json` already
+ * carries everything an offer needs: `object_key`, `content_hash_sha256`,
+ * `byte_size`, `media_type` and `redacted_summary` per file, and the pack's
+ * `work_context_id` / `session_id` / `operator_id` / `repo_basename` / `branch`
+ * beside them. This module rebuilds a `RawEvidenceUploadFile` from those
+ * recorded facts and NEVER re-derives them: the bytes on disk were sanitized
+ * when they were staged, and re-reading the original source would both cost the
+ * IO and risk offering content the redactor of the day would have masked
+ * differently.
+ *
+ * Metadata only, as everywhere in this family: pack ids, pack-relative paths,
+ * hashes, byte counts, reason labels. Never content, never an operator path.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { RAW_EVIDENCE_BUCKET, RAW_EVIDENCE_RETENTION_MODE, } from "./adapters/raw-evidence-manifest.js";
+import { RAW_EVIDENCE_DIR } from "./disk-usage.js";
+import { describeError } from "./health-detail.js";
+import { LOCAL_COLLECTOR_VERSION } from "./local-state.js";
+import { deliveryHold, } from "./raw-evidence-staging.js";
+/**
+ * How much undelivered backlog one tick may re-offer.
+ *
+ * A tick is fifteen minutes and the backlog is measured in gigabytes, so this
+ * is a drain rate, not a ceiling on what will eventually be sent: 512 MB a tick
+ * clears the reference Mac's 1.9 GB in four ticks (one hour) without competing
+ * with the tick's own collection, which owns a separate 2 GB budget. The object
+ * count bounds the number of `begin` round-trips for the opposite case — a
+ * backlog of thousands of small objects.
+ */
+export const REDELIVERY_MAX_BYTES_PER_TICK = 512 * 1024 * 1024;
+export const REDELIVERY_MAX_OBJECTS_PER_TICK = 64;
+/**
+ * Which undelivered objects this tick will re-offer, oldest bytes first.
+ *
+ * Oldest first is deliberate: the objects that have been undelivered longest
+ * are the ones most likely to be the only remaining copy of a session whose
+ * source transcript has since been rotated away by its agent host, and they are
+ * also the ones the disk cap will start refusing to keep. A newest-first drain
+ * would leave a 45-day-old orphan permanently at the back of the queue.
+ *
+ * `unknown` objects are deliberately NOT planned. "The ledger cannot say" is
+ * `cockpit clean --reconcile`'s question, it already runs one bounded batch per
+ * tick ahead of this, and it turns each answer into `committed` or
+ * `uncommitted` — so an unknown object reaches this planner on a later tick,
+ * with the server's own answer behind it, rather than being uploaded again on a
+ * guess.
+ */
+export async function planEvidenceRedelivery(options) {
+    const plan = {
+        packs: [],
+        object_count: 0,
+        bytes: 0,
+        held_count: 0,
+        held_bytes: 0,
+        unplannable: {},
+        deferred_count: 0,
+        deferred_bytes: 0,
+    };
+    const maxBytes = options.maxBytes ?? REDELIVERY_MAX_BYTES_PER_TICK;
+    const maxObjects = options.maxObjects ?? REDELIVERY_MAX_OBJECTS_PER_TICK;
+    const root = path.join(options.stateDir, RAW_EVIDENCE_DIR);
+    const candidates = uncommittedOldestFirst(options.inventory);
+    const manifests = new Map();
+    const packPlans = new Map();
+    for (const object of candidates) {
+        if (!object.content_hash) {
+            countUnplannable(plan, "no_content_hash");
+            continue;
+        }
+        if (deliveryHold(options.staging, object.content_hash, options.now)) {
+            plan.held_count += 1;
+            plan.held_bytes += object.byte_size;
+            continue;
+        }
+        if (plan.object_count >= maxObjects ||
+            plan.bytes + object.byte_size > maxBytes) {
+            plan.deferred_count += 1;
+            plan.deferred_bytes += object.byte_size;
+            continue;
+        }
+        if (!manifests.has(object.pack_id)) {
+            manifests.set(object.pack_id, await readPackManifest(path.join(root, object.pack_id), object.pack_id));
+        }
+        const manifest = manifests.get(object.pack_id) ?? null;
+        if (!manifest) {
+            countUnplannable(plan, "manifest_unreadable");
+            continue;
+        }
+        const row = manifest.files.get(object.relative_path);
+        if (!row) {
+            countUnplannable(plan, "no_manifest_row");
+            continue;
+        }
+        let packPlan = packPlans.get(object.pack_id);
+        if (!packPlan) {
+            packPlan = {
+                pack_id: object.pack_id,
+                provenance: manifest.provenance,
+                files: [],
+                bytes: 0,
+            };
+            packPlans.set(object.pack_id, packPlan);
+            plan.packs.push(packPlan);
+        }
+        packPlan.files.push({
+            pointer: {
+                raw_evidence_pointer_id: row.object_key,
+                privacy_classification: "remote_durable_raw_evidence",
+                retention_policy: {
+                    mode: RAW_EVIDENCE_RETENTION_MODE,
+                    privacy_classification: "remote_durable_raw_evidence",
+                },
+                storage_scope: "remote_object",
+                storage_bucket: RAW_EVIDENCE_BUCKET,
+                object_key: row.object_key,
+                content_hash_sha256: row.content_hash_sha256,
+                byte_size: row.byte_size,
+                media_type: row.media_type,
+                redacted_summary: row.redacted_summary,
+            },
+            local_path: path.join(root, object.pack_id, ...object.relative_path.split("/").filter(Boolean)),
+            kind: row.kind,
+            codex_session_id: null,
+        });
+        packPlan.bytes += object.byte_size;
+        plan.object_count += 1;
+        plan.bytes += object.byte_size;
+    }
+    return plan;
+}
+/**
+ * Every uncommitted object, oldest staged bytes first.
+ *
+ * Sorted on `disk_age_ms` (the file's own mtime) rather than `age_ms`, because
+ * `age_ms` for a reconciled object is the age of the reconcile ANSWER — the
+ * same confusion that had 45-day-old orphans reading as one day old.
+ */
+function uncommittedOldestFirst(inventory) {
+    return inventory.packs
+        .flatMap((pack) => pack.objects)
+        .filter((object) => object.state === "uncommitted")
+        .sort((left, right) => (right.disk_age_ms || right.age_ms) - (left.disk_age_ms || left.age_ms));
+}
+function countUnplannable(plan, reason) {
+    plan.unplannable[reason] = (plan.unplannable[reason] ?? 0) + 1;
+}
+/**
+ * A pack's recorded provenance and file rows.
+ *
+ * Every field is read back from what the collector WROTE when it staged the
+ * pack; nothing is re-derived from the machine's current state. A pack staged
+ * six weeks ago under a work context that no longer exists still knows which
+ * repo, branch, operator and work context it belonged to, and that is the only
+ * honest label for those bytes.
+ */
+async function readPackManifest(dir, packId) {
+    const raw = await fs
+        .readFile(path.join(dir, "manifest.json"), "utf8")
+        .catch(() => null);
+    if (!raw)
+        return null;
+    try {
+        const parsed = JSON.parse(raw);
+        const provenance = provenanceFromManifest(parsed, packId);
+        if (!provenance)
+            return null;
+        const files = new Map();
+        for (const entry of asArray(parsed["files"])) {
+            const row = manifestFileRow(entry);
+            if (row)
+                files.set(row.relative_path, row.value);
+        }
+        return { provenance, files };
+    }
+    catch (error) {
+        console.error("[evidence-redelivery] pack manifest unreadable; its objects cannot be re-offered", JSON.stringify({
+            reason: "manifest_unreadable",
+            pack_id: packId,
+            ...describeError(error),
+        }));
+        return null;
+    }
+}
+function provenanceFromManifest(manifest, packId) {
+    const workContextId = asText(manifest["work_context_id"]);
+    const sessionId = asText(manifest["session_id"]);
+    const operatorId = asText(manifest["operator_id"]);
+    const repo = asText(manifest["repo_basename"]) ?? asText(manifest["repo_label"]);
+    if (!workContextId || !sessionId || !operatorId || !repo) {
+        console.error("[evidence-redelivery] pack manifest names no work context; its objects cannot be re-offered", JSON.stringify({
+            reason: "manifest_provenance_incomplete",
+            pack_id: packId,
+            has_work_context: Boolean(workContextId),
+            has_session: Boolean(sessionId),
+            has_operator: Boolean(operatorId),
+            has_repo: Boolean(repo),
+        }));
+        return null;
+    }
+    const repoLabel = asText(manifest["repo_label"]);
+    const worktreeLabel = asText(manifest["worktree_label"]);
+    return {
+        capture_source: "collector_runtime",
+        // The version that is re-offering, not the one that staged: this IS a
+        // delivery by today's collector, and pretending otherwise would misreport
+        // which build put the object in the bucket.
+        capture_adapter_version: LOCAL_COLLECTOR_VERSION,
+        collector_version: LOCAL_COLLECTOR_VERSION,
+        repo,
+        // A pack staged on a detached HEAD or a deleted worktree can have no branch
+        // recorded. `unknown` is a label, not a guess at which branch it was.
+        branch: asText(manifest["branch"]) ?? "unknown",
+        ...(repoLabel ? { repo_label: repoLabel } : {}),
+        ...(worktreeLabel ? { worktree_label: worktreeLabel } : {}),
+        operator_id: operatorId,
+        session_id: sessionId,
+        work_context_id: workContextId,
+    };
+}
+function manifestFileRow(entry) {
+    if (!entry || typeof entry !== "object")
+        return null;
+    const row = entry;
+    const relativePath = asText(row["relative_path"]);
+    const objectKey = asText(row["object_key"]);
+    const hash = asText(row["content_hash_sha256"]);
+    const mediaType = asText(row["media_type"]);
+    const summary = asText(row["redacted_summary"]);
+    const byteSize = row["byte_size"];
+    if (!relativePath ||
+        !objectKey ||
+        !hash ||
+        hash.length !== 64 ||
+        !mediaType ||
+        !summary ||
+        typeof byteSize !== "number") {
+        return null;
+    }
+    return {
+        relative_path: relativePath,
+        value: {
+            object_key: objectKey,
+            content_hash_sha256: hash,
+            byte_size: byteSize,
+            media_type: mediaType,
+            redacted_summary: summary,
+            kind: asText(row["kind"]) ?? "unknown",
+        },
+    };
+}
+function asArray(value) {
+    return Array.isArray(value) ? value : [];
+}
+function asText(value) {
+    return typeof value === "string" && value.trim() ? value : null;
+}