@bli-cockpit/cli 0.2.70 → 0.2.74

package/dist/evidence-redelivery.js

@@ -0,0 +1,280 @@
+/**
+ * Offering undelivered staged evidence again, on the next tick and every tick
+ * after, until it lands or names why it cannot.
+ *
+ * BLI-3797. `evidence-redelivery-plan.ts` says WHICH bytes and rebuilds the
+ * offer; this file spends it against the same chunked upload client the live
+ * sync uses, writes each success into the commit ledger and each failure into
+ * the delivery-attempt history (so the existing 15-min → 6-h backoff applies
+ * and one doomed object is not re-offered 96 times a day), and says what
+ * happened on every branch.
+ *
+ * Four properties, the same ones the reconcile and prune follow-ups keep:
+ *
+ * - **It never throws and never fails a tick.** A drain that cannot run is a
+ *   log line and a receipt, never a reason collection did not happen.
+ * - **It never deletes.** Nothing here removes a staged byte; the prune that
+ *   follows it decides that, and it only ever takes what the ledger vouches
+ *   for — which now includes whatever this just delivered.
+ * - **It is bounded.** One drain rate per tick (see the plan module), so the
+ *   backlog clears in hours without competing with the tick's own collection.
+ * - **It says so on both branches.** A tick with nothing undelivered is silent
+ *   (the steady state, 96 times a day); every other outcome, including a clean
+ *   drain, gets one `[evidence-redelivery]` line with counts and reasons.
+ */
+import { uploadRawEvidenceFilesChunked, } from "./evidence-upload-client.js";
+import { planEvidenceRedelivery, } from "./evidence-redelivery-plan.js";
+import { markObjectCommitted, readRawEvidenceCursor, writeRawEvidenceCursor, } from "./cursors/raw-evidence-cursor.js";
+import { readStagingInventory } from "./disk-usage.js";
+import { describeError } from "./health-detail.js";
+import { getCollectorRuntimePaths, readLocalCollectorSessionFile, } from "./local-state.js";
+import { clearDeliveryAttempt, readRawEvidenceStagingState, recordDeliveryFailure, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
+/**
+ * The drain the sync tick runs and `cockpit doctor`'s disk fix reuses.
+ * Never throws.
+ */
+export async function runEvidenceRedelivery(options) {
+    const env = options.env ?? process.env;
+    const now = options.now ?? new Date();
+    if (env["COCKPIT_DISABLE_EVIDENCE_REDELIVERY"] === "1") {
+        return emptyResult("disabled", "skipped");
+    }
+    const paths = getCollectorRuntimePaths(options.homeDir);
+    try {
+        const inventory = options.inventory ?? (await readStagingInventory(paths, now));
+        if (inventory.uncommitted_count === 0) {
+            // The steady state, 96 ticks a day. Saying so every time would bury the
+            // ticks that matter; `cockpit doctor`'s disk row still says it out loud
+            // to a person who asks.
+            return emptyResult("nothing_uncommitted", "ok");
+        }
+        const staging = await readRawEvidenceStagingState(paths.state_dir);
+        const plan = await planEvidenceRedelivery({
+            stateDir: paths.state_dir,
+            inventory,
+            staging,
+            now,
+            ...(options.maxBytes != null ? { maxBytes: options.maxBytes } : {}),
+            ...(options.maxObjects != null ? { maxObjects: options.maxObjects } : {}),
+        });
+        if (plan.object_count === 0) {
+            const result = emptyPlanResult(plan);
+            reportRedelivery(result, plan);
+            return result;
+        }
+        const session = await readLocalCollectorSessionFile(paths).catch(() => null);
+        if (!session ||
+            session.session_state !== "valid" ||
+            typeof session.device_token !== "string" ||
+            !session.device_token) {
+            // Not a failure of the backlog: this machine simply cannot speak to the
+            // dashboard right now. The bytes stay, the reason is named, and the next
+            // tick after a `cockpit login` picks them straight back up.
+            const result = {
+                ...emptyPlanResult(plan),
+                status: "skipped",
+                reason: "no_device_session",
+            };
+            reportRedelivery(result, plan);
+            return result;
+        }
+        const outcome = await deliverPlan({
+            plan,
+            dashboardUrl: options.dashboardUrl,
+            deviceToken: session.device_token,
+            fetchImpl: options.fetch ?? fetch,
+            now,
+        });
+        await recordOutcomes(paths, staging, outcome.outcomes, now);
+        const result = {
+            status: outcome.failed > 0 && outcome.uploaded === 0 ? "fail" : "ok",
+            reason: "redelivered",
+            offered: plan.object_count,
+            offered_bytes: plan.bytes,
+            uploaded: outcome.uploaded,
+            uploaded_bytes: outcome.uploadedBytes,
+            reused: outcome.reused,
+            failed: outcome.failed,
+            held: plan.held_count,
+            deferred: plan.deferred_count,
+            deferred_bytes: plan.deferred_bytes,
+            packs: plan.packs.length,
+            failure_reasons: outcome.failureReasons,
+        };
+        reportRedelivery(result, plan);
+        return result;
+    }
+    catch (error) {
+        console.error("[evidence-redelivery] the drain could not run; nothing was re-offered", JSON.stringify({ reason: "redelivery_threw", ...describeError(error) }));
+        return { ...emptyResult("redelivery_threw", "fail") };
+    }
+}
+/**
+ * One `uploadRawEvidenceFilesChunked` call per pack, because provenance is a
+ * PACK fact: the repo, branch, operator, session and work context these bytes
+ * belonged to were recorded when they were staged, and sending one pack's
+ * objects under another pack's labels would file six-week-old evidence against
+ * today's ticket.
+ */
+async function deliverPlan(options) {
+    const result = {
+        outcomes: [],
+        uploaded: 0,
+        uploadedBytes: 0,
+        reused: 0,
+        failed: 0,
+        failureReasons: [],
+    };
+    for (const pack of options.plan.packs) {
+        const uploaded = await deliverOnePack(pack, options);
+        for (const outcome of uploaded) {
+            result.outcomes.push(outcome);
+            if (outcome.upload_state === "uploaded") {
+                result.uploaded += 1;
+                result.uploadedBytes += outcome.pointer.byte_size ?? 0;
+                continue;
+            }
+            if (outcome.upload_state === "reused_existing") {
+                result.reused += 1;
+                continue;
+            }
+            result.failed += 1;
+            const reason = outcome.reason ?? "unknown";
+            if (!result.failureReasons.includes(reason)) {
+                result.failureReasons.push(reason);
+            }
+        }
+    }
+    return result;
+}
+async function deliverOnePack(pack, options) {
+    try {
+        const uploaded = await uploadRawEvidenceFilesChunked({
+            fetchImpl: options.fetchImpl,
+            dashboardUrl: options.dashboardUrl,
+            deviceToken: options.deviceToken,
+            provenance: pack.provenance,
+            generatedAt: options.now.toISOString(),
+            files: pack.files,
+        });
+        return uploaded.outcomes;
+    }
+    catch (error) {
+        // The upload client already reports its own transport failures; this is the
+        // net for anything it does not, and it must not abandon the packs behind
+        // this one. Each file is reported failed so the backoff history records the
+        // attempt rather than re-offering all of them again in fifteen minutes.
+        console.error("[evidence-redelivery] a pack's re-offer threw; the rest of the drain continues", JSON.stringify({
+            reason: "pack_redelivery_threw",
+            pack_id: pack.pack_id,
+            object_count: pack.files.length,
+            ...describeError(error),
+        }));
+        return pack.files.map((file) => ({
+            pointer: file.pointer,
+            object_key: file.pointer.object_key ?? "",
+            codex_session_id: null,
+            kind: file.kind ?? "unknown",
+            upload_state: "upload_failed",
+            reason: "pack_redelivery_threw",
+            uploaded_chunk_count: 0,
+        }));
+    }
+}
+/**
+ * Write what happened where the next tick will read it: the commit ledger for
+ * anything that landed, the delivery-attempt history for anything that did not.
+ *
+ * Both writes happen even when the run partly failed, and the ledger is written
+ * FIRST — a delivered object whose ledger row was lost would be re-offered
+ * forever, which is the failure this whole module exists to end.
+ */
+async function recordOutcomes(paths, staging, outcomes, now) {
+    const cursor = await readRawEvidenceCursor(paths);
+    let ledgerChanged = false;
+    let stagingChanged = false;
+    for (const outcome of outcomes) {
+        const hash = outcome.pointer.content_hash_sha256;
+        if (!hash)
+            continue;
+        if (outcome.upload_state === "upload_failed") {
+            recordDeliveryFailure(staging, hash, {
+                reason: outcome.reason ?? "unknown",
+                attemptedAt: now,
+                byteSize: outcome.pointer.byte_size ?? 0,
+            });
+            stagingChanged = true;
+            continue;
+        }
+        markObjectCommitted(cursor, hash, {
+            object_key: outcome.object_key,
+            byte_size: outcome.pointer.byte_size ?? 0,
+            committed_at: now.toISOString(),
+        });
+        ledgerChanged = true;
+        if (clearDeliveryAttempt(staging, hash))
+            stagingChanged = true;
+    }
+    if (ledgerChanged)
+        await writeRawEvidenceCursor(paths, cursor);
+    if (stagingChanged) {
+        await writeRawEvidenceStagingState(paths.state_dir, staging);
+    }
+}
+/**
+ * One line per non-steady-state tick. Counts, byte totals and reason labels
+ * only — no pack id, no hash, no path, per the logging contract.
+ */
+function reportRedelivery(result, plan) {
+    const fields = {
+        reason: result.reason,
+        status: result.status,
+        offered: result.offered,
+        offered_bytes: result.offered_bytes,
+        uploaded: result.uploaded,
+        uploaded_bytes: result.uploaded_bytes,
+        reused: result.reused,
+        failed: result.failed,
+        held_by_backoff: result.held,
+        deferred_to_next_tick: result.deferred,
+        deferred_bytes: result.deferred_bytes,
+        packs: result.packs,
+        unplannable: plan.unplannable,
+        failure_reasons: result.failure_reasons,
+    };
+    if (result.status === "fail") {
+        console.error("[evidence-redelivery] every re-offered object was refused", JSON.stringify(fields));
+        return;
+    }
+    console.error(result.uploaded > 0
+        ? "[evidence-redelivery] undelivered staged evidence landed"
+        : "[evidence-redelivery] undelivered staged evidence was not re-offered this tick", JSON.stringify(fields));
+}
+function emptyPlanResult(plan) {
+    const nothingPlannable = plan.held_count > 0 && plan.deferred_count === 0
+        ? "all_held_by_backoff"
+        : "nothing_plannable";
+    return {
+        ...emptyResult(nothingPlannable, "ok"),
+        held: plan.held_count,
+        deferred: plan.deferred_count,
+        deferred_bytes: plan.deferred_bytes,
+    };
+}
+function emptyResult(reason, status) {
+    return {
+        status,
+        reason,
+        offered: 0,
+        offered_bytes: 0,
+        uploaded: 0,
+        uploaded_bytes: 0,
+        reused: 0,
+        failed: 0,
+        held: 0,
+        deferred: 0,
+        deferred_bytes: 0,
+        packs: 0,
+        failure_reasons: [],
+    };
+}

package/dist/local-state-status.js

@@ -1,5 +1,7 @@
 import os from "node:os";
 import path from "node:path";
+import { readStagingInventory } from "./disk-usage.js";
+import { describeError } from "./health-detail.js";
 import { summarizeLocalUploadSpool } from "./spool/local-spool.js";
 import { summarizeInstallEventOutbox } from "./spool/install-event-outbox.js";
 import { readRawEvidenceStagingState, summarizeStuckEvidence, } from "./raw-evidence-staging.js";
@@ -55,6 +57,10 @@ export async function inspectLocalCollectorStatus(options = {}) {
         stuck_evidence_max_attempts: stuckEvidence.max_attempts,
         stuck_evidence_oldest_failure_at: stuckEvidence.oldest_first_failed_at,
         stuck_evidence_reasons: stuckEvidence.reasons,
+        staging_uncommitted_count: reading.staging?.uncommitted_count ?? null,
+        staging_uncommitted_bytes: reading.staging?.uncommitted_bytes ?? null,
+        staging_uncommitted_reason: reading.staging?.dominant_uncommitted_reason?.reason ?? null,
+        staging_oldest_uncommitted_at: reading.staging?.oldest_uncommitted_at ?? null,
         details: describeCollectorStatus(reading),
     };
 }
@@ -76,7 +82,20 @@ async function readCollectorStatus(options, now) {
         readRawEvidenceStagingState(paths.state_dir),
     ]);
     const stuckEvidence = summarizeStuckEvidence(stagingState, now);
+    // Best-effort by construction: a status read must never fail because the
+    // staging tree could not be walked, and an absent backlog reads as "this read
+    // did not ask", never as "there is none".
+    const staging = options.includeStagingBacklog
+        ? await readStagingInventory(paths, now).catch((error) => {
+            console.error("[collector status] the staging tree could not be read; the undelivered backlog is unknown", JSON.stringify({
+                reason: "staging_inventory_unreadable",
+                ...describeError(error),
+            }));
+            return null;
+        })
+        : null;
     return {
+        staging,
         paths,
         config,
         session,
@@ -134,6 +153,18 @@ function describeCollectorStatus(reading) {
     if (healthOutbox.pending_count > 0) {
         details.push(`Collector health retry pending: ${healthOutbox.pending_count} sanitized receipt(s) queued since ${healthOutbox.oldest_created_at ?? "unknown"}.`);
     }
+    // BLI-3797: the line above only sees objects whose delivery FAILED. This one
+    // sees evidence that is on this disk and not in the bucket, however it got
+    // that way — which is the only question that answers "has everything landed?".
+    if (reading.staging && reading.staging.uncommitted_count > 0) {
+        const dominant = reading.staging.dominant_uncommitted_reason;
+        details.push(`Staged evidence not yet accepted: ${reading.staging.uncommitted_count} object(s), ${mib(reading.staging.uncommitted_bytes)} MB, oldest staged ${reading.staging.oldest_uncommitted_at ?? "unknown"}${dominant
+            ? ` — biggest reason ${dominant.reason} on ${dominant.count} object(s)`
+            : ""}. \`cockpit sync\` re-offers them each tick; \`cockpit doctor --fix\` does it now.`);
+    }
+    else if (reading.staging) {
+        details.push("Staged evidence: every object on this disk is accepted.");
+    }
     if (stuckEvidence.stuck_object_count > 0) {
         details.push(`Raw evidence stuck: ${stuckEvidence.stuck_object_count} object(s) have never been accepted (${stuckEvidence.held_object_count} waiting on backoff, worst ${stuckEvidence.max_attempts} attempt(s) since ${stuckEvidence.oldest_first_failed_at ?? "unknown"}) — reasons: ${stuckEvidence.reasons.join(", ") || "unknown"}.`);
     }
@@ -155,6 +186,9 @@ export function classifyCollectorFreshness(context, lastUploadSuccessAt, now) {
         return "stale";
     return now.getTime() - latestActivityAt <= 5 * 60 * 1000 ? "fresh" : "stale";
 }
+function mib(bytes) {
+    return (bytes / (1024 * 1024)).toFixed(1);
+}
 function latestTimestamp(values) {
     const timestamps = values
         .map((value) => Date.parse(value ?? ""))

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bli-cockpit/cli",
-  "version": "0.2.70",
+  "version": "0.2.74",
   "private": false,
   "type": "module",
   "bin": {
@@ -24,11 +24,11 @@
     "pretypecheck": "node ../../scripts/build-workspace-dep.mjs @bli-cockpit/cli",
     "typecheck": "node -e \"await import('./dist/commands/public-root.js')\"",
     "pretest": "node ../../scripts/build-workspace-dep.mjs @bli-cockpit/cli",
-    "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
+    "test": "node dist/cli.js --help && node ../../scripts/assert-public-cli-routing.mjs && node ../../scripts/assert-public-cli-verb-help.mjs && node ../../scripts/assert-public-cli-runtime-files.mjs && node ../../scripts/assert-public-cli-no-fleet-posts.mjs && node ../../scripts/assert-public-package-pack.mjs --workspace=@bli-cockpit/cli"
   },
   "dependencies": {
-    "@bli-cockpit/memory-mcp": "0.1.10",
-    "@bli-cockpit/mcp": "0.1.11",
-    "@bli-cockpit/telemetry-core": "0.1.31"
+    "@bli-cockpit/memory-mcp": "0.1.11",
+    "@bli-cockpit/mcp": "0.1.13",
+    "@bli-cockpit/telemetry-core": "0.1.32"
   }
 }