@bli-cockpit/cli 0.2.51 → 0.2.53

package/dist/commands/sync-followups.js

@@ -1,8 +1,8 @@
 /**
  * What the sync tick does AFTER collection's own outcome is decided and
  * reported: keep this machine's CLI current on npm `latest` (BLI-2601), put a
- * broken scheduler registration back (BLI-2721), and keep BLI Memory
- * registered with both agent hosts (BLI-3580).
+ * broken scheduler registration back (BLI-2721), keep BLI Memory registered
+ * with both agent hosts (BLI-3580), and stop the laptop filling up (BLI-3619).
  *
  * Split out of commands/sync.ts (BLI-3578), moved verbatim. They belong
  * together because they share one rule, and it is the reason both are called
@@ -18,7 +18,11 @@ import { redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./insta
 import { runSelfUpdate, SelfUpdateError } from "./install-update.js";
 import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
 import { runAutostartSelfHeal, } from "../autostart-self-heal.js";
+import { runStagingPrune } from "../disk-prune.js";
+import { runEvidenceReconcile, } from "../evidence-reconcile-client.js";
 import { envWithNodeRuntimeOnPath, runScheduledSelfUpdate, } from "../scheduled-self-update.js";
+/** How many reconcile batches (of up to 500 hashes each) one sync tick may spend. */
+const RECONCILE_BATCHES_PER_TICK = 1;
 /**
  * BLI-2721: after the tick's collection and self-update are done and
  * reported, repair a broken/legacy autostart registration in place (Windows
@@ -183,6 +187,140 @@ function memoryInstallEvent(outcome) {
     }
     return { step: "memory_install", status: "ok", error_detail: detail };
 }
+/**
+ * BLI-3619: staged raw evidence Tower has already accepted stops living on the
+ * laptop forever.
+ *
+ * Same shape as the memory install above and for the same reasons: it runs only
+ * once collection's own outcome is decided and reported, at most once a day, it
+ * cannot throw, and its outcome is its own named receipt rather than a sync
+ * failure. A machine that cannot prune is a machine short of disk, and that
+ * must never also be a machine that stops collecting.
+ *
+ * The rule itself is `disk-retention.ts`: only objects the upload ledger
+ * vouches for are ever deleted, and anything undelivered is counted, aged and
+ * named instead.
+ *
+ * BLI-3619's second half runs FIRST, every tick, bounded to
+ * `RECONCILE_BATCHES_PER_TICK` (one batch of up to 500 hashes): the local
+ * ledger's own memory is capped, so a laptop that keeps accumulating
+ * `unknown` state needs SOMETHING asking the server on a schedule, not only
+ * when a person happens to type `cockpit clean --reconcile`. One batch a tick
+ * is not throttled to once a day like the prune below — an `unknown` backlog
+ * drains at up to 500 hashes/15 min, and the ordinary case (nothing unknown)
+ * costs one cheap local read and no network call at all, so it never competes
+ * with collection for the tick's time. A reconcile failure never blocks the
+ * prune that follows it.
+ */
+export async function runStagingPruneAfterSync(command, io, dashboardUrl, options = {}) {
+    const events = [];
+    const reconciled = await runReconcileFollowUp(command, io, dashboardUrl, options);
+    if (reconciled)
+        events.push(reconcileEvent(reconciled));
+    let result;
+    try {
+        result = await runStagingPrune(getCollectorRuntimePaths(command.homeDir), {
+            env: io.env,
+            ...(options.now ? { now: options.now } : {}),
+        });
+    }
+    catch (error) {
+        // runStagingPrune already catches everything it can reach; this is the
+        // last-resort net so a prune crash truly cannot touch the sync result.
+        result = { ...prunedNothing(), reason: "prune_threw", status: "fail" };
+        console.error("[collector prune] the prune follow-up threw", JSON.stringify({
+            reason: "prune_followup_threw",
+            detail: redactedSyncErrorDetail(error),
+        }));
+    }
+    // The daily throttle is the steady state — reporting it would post a receipt
+    // on 95 of every 96 ticks for no new information.
+    if (result.reason !== "throttled_recent_run") {
+        events.push(stagingPruneEvent(result));
+    }
+    if (events.length === 0)
+        return;
+    await reportInstallEventsBestEffort({
+        homeDir: command.homeDir,
+        dashboardUrl,
+        command: "sync",
+        events,
+        json: command.json,
+        io,
+    });
+}
+/**
+ * Never throws (`runEvidenceReconcile` already never does); returns `null`
+ * for the boring, common case — nothing this tick was `unknown` — so that
+ * case costs no receipt either, the same rule the prune's own daily throttle
+ * follows above.
+ */
+async function runReconcileFollowUp(command, io, dashboardUrl, options) {
+    const result = await runEvidenceReconcile({
+        homeDir: command.homeDir,
+        dashboardUrl,
+        maxBatches: RECONCILE_BATCHES_PER_TICK,
+        fetch: io.fetch,
+        ...(options.now ? { now: options.now } : {}),
+    });
+    return result.reason === "nothing_unknown" ? null : result;
+}
+function prunedNothing() {
+    return {
+        status: "skipped",
+        reason: "prune_threw",
+        deleted_files: 0,
+        deleted_bytes: 0,
+        removed_packs: 0,
+        kept_uncommitted: 0,
+        kept_uncommitted_bytes: 0,
+        oldest_uncommitted_age_ms: 0,
+        kept_unknown: 0,
+        kept_unknown_bytes: 0,
+        kept_in_window: 0,
+        cap_bytes: 0,
+        bytes_before: 0,
+        bytes_after: 0,
+        cap_blocked_by_uncommitted: false,
+        cap_blocked_count: 0,
+        failed_deletions: 0,
+    };
+}
+/** Counts and byte totals only; no pack id and no path travels in a receipt. */
+function stagingPruneEvent(result) {
+    const detail = [
+        `freed ${result.deleted_bytes}B in ${result.deleted_files} file(s)`,
+        `held ${result.kept_uncommitted_bytes}B uncommitted`,
+        result.cap_blocked_by_uncommitted
+            ? `staging_cap_blocked_by_uncommitted ${result.cap_blocked_count}`
+            : null,
+        result.failed_deletions > 0
+            ? `failed_deletions ${result.failed_deletions}`
+            : null,
+    ]
+        .filter((part) => Boolean(part))
+        .join("; ");
+    if (result.status === "fail") {
+        return {
+            step: "staging_prune",
+            status: "fail",
+            error_code: result.reason,
+            error_detail: detail,
+        };
+    }
+    if (result.status === "skipped") {
+        return { step: "staging_prune", status: "skipped", error_code: result.reason };
+    }
+    return { step: "staging_prune", status: "ok", error_detail: detail };
+}
+/** Counts only; no hash and no pack id travels in a receipt. */
+function reconcileEvent(result) {
+    const detail = `asked ${result.asked} hash(es) across ${result.batches}/${result.total_batches} batch(es); committed ${result.committed}, not_committed ${result.not_committed}, unknown_to_server ${result.unknown_to_server}, failed_batches ${result.failed_batches}`;
+    if (result.status === "fail") {
+        return { step: "evidence_reconcile", status: "fail", error_code: result.reason, error_detail: detail };
+    }
+    return { step: "evidence_reconcile", status: "ok", error_detail: detail };
+}
 /**
  * BLI-2601: the fleet keeps itself current on npm `latest` without anyone
  * re-running `npm i -g @bli-cockpit/cli` by hand after day 0. This always

package/dist/commands/sync.js

@@ -5,7 +5,7 @@ import { discoverCommandWorktrees } from "./local-discovery.js";
 import { sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
 import { classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
 import { runAttributedWorktreeSync, } from "./session-sync.js";
-import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, } from "./sync-followups.js";
+import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, runStagingPruneAfterSync, } from "./sync-followups.js";
 import { describeError } from "../health-detail.js";
 import { inspectBackfillLock } from "../backfill-lock.js";
 import { rotateCollectorLogsBestEffort } from "../log-rotation.js";
@@ -52,6 +52,9 @@ export async function runSync(command, io) {
         // BLI-3580: BLI Memory's registration converges the same way — after
         // collection, at most once a day, its own receipt either way.
         await runMemoryInstallAfterSync(command, io, dashboardUrl);
+        // BLI-3619: and the disk stops growing without bound — same daily cadence,
+        // same rule that a follow-up never blocks or fails collection.
+        await runStagingPruneAfterSync(command, io, dashboardUrl);
         return result.exitCode;
     }
     catch (error) {

package/dist/cursors/raw-evidence-reconcile-cursor.js

@@ -0,0 +1,132 @@
+/**
+ * The server's own answer, kept durably on this machine.
+ *
+ * BLI-3619. `cursors/raw-evidence.json` `objects` answers "did this content
+ * hash commit?" but is capped at 5,000 rows and prunes the oldest, so a pack
+ * staged before its own memory begins reads `unknown` — the ledger cannot say,
+ * not that it says no. `cockpit clean --reconcile` asks the dashboard once per
+ * hash and this file is where that answer lives afterward, so the same
+ * question is never asked twice for the same hash and a `--dry-run` run can
+ * read a reconciled state from an earlier `--reconcile` run without spending
+ * another round trip.
+ *
+ * Keyed on `content_hash_sha256`, same as the commit ledger and for the same
+ * reason: `pack_id` is a local directory name the server has never seen.
+ *
+ * Metadata only: hashes, a verdict label, and two timestamps. Never a path,
+ * never a byte of content.
+ */
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { describeError, isMissingFileFailure } from "../health-detail.js";
+export const RECONCILE_CURSOR_FILENAME = "raw-evidence-reconcile.json";
+// Same order of magnitude as the commit ledger's own cap (D4): entries are
+// small, and the question this file answers is bounded by how many hashes a
+// laptop can hold `unknown` at once, which the byte cap already keeps small.
+const MAX_TRACKED_RESULTS = 5_000;
+export function emptyReconcileCursorState() {
+    return {
+        schema_version: "cockpit-raw-evidence-reconcile-cursor.v1",
+        updated_at: null,
+        results: {},
+    };
+}
+export async function readReconcileCursor(paths) {
+    try {
+        const raw = JSON.parse(await fs.readFile(reconcileCursorPath(paths), "utf8"));
+        return parseReconcileCursorState(raw);
+    }
+    catch (error) {
+        if (!isMissingFileFailure(error)) {
+            console.error("[collector reconcile] reconcile cursor unreadable, starting from empty", JSON.stringify({
+                reason: "reconcile_cursor_unreadable",
+                ...describeError(error),
+            }));
+        }
+        return emptyReconcileCursorState();
+    }
+}
+export async function writeReconcileCursor(paths, state) {
+    const filePath = reconcileCursorPath(paths);
+    await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 });
+    const pruned = pruneReconcileState(state);
+    const serialized = `${JSON.stringify(pruned, null, 2)}\n`;
+    const tempPath = `${filePath}.tmp-${process.pid}-${crypto.randomUUID()}`;
+    let handle = null;
+    try {
+        handle = await fs.open(tempPath, "w", 0o600);
+        await handle.writeFile(serialized);
+        await handle.sync();
+    }
+    finally {
+        await handle?.close();
+    }
+    try {
+        await fs.rename(tempPath, filePath);
+    }
+    catch (error) {
+        await fs.rm(tempPath, { force: true }).catch(() => undefined);
+        throw error;
+    }
+    if (process.platform !== "win32") {
+        await fs.chmod(filePath, 0o600).catch(() => undefined);
+    }
+}
+export function recordReconcileResult(state, contentHash, entry) {
+    state.results[contentHash] = entry;
+}
+function pruneReconcileState(state) {
+    const entries = Object.entries(state.results);
+    if (entries.length <= MAX_TRACKED_RESULTS)
+        return state;
+    entries.sort((a, b) => b[1].checked_at.localeCompare(a[1].checked_at));
+    return {
+        ...state,
+        results: Object.fromEntries(entries.slice(0, MAX_TRACKED_RESULTS)),
+    };
+}
+function reconcileCursorPath(paths) {
+    return path.join(paths.cursors_dir, RECONCILE_CURSOR_FILENAME);
+}
+function parseReconcileCursorState(value) {
+    if (!value || typeof value !== "object")
+        return emptyReconcileCursorState();
+    const record = value;
+    const results = {};
+    const rawResults = record["results"];
+    if (rawResults && typeof rawResults === "object") {
+        for (const [hash, entry] of Object.entries(rawResults)) {
+            const parsed = parseReconcileEntry(entry);
+            if (parsed)
+                results[hash] = parsed;
+        }
+    }
+    return {
+        schema_version: "cockpit-raw-evidence-reconcile-cursor.v1",
+        updated_at: optionalString(record["updated_at"]),
+        results,
+    };
+}
+function parseReconcileEntry(value) {
+    if (!value || typeof value !== "object")
+        return null;
+    const record = value;
+    const verdict = record["verdict"];
+    const checkedAt = optionalString(record["checked_at"]);
+    if (!checkedAt)
+        return null;
+    if (verdict !== "committed" &&
+        verdict !== "not_committed" &&
+        verdict !== "unknown_to_server") {
+        return null;
+    }
+    return {
+        verdict,
+        checked_at: checkedAt,
+        server_committed_at: optionalString(record["server_committed_at"]),
+    };
+}
+function optionalString(value) {
+    return typeof value === "string" && value.trim() ? value : null;
+}

package/dist/disk-prune.js

@@ -0,0 +1,246 @@
+/**
+ * Carrying out the retention plan, and saying what it did.
+ *
+ * BLI-3619. The decision is `disk-retention.ts` and the reading is
+ * `disk-usage.ts`; this file is the only one that removes a staged byte. Three
+ * properties it must keep:
+ *
+ * - **It never fails the tick.** Every path is caught. A prune that cannot run
+ *   is a log line, never a reason collection did not happen — the same rule the
+ *   log rotation and the two self-heal follow-ups already live under.
+ * - **It never deletes what the ledger did not vouch for.** The plan decides
+ *   that; this file does not second-guess it, and re-reads nothing.
+ * - **It says so on both branches.** `[collector prune]` fires on the boring
+ *   run too, because "0 deleted, 2.1 GB held, all committed" is the only
+ *   evidence that the drain is still running at all (the BLI-2528 lesson: what
+ *   only speaks on failure cannot answer "did anything work today?").
+ *
+ * Throttled to once a day by a marker file, the same idiom the raw-evidence GC,
+ * the scheduled self-update and the BLI Memory install use. Walking a pack tree
+ * is cheap; doing it every fifteen minutes for a decision that changes on the
+ * scale of hours is not.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { planStagingRetention, retentionOptionsFromEnv, } from "./disk-retention.js";
+import { RAW_EVIDENCE_DIR, readStagingInventory } from "./disk-usage.js";
+import { describeError } from "./health-detail.js";
+import { readRawEvidenceStagingState, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
+export const STAGING_PRUNE_THROTTLE_MARKER = ".last-staging-prune";
+export const STAGING_PRUNE_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
+/**
+ * The prune the sync tick runs and `cockpit clean` / `cockpit doctor` reuse.
+ * Never throws.
+ */
+export async function runStagingPrune(paths, options = {}) {
+    const env = options.env ?? process.env;
+    const now = options.now ?? new Date();
+    if (env["COCKPIT_DISABLE_GC"] === "1") {
+        return skippedResult("disabled");
+    }
+    if (!options.force && (await prunedRecently(paths, now))) {
+        return skippedResult("throttled_recent_run");
+    }
+    try {
+        if (!options.dryRun && !options.force)
+            await markPruned(paths, now);
+        const plan = await planFromDisk(paths, env, options, now);
+        const result = options.dryRun
+            ? dryRunResult(plan)
+            : await applyPlan(paths, plan);
+        reportPrune(result, { dryRun: Boolean(options.dryRun) });
+        return result;
+    }
+    catch (error) {
+        console.error("[collector prune] the prune could not run; nothing was deleted", JSON.stringify({ reason: "prune_threw", ...describeError(error) }));
+        return { ...skippedResult("prune_threw"), status: "fail" };
+    }
+}
+export async function planFromDisk(paths, env, options, now) {
+    const fromEnv = retentionOptionsFromEnv(env);
+    const inventory = await readStagingInventory(paths, now);
+    return planStagingRetention(inventory, {
+        retentionMs: options.retentionMs ?? fromEnv.retentionMs,
+        capBytes: options.capBytes ?? fromEnv.capBytes,
+        allCommitted: options.allCommitted,
+    });
+}
+async function applyPlan(paths, plan) {
+    const root = path.join(paths.state_dir, RAW_EVIDENCE_DIR);
+    const tally = { files: 0, bytes: 0, packs: 0, failed: 0, hashes: [] };
+    const removedPackIds = await removeEmptyPacks(root, plan, tally);
+    await removeRemainingFiles(root, plan, removedPackIds, tally);
+    await forgetStagedObjects(paths, tally.hashes);
+    return {
+        ...dryRunResult(plan),
+        status: "ok",
+        reason: tally.files > 0 ? "pruned" : "nothing_eligible",
+        deleted_files: tally.files,
+        deleted_bytes: tally.bytes,
+        removed_packs: tally.packs,
+        failed_deletions: tally.failed,
+    };
+}
+/**
+ * A pack whose every payload file is going is removed whole, in one call, so
+ * `manifest.json` goes with it rather than surviving as a description of files
+ * that are not there.
+ */
+async function removeEmptyPacks(root, plan, tally) {
+    const removed = new Set();
+    for (const pack of plan.empty_packs) {
+        const gone = await rmQuietly(path.join(root, pack.pack_id), true);
+        if (!gone) {
+            tally.failed += 1;
+            continue;
+        }
+        removed.add(pack.pack_id);
+        tally.packs += 1;
+        tally.bytes += pack.manifest_bytes;
+    }
+    return removed;
+}
+async function removeRemainingFiles(root, plan, removedPackIds, tally) {
+    const emptyPackIds = new Set(plan.empty_packs.map((pack) => pack.pack_id));
+    for (const entry of plan.delete) {
+        if (emptyPackIds.has(entry.pack_id)) {
+            // The pack removal above already took these bytes — or failed, in which
+            // case the file is still on disk and must be counted as such.
+            if (!removedPackIds.has(entry.pack_id))
+                tally.failed += 1;
+            else
+                countRemoved(tally, entry);
+            continue;
+        }
+        const file = path.join(root, entry.pack_id, ...entry.relative_path.split("/").filter(Boolean));
+        if (await rmQuietly(file, false))
+            countRemoved(tally, entry);
+        else
+            tally.failed += 1;
+    }
+}
+function countRemoved(tally, entry) {
+    tally.files += 1;
+    tally.bytes += entry.byte_size;
+    if (entry.content_hash)
+        tally.hashes.push(entry.content_hash);
+}
+async function rmQuietly(target, recursive) {
+    return fs.rm(target, { recursive, force: true }).then(() => true, () => false);
+}
+/**
+ * The staged-object index points at bytes that are no longer there. Left alone
+ * it self-heals — `resolveStagedObject` stats the file and drops the entry —
+ * but the index is capped at 5,000 rows, so stale entries evict live ones and
+ * a real staged copy stops being found. One atomic write clears them together.
+ */
+async function forgetStagedObjects(paths, hashes) {
+    if (hashes.length === 0)
+        return;
+    try {
+        const staging = await readRawEvidenceStagingState(paths.state_dir);
+        let removed = 0;
+        for (const hash of hashes) {
+            if (staging.staged[hash]) {
+                delete staging.staged[hash];
+                removed += 1;
+            }
+        }
+        if (removed === 0)
+            return;
+        staging.updated_at = new Date().toISOString();
+        await writeRawEvidenceStagingState(paths.state_dir, staging);
+    }
+    catch (error) {
+        // Not fatal and not silent: the bytes are gone either way, and the index
+        // heals on the next read. What must not happen is nobody knowing the index
+        // is carrying rows for files that no longer exist.
+        console.error("[collector prune] staged index not updated after the prune", JSON.stringify({
+            reason: "staged_index_write_failed",
+            forgotten_count: hashes.length,
+            ...describeError(error),
+        }));
+    }
+}
+function dryRunResult(plan) {
+    return {
+        status: "ok",
+        reason: plan.delete.length > 0 ? "would_prune" : "nothing_eligible",
+        deleted_files: plan.delete.length,
+        deleted_bytes: plan.deleted_bytes + plan.empty_pack_manifest_bytes,
+        removed_packs: plan.empty_packs.length,
+        kept_uncommitted: plan.kept_uncommitted.count,
+        kept_uncommitted_bytes: plan.kept_uncommitted.bytes,
+        oldest_uncommitted_age_ms: plan.kept_uncommitted.oldest_age_ms,
+        kept_unknown: plan.kept_unknown.count,
+        kept_unknown_bytes: plan.kept_unknown.bytes,
+        kept_in_window: plan.kept_in_window.count,
+        cap_bytes: plan.cap_bytes,
+        bytes_before: plan.bytes_before,
+        bytes_after: plan.bytes_after,
+        cap_blocked_by_uncommitted: plan.cap_blocked_by_uncommitted,
+        cap_blocked_count: plan.cap_blocked_count,
+        failed_deletions: 0,
+    };
+}
+function skippedResult(reason) {
+    return {
+        status: "skipped",
+        reason,
+        deleted_files: 0,
+        deleted_bytes: 0,
+        removed_packs: 0,
+        kept_uncommitted: 0,
+        kept_uncommitted_bytes: 0,
+        oldest_uncommitted_age_ms: 0,
+        kept_unknown: 0,
+        kept_unknown_bytes: 0,
+        kept_in_window: 0,
+        cap_bytes: 0,
+        bytes_before: 0,
+        bytes_after: 0,
+        cap_blocked_by_uncommitted: false,
+        cap_blocked_count: 0,
+        failed_deletions: 0,
+    };
+}
+/**
+ * The receipt, on stderr, which launchd captures to `sync.err.log`. Counts,
+ * byte totals and reason labels only — never a pack id, never a path.
+ */
+export function reportPrune(result, options = { dryRun: false }) {
+    console.error(options.dryRun ? "[collector prune] dry run" : "[collector prune] swept", JSON.stringify({
+        reason: result.reason,
+        deleted_files: result.deleted_files,
+        deleted_bytes: result.deleted_bytes,
+        removed_packs: result.removed_packs,
+        kept_uncommitted: result.kept_uncommitted,
+        staging_uncommitted_bytes: result.kept_uncommitted_bytes,
+        oldest_uncommitted_age_ms: result.oldest_uncommitted_age_ms,
+        kept_unknown: result.kept_unknown,
+        kept_in_window: result.kept_in_window,
+        cap_bytes: result.cap_bytes,
+        bytes_after: result.bytes_after,
+        failed_deletions: result.failed_deletions,
+    }));
+    if (!result.cap_blocked_by_uncommitted)
+        return;
+    console.error("[collector prune] still over the cap and nothing committed is left to take", JSON.stringify({
+        reason: "staging_cap_blocked_by_uncommitted",
+        blocked_object_count: result.cap_blocked_count,
+        staging_uncommitted_bytes: result.kept_uncommitted_bytes,
+        unknown_bytes: result.kept_unknown_bytes,
+        cap_bytes: result.cap_bytes,
+        bytes_after: result.bytes_after,
+    }));
+}
+async function prunedRecently(paths, now) {
+    const marker = path.join(paths.state_dir, STAGING_PRUNE_THROTTLE_MARKER);
+    const info = await fs.stat(marker).catch(() => null);
+    return Boolean(info && now.getTime() - info.mtimeMs < STAGING_PRUNE_MIN_INTERVAL_MS);
+}
+async function markPruned(paths, now) {
+    const marker = path.join(paths.state_dir, STAGING_PRUNE_THROTTLE_MARKER);
+    await fs.mkdir(paths.state_dir, { recursive: true }).catch(() => undefined);
+    await fs.writeFile(marker, now.toISOString()).catch(() => undefined);
+}