@bli-cockpit/cli 0.2.51 → 0.2.52

package/dist/commands/sync-followups.js

@@ -1,8 +1,8 @@
 /**
  * What the sync tick does AFTER collection's own outcome is decided and
  * reported: keep this machine's CLI current on npm `latest` (BLI-2601), put a
- * broken scheduler registration back (BLI-2721), and keep BLI Memory
- * registered with both agent hosts (BLI-3580).
+ * broken scheduler registration back (BLI-2721), keep BLI Memory registered
+ * with both agent hosts (BLI-3580), and stop the laptop filling up (BLI-3619).
  *
  * Split out of commands/sync.ts (BLI-3578), moved verbatim. They belong
  * together because they share one rule, and it is the reason both are called
@@ -18,6 +18,7 @@ import { redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./insta
 import { runSelfUpdate, SelfUpdateError } from "./install-update.js";
 import { getCollectorRuntimePaths, LOCAL_COLLECTOR_VERSION, } from "../local-state.js";
 import { runAutostartSelfHeal, } from "../autostart-self-heal.js";
+import { runStagingPrune } from "../disk-prune.js";
 import { envWithNodeRuntimeOnPath, runScheduledSelfUpdate, } from "../scheduled-self-update.js";
 /**
  * BLI-2721: after the tick's collection and self-update are done and
@@ -183,6 +184,98 @@ function memoryInstallEvent(outcome) {
     }
     return { step: "memory_install", status: "ok", error_detail: detail };
 }
+/**
+ * BLI-3619: staged raw evidence Tower has already accepted stops living on the
+ * laptop forever.
+ *
+ * Same shape as the memory install above and for the same reasons: it runs only
+ * once collection's own outcome is decided and reported, at most once a day, it
+ * cannot throw, and its outcome is its own named receipt rather than a sync
+ * failure. A machine that cannot prune is a machine short of disk, and that
+ * must never also be a machine that stops collecting.
+ *
+ * The rule itself is `disk-retention.ts`: only objects the upload ledger
+ * vouches for are ever deleted, and anything undelivered is counted, aged and
+ * named instead.
+ */
+export async function runStagingPruneAfterSync(command, io, dashboardUrl, options = {}) {
+    let result;
+    try {
+        result = await runStagingPrune(getCollectorRuntimePaths(command.homeDir), {
+            env: io.env,
+            ...(options.now ? { now: options.now } : {}),
+        });
+    }
+    catch (error) {
+        // runStagingPrune already catches everything it can reach; this is the
+        // last-resort net so a prune crash truly cannot touch the sync result.
+        result = { ...prunedNothing(), reason: "prune_threw", status: "fail" };
+        console.error("[collector prune] the prune follow-up threw", JSON.stringify({
+            reason: "prune_followup_threw",
+            detail: redactedSyncErrorDetail(error),
+        }));
+    }
+    // The daily throttle is the steady state — reporting it would post a receipt
+    // on 95 of every 96 ticks for no new information.
+    if (result.reason === "throttled_recent_run")
+        return;
+    await reportInstallEventsBestEffort({
+        homeDir: command.homeDir,
+        dashboardUrl,
+        command: "sync",
+        events: [stagingPruneEvent(result)],
+        json: command.json,
+        io,
+    });
+}
+function prunedNothing() {
+    return {
+        status: "skipped",
+        reason: "prune_threw",
+        deleted_files: 0,
+        deleted_bytes: 0,
+        removed_packs: 0,
+        kept_uncommitted: 0,
+        kept_uncommitted_bytes: 0,
+        oldest_uncommitted_age_ms: 0,
+        kept_unknown: 0,
+        kept_unknown_bytes: 0,
+        kept_in_window: 0,
+        cap_bytes: 0,
+        bytes_before: 0,
+        bytes_after: 0,
+        cap_blocked_by_uncommitted: false,
+        cap_blocked_count: 0,
+        failed_deletions: 0,
+    };
+}
+/** Counts and byte totals only; no pack id and no path travels in a receipt. */
+function stagingPruneEvent(result) {
+    const detail = [
+        `freed ${result.deleted_bytes}B in ${result.deleted_files} file(s)`,
+        `held ${result.kept_uncommitted_bytes}B uncommitted`,
+        result.cap_blocked_by_uncommitted
+            ? `staging_cap_blocked_by_uncommitted ${result.cap_blocked_count}`
+            : null,
+        result.failed_deletions > 0
+            ? `failed_deletions ${result.failed_deletions}`
+            : null,
+    ]
+        .filter((part) => Boolean(part))
+        .join("; ");
+    if (result.status === "fail") {
+        return {
+            step: "staging_prune",
+            status: "fail",
+            error_code: result.reason,
+            error_detail: detail,
+        };
+    }
+    if (result.status === "skipped") {
+        return { step: "staging_prune", status: "skipped", error_code: result.reason };
+    }
+    return { step: "staging_prune", status: "ok", error_detail: detail };
+}
 /**
  * BLI-2601: the fleet keeps itself current on npm `latest` without anyone
  * re-running `npm i -g @bli-cockpit/cli` by hand after day 0. This always

package/dist/commands/sync.js

@@ -5,7 +5,7 @@ import { discoverCommandWorktrees } from "./local-discovery.js";
 import { sendCollectorHeartbeatBestEffort, } from "./heartbeat.js";
 import { classifySyncFailureRecords, classifySyncHealthError, redactedSyncErrorDetail, reportInstallEventsBestEffort, } from "./install-receipts.js";
 import { runAttributedWorktreeSync, } from "./session-sync.js";
-import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, } from "./sync-followups.js";
+import { runAutostartSelfHealAfterSync, runMemoryInstallAfterSync, runScheduledSelfUpdateAfterSync, runStagingPruneAfterSync, } from "./sync-followups.js";
 import { describeError } from "../health-detail.js";
 import { inspectBackfillLock } from "../backfill-lock.js";
 import { rotateCollectorLogsBestEffort } from "../log-rotation.js";
@@ -52,6 +52,9 @@ export async function runSync(command, io) {
         // BLI-3580: BLI Memory's registration converges the same way — after
         // collection, at most once a day, its own receipt either way.
         await runMemoryInstallAfterSync(command, io, dashboardUrl);
+        // BLI-3619: and the disk stops growing without bound — same daily cadence,
+        // same rule that a follow-up never blocks or fails collection.
+        await runStagingPruneAfterSync(command, io, dashboardUrl);
         return result.exitCode;
     }
     catch (error) {

package/dist/disk-prune.js

@@ -0,0 +1,246 @@
+/**
+ * Carrying out the retention plan, and saying what it did.
+ *
+ * BLI-3619. The decision is `disk-retention.ts` and the reading is
+ * `disk-usage.ts`; this file is the only one that removes a staged byte. Three
+ * properties it must keep:
+ *
+ * - **It never fails the tick.** Every path is caught. A prune that cannot run
+ *   is a log line, never a reason collection did not happen — the same rule the
+ *   log rotation and the two self-heal follow-ups already live under.
+ * - **It never deletes what the ledger did not vouch for.** The plan decides
+ *   that; this file does not second-guess it, and re-reads nothing.
+ * - **It says so on both branches.** `[collector prune]` fires on the boring
+ *   run too, because "0 deleted, 2.1 GB held, all committed" is the only
+ *   evidence that the drain is still running at all (the BLI-2528 lesson: what
+ *   only speaks on failure cannot answer "did anything work today?").
+ *
+ * Throttled to once a day by a marker file, the same idiom the raw-evidence GC,
+ * the scheduled self-update and the BLI Memory install use. Walking a pack tree
+ * is cheap; doing it every fifteen minutes for a decision that changes on the
+ * scale of hours is not.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { planStagingRetention, retentionOptionsFromEnv, } from "./disk-retention.js";
+import { RAW_EVIDENCE_DIR, readStagingInventory } from "./disk-usage.js";
+import { describeError } from "./health-detail.js";
+import { readRawEvidenceStagingState, writeRawEvidenceStagingState, } from "./raw-evidence-staging.js";
+export const STAGING_PRUNE_THROTTLE_MARKER = ".last-staging-prune";
+export const STAGING_PRUNE_MIN_INTERVAL_MS = 24 * 60 * 60 * 1000;
+/**
+ * The prune the sync tick runs and `cockpit clean` / `cockpit doctor` reuse.
+ * Never throws.
+ */
+export async function runStagingPrune(paths, options = {}) {
+    const env = options.env ?? process.env;
+    const now = options.now ?? new Date();
+    if (env["COCKPIT_DISABLE_GC"] === "1") {
+        return skippedResult("disabled");
+    }
+    if (!options.force && (await prunedRecently(paths, now))) {
+        return skippedResult("throttled_recent_run");
+    }
+    try {
+        if (!options.dryRun && !options.force)
+            await markPruned(paths, now);
+        const plan = await planFromDisk(paths, env, options, now);
+        const result = options.dryRun
+            ? dryRunResult(plan)
+            : await applyPlan(paths, plan);
+        reportPrune(result, { dryRun: Boolean(options.dryRun) });
+        return result;
+    }
+    catch (error) {
+        console.error("[collector prune] the prune could not run; nothing was deleted", JSON.stringify({ reason: "prune_threw", ...describeError(error) }));
+        return { ...skippedResult("prune_threw"), status: "fail" };
+    }
+}
+export async function planFromDisk(paths, env, options, now) {
+    const fromEnv = retentionOptionsFromEnv(env);
+    const inventory = await readStagingInventory(paths, now);
+    return planStagingRetention(inventory, {
+        retentionMs: options.retentionMs ?? fromEnv.retentionMs,
+        capBytes: options.capBytes ?? fromEnv.capBytes,
+        allCommitted: options.allCommitted,
+    });
+}
+async function applyPlan(paths, plan) {
+    const root = path.join(paths.state_dir, RAW_EVIDENCE_DIR);
+    const tally = { files: 0, bytes: 0, packs: 0, failed: 0, hashes: [] };
+    const removedPackIds = await removeEmptyPacks(root, plan, tally);
+    await removeRemainingFiles(root, plan, removedPackIds, tally);
+    await forgetStagedObjects(paths, tally.hashes);
+    return {
+        ...dryRunResult(plan),
+        status: "ok",
+        reason: tally.files > 0 ? "pruned" : "nothing_eligible",
+        deleted_files: tally.files,
+        deleted_bytes: tally.bytes,
+        removed_packs: tally.packs,
+        failed_deletions: tally.failed,
+    };
+}
+/**
+ * A pack whose every payload file is going is removed whole, in one call, so
+ * `manifest.json` goes with it rather than surviving as a description of files
+ * that are not there.
+ */
+async function removeEmptyPacks(root, plan, tally) {
+    const removed = new Set();
+    for (const pack of plan.empty_packs) {
+        const gone = await rmQuietly(path.join(root, pack.pack_id), true);
+        if (!gone) {
+            tally.failed += 1;
+            continue;
+        }
+        removed.add(pack.pack_id);
+        tally.packs += 1;
+        tally.bytes += pack.manifest_bytes;
+    }
+    return removed;
+}
+async function removeRemainingFiles(root, plan, removedPackIds, tally) {
+    const emptyPackIds = new Set(plan.empty_packs.map((pack) => pack.pack_id));
+    for (const entry of plan.delete) {
+        if (emptyPackIds.has(entry.pack_id)) {
+            // The pack removal above already took these bytes — or failed, in which
+            // case the file is still on disk and must be counted as such.
+            if (!removedPackIds.has(entry.pack_id))
+                tally.failed += 1;
+            else
+                countRemoved(tally, entry);
+            continue;
+        }
+        const file = path.join(root, entry.pack_id, ...entry.relative_path.split("/").filter(Boolean));
+        if (await rmQuietly(file, false))
+            countRemoved(tally, entry);
+        else
+            tally.failed += 1;
+    }
+}
+function countRemoved(tally, entry) {
+    tally.files += 1;
+    tally.bytes += entry.byte_size;
+    if (entry.content_hash)
+        tally.hashes.push(entry.content_hash);
+}
+async function rmQuietly(target, recursive) {
+    return fs.rm(target, { recursive, force: true }).then(() => true, () => false);
+}
+/**
+ * The staged-object index points at bytes that are no longer there. Left alone
+ * it self-heals — `resolveStagedObject` stats the file and drops the entry —
+ * but the index is capped at 5,000 rows, so stale entries evict live ones and
+ * a real staged copy stops being found. One atomic write clears them together.
+ */
+async function forgetStagedObjects(paths, hashes) {
+    if (hashes.length === 0)
+        return;
+    try {
+        const staging = await readRawEvidenceStagingState(paths.state_dir);
+        let removed = 0;
+        for (const hash of hashes) {
+            if (staging.staged[hash]) {
+                delete staging.staged[hash];
+                removed += 1;
+            }
+        }
+        if (removed === 0)
+            return;
+        staging.updated_at = new Date().toISOString();
+        await writeRawEvidenceStagingState(paths.state_dir, staging);
+    }
+    catch (error) {
+        // Not fatal and not silent: the bytes are gone either way, and the index
+        // heals on the next read. What must not happen is nobody knowing the index
+        // is carrying rows for files that no longer exist.
+        console.error("[collector prune] staged index not updated after the prune", JSON.stringify({
+            reason: "staged_index_write_failed",
+            forgotten_count: hashes.length,
+            ...describeError(error),
+        }));
+    }
+}
+function dryRunResult(plan) {
+    return {
+        status: "ok",
+        reason: plan.delete.length > 0 ? "would_prune" : "nothing_eligible",
+        deleted_files: plan.delete.length,
+        deleted_bytes: plan.deleted_bytes + plan.empty_pack_manifest_bytes,
+        removed_packs: plan.empty_packs.length,
+        kept_uncommitted: plan.kept_uncommitted.count,
+        kept_uncommitted_bytes: plan.kept_uncommitted.bytes,
+        oldest_uncommitted_age_ms: plan.kept_uncommitted.oldest_age_ms,
+        kept_unknown: plan.kept_unknown.count,
+        kept_unknown_bytes: plan.kept_unknown.bytes,
+        kept_in_window: plan.kept_in_window.count,
+        cap_bytes: plan.cap_bytes,
+        bytes_before: plan.bytes_before,
+        bytes_after: plan.bytes_after,
+        cap_blocked_by_uncommitted: plan.cap_blocked_by_uncommitted,
+        cap_blocked_count: plan.cap_blocked_count,
+        failed_deletions: 0,
+    };
+}
+function skippedResult(reason) {
+    return {
+        status: "skipped",
+        reason,
+        deleted_files: 0,
+        deleted_bytes: 0,
+        removed_packs: 0,
+        kept_uncommitted: 0,
+        kept_uncommitted_bytes: 0,
+        oldest_uncommitted_age_ms: 0,
+        kept_unknown: 0,
+        kept_unknown_bytes: 0,
+        kept_in_window: 0,
+        cap_bytes: 0,
+        bytes_before: 0,
+        bytes_after: 0,
+        cap_blocked_by_uncommitted: false,
+        cap_blocked_count: 0,
+        failed_deletions: 0,
+    };
+}
+/**
+ * The receipt, on stderr, which launchd captures to `sync.err.log`. Counts,
+ * byte totals and reason labels only — never a pack id, never a path.
+ */
+export function reportPrune(result, options = { dryRun: false }) {
+    console.error(options.dryRun ? "[collector prune] dry run" : "[collector prune] swept", JSON.stringify({
+        reason: result.reason,
+        deleted_files: result.deleted_files,
+        deleted_bytes: result.deleted_bytes,
+        removed_packs: result.removed_packs,
+        kept_uncommitted: result.kept_uncommitted,
+        staging_uncommitted_bytes: result.kept_uncommitted_bytes,
+        oldest_uncommitted_age_ms: result.oldest_uncommitted_age_ms,
+        kept_unknown: result.kept_unknown,
+        kept_in_window: result.kept_in_window,
+        cap_bytes: result.cap_bytes,
+        bytes_after: result.bytes_after,
+        failed_deletions: result.failed_deletions,
+    }));
+    if (!result.cap_blocked_by_uncommitted)
+        return;
+    console.error("[collector prune] still over the cap and nothing committed is left to take", JSON.stringify({
+        reason: "staging_cap_blocked_by_uncommitted",
+        blocked_object_count: result.cap_blocked_count,
+        staging_uncommitted_bytes: result.kept_uncommitted_bytes,
+        unknown_bytes: result.kept_unknown_bytes,
+        cap_bytes: result.cap_bytes,
+        bytes_after: result.bytes_after,
+    }));
+}
+async function prunedRecently(paths, now) {
+    const marker = path.join(paths.state_dir, STAGING_PRUNE_THROTTLE_MARKER);
+    const info = await fs.stat(marker).catch(() => null);
+    return Boolean(info && now.getTime() - info.mtimeMs < STAGING_PRUNE_MIN_INTERVAL_MS);
+}
+async function markPruned(paths, now) {
+    const marker = path.join(paths.state_dir, STAGING_PRUNE_THROTTLE_MARKER);
+    await fs.mkdir(paths.state_dir, { recursive: true }).catch(() => undefined);
+    await fs.writeFile(marker, now.toISOString()).catch(() => undefined);
+}

package/dist/disk-retention.js

@@ -0,0 +1,157 @@
+/** How long a committed spare is kept so a retry still has local bytes. */
+export const DEFAULT_STAGING_RETENTION_MS = 48 * 60 * 60 * 1000;
+/** Ceiling on everything under `raw-evidence`, manifests included. */
+export const DEFAULT_STAGING_CAP_BYTES = 2 * 1024 * 1024 * 1024;
+/** Local knobs, read from the environment like `COCKPIT_DISABLE_GC`. */
+export const STAGING_CAP_ENV = "COCKPIT_STAGING_CAP_BYTES";
+export const STAGING_RETENTION_ENV = "COCKPIT_STAGING_RETENTION_HOURS";
+/** The two knobs, from the environment, falling back to the defaults above. */
+export function retentionOptionsFromEnv(env) {
+    const hours = Number(env[STAGING_RETENTION_ENV]);
+    const cap = Number(env[STAGING_CAP_ENV]);
+    return {
+        retentionMs: Number.isFinite(hours) && hours >= 0
+            ? hours * 60 * 60 * 1000
+            : DEFAULT_STAGING_RETENTION_MS,
+        capBytes: Number.isFinite(cap) && cap > 0 ? cap : DEFAULT_STAGING_CAP_BYTES,
+    };
+}
+export function planStagingRetention(inventory, options = {}) {
+    const retentionMs = options.retentionMs ?? DEFAULT_STAGING_RETENTION_MS;
+    const capBytes = options.capBytes ?? DEFAULT_STAGING_CAP_BYTES;
+    const plan = emptyPlan(capBytes, retentionMs, inventory.total_bytes);
+    const committedInWindow = [];
+    for (const pack of inventory.packs) {
+        for (const object of pack.objects) {
+            if (object.state !== "committed") {
+                keepUnvouched(plan, object);
+                continue;
+            }
+            if (options.allCommitted || object.age_ms >= retentionMs) {
+                pushDeletion(plan, object, "committed_past_retry_window");
+                continue;
+            }
+            committedInWindow.push(object);
+            addToBucket(plan.kept_in_window, object);
+        }
+    }
+    applyByteCap(plan, committedInWindow, capBytes, inventory);
+    markEmptyPacks(plan, inventory.packs);
+    plan.bytes_after =
+        inventory.total_bytes - plan.deleted_bytes - plan.empty_pack_manifest_bytes;
+    // Over the cap is only "blocked by uncommitted" when uncommitted bytes are
+    // what is left. Over the cap with nothing unvouched on disk is a different
+    // sentence — the remaining manifests alone exceed it — and saying the wrong
+    // one would send an operator hunting evidence that is not missing.
+    const unvouched = plan.kept_uncommitted.count + plan.kept_unknown.count;
+    plan.cap_blocked_by_uncommitted = plan.bytes_after > capBytes && unvouched > 0;
+    plan.cap_blocked_count = plan.cap_blocked_by_uncommitted ? unvouched : 0;
+    return plan;
+}
+/**
+ * The oldest committed spares go until the total fits, newest kept last. Sorted
+ * by the ledger's `committed_at` age rather than a file mtime: mtime moves when
+ * a pack is refilled, and the question here is how long ago the bytes became
+ * safe elsewhere.
+ *
+ * It does not run at all when the floor is already above the cap. That branch
+ * is not theoretical: on the reference Mac (2026-09-05) 8.0 GB of staging is
+ * evidence no ledger vouches for, against a 2 GB cap, so spending the retry
+ * window on 1,027 committed spares would have deleted a day of local retry
+ * copies and still ended over the cap. The cap takes a spare only when taking
+ * it can actually get this machine under the cap.
+ */
+function applyByteCap(plan, committedInWindow, capBytes, inventory) {
+    let projected = plan.bytes_before - plan.deleted_bytes;
+    if (projected <= capBytes)
+        return;
+    if (unremovableFloor(inventory) >= capBytes)
+        return;
+    const oldestFirst = [...committedInWindow].sort((a, b) => b.age_ms - a.age_ms);
+    for (const object of oldestFirst) {
+        if (projected <= capBytes)
+            break;
+        pushDeletion(plan, object, "cap_committed_oldest_first");
+        removeFromBucket(plan.kept_in_window, object);
+        projected -= object.byte_size;
+    }
+}
+/**
+ * Bytes no retention rule may take: every object the ledger does not vouch for,
+ * plus the `manifest.json` of each pack holding one, because that pack survives
+ * whatever else is deleted from it.
+ */
+function unremovableFloor(inventory) {
+    let floor = 0;
+    for (const pack of inventory.packs) {
+        const unvouched = pack.objects.filter((object) => object.state !== "committed");
+        if (unvouched.length === 0)
+            continue;
+        floor += pack.manifest_bytes;
+        for (const object of unvouched)
+            floor += object.byte_size;
+    }
+    return floor;
+}
+/**
+ * A pack with no payload left is deleted whole, `manifest.json` included. The
+ * manifest is pack metadata — its only job is to describe files that are no
+ * longer there — and on the reference Mac those manifests were 836 MB across
+ * 1,612 packs, so leaving them behind would keep almost a gigabyte of
+ * descriptions of nothing.
+ */
+function markEmptyPacks(plan, packs) {
+    const deleted = new Set(plan.delete.map((entry) => `${entry.pack_id}/${entry.relative_path}`));
+    for (const pack of packs) {
+        const survives = pack.objects.some((object) => !deleted.has(`${pack.pack_id}/${object.relative_path}`));
+        if (survives)
+            continue;
+        plan.empty_packs.push({
+            pack_id: pack.pack_id,
+            manifest_bytes: pack.manifest_bytes,
+        });
+        plan.empty_pack_manifest_bytes += pack.manifest_bytes;
+    }
+}
+function keepUnvouched(plan, object) {
+    const bucket = object.state === "unknown" ? plan.kept_unknown : plan.kept_uncommitted;
+    addToBucket(bucket, object);
+}
+function pushDeletion(plan, object, reason) {
+    plan.delete.push({
+        pack_id: object.pack_id,
+        relative_path: object.relative_path,
+        byte_size: object.byte_size,
+        content_hash: object.content_hash,
+        reason,
+        age_ms: object.age_ms,
+    });
+    plan.deleted_bytes += object.byte_size;
+}
+function addToBucket(bucket, object) {
+    bucket.count += 1;
+    bucket.bytes += object.byte_size;
+    bucket.oldest_age_ms = Math.max(bucket.oldest_age_ms, object.age_ms);
+}
+function removeFromBucket(bucket, object) {
+    bucket.count -= 1;
+    bucket.bytes -= object.byte_size;
+}
+function emptyPlan(capBytes, retentionMs, bytesBefore) {
+    const bucket = () => ({ count: 0, bytes: 0, oldest_age_ms: 0 });
+    return {
+        delete: [],
+        deleted_bytes: 0,
+        empty_packs: [],
+        empty_pack_manifest_bytes: 0,
+        kept_uncommitted: bucket(),
+        kept_unknown: bucket(),
+        kept_in_window: bucket(),
+        cap_bytes: capBytes,
+        retention_ms: retentionMs,
+        bytes_before: bytesBefore,
+        bytes_after: bytesBefore,
+        cap_blocked_by_uncommitted: false,
+        cap_blocked_count: 0,
+    };
+}