@bli-cockpit/cli 0.2.94 → 0.2.96

package/dist/disk-usage-files.js

@@ -0,0 +1,99 @@
+/**
+ * The only four things in this family that touch a file for its own sake:
+ * list a pack's files, add up a directory's bytes, read a pack's manifest, and
+ * hash a file the manifest does not name.
+ *
+ * They are together because they share one rule — a daily inventory must not
+ * re-read 20 GB. Sizes come from `stat`, hashes come from the manifest each
+ * pack already wrote, and the one path that must actually read bytes
+ * (`hashUnnamedFile`) spends a budget and stops.
+ */
+import crypto from "node:crypto";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { describeError } from "./health-detail.js";
+/** Bytes this inventory may spend hashing files no manifest names. */
+export const UNNAMED_FILE_HASH_BUDGET_BYTES = 256 * 1024 * 1024;
+/**
+ * A file its own manifest does not name gets hashed — but only while the run's
+ * byte budget lasts. Past it the object stays `unknown` and says which budget
+ * ran out, because reading 20 GB to answer a housekeeping question is how the
+ * old GC earned its once-a-day throttle.
+ */
+export async function hashUnnamedFile(file, size, budget) {
+    if (size > budget.remaining)
+        return null;
+    const bytes = await fs.readFile(file).catch(() => null);
+    if (!bytes)
+        return null;
+    budget.remaining -= bytes.byteLength;
+    return crypto.createHash("sha256").update(bytes).digest("hex");
+}
+/**
+ * A pack's own record of what it staged: pack-relative path -> content hash.
+ * This is where a classification gets its hash without reading a payload.
+ */
+export async function readManifestIndex(dir) {
+    const raw = await fs
+        .readFile(path.join(dir, "manifest.json"), "utf8")
+        .catch(() => null);
+    if (!raw)
+        return null;
+    try {
+        const parsed = JSON.parse(raw);
+        const index = new Map();
+        for (const file of parsed.files ?? []) {
+            if (typeof file.relative_path === "string" &&
+                typeof file.content_hash_sha256 === "string" &&
+                file.content_hash_sha256.length === 64) {
+                index.set(file.relative_path, file.content_hash_sha256);
+            }
+        }
+        return index;
+    }
+    catch (error) {
+        // Named, not swallowed: an unreadable manifest turns every file in the
+        // pack into `unknown`, which is the difference between a pack that gets
+        // pruned and one that is kept forever (BLI-3238's rule, BLI-3619's cost).
+        console.error("[collector disk] pack manifest unreadable, its files cannot be classified", JSON.stringify({
+            reason: "manifest_unreadable",
+            pack_id: path.basename(dir),
+            ...describeError(error),
+        }));
+        return null;
+    }
+}
+/** Every file under a directory, as POSIX paths relative to it. */
+export async function listPackFiles(dir) {
+    const files = [];
+    const stack = [{ dir, prefix: "" }];
+    while (stack.length > 0) {
+        const current = stack.pop();
+        if (!current)
+            continue;
+        const entries = await fs
+            .readdir(current.dir, { withFileTypes: true })
+            .catch(() => []);
+        for (const entry of entries) {
+            const relative = current.prefix
+                ? `${current.prefix}/${entry.name}`
+                : entry.name;
+            if (entry.isDirectory()) {
+                stack.push({ dir: path.join(current.dir, entry.name), prefix: relative });
+            }
+            else if (entry.isFile()) {
+                files.push(relative);
+            }
+        }
+    }
+    return files;
+}
+/** What a directory weighs, for the parts of the footprint that have no packs. */
+export async function directoryBytes(dir) {
+    let total = 0;
+    for (const relative of await listPackFiles(dir)) {
+        const info = await fs.stat(path.join(dir, relative)).catch(() => null);
+        total += info?.isFile() ? info.size : 0;
+    }
+    return total;
+}

package/dist/disk-usage-footprint.js

@@ -0,0 +1,45 @@
+/**
+ * The whole-machine read: staging, rotated logs, the spool, and the vaults a
+ * person made by hand — the four places Cockpit ever writes bytes on a laptop.
+ *
+ * The staging half is the rest of this family; what lives here is the other
+ * three and the one function that puts all four together.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { MANUAL_VAULT_PREFIX, } from "./disk-usage-facts.js";
+import { directoryBytes, listPackFiles } from "./disk-usage-files.js";
+import { readStagingInventory } from "./disk-usage-scan.js";
+import { rotatedLogFootprint } from "./log-rotation.js";
+/**
+ * Everything on this machine Cockpit put there, in one read. Used by the sync
+ * tick's prune, by `cockpit clean` and by `cockpit doctor`'s disk row, so all
+ * three quote the same numbers rather than three near-misses.
+ */
+export async function readDiskFootprint(paths, now = new Date()) {
+    return {
+        staging: await readStagingInventory(paths, now),
+        logs: await rotatedLogFootprint(paths),
+        spool_bytes: await directoryBytes(paths.spool_dir),
+        vaults: await readManualVaults(paths),
+    };
+}
+/** The hand-made vaults: counted and named so nothing deletes them silently. */
+async function readManualVaults(paths) {
+    const entries = await fs
+        .readdir(paths.state_dir, { withFileTypes: true })
+        .catch(() => []);
+    const vaults = [];
+    for (const entry of entries) {
+        if (!entry.isDirectory() || !entry.name.startsWith(MANUAL_VAULT_PREFIX)) {
+            continue;
+        }
+        const dir = path.join(paths.state_dir, entry.name);
+        vaults.push({
+            name: entry.name,
+            byte_size: await directoryBytes(dir),
+            file_count: (await listPackFiles(dir)).length,
+        });
+    }
+    return vaults;
+}

package/dist/disk-usage-ledger.js

@@ -0,0 +1,49 @@
+/**
+ * The three records this machine keeps about staged bytes, read before the
+ * walk starts so every object is classified against the same answers: what the
+ * upload ledger committed, what the server said when it was asked, and when
+ * each object was staged.
+ *
+ * `reachesBackTo` is the load-bearing one. The commit ledger's `objects` map is
+ * capped and prunes the oldest, so its silence about an object staged before
+ * that horizon is "I forgot", not "it never landed" — and the classifier can
+ * only tell those apart if it knows how far back the ledger still remembers.
+ */
+import { readRawEvidenceCursor } from "./cursors/raw-evidence-cursor.js";
+import { readReconcileCursor, } from "./cursors/raw-evidence-reconcile-cursor.js";
+import { readRawEvidenceStagingState } from "./raw-evidence-staging.js";
+/**
+ * The commit record, read from the one place that writes it: the raw-evidence
+ * cursor's `objects` map (`upload.ts` → `markObjectCommitted`, only ever for an
+ * outcome that was not `upload_failed`).
+ */
+export async function readCommitLedger(paths) {
+    const cursor = await readRawEvidenceCursor(paths);
+    const committed = new Map();
+    let oldest = null;
+    for (const [hash, entry] of Object.entries(cursor.objects)) {
+        committed.set(hash, entry.committed_at);
+        if (!oldest || entry.committed_at.localeCompare(oldest) < 0) {
+            oldest = entry.committed_at;
+        }
+    }
+    return { committed, reachesBackTo: oldest };
+}
+/**
+ * The reconcile cursor as a lookup map. Read alongside the commit ledger so an
+ * object the server confirmed committed classifies exactly like one this
+ * machine's own ledger still remembers.
+ */
+export async function readReconcileLedger(paths) {
+    const cursor = await readReconcileCursor(paths);
+    return new Map(Object.entries(cursor.results));
+}
+/** `pack_id/relative_path` → when this machine staged those bytes. */
+export async function readStagedAtIndex(paths) {
+    const staging = await readRawEvidenceStagingState(paths.state_dir);
+    const index = new Map();
+    for (const entry of Object.values(staging.staged)) {
+        index.set(`${entry.pack_id}/${entry.relative_path}`, entry.staged_at);
+    }
+    return index;
+}

package/dist/disk-usage-scan.js

@@ -0,0 +1,80 @@
+/**
+ * The walk itself: read the three records once, then visit every pack under
+ * `raw-evidence`, classify each file in it, and add it to the totals.
+ *
+ * This is the entry the prune, `cockpit clean`, the redelivery plan and the
+ * heartbeat all spend, so the pass is deliberately cheap — one `stat` per file
+ * for two facts (its size and when its bytes were written here), hashes off
+ * each pack's own manifest, and a single byte budget shared across the run.
+ */
+import fs from "node:fs/promises";
+import path from "node:path";
+import { classifyObject, } from "./disk-usage-classify.js";
+import { RAW_EVIDENCE_DIR, } from "./disk-usage-facts.js";
+import { directoryBytes, listPackFiles, readManifestIndex, UNNAMED_FILE_HASH_BUDGET_BYTES, } from "./disk-usage-files.js";
+import { readCommitLedger, readReconcileLedger, readStagedAtIndex, } from "./disk-usage-ledger.js";
+import { accumulatePack, emptyStagingInventory, rankUncommittedReasons, } from "./disk-usage-totals.js";
+export async function readStagingInventory(paths, now = new Date()) {
+    const root = path.join(paths.state_dir, RAW_EVIDENCE_DIR);
+    const ledger = await readCommitLedger(paths);
+    const reconciled = await readReconcileLedger(paths);
+    const stagedAt = await readStagedAtIndex(paths);
+    const entries = await fs
+        .readdir(root, { withFileTypes: true })
+        .catch(() => []);
+    const inventory = emptyStagingInventory();
+    const budget = { remaining: UNNAMED_FILE_HASH_BUDGET_BYTES };
+    for (const entry of entries) {
+        if (!entry.isDirectory())
+            continue;
+        const dir = path.join(root, entry.name);
+        if (entry.name.startsWith(".staging-")) {
+            inventory.orphan_staging_dirs += 1;
+            inventory.orphan_staging_bytes += await directoryBytes(dir);
+            continue;
+        }
+        if (!entry.name.startsWith("work-"))
+            continue;
+        const pack = await readPack(dir, entry.name, {
+            ledger,
+            reconciled,
+            stagedAt,
+            now,
+            budget,
+        });
+        inventory.packs.push(pack);
+        accumulatePack(inventory, pack);
+    }
+    rankUncommittedReasons(inventory);
+    return inventory;
+}
+/** One pack: its manifest's own bytes, and a classified fact per payload file. */
+async function readPack(dir, packId, options) {
+    const manifest = await readManifestIndex(dir);
+    const pack = {
+        pack_id: packId,
+        objects: [],
+        manifest_bytes: 0,
+        manifest_present: manifest !== null,
+    };
+    for (const relative of await listPackFiles(dir)) {
+        // One stat, two facts: how big the staged copy is and WHEN it was written.
+        // The mtime is the only unfalsifiable record of how long undelivered
+        // evidence has been sitting here (BLI-3797) — every other timestamp on the
+        // machine records when something last had an opinion about it.
+        const info = await fs.stat(path.join(dir, relative)).catch(() => null);
+        const size = info?.isFile() ? info.size : 0;
+        if (relative === "manifest.json") {
+            pack.manifest_bytes += size;
+            continue;
+        }
+        const stagedOnDiskAt = info && Number.isFinite(info.mtimeMs)
+            ? new Date(info.mtimeMs).toISOString()
+            : null;
+        pack.objects.push(await classifyObject(dir, packId, relative, size, manifest, {
+            ...options,
+            stagedOnDiskAt,
+        }));
+    }
+    return pack;
+}

package/dist/disk-usage-totals.js

@@ -0,0 +1,83 @@
+export function emptyStagingInventory() {
+    return {
+        pack_count: 0,
+        object_count: 0,
+        total_bytes: 0,
+        committed_count: 0,
+        committed_bytes: 0,
+        uncommitted_count: 0,
+        uncommitted_bytes: 0,
+        oldest_uncommitted_at: null,
+        uncommitted_reasons: [],
+        dominant_uncommitted_reason: null,
+        unknown_count: 0,
+        unknown_bytes: 0,
+        manifest_bytes: 0,
+        orphan_staging_dirs: 0,
+        orphan_staging_bytes: 0,
+        packs: [],
+    };
+}
+/** One pack's objects against the running totals, each in its own state. */
+export function accumulatePack(inventory, pack) {
+    inventory.pack_count += 1;
+    inventory.manifest_bytes += pack.manifest_bytes;
+    inventory.total_bytes += pack.manifest_bytes;
+    for (const object of pack.objects) {
+        inventory.object_count += 1;
+        inventory.total_bytes += object.byte_size;
+        if (object.state === "committed") {
+            inventory.committed_count += 1;
+            inventory.committed_bytes += object.byte_size;
+            continue;
+        }
+        if (object.state === "unknown") {
+            inventory.unknown_count += 1;
+            inventory.unknown_bytes += object.byte_size;
+            continue;
+        }
+        inventory.uncommitted_count += 1;
+        inventory.uncommitted_bytes += object.byte_size;
+        tallyUncommittedReason(inventory, object);
+        // The age of the BYTES, not of the answer about them (BLI-3797). Falls back
+        // to `decided_at` only when the file has no readable mtime, which is the
+        // one case where the disk itself cannot say.
+        const stagedAt = object.staged_on_disk_at ?? object.decided_at;
+        if (stagedAt &&
+            (!inventory.oldest_uncommitted_at ||
+                stagedAt.localeCompare(inventory.oldest_uncommitted_at) < 0)) {
+            inventory.oldest_uncommitted_at = stagedAt;
+        }
+    }
+}
+/**
+ * One more object against its reason label. Kept as a mutable array rather than
+ * a Map so the inventory stays a plain serializable value — it rides `--json`
+ * output and a receipt, and a Map would silently become `{}` in both.
+ */
+function tallyUncommittedReason(inventory, object) {
+    const existing = inventory.uncommitted_reasons.find((tally) => tally.reason === object.reason);
+    const tally = existing ?? {
+        reason: object.reason,
+        count: 0,
+        bytes: 0,
+        oldest_disk_age_ms: 0,
+    };
+    tally.count += 1;
+    tally.bytes += object.byte_size;
+    tally.oldest_disk_age_ms = Math.max(tally.oldest_disk_age_ms, object.disk_age_ms || object.age_ms);
+    if (!existing)
+        inventory.uncommitted_reasons.push(tally);
+}
+/**
+ * Rank the reasons by BYTES, then name the biggest one.
+ *
+ * Bytes, not object count, because the question the ranking answers is "what is
+ * holding this laptop over its cap" — 234 objects at 1,836 MB and 17 at 81 MB
+ * are not the same finding, and counting rows would have called them close.
+ */
+export function rankUncommittedReasons(inventory) {
+    inventory.uncommitted_reasons.sort((left, right) => right.bytes - left.bytes || right.count - left.count);
+    inventory.dominant_uncommitted_reason =
+        inventory.uncommitted_reasons[0] ?? null;
+}