@celilo/cli 0.16.2 → 0.18.0

package/src/services/backup-metadata.ts

@@ -43,6 +43,10 @@ export function createBackupRecord(params: CreateBackupParams): BackupRecord {
     metadata: {},
     status: 'in_progress' as BackupStatus,
     errorMessage: null,
+    // Recorded so the staging reaper can distinguish a live backup from one
+    // whose process was killed before its `finally` ran — see
+    // services/backup-staging.ts.
+    pid: process.pid,
     startedAt: now,
     completedAt: null,
   };

package/src/services/backup-staging.test.ts

@@ -0,0 +1,134 @@
+import { describe, expect, test } from 'bun:test';
+import {
+  type ReapStagingDeps,
+  STAGING_PREFIX,
+  STAGING_TTL_MS,
+  type StagingOwner,
+  reapOrphanedStaging,
+  stagingDirFor,
+} from './backup-staging';
+
+const NOW = new Date('2026-08-06T12:00:00Z').getTime();
+
+function deps(
+  owners: Record<string, StagingOwner | null>,
+  options: {
+    dirs?: string[];
+    runnablePids?: number[];
+    unremovable?: string[];
+  } = {},
+): ReapStagingDeps & { removed: string[] } {
+  const removed: string[] = [];
+  return {
+    removed,
+    listStagingDirs: () => options.dirs ?? Object.keys(owners).map((id) => stagingDirFor(id)),
+    lookupOwner: (id) => owners[id] ?? null,
+    isPidRunnable: (pid) => (options.runnablePids ?? []).includes(pid),
+    remove: (path) => {
+      if (options.unremovable?.includes(path)) throw new Error('EACCES');
+      removed.push(path);
+    },
+    now: () => NOW,
+  };
+}
+
+function owner(over: Partial<StagingOwner> = {}): StagingOwner {
+  return { status: 'in_progress', pid: 111, startedAt: new Date(NOW - 60_000), ...over };
+}
+
+describe('reapOrphanedStaging', () => {
+  test('reclaims staging whose backup record no longer exists', () => {
+    const d = deps({ 'gone-id': null });
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed).toHaveLength(1);
+    expect(report.reclaimed[0]?.reason).toBe('record-absent');
+    expect(d.removed).toEqual([stagingDirFor('gone-id')]);
+  });
+
+  test.each([['completed'], ['failed']])('reclaims staging for a %s record', (status) => {
+    const d = deps({ 'done-id': owner({ status }) });
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed[0]?.reason).toBe('record-terminal');
+    expect(d.removed).toEqual([stagingDirFor('done-id')]);
+  });
+
+  test('reclaims staging whose owning process is dead', () => {
+    // in_progress, inside the TTL — only the liveness probe can tell.
+    const d = deps({ 'dead-id': owner({ pid: 999 }) }, { runnablePids: [] });
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed[0]?.reason).toBe('process-dead');
+    expect(d.removed).toEqual([stagingDirFor('dead-id')]);
+  });
+
+  // The one that must never regress: a running backup writing gigabytes into
+  // its staging dir must survive a concurrent sweep.
+  test('KEEPS staging owned by a live backup', () => {
+    const d = deps({ 'live-id': owner({ pid: 111 }) }, { runnablePids: [111] });
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed).toHaveLength(0);
+    expect(report.kept).toEqual([stagingDirFor('live-id')]);
+    expect(d.removed).toEqual([]);
+  });
+
+  test('reclaims a record past the TTL even when its pid looks alive', () => {
+    // Guards pid reuse: after the pid space wraps, a stale record's pid names
+    // an unrelated live process and liveness alone would strand this forever.
+    const d = deps(
+      { 'stale-id': owner({ pid: 111, startedAt: new Date(NOW - STAGING_TTL_MS - 1) }) },
+      { runnablePids: [111] },
+    );
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed[0]?.reason).toBe('expired');
+  });
+
+  test('keeps a pid-less record until it expires, then reclaims it', () => {
+    const fresh = deps({ 'old-fmt': owner({ pid: null }) });
+    expect(reapOrphanedStaging(fresh).kept).toHaveLength(1);
+
+    const expired = deps({
+      'old-fmt': owner({ pid: null, startedAt: new Date(NOW - STAGING_TTL_MS - 1) }),
+    });
+    expect(reapOrphanedStaging(expired).reclaimed[0]?.reason).toBe('expired');
+  });
+
+  test('ignores directories that are not staging, rather than deleting them', () => {
+    const d = deps(
+      {},
+      {
+        dirs: [
+          '/tmp/something-else',
+          '/tmp/celilo-unrelated',
+          `/tmp/${STAGING_PREFIX}`, // prefix with no record id
+        ],
+      },
+    );
+    const report = reapOrphanedStaging(d);
+
+    expect(report.ignored).toHaveLength(3);
+    expect(report.reclaimed).toHaveLength(0);
+    expect(d.removed).toEqual([]);
+  });
+
+  test('an undeletable directory is reported as kept, not as reclaimed space', () => {
+    const path = stagingDirFor('locked-id');
+    const d = deps({ 'locked-id': null }, { unremovable: [path] });
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed).toHaveLength(0);
+    expect(report.kept).toEqual([path]);
+  });
+
+  test('one undeletable directory does not stop the rest of the pass', () => {
+    const locked = stagingDirFor('a-locked');
+    const d = deps({ 'a-locked': null, 'b-orphan': null }, { unremovable: [locked] });
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed.map((r) => r.recordId)).toEqual(['b-orphan']);
+    expect(report.kept).toEqual([locked]);
+  });
+});

package/src/services/backup-staging.ts

@@ -0,0 +1,192 @@
+/**
+ * Reclaiming backup staging directories whose owner is gone.
+ *
+ * `backup-create.ts` assembles every envelope in a temp directory and removes
+ * it in a `finally`. That is correct and it is not enough: a `finally` does not
+ * run when the process is killed by a signal it cannot intercept — a dispatcher
+ * timeout, an OOM, an operator's Ctrl-C, a host reboot. Those are exactly the
+ * cases that strand the LARGEST directories, because the longer a backup has
+ * run the more it has written.
+ *
+ * Measured on celilo-mgr 2026-08-05: the scheduled sweep inherited the event
+ * bus's 60s default timeout against a forgejo backup that needs ~5.5 minutes,
+ * so every attempt was SIGTERMed at ~1.4 GB of staging, three times an hour,
+ * for days. 15 GB stranded in 26 hours, then 27 GB in the next 5.7. Nothing
+ * reclaimed it, because the only cleanup was the `finally` that never ran.
+ *
+ * So reclamation must not be a property of how a backup ends. This pass asks a
+ * different question — "is anyone still using this directory?" — and answers it
+ * from state that outlives the process.
+ *
+ * Identity comes free: a staging directory is named `celilo-backup-<record.id>`,
+ * so the directory names its own backup record. No lockfile, no marker, no
+ * second source of truth that could itself be stranded — and notably nothing
+ * written BY the process whose death is the problem.
+ *
+ * Dependencies are injected so the decision logic tests with no filesystem and
+ * no database.
+ */
+
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+/** Every staging directory starts with this. The rest of the name is the record id. */
+export const STAGING_PREFIX = 'celilo-backup-';
+
+/**
+ * Where a backup record's staging lives. Single source of truth — `backup-create.ts`
+ * builds its temp dir from this so the reaper can never drift from the writer.
+ */
+export function stagingDirFor(recordId: string): string {
+  return join(tmpdir(), `${STAGING_PREFIX}${recordId}`);
+}
+
+/**
+ * How long an `in_progress` backup record may vouch for its staging before the
+ * directory is reclaimed regardless of what its pid appears to be doing.
+ *
+ * This is not redundancy on the liveness check — it is the only check that
+ * survives pid reuse, for the same reason `OPERATION_TTL_MS` exists in
+ * `module-operations.ts`. A pid is a recycled number: once the pid space wraps,
+ * a stale record's pid names an unrelated live process and the liveness probe
+ * reports "still running" forever, stranding the directory permanently.
+ *
+ * Six hours is comfortably longer than any real backup (the largest measured is
+ * ~5.5 minutes) and short enough that a wedged record costs one cycle rather
+ * than a filesystem.
+ */
+export const STAGING_TTL_MS = 6 * 60 * 60 * 1000;
+
+/** What the backup record says about the process that owns a staging directory. */
+export interface StagingOwner {
+  status: string;
+  /** Null for records written before backups recorded their pid. */
+  pid: number | null;
+  startedAt: Date;
+}
+
+export interface ReapStagingDeps {
+  /** Absolute paths of every `celilo-backup-*` entry in the temp dir. */
+  listStagingDirs(): string[];
+  /** The backup record for this id, or null when it no longer exists. */
+  lookupOwner(recordId: string): StagingOwner | null;
+  isPidRunnable(pid: number): boolean;
+  remove(path: string): void;
+  now(): number;
+}
+
+export interface ReapedStaging {
+  path: string;
+  recordId: string;
+  /** Why it was reclaimable — surfaced so a sweep can explain itself. */
+  reason: 'record-absent' | 'record-terminal' | 'process-dead' | 'expired';
+}
+
+/**
+ * Written to a backup record reclaimed while it still claimed to be running.
+ *
+ * A record left `in_progress` after its process died misreports the system
+ * twice: `celilo backup list` shows work apparently underway, and the `backups`
+ * drift check can read a module as recently backed up when every attempt in
+ * fact died. Nine such rows were live on celilo-mgr while forgejo had no usable
+ * backup at all.
+ */
+export const ABANDONED_BACKUP_MESSAGE =
+  'abandoned — the backup process ended without recording an outcome';
+
+/**
+ * Whether reclaiming this directory also means its record was lying about
+ * being in progress.
+ *
+ * Only the two reasons reached from the `in_progress` branch qualify:
+ * `record-terminal` already has an outcome and `record-absent` has no row to
+ * correct.
+ */
+export function impliesAbandonedRecord(reason: ReapedStaging['reason']): boolean {
+  return reason === 'process-dead' || reason === 'expired';
+}
+
+export interface ReapStagingReport {
+  reclaimed: ReapedStaging[];
+  /** Left alone because a live backup is using it. */
+  kept: string[];
+  /** Names that did not look like staging at all. Never touched. */
+  ignored: string[];
+}
+
+/**
+ * Decide, for one staging directory, whether its owner is gone.
+ *
+ * Returns null when the directory must be left alone. Every branch that
+ * reclaims must be able to say why; "I could not prove it is alive" is not a
+ * reason to delete, which is why an unrecognised state keeps the directory.
+ */
+function reclaimReason(
+  owner: StagingOwner | null,
+  deps: Pick<ReapStagingDeps, 'isPidRunnable' | 'now'>,
+): ReapedStaging['reason'] | null {
+  // The record was deleted, or never committed. Nothing will ever finish this.
+  if (!owner) return 'record-absent';
+
+  // Completed or failed: the writer reached an ending and either cleaned up
+  // already (in which case we will not see the directory) or was killed after
+  // recording its outcome.
+  if (owner.status !== 'in_progress') return 'record-terminal';
+
+  const age = deps.now() - owner.startedAt.getTime();
+  if (age > STAGING_TTL_MS) return 'expired';
+
+  // A record from before backups carried a pid. Age is the only signal
+  // available, and it has not expired — leave it.
+  if (owner.pid === null) return null;
+
+  return deps.isPidRunnable(owner.pid) ? null : 'process-dead';
+}
+
+/**
+ * Reclaim every staging directory whose owning backup is no longer running.
+ *
+ * Conservative by construction: a directory is removed only when its owner is
+ * provably gone. A running backup — `in_progress` record, live pid, inside the
+ * TTL — is always kept, so a long-running or concurrent backup can never be
+ * destroyed by this pass.
+ *
+ * A name that is not `celilo-backup-<id>` is ignored rather than removed. This
+ * runs against a shared temp directory as a privileged user; deleting something
+ * it does not understand is not its job.
+ */
+export function reapOrphanedStaging(deps: ReapStagingDeps): ReapStagingReport {
+  const report: ReapStagingReport = { reclaimed: [], kept: [], ignored: [] };
+
+  for (const path of deps.listStagingDirs()) {
+    const name = path.slice(path.lastIndexOf('/') + 1);
+    if (!name.startsWith(STAGING_PREFIX)) {
+      report.ignored.push(path);
+      continue;
+    }
+
+    const recordId = name.slice(STAGING_PREFIX.length);
+    if (recordId.length === 0) {
+      report.ignored.push(path);
+      continue;
+    }
+
+    const reason = reclaimReason(deps.lookupOwner(recordId), deps);
+    if (!reason) {
+      report.kept.push(path);
+      continue;
+    }
+
+    // A directory that cannot be removed is not fatal: the next pass retries,
+    // and failing the whole sweep over one undeletable path would stop backups
+    // entirely. Treated as kept so the report never claims space it did not free.
+    try {
+      deps.remove(path);
+      report.reclaimed.push({ path, recordId, reason });
+    } catch {
+      report.kept.push(path);
+    }
+  }
+
+  return report;
+}

package/src/services/backup-storage.ts

@@ -104,6 +104,35 @@ export async function addBackupStorage(params: {
   };
 }
 
+/**
+ * Replace a storage destination's credentials.
+ *
+ * Clears the verification stamp in the same statement. A `verified`
+ * flag describes the destination the credentials pointed at; once they
+ * change it describes somewhere else, and carrying it forward is how
+ * celilo-mgr ended up reporting `✓ Verified` for a macOS path on a
+ * Linux host (#566). Callers re-verify against the new destination.
+ */
+export async function updateStorageCredentials(
+  id: string,
+  credentials: Record<string, unknown>,
+): Promise<void> {
+  const masterKey = await getOrCreateMasterKey();
+  const encrypted = encryptSecret(JSON.stringify(credentials), masterKey);
+
+  getDb()
+    .update(backupStorages)
+    .set({
+      credentialsEncrypted: JSON.stringify(encrypted),
+      verified: false,
+      verifiedAt: null,
+      verificationError: null,
+      updatedAt: new Date(),
+    })
+    .where(eq(backupStorages.id, id))
+    .run();
+}
+
 /**
  * Get backup storage by storage ID (user-facing identifier)
  */

package/src/services/backup-sweep.test.ts

@@ -2,8 +2,10 @@ import { describe, expect, test } from 'bun:test';
 import type { ModuleManifest } from '../manifest/schema';
 import type { BackupSchedule } from './backup-schedule';
 import {
+  BACKUP_SWEEP_MAX_ATTEMPTS,
   BACKUP_SWEEP_PATTERN,
   BACKUP_SWEEP_SUBSCRIBER,
+  BACKUP_SWEEP_TIMEOUT_MS,
   type BackupSweepDeps,
   type BackupSweepModule,
   ensureBackupSweepSubscriber,
@@ -33,11 +35,54 @@ function deps(
     prune: async ({ id }) => {
       pruned.push(id);
     },
+    reapStaging: () => ({ reclaimed: [], kept: [], ignored: [] }),
     ...overrides,
   };
 }
 
 describe('runBackupSweep', () => {
+  // Ordering is the point, not just that it happens: this pass is the only
+  // thing creating staging on a schedule, and a box already short on disk
+  // needs the space back BEFORE another few GB are requested.
+  test('reclaims orphaned staging before backing anything up', async () => {
+    const order: string[] = [];
+    const report = await runBackupSweep(
+      deps([moduleWith('forgejo', 'daily')], {
+        reapStaging: () => {
+          order.push('reap');
+          return {
+            reclaimed: [{ path: '/tmp/celilo-backup-x', recordId: 'x', reason: 'process-dead' }],
+            kept: [],
+            ignored: [],
+          };
+        },
+        backup: async () => {
+          order.push('backup');
+          return { success: true };
+        },
+      }),
+    );
+
+    expect(order).toEqual(['reap', 'backup']);
+    expect(report.staging.reclaimed).toHaveLength(1);
+  });
+
+  test('reclaims staging even when no module is due to back up', async () => {
+    const report = await runBackupSweep(
+      deps([moduleWith('forgejo', 'daily')], {
+        isDue: () => false,
+        reapStaging: () => ({
+          reclaimed: [{ path: '/tmp/celilo-backup-y', recordId: 'y', reason: 'record-absent' }],
+          kept: [],
+          ignored: [],
+        }),
+      }),
+    );
+
+    expect(report.backedUp).toEqual([]);
+    expect(report.staging.reclaimed).toHaveLength(1);
+  });
+
   test('backs up a module whose declared cadence is due', async () => {
     const d = deps([moduleWith('authentik', 'daily')]);
     const report = await runBackupSweep(d);
@@ -129,6 +174,8 @@ describe('ensureBackupSweepSubscriber', () => {
       pattern: string;
       handler: string;
       registeredBy?: string;
+      maxAttempts?: number;
+      timeoutMs?: number;
     }> = [];
     ensureBackupSweepSubscriber({
       subscribe: (options) => calls.push(options),
@@ -140,9 +187,30 @@ describe('ensureBackupSweepSubscriber', () => {
         pattern: BACKUP_SWEEP_PATTERN,
         handler: 'celilo backup sweep',
         registeredBy: 'celilo-backup',
+        maxAttempts: BACKUP_SWEEP_MAX_ATTEMPTS,
+        timeoutMs: BACKUP_SWEEP_TIMEOUT_MS,
       },
     ]);
     // 1h is the coarsest tick that can still serve an `hourly` cadence.
     expect(BACKUP_SWEEP_PATTERN).toBe('timer.tick.1h');
   });
+
+  // Both values are the bug. Registering without them inherits the bus
+  // defaults of 60000ms / 3 attempts, which cannot finish a backup that needs
+  // ~5.5 minutes and then retries the impossible twice more per tick — three
+  // killed backups and three stranded staging directories every hour.
+  test('states the budget explicitly rather than inheriting the bus defaults', () => {
+    let registered: { maxAttempts?: number; timeoutMs?: number } | undefined;
+    ensureBackupSweepSubscriber({
+      subscribe: (options) => {
+        registered = options;
+        return options;
+      },
+    });
+
+    expect(registered?.timeoutMs).toBeDefined();
+    expect(registered?.maxAttempts).toBeDefined();
+    expect(registered?.timeoutMs).toBeGreaterThan(60_000);
+    expect(registered?.maxAttempts).toBe(1);
+  });
 });

package/src/services/backup-sweep.ts

@@ -18,23 +18,72 @@
 
 import type { ModuleManifest } from '../manifest/schema';
 import { type BackupSchedule, effectiveBackupSchedule } from './backup-schedule';
+import type { ReapStagingReport } from './backup-staging';
 import { InFlightError } from './module-operations';
 
 export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep';
 export const BACKUP_SWEEP_PATTERN = 'timer.tick.1h';
 
+/**
+ * How long the sweep may run before the dispatcher kills it.
+ *
+ * Set EXPLICITLY because the bus default is 60 seconds and this pass cannot
+ * finish in 60 seconds. A single forgejo backup measured 2026-08-06 takes ~5.5
+ * minutes — snapshotting SQLite, archiving repositories, and streaming ~1.3 GB
+ * over SSH — and the sweep runs every due module serially, so the budget covers
+ * the sum rather than the slowest one.
+ *
+ * Inheriting the default made every scheduled forgejo backup structurally
+ * impossible: killed at 60s, mid-encrypt, three times an hour, for days.
+ *
+ * Four hours is generous on purpose. The cost of it being too large is one
+ * delayed reclamation of a wedged sweep; the cost of it being too small is a
+ * backup that can never succeed. `celilo-mgmt.registry-poll` set the precedent
+ * at 30 minutes for the same reason.
+ *
+ * ponytail: one flat number for a serial sweep. If the fleet grows enough that
+ * the sum stops fitting, the upgrade is for the sweep to emit a per-module
+ * backup event carrying its own budget.
+ */
+export const BACKUP_SWEEP_TIMEOUT_MS = 4 * 60 * 60 * 1000;
+
+/**
+ * One attempt per tick.
+ *
+ * The bus default of 3 is right for a transient fault and wrong for this pass.
+ * A sweep that cannot finish inside its budget will not finish on the retry
+ * either — retrying it produced three killed backups and three stranded staging
+ * directories per hour instead of one, which is how 27 GB accumulated in 5.7
+ * hours. The hourly tick is already the retry.
+ */
+export const BACKUP_SWEEP_MAX_ATTEMPTS = 1;
+
 export interface SubscriberRegistrar {
   subscribe(options: {
     name: string;
     pattern: string;
     handler: string;
     registeredBy?: string;
+    maxAttempts?: number;
+    timeoutMs?: number;
   }): unknown;
 }
 
 /**
  * Idempotent: `bus.subscribe` upserts by name, so this is safe to call on
  * every module install and update.
+ *
+ * Also called from `celilo system migrate`, which the .deb postinst runs on
+ * every apt upgrade. Registering only from module install/update meant a
+ * corrected budget would not reach an existing fleet until some module happened
+ * to be touched next — indistinguishable from a fix that shipped and silently
+ * did nothing. celilo-mgr sat at the 60s default with the row already present.
+ *
+ * This re-arms a deliberately paused sweep, and that is intentional: the pause
+ * only ever existed because staging leaked, and the reaper that makes it not
+ * leak ships in this same binary. Re-arming without the reaper present is the
+ * failure this comment exists to prevent — do not lift this call into a release
+ * that does not carry `reapOrphanedStaging`.
  */
 export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void {
   bus.subscribe({
@@ -42,6 +91,8 @@ export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void {
     pattern: BACKUP_SWEEP_PATTERN,
     handler: 'celilo backup sweep',
     registeredBy: 'celilo-backup',
+    maxAttempts: BACKUP_SWEEP_MAX_ATTEMPTS,
+    timeoutMs: BACKUP_SWEEP_TIMEOUT_MS,
   });
 }
 
@@ -57,9 +108,13 @@ export interface BackupSweepDeps {
   backup(moduleId: string): Promise<{ success: boolean; error?: string }>;
   /** Apply the module's declared retention. No-op when it declares none. */
   prune(module: BackupSweepModule): Promise<void>;
+  /** Reclaim staging left by backups whose process died. See backup-staging.ts. */
+  reapStaging(): ReapStagingReport;
 }
 
 export interface BackupSweepReport {
+  /** Staging reclaimed before this pass created any of its own. */
+  staging: ReapStagingReport;
   backedUp: string[];
   /** Explicit `schedule: manual` — the author opted out. */
   skippedManual: string[];
@@ -70,7 +125,14 @@ export interface BackupSweepReport {
 }
 
 export async function runBackupSweep(deps: BackupSweepDeps): Promise<BackupSweepReport> {
+  // Reclaim BEFORE backing anything up, not after. This pass is the only thing
+  // that creates staging on a schedule, so it is where the orphans come from —
+  // and a box already short on disk needs the space freed before we ask for
+  // another ~4 GB of it, not once we are finished with it.
+  const staging = deps.reapStaging();
+
   const report: BackupSweepReport = {
+    staging,
     backedUp: [],
     skippedManual: [],
     skippedNotDue: [],

package/src/services/config-interview.ts

@@ -260,7 +260,7 @@ export async function autoDeriveMachineConfig(
     upsertModuleConfig(db, moduleId, variable.name, derived);
     configured.push(variable.name);
 
-    // Handle per_selection follow-ups (e.g., zone_ip_* from zone list)
+    // Handle per_selection follow-ups (e.g., `zone.<zone>.ip` from the zone list)
     if (variable.options && variable.per_selection) {
       for (const selectedVal of derived.split(',')) {
         const followUpKey = variable.per_selection.key_pattern.replace('{value}', selectedVal);

package/src/services/deploy-ansible.ts

@@ -22,7 +22,6 @@ export interface AnsibleResult {
  * Parse raw Ansible output lines into concise human-readable status.
  * Returns null for lines that should be suppressed (decorative separators, etc.)
  */
-// biome-ignore lint/suspicious/noControlCharactersInRegex: intentionally stripping ANSI escape codes
 const ANSI_ESCAPE = /\x1b\[[0-9;]*m/g;
 
 /**

package/src/services/disk-probe.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, test } from 'bun:test';
+import { parseDfOutput, percentUsed } from './disk-probe';
+
+describe('percentUsed', () => {
+  // Checked against what celilo-mgr's df actually printed, because an operator
+  // comparing an alert to their own df must see the same figure.
+  //
+  // 117G root, 11G used, 102G available — after the staging was cleared.
+  test('matches df for celilo-mgr at 10% used', () => {
+    expect(percentUsed(122_683_392, 111_149_056, 106_954_752)).toBe(10);
+  });
+
+  // Same filesystem at 38G used / 75G available — what df reported while the
+  // leak was running.
+  test('matches df for celilo-mgr at 34% used', () => {
+    expect(percentUsed(122_683_392, 82_837_504, 78_643_200)).toBe(34);
+  });
+
+  // Not 1 - bavail/blocks. A filesystem reserves blocks for root, so
+  // free-to-root and free-to-everyone differ; df computes capacity against
+  // what an ordinary process can use. With a large reserve the naive formula
+  // over-reports and would page early forever.
+  test('excludes root-reserved blocks, as df does', () => {
+    // 1000 total, 100 free to root, only 50 usable by others → 900 used of 950.
+    expect(percentUsed(1000, 100, 50)).toBe(95);
+    // The naive 1 - bavail/blocks would say 95% here too by coincidence, so
+    // use a case where they diverge: 1000 total, 500 free, 200 available.
+    expect(percentUsed(1000, 500, 200)).toBe(71); // 500 used of 700 usable
+  });
+
+  test('a full filesystem reports 100', () => {
+    expect(percentUsed(1000, 0, 0)).toBe(100);
+  });
+
+  test('a degenerate zero-block filesystem does not divide by zero', () => {
+    expect(percentUsed(0, 0, 0)).toBe(0);
+  });
+});
+
+describe('parseDfOutput', () => {
+  test('parses real df -P output', () => {
+    const out = [
+      'Filesystem     1024-blocks     Used Available Capacity Mounted on',
+      '/dev/mmcblk0p2   120699413 11534336 106954752      10% /',
+    ].join('\n');
+
+    expect(parseDfOutput(out)).toEqual({
+      usedPercent: 10,
+      availableBytes: 106_954_752 * 1024,
+    });
+  });
+
+  test('parses a nearly-full filesystem', () => {
+    const out = [
+      'Filesystem     1024-blocks      Used Available Capacity Mounted on',
+      '/dev/sda1         41284928  39220684    966140      98% /',
+    ].join('\n');
+
+    expect(parseDfOutput(out)?.usedPercent).toBe(98);
+  });
+
+  test('returns null rather than guessing on unusable output', () => {
+    expect(parseDfOutput('')).toBeNull();
+    expect(parseDfOutput('Filesystem 1024-blocks Used Available Capacity Mounted on')).toBeNull();
+    expect(parseDfOutput('header\ntoo few fields')).toBeNull();
+    expect(parseDfOutput('header\n/dev/sda1 a b c d /')).toBeNull();
+  });
+
+  test('tolerates leading whitespace and extra columns', () => {
+    const out =
+      '  Filesystem 1024-blocks Used Available Capacity Mounted on\n  /dev/sda1 100 50 40 56% / extra';
+    expect(parseDfOutput(out)?.usedPercent).toBe(56);
+  });
+});