@celilo/cli 0.17.0 → 0.18.0

package/src/services/audit/index.ts

@@ -10,6 +10,10 @@
  * truth for what audit can detect is this file's `AuditDeps` shape.
  */
 
+import {
+  type AbandonedOperationsAuditDeps,
+  auditAbandonedOperations,
+} from './abandoned-operations';
 import { type BackupsAuditDeps, auditBackups } from './backups';
 import { type CapabilityAbiAuditDeps, auditCapabilityAbi } from './capability-abi';
 import { type CliVersionAuditDeps, auditCliVersion } from './cli-version';
@@ -48,6 +52,7 @@ export interface AuditDeps {
   moduleConfigs: ModuleConfigsAuditDeps;
   health: HealthAuditDeps;
   backups: BackupsAuditDeps;
+  abandonedOperations: AbandonedOperationsAuditDeps;
   undeployedModules: UndeployedModulesAuditDeps;
   unconfiguredModules: UnconfiguredModulesAuditDeps;
   servicesCredentials: ServicesCredentialsAuditDeps;
@@ -99,6 +104,10 @@ export async function runAudit(
     wrap('module_configs', auditModuleConfigs(deps.moduleConfigs)),
     wrap('health', auditHealth(deps.health)),
     wrap('backups', auditBackups(deps.backups)),
+    wrap(
+      'abandoned_operations',
+      Promise.resolve(auditAbandonedOperations(deps.abandonedOperations)),
+    ),
     wrap('undeployed_modules', auditUndeployedModules(deps.undeployedModules)),
     wrap('unconfigured_modules', auditUnconfiguredModules(deps.unconfiguredModules)),
     wrap('services_credentials', auditServicesCredentials(deps.servicesCredentials)),

package/src/services/audit/types.ts

@@ -30,12 +30,14 @@ export type DriftCategory =
   | 'module_configs'
   | 'health'
   | 'backups'
+  | 'abandoned_operations'
   | 'undeployed_modules'
   | 'unconfigured_modules'
   | 'services_credentials'
   | 'secrets_decryptable'
   | 'services_reachable'
   | 'machines_reachable'
+  | 'disk_space'
   | 'transport_reads'
   | 'trusted_sources';
 

package/src/services/backup-create.ts

@@ -4,7 +4,6 @@
  */
 
 import { copyFileSync, existsSync, mkdirSync, rmSync, statSync, writeFileSync } from 'node:fs';
-import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { eq } from 'drizzle-orm';
 import { getDbPath } from '../config/paths';
@@ -20,6 +19,7 @@ import { encryptFileToFile } from './backup-cipher';
 import { buildManifest } from './backup-manifest';
 import { completeBackup, createBackupRecord, failBackup, listBackups } from './backup-metadata';
 import type { BackupSchedule } from './backup-schedule';
+import { stagingDirFor } from './backup-staging';
 import {
   createStorageProvider,
   getBackupStorage,
@@ -113,7 +113,7 @@ export async function createSystemStateBackup(
   });
   const opId = startOperation('__system__', 'backup');
 
-  const tempDir = join(tmpdir(), `celilo-backup-${record.id}`);
+  const tempDir = stagingDirFor(record.id);
 
   try {
     mkdirSync(tempDir, { recursive: true });
@@ -328,7 +328,7 @@ export async function createModuleBackup(
   // Envelope layout (v1.0):
   //   manifest.json   - schema version, host, moduleId, dataSchemaVersion
   //   data/           - on_backup hook artifacts (the hook treats this as backup_dir)
-  const tempDir = join(tmpdir(), `celilo-backup-${record.id}`);
+  const tempDir = stagingDirFor(record.id);
   const envelopeDir = join(tempDir, 'envelope');
   const dataDir = join(envelopeDir, 'data');
 
@@ -496,7 +496,7 @@ export async function importModuleBackup(
     moduleVersion: manifest.version,
   });
 
-  const tempDir = join(tmpdir(), `celilo-backup-${record.id}`);
+  const tempDir = stagingDirFor(record.id);
   const artifactDir = join(tempDir, 'artifacts');
 
   try {

package/src/services/backup-in-flight-refusal.test.ts

@@ -129,6 +129,7 @@ describe('backup/restore in-flight refusal', () => {
         status: 'completed' as const,
         errorMessage: null,
         name: null,
+        pid: process.pid,
         startedAt: new Date(),
         completedAt: null,
       };
@@ -152,6 +153,7 @@ describe('backup/restore in-flight refusal', () => {
         status: 'completed' as const,
         errorMessage: null,
         name: null,
+        pid: process.pid,
         startedAt: new Date(),
         completedAt: null,
       };

package/src/services/backup-metadata.ts

@@ -43,6 +43,10 @@ export function createBackupRecord(params: CreateBackupParams): BackupRecord {
     metadata: {},
     status: 'in_progress' as BackupStatus,
     errorMessage: null,
+    // Recorded so the staging reaper can distinguish a live backup from one
+    // whose process was killed before its `finally` ran — see
+    // services/backup-staging.ts.
+    pid: process.pid,
     startedAt: now,
     completedAt: null,
   };

package/src/services/backup-staging.test.ts

@@ -0,0 +1,134 @@
+import { describe, expect, test } from 'bun:test';
+import {
+  type ReapStagingDeps,
+  STAGING_PREFIX,
+  STAGING_TTL_MS,
+  type StagingOwner,
+  reapOrphanedStaging,
+  stagingDirFor,
+} from './backup-staging';
+
+const NOW = new Date('2026-08-06T12:00:00Z').getTime();
+
+function deps(
+  owners: Record<string, StagingOwner | null>,
+  options: {
+    dirs?: string[];
+    runnablePids?: number[];
+    unremovable?: string[];
+  } = {},
+): ReapStagingDeps & { removed: string[] } {
+  const removed: string[] = [];
+  return {
+    removed,
+    listStagingDirs: () => options.dirs ?? Object.keys(owners).map((id) => stagingDirFor(id)),
+    lookupOwner: (id) => owners[id] ?? null,
+    isPidRunnable: (pid) => (options.runnablePids ?? []).includes(pid),
+    remove: (path) => {
+      if (options.unremovable?.includes(path)) throw new Error('EACCES');
+      removed.push(path);
+    },
+    now: () => NOW,
+  };
+}
+
+function owner(over: Partial<StagingOwner> = {}): StagingOwner {
+  return { status: 'in_progress', pid: 111, startedAt: new Date(NOW - 60_000), ...over };
+}
+
+describe('reapOrphanedStaging', () => {
+  test('reclaims staging whose backup record no longer exists', () => {
+    const d = deps({ 'gone-id': null });
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed).toHaveLength(1);
+    expect(report.reclaimed[0]?.reason).toBe('record-absent');
+    expect(d.removed).toEqual([stagingDirFor('gone-id')]);
+  });
+
+  test.each([['completed'], ['failed']])('reclaims staging for a %s record', (status) => {
+    const d = deps({ 'done-id': owner({ status }) });
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed[0]?.reason).toBe('record-terminal');
+    expect(d.removed).toEqual([stagingDirFor('done-id')]);
+  });
+
+  test('reclaims staging whose owning process is dead', () => {
+    // in_progress, inside the TTL — only the liveness probe can tell.
+    const d = deps({ 'dead-id': owner({ pid: 999 }) }, { runnablePids: [] });
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed[0]?.reason).toBe('process-dead');
+    expect(d.removed).toEqual([stagingDirFor('dead-id')]);
+  });
+
+  // The one that must never regress: a running backup writing gigabytes into
+  // its staging dir must survive a concurrent sweep.
+  test('KEEPS staging owned by a live backup', () => {
+    const d = deps({ 'live-id': owner({ pid: 111 }) }, { runnablePids: [111] });
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed).toHaveLength(0);
+    expect(report.kept).toEqual([stagingDirFor('live-id')]);
+    expect(d.removed).toEqual([]);
+  });
+
+  test('reclaims a record past the TTL even when its pid looks alive', () => {
+    // Guards pid reuse: after the pid space wraps, a stale record's pid names
+    // an unrelated live process and liveness alone would strand this forever.
+    const d = deps(
+      { 'stale-id': owner({ pid: 111, startedAt: new Date(NOW - STAGING_TTL_MS - 1) }) },
+      { runnablePids: [111] },
+    );
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed[0]?.reason).toBe('expired');
+  });
+
+  test('keeps a pid-less record until it expires, then reclaims it', () => {
+    const fresh = deps({ 'old-fmt': owner({ pid: null }) });
+    expect(reapOrphanedStaging(fresh).kept).toHaveLength(1);
+
+    const expired = deps({
+      'old-fmt': owner({ pid: null, startedAt: new Date(NOW - STAGING_TTL_MS - 1) }),
+    });
+    expect(reapOrphanedStaging(expired).reclaimed[0]?.reason).toBe('expired');
+  });
+
+  test('ignores directories that are not staging, rather than deleting them', () => {
+    const d = deps(
+      {},
+      {
+        dirs: [
+          '/tmp/something-else',
+          '/tmp/celilo-unrelated',
+          `/tmp/${STAGING_PREFIX}`, // prefix with no record id
+        ],
+      },
+    );
+    const report = reapOrphanedStaging(d);
+
+    expect(report.ignored).toHaveLength(3);
+    expect(report.reclaimed).toHaveLength(0);
+    expect(d.removed).toEqual([]);
+  });
+
+  test('an undeletable directory is reported as kept, not as reclaimed space', () => {
+    const path = stagingDirFor('locked-id');
+    const d = deps({ 'locked-id': null }, { unremovable: [path] });
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed).toHaveLength(0);
+    expect(report.kept).toEqual([path]);
+  });
+
+  test('one undeletable directory does not stop the rest of the pass', () => {
+    const locked = stagingDirFor('a-locked');
+    const d = deps({ 'a-locked': null, 'b-orphan': null }, { unremovable: [locked] });
+    const report = reapOrphanedStaging(d);
+
+    expect(report.reclaimed.map((r) => r.recordId)).toEqual(['b-orphan']);
+    expect(report.kept).toEqual([locked]);
+  });
+});

package/src/services/backup-staging.ts

@@ -0,0 +1,192 @@
+/**
+ * Reclaiming backup staging directories whose owner is gone.
+ *
+ * `backup-create.ts` assembles every envelope in a temp directory and removes
+ * it in a `finally`. That is correct and it is not enough: a `finally` does not
+ * run when the process is killed by a signal it cannot intercept — a dispatcher
+ * timeout, an OOM, an operator's Ctrl-C, a host reboot. Those are exactly the
+ * cases that strand the LARGEST directories, because the longer a backup has
+ * run the more it has written.
+ *
+ * Measured on celilo-mgr 2026-08-05: the scheduled sweep inherited the event
+ * bus's 60s default timeout against a forgejo backup that needs ~5.5 minutes,
+ * so every attempt was SIGTERMed at ~1.4 GB of staging, three times an hour,
+ * for days. 15 GB stranded in 26 hours, then 27 GB in the next 5.7. Nothing
+ * reclaimed it, because the only cleanup was the `finally` that never ran.
+ *
+ * So reclamation must not be a property of how a backup ends. This pass asks a
+ * different question — "is anyone still using this directory?" — and answers it
+ * from state that outlives the process.
+ *
+ * Identity comes free: a staging directory is named `celilo-backup-<record.id>`,
+ * so the directory names its own backup record. No lockfile, no marker, no
+ * second source of truth that could itself be stranded — and notably nothing
+ * written BY the process whose death is the problem.
+ *
+ * Dependencies are injected so the decision logic tests with no filesystem and
+ * no database.
+ */
+
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+/** Every staging directory starts with this. The rest of the name is the record id. */
+export const STAGING_PREFIX = 'celilo-backup-';
+
+/**
+ * Where a backup record's staging lives. Single source of truth — `backup-create.ts`
+ * builds its temp dir from this so the reaper can never drift from the writer.
+ */
+export function stagingDirFor(recordId: string): string {
+  return join(tmpdir(), `${STAGING_PREFIX}${recordId}`);
+}
+
+/**
+ * How long an `in_progress` backup record may vouch for its staging before the
+ * directory is reclaimed regardless of what its pid appears to be doing.
+ *
+ * This is not redundancy on the liveness check — it is the only check that
+ * survives pid reuse, for the same reason `OPERATION_TTL_MS` exists in
+ * `module-operations.ts`. A pid is a recycled number: once the pid space wraps,
+ * a stale record's pid names an unrelated live process and the liveness probe
+ * reports "still running" forever, stranding the directory permanently.
+ *
+ * Six hours is comfortably longer than any real backup (the largest measured is
+ * ~5.5 minutes) and short enough that a wedged record costs one cycle rather
+ * than a filesystem.
+ */
+export const STAGING_TTL_MS = 6 * 60 * 60 * 1000;
+
+/** What the backup record says about the process that owns a staging directory. */
+export interface StagingOwner {
+  status: string;
+  /** Null for records written before backups recorded their pid. */
+  pid: number | null;
+  startedAt: Date;
+}
+
+export interface ReapStagingDeps {
+  /** Absolute paths of every `celilo-backup-*` entry in the temp dir. */
+  listStagingDirs(): string[];
+  /** The backup record for this id, or null when it no longer exists. */
+  lookupOwner(recordId: string): StagingOwner | null;
+  isPidRunnable(pid: number): boolean;
+  remove(path: string): void;
+  now(): number;
+}
+
+export interface ReapedStaging {
+  path: string;
+  recordId: string;
+  /** Why it was reclaimable — surfaced so a sweep can explain itself. */
+  reason: 'record-absent' | 'record-terminal' | 'process-dead' | 'expired';
+}
+
+/**
+ * Written to a backup record reclaimed while it still claimed to be running.
+ *
+ * A record left `in_progress` after its process died misreports the system
+ * twice: `celilo backup list` shows work apparently underway, and the `backups`
+ * drift check can read a module as recently backed up when every attempt in
+ * fact died. Nine such rows were live on celilo-mgr while forgejo had no usable
+ * backup at all.
+ */
+export const ABANDONED_BACKUP_MESSAGE =
+  'abandoned — the backup process ended without recording an outcome';
+
+/**
+ * Whether reclaiming this directory also means its record was lying about
+ * being in progress.
+ *
+ * Only the two reasons reached from the `in_progress` branch qualify:
+ * `record-terminal` already has an outcome and `record-absent` has no row to
+ * correct.
+ */
+export function impliesAbandonedRecord(reason: ReapedStaging['reason']): boolean {
+  return reason === 'process-dead' || reason === 'expired';
+}
+
+export interface ReapStagingReport {
+  reclaimed: ReapedStaging[];
+  /** Left alone because a live backup is using it. */
+  kept: string[];
+  /** Names that did not look like staging at all. Never touched. */
+  ignored: string[];
+}
+
+/**
+ * Decide, for one staging directory, whether its owner is gone.
+ *
+ * Returns null when the directory must be left alone. Every branch that
+ * reclaims must be able to say why; "I could not prove it is alive" is not a
+ * reason to delete, which is why an unrecognised state keeps the directory.
+ */
+function reclaimReason(
+  owner: StagingOwner | null,
+  deps: Pick<ReapStagingDeps, 'isPidRunnable' | 'now'>,
+): ReapedStaging['reason'] | null {
+  // The record was deleted, or never committed. Nothing will ever finish this.
+  if (!owner) return 'record-absent';
+
+  // Completed or failed: the writer reached an ending and either cleaned up
+  // already (in which case we will not see the directory) or was killed after
+  // recording its outcome.
+  if (owner.status !== 'in_progress') return 'record-terminal';
+
+  const age = deps.now() - owner.startedAt.getTime();
+  if (age > STAGING_TTL_MS) return 'expired';
+
+  // A record from before backups carried a pid. Age is the only signal
+  // available, and it has not expired — leave it.
+  if (owner.pid === null) return null;
+
+  return deps.isPidRunnable(owner.pid) ? null : 'process-dead';
+}
+
+/**
+ * Reclaim every staging directory whose owning backup is no longer running.
+ *
+ * Conservative by construction: a directory is removed only when its owner is
+ * provably gone. A running backup — `in_progress` record, live pid, inside the
+ * TTL — is always kept, so a long-running or concurrent backup can never be
+ * destroyed by this pass.
+ *
+ * A name that is not `celilo-backup-<id>` is ignored rather than removed. This
+ * runs against a shared temp directory as a privileged user; deleting something
+ * it does not understand is not its job.
+ */
+export function reapOrphanedStaging(deps: ReapStagingDeps): ReapStagingReport {
+  const report: ReapStagingReport = { reclaimed: [], kept: [], ignored: [] };
+
+  for (const path of deps.listStagingDirs()) {
+    const name = path.slice(path.lastIndexOf('/') + 1);
+    if (!name.startsWith(STAGING_PREFIX)) {
+      report.ignored.push(path);
+      continue;
+    }
+
+    const recordId = name.slice(STAGING_PREFIX.length);
+    if (recordId.length === 0) {
+      report.ignored.push(path);
+      continue;
+    }
+
+    const reason = reclaimReason(deps.lookupOwner(recordId), deps);
+    if (!reason) {
+      report.kept.push(path);
+      continue;
+    }
+
+    // A directory that cannot be removed is not fatal: the next pass retries,
+    // and failing the whole sweep over one undeletable path would stop backups
+    // entirely. Treated as kept so the report never claims space it did not free.
+    try {
+      deps.remove(path);
+      report.reclaimed.push({ path, recordId, reason });
+    } catch {
+      report.kept.push(path);
+    }
+  }
+
+  return report;
+}

package/src/services/backup-sweep.test.ts

@@ -2,8 +2,10 @@ import { describe, expect, test } from 'bun:test';
 import type { ModuleManifest } from '../manifest/schema';
 import type { BackupSchedule } from './backup-schedule';
 import {
+  BACKUP_SWEEP_MAX_ATTEMPTS,
   BACKUP_SWEEP_PATTERN,
   BACKUP_SWEEP_SUBSCRIBER,
+  BACKUP_SWEEP_TIMEOUT_MS,
   type BackupSweepDeps,
   type BackupSweepModule,
   ensureBackupSweepSubscriber,
@@ -33,11 +35,54 @@ function deps(
     prune: async ({ id }) => {
       pruned.push(id);
     },
+    reapStaging: () => ({ reclaimed: [], kept: [], ignored: [] }),
     ...overrides,
   };
 }
 
 describe('runBackupSweep', () => {
+  // Ordering is the point, not just that it happens: this pass is the only
+  // thing creating staging on a schedule, and a box already short on disk
+  // needs the space back BEFORE another few GB are requested.
+  test('reclaims orphaned staging before backing anything up', async () => {
+    const order: string[] = [];
+    const report = await runBackupSweep(
+      deps([moduleWith('forgejo', 'daily')], {
+        reapStaging: () => {
+          order.push('reap');
+          return {
+            reclaimed: [{ path: '/tmp/celilo-backup-x', recordId: 'x', reason: 'process-dead' }],
+            kept: [],
+            ignored: [],
+          };
+        },
+        backup: async () => {
+          order.push('backup');
+          return { success: true };
+        },
+      }),
+    );
+
+    expect(order).toEqual(['reap', 'backup']);
+    expect(report.staging.reclaimed).toHaveLength(1);
+  });
+
+  test('reclaims staging even when no module is due to back up', async () => {
+    const report = await runBackupSweep(
+      deps([moduleWith('forgejo', 'daily')], {
+        isDue: () => false,
+        reapStaging: () => ({
+          reclaimed: [{ path: '/tmp/celilo-backup-y', recordId: 'y', reason: 'record-absent' }],
+          kept: [],
+          ignored: [],
+        }),
+      }),
+    );
+
+    expect(report.backedUp).toEqual([]);
+    expect(report.staging.reclaimed).toHaveLength(1);
+  });
+
   test('backs up a module whose declared cadence is due', async () => {
     const d = deps([moduleWith('authentik', 'daily')]);
     const report = await runBackupSweep(d);
@@ -129,6 +174,8 @@ describe('ensureBackupSweepSubscriber', () => {
       pattern: string;
       handler: string;
       registeredBy?: string;
+      maxAttempts?: number;
+      timeoutMs?: number;
     }> = [];
     ensureBackupSweepSubscriber({
       subscribe: (options) => calls.push(options),
@@ -140,9 +187,30 @@ describe('ensureBackupSweepSubscriber', () => {
         pattern: BACKUP_SWEEP_PATTERN,
         handler: 'celilo backup sweep',
         registeredBy: 'celilo-backup',
+        maxAttempts: BACKUP_SWEEP_MAX_ATTEMPTS,
+        timeoutMs: BACKUP_SWEEP_TIMEOUT_MS,
       },
     ]);
     // 1h is the coarsest tick that can still serve an `hourly` cadence.
     expect(BACKUP_SWEEP_PATTERN).toBe('timer.tick.1h');
   });
+
+  // Both values are the bug. Registering without them inherits the bus
+  // defaults of 60000ms / 3 attempts, which cannot finish a backup that needs
+  // ~5.5 minutes and then retries the impossible twice more per tick — three
+  // killed backups and three stranded staging directories every hour.
+  test('states the budget explicitly rather than inheriting the bus defaults', () => {
+    let registered: { maxAttempts?: number; timeoutMs?: number } | undefined;
+    ensureBackupSweepSubscriber({
+      subscribe: (options) => {
+        registered = options;
+        return options;
+      },
+    });
+
+    expect(registered?.timeoutMs).toBeDefined();
+    expect(registered?.maxAttempts).toBeDefined();
+    expect(registered?.timeoutMs).toBeGreaterThan(60_000);
+    expect(registered?.maxAttempts).toBe(1);
+  });
 });

package/src/services/backup-sweep.ts

@@ -18,23 +18,72 @@
 
 import type { ModuleManifest } from '../manifest/schema';
 import { type BackupSchedule, effectiveBackupSchedule } from './backup-schedule';
+import type { ReapStagingReport } from './backup-staging';
 import { InFlightError } from './module-operations';
 
 export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep';
 export const BACKUP_SWEEP_PATTERN = 'timer.tick.1h';
 
+/**
+ * How long the sweep may run before the dispatcher kills it.
+ *
+ * Set EXPLICITLY because the bus default is 60 seconds and this pass cannot
+ * finish in 60 seconds. A single forgejo backup measured 2026-08-06 takes ~5.5
+ * minutes — snapshotting SQLite, archiving repositories, and streaming ~1.3 GB
+ * over SSH — and the sweep runs every due module serially, so the budget covers
+ * the sum rather than the slowest one.
+ *
+ * Inheriting the default made every scheduled forgejo backup structurally
+ * impossible: killed at 60s, mid-encrypt, three times an hour, for days.
+ *
+ * Four hours is generous on purpose. The cost of it being too large is one
+ * delayed reclamation of a wedged sweep; the cost of it being too small is a
+ * backup that can never succeed. `celilo-mgmt.registry-poll` set the precedent
+ * at 30 minutes for the same reason.
+ *
+ * ponytail: one flat number for a serial sweep. If the fleet grows enough that
+ * the sum stops fitting, the upgrade is for the sweep to emit a per-module
+ * backup event carrying its own budget.
+ */
+export const BACKUP_SWEEP_TIMEOUT_MS = 4 * 60 * 60 * 1000;
+
+/**
+ * One attempt per tick.
+ *
+ * The bus default of 3 is right for a transient fault and wrong for this pass.
+ * A sweep that cannot finish inside its budget will not finish on the retry
+ * either — retrying it produced three killed backups and three stranded staging
+ * directories per hour instead of one, which is how 27 GB accumulated in 5.7
+ * hours. The hourly tick is already the retry.
+ */
+export const BACKUP_SWEEP_MAX_ATTEMPTS = 1;
+
 export interface SubscriberRegistrar {
   subscribe(options: {
     name: string;
     pattern: string;
     handler: string;
     registeredBy?: string;
+    maxAttempts?: number;
+    timeoutMs?: number;
   }): unknown;
 }
 
 /**
  * Idempotent: `bus.subscribe` upserts by name, so this is safe to call on
  * every module install and update.
+ *
+ * Also called from `celilo system migrate`, which the .deb postinst runs on
+ * every apt upgrade. Registering only from module install/update meant a
+ * corrected budget would not reach an existing fleet until some module happened
+ * to be touched next — indistinguishable from a fix that shipped and silently
+ * did nothing. celilo-mgr sat at the 60s default with the row already present.
+ *
+ * This re-arms a deliberately paused sweep, and that is intentional: the pause
+ * only ever existed because staging leaked, and the reaper that makes it not
+ * leak ships in this same binary. Re-arming without the reaper present is the
+ * failure this comment exists to prevent — do not lift this call into a release
+ * that does not carry `reapOrphanedStaging`.
  */
 export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void {
   bus.subscribe({
@@ -42,6 +91,8 @@ export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void {
     pattern: BACKUP_SWEEP_PATTERN,
     handler: 'celilo backup sweep',
     registeredBy: 'celilo-backup',
+    maxAttempts: BACKUP_SWEEP_MAX_ATTEMPTS,
+    timeoutMs: BACKUP_SWEEP_TIMEOUT_MS,
   });
 }
 
@@ -57,9 +108,13 @@ export interface BackupSweepDeps {
   backup(moduleId: string): Promise<{ success: boolean; error?: string }>;
   /** Apply the module's declared retention. No-op when it declares none. */
   prune(module: BackupSweepModule): Promise<void>;
+  /** Reclaim staging left by backups whose process died. See backup-staging.ts. */
+  reapStaging(): ReapStagingReport;
 }
 
 export interface BackupSweepReport {
+  /** Staging reclaimed before this pass created any of its own. */
+  staging: ReapStagingReport;
   backedUp: string[];
   /** Explicit `schedule: manual` — the author opted out. */
   skippedManual: string[];
@@ -70,7 +125,14 @@ export interface BackupSweepReport {
 }
 
 export async function runBackupSweep(deps: BackupSweepDeps): Promise<BackupSweepReport> {
+  // Reclaim BEFORE backing anything up, not after. This pass is the only thing
+  // that creates staging on a schedule, so it is where the orphans come from —
+  // and a box already short on disk needs the space freed before we ask for
+  // another ~4 GB of it, not once we are finished with it.
+  const staging = deps.reapStaging();
+
   const report: BackupSweepReport = {
+    staging,
     backedUp: [],
     skippedManual: [],
     skippedNotDue: [],

package/src/services/config-interview.ts

@@ -260,7 +260,7 @@ export async function autoDeriveMachineConfig(
     upsertModuleConfig(db, moduleId, variable.name, derived);
     configured.push(variable.name);
 
-    // Handle per_selection follow-ups (e.g., zone_ip_* from zone list)
+    // Handle per_selection follow-ups (e.g., `zone.<zone>.ip` from the zone list)
     if (variable.options && variable.per_selection) {
       for (const selectedVal of derived.split(',')) {
         const followUpKey = variable.per_selection.key_pattern.replace('{value}', selectedVal);

package/src/services/deploy-ansible.ts

@@ -22,7 +22,6 @@ export interface AnsibleResult {
  * Parse raw Ansible output lines into concise human-readable status.
  * Returns null for lines that should be suppressed (decorative separators, etc.)
  */
-// biome-ignore lint/suspicious/noControlCharactersInRegex: intentionally stripping ANSI escape codes
 const ANSI_ESCAPE = /\x1b\[[0-9;]*m/g;
 
 /**