@celilo/cli 0.17.0 → 0.19.0

package/src/services/backup-sweep.test.ts

@@ -2,8 +2,10 @@ import { describe, expect, test } from 'bun:test';
 import type { ModuleManifest } from '../manifest/schema';
 import type { BackupSchedule } from './backup-schedule';
 import {
+  BACKUP_SWEEP_MAX_ATTEMPTS,
   BACKUP_SWEEP_PATTERN,
   BACKUP_SWEEP_SUBSCRIBER,
+  BACKUP_SWEEP_TIMEOUT_MS,
   type BackupSweepDeps,
   type BackupSweepModule,
   ensureBackupSweepSubscriber,
@@ -33,11 +35,54 @@ function deps(
     prune: async ({ id }) => {
       pruned.push(id);
     },
+    reapStaging: () => ({ reclaimed: [], kept: [], ignored: [] }),
     ...overrides,
   };
 }
 
 describe('runBackupSweep', () => {
+  // Ordering is the point, not just that it happens: this pass is the only
+  // thing creating staging on a schedule, and a box already short on disk
+  // needs the space back BEFORE another few GB are requested.
+  test('reclaims orphaned staging before backing anything up', async () => {
+    const order: string[] = [];
+    const report = await runBackupSweep(
+      deps([moduleWith('forgejo', 'daily')], {
+        reapStaging: () => {
+          order.push('reap');
+          return {
+            reclaimed: [{ path: '/tmp/celilo-backup-x', recordId: 'x', reason: 'process-dead' }],
+            kept: [],
+            ignored: [],
+          };
+        },
+        backup: async () => {
+          order.push('backup');
+          return { success: true };
+        },
+      }),
+    );
+
+    expect(order).toEqual(['reap', 'backup']);
+    expect(report.staging.reclaimed).toHaveLength(1);
+  });
+
+  test('reclaims staging even when no module is due to back up', async () => {
+    const report = await runBackupSweep(
+      deps([moduleWith('forgejo', 'daily')], {
+        isDue: () => false,
+        reapStaging: () => ({
+          reclaimed: [{ path: '/tmp/celilo-backup-y', recordId: 'y', reason: 'record-absent' }],
+          kept: [],
+          ignored: [],
+        }),
+      }),
+    );
+
+    expect(report.backedUp).toEqual([]);
+    expect(report.staging.reclaimed).toHaveLength(1);
+  });
+
   test('backs up a module whose declared cadence is due', async () => {
     const d = deps([moduleWith('authentik', 'daily')]);
     const report = await runBackupSweep(d);
@@ -129,6 +174,8 @@ describe('ensureBackupSweepSubscriber', () => {
       pattern: string;
       handler: string;
       registeredBy?: string;
+      maxAttempts?: number;
+      timeoutMs?: number;
     }> = [];
     ensureBackupSweepSubscriber({
       subscribe: (options) => calls.push(options),
@@ -140,9 +187,30 @@ describe('ensureBackupSweepSubscriber', () => {
         pattern: BACKUP_SWEEP_PATTERN,
         handler: 'celilo backup sweep',
         registeredBy: 'celilo-backup',
+        maxAttempts: BACKUP_SWEEP_MAX_ATTEMPTS,
+        timeoutMs: BACKUP_SWEEP_TIMEOUT_MS,
       },
     ]);
     // 1h is the coarsest tick that can still serve an `hourly` cadence.
     expect(BACKUP_SWEEP_PATTERN).toBe('timer.tick.1h');
   });
+
+  // Both values are the bug. Registering without them inherits the bus
+  // defaults of 60000ms / 3 attempts, which cannot finish a backup that needs
+  // ~5.5 minutes and then retries the impossible twice more per tick — three
+  // killed backups and three stranded staging directories every hour.
+  test('states the budget explicitly rather than inheriting the bus defaults', () => {
+    let registered: { maxAttempts?: number; timeoutMs?: number } | undefined;
+    ensureBackupSweepSubscriber({
+      subscribe: (options) => {
+        registered = options;
+        return options;
+      },
+    });
+
+    expect(registered?.timeoutMs).toBeDefined();
+    expect(registered?.maxAttempts).toBeDefined();
+    expect(registered?.timeoutMs).toBeGreaterThan(60_000);
+    expect(registered?.maxAttempts).toBe(1);
+  });
 });

package/src/services/backup-sweep.ts

@@ -18,23 +18,72 @@
 
 import type { ModuleManifest } from '../manifest/schema';
 import { type BackupSchedule, effectiveBackupSchedule } from './backup-schedule';
+import type { ReapStagingReport } from './backup-staging';
 import { InFlightError } from './module-operations';
 
 export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep';
 export const BACKUP_SWEEP_PATTERN = 'timer.tick.1h';
 
+/**
+ * How long the sweep may run before the dispatcher kills it.
+ *
+ * Set EXPLICITLY because the bus default is 60 seconds and this pass cannot
+ * finish in 60 seconds. A single forgejo backup measured 2026-08-06 takes ~5.5
+ * minutes — snapshotting SQLite, archiving repositories, and streaming ~1.3 GB
+ * over SSH — and the sweep runs every due module serially, so the budget covers
+ * the sum rather than the slowest one.
+ *
+ * Inheriting the default made every scheduled forgejo backup structurally
+ * impossible: killed at 60s, mid-encrypt, three times an hour, for days.
+ *
+ * Four hours is generous on purpose. The cost of it being too large is one
+ * delayed reclamation of a wedged sweep; the cost of it being too small is a
+ * backup that can never succeed. `celilo-mgmt.registry-poll` set the precedent
+ * at 30 minutes for the same reason.
+ *
+ * ponytail: one flat number for a serial sweep. If the fleet grows enough that
+ * the sum stops fitting, the upgrade is for the sweep to emit a per-module
+ * backup event carrying its own budget.
+ */
+export const BACKUP_SWEEP_TIMEOUT_MS = 4 * 60 * 60 * 1000;
+
+/**
+ * One attempt per tick.
+ *
+ * The bus default of 3 is right for a transient fault and wrong for this pass.
+ * A sweep that cannot finish inside its budget will not finish on the retry
+ * either — retrying it produced three killed backups and three stranded staging
+ * directories per hour instead of one, which is how 27 GB accumulated in 5.7
+ * hours. The hourly tick is already the retry.
+ */
+export const BACKUP_SWEEP_MAX_ATTEMPTS = 1;
+
 export interface SubscriberRegistrar {
   subscribe(options: {
     name: string;
     pattern: string;
     handler: string;
     registeredBy?: string;
+    maxAttempts?: number;
+    timeoutMs?: number;
   }): unknown;
 }
 
 /**
  * Idempotent: `bus.subscribe` upserts by name, so this is safe to call on
  * every module install and update.
+ *
+ * Also called from `celilo system migrate`, which the .deb postinst runs on
+ * every apt upgrade. Registering only from module install/update meant a
+ * corrected budget would not reach an existing fleet until some module happened
+ * to be touched next — indistinguishable from a fix that shipped and silently
+ * did nothing. celilo-mgr sat at the 60s default with the row already present.
+ *
+ * This re-arms a deliberately paused sweep, and that is intentional: the pause
+ * only ever existed because staging leaked, and the reaper that makes it not
+ * leak ships in this same binary. Re-arming without the reaper present is the
+ * failure this comment exists to prevent — do not lift this call into a release
+ * that does not carry `reapOrphanedStaging`.
  */
 export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void {
   bus.subscribe({
@@ -42,6 +91,8 @@ export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void {
     pattern: BACKUP_SWEEP_PATTERN,
     handler: 'celilo backup sweep',
     registeredBy: 'celilo-backup',
+    maxAttempts: BACKUP_SWEEP_MAX_ATTEMPTS,
+    timeoutMs: BACKUP_SWEEP_TIMEOUT_MS,
   });
 }
 
@@ -57,9 +108,13 @@ export interface BackupSweepDeps {
   backup(moduleId: string): Promise<{ success: boolean; error?: string }>;
   /** Apply the module's declared retention. No-op when it declares none. */
   prune(module: BackupSweepModule): Promise<void>;
+  /** Reclaim staging left by backups whose process died. See backup-staging.ts. */
+  reapStaging(): ReapStagingReport;
 }
 
 export interface BackupSweepReport {
+  /** Staging reclaimed before this pass created any of its own. */
+  staging: ReapStagingReport;
   backedUp: string[];
   /** Explicit `schedule: manual` — the author opted out. */
   skippedManual: string[];
@@ -70,7 +125,14 @@ export interface BackupSweepReport {
 }
 
 export async function runBackupSweep(deps: BackupSweepDeps): Promise<BackupSweepReport> {
+  // Reclaim BEFORE backing anything up, not after. This pass is the only thing
+  // that creates staging on a schedule, so it is where the orphans come from —
+  // and a box already short on disk needs the space freed before we ask for
+  // another ~4 GB of it, not once we are finished with it.
+  const staging = deps.reapStaging();
+
   const report: BackupSweepReport = {
+    staging,
     backedUp: [],
     skippedManual: [],
     skippedNotDue: [],

package/src/services/bus-interview.ts

@@ -14,6 +14,7 @@
 
 import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
 import { getEventBusPath } from '../config/paths';
+import { InterviewUnansweredError } from './interview-errors';
 import { ensureResponderForInterview } from './responder-probe';
 
 const NO_SCHEMAS = defineEvents({});
@@ -239,6 +240,13 @@ export interface InterviewRequiredPayload {
  */
 export interface InterviewReply {
   value: unknown;
+  /**
+   * Set instead of `value` when the responder could not reach a decider (e.g.
+   * the remote client has no TTY and no pre-staged answer). `askInterview`
+   * turns it into an `InterviewUnansweredError` so the question fails loudly
+   * rather than silently resolving to `defaultValue`.
+   */
+  error?: string;
 }
 
 /**
@@ -314,11 +322,9 @@ export async function askInterview(
   payload: InterviewRequiredPayload,
   ownerBus?: Bus,
 ): Promise<unknown> {
-  const reply = await busInterviewGuarded<InterviewReply>(
-    EVENT_TYPES.interviewRequired(payload.scope, payload.key),
-    payload,
-    ownerBus,
-  );
+  const type = EVENT_TYPES.interviewRequired(payload.scope, payload.key);
+  const reply = await busInterviewGuarded<InterviewReply>(type, payload, ownerBus);
+  if (reply.error) throw new InterviewUnansweredError(type, reply.error);
   return reply.value;
 }
 

package/src/services/config-interview.ts

@@ -260,7 +260,7 @@ export async function autoDeriveMachineConfig(
     upsertModuleConfig(db, moduleId, variable.name, derived);
     configured.push(variable.name);
 
-    // Handle per_selection follow-ups (e.g., zone_ip_* from zone list)
+    // Handle per_selection follow-ups (e.g., `zone.<zone>.ip` from the zone list)
     if (variable.options && variable.per_selection) {
       for (const selectedVal of derived.split(',')) {
         const followUpKey = variable.per_selection.key_pattern.replace('{value}', selectedVal);

package/src/services/deploy-ansible.ts

@@ -22,7 +22,6 @@ export interface AnsibleResult {
  * Parse raw Ansible output lines into concise human-readable status.
  * Returns null for lines that should be suppressed (decorative separators, etc.)
  */
-// biome-ignore lint/suspicious/noControlCharactersInRegex: intentionally stripping ANSI escape codes
 const ANSI_ESCAPE = /\x1b\[[0-9;]*m/g;
 
 /**

package/src/services/disk-probe.test.ts

@@ -0,0 +1,74 @@
+import { describe, expect, test } from 'bun:test';
+import { parseDfOutput, percentUsed } from './disk-probe';
+
+describe('percentUsed', () => {
+  // Checked against what celilo-mgr's df actually printed, because an operator
+  // comparing an alert to their own df must see the same figure.
+  //
+  // 117G root, 11G used, 102G available — after the staging was cleared.
+  test('matches df for celilo-mgr at 10% used', () => {
+    expect(percentUsed(122_683_392, 111_149_056, 106_954_752)).toBe(10);
+  });
+
+  // Same filesystem at 38G used / 75G available — what df reported while the
+  // leak was running.
+  test('matches df for celilo-mgr at 34% used', () => {
+    expect(percentUsed(122_683_392, 82_837_504, 78_643_200)).toBe(34);
+  });
+
+  // Not 1 - bavail/blocks. A filesystem reserves blocks for root, so
+  // free-to-root and free-to-everyone differ; df computes capacity against
+  // what an ordinary process can use. With a large reserve the naive formula
+  // over-reports and would page early forever.
+  test('excludes root-reserved blocks, as df does', () => {
+    // 1000 total, 100 free to root, only 50 usable by others → 900 used of 950.
+    expect(percentUsed(1000, 100, 50)).toBe(95);
+    // The naive 1 - bavail/blocks would say 95% here too by coincidence, so
+    // use a case where they diverge: 1000 total, 500 free, 200 available.
+    expect(percentUsed(1000, 500, 200)).toBe(71); // 500 used of 700 usable
+  });
+
+  test('a full filesystem reports 100', () => {
+    expect(percentUsed(1000, 0, 0)).toBe(100);
+  });
+
+  test('a degenerate zero-block filesystem does not divide by zero', () => {
+    expect(percentUsed(0, 0, 0)).toBe(0);
+  });
+});
+
+describe('parseDfOutput', () => {
+  test('parses real df -P output', () => {
+    const out = [
+      'Filesystem     1024-blocks     Used Available Capacity Mounted on',
+      '/dev/mmcblk0p2   120699413 11534336 106954752      10% /',
+    ].join('\n');
+
+    expect(parseDfOutput(out)).toEqual({
+      usedPercent: 10,
+      availableBytes: 106_954_752 * 1024,
+    });
+  });
+
+  test('parses a nearly-full filesystem', () => {
+    const out = [
+      'Filesystem     1024-blocks      Used Available Capacity Mounted on',
+      '/dev/sda1         41284928  39220684    966140      98% /',
+    ].join('\n');
+
+    expect(parseDfOutput(out)?.usedPercent).toBe(98);
+  });
+
+  test('returns null rather than guessing on unusable output', () => {
+    expect(parseDfOutput('')).toBeNull();
+    expect(parseDfOutput('Filesystem 1024-blocks Used Available Capacity Mounted on')).toBeNull();
+    expect(parseDfOutput('header\ntoo few fields')).toBeNull();
+    expect(parseDfOutput('header\n/dev/sda1 a b c d /')).toBeNull();
+  });
+
+  test('tolerates leading whitespace and extra columns', () => {
+    const out =
+      '  Filesystem 1024-blocks Used Available Capacity Mounted on\n  /dev/sda1 100 50 40 56% / extra';
+    expect(parseDfOutput(out)?.usedPercent).toBe(56);
+  });
+});

package/src/services/disk-probe.ts

@@ -0,0 +1,145 @@
+/**
+ * Disk-usage probe for the machine pool.
+ *
+ * Structurally a sibling of `machine-probe.ts` — same pool, same SSH bounding —
+ * with one deliberate difference that is the whole reason this file has its own
+ * comment.
+ *
+ * ⚠️ THE LOCAL BOX IS MEASURED, NOT EXEMPTED.
+ *
+ * `probeMachines()` reports the management server reachable WITHOUT probing it,
+ * and that is correct there: celilo runs as a user with no SSH key for itself,
+ * so `ssh root@127.0.0.1` fails on a perfectly healthy host — and the question
+ * is meaningless anyway, since if this box were unreachable this code would not
+ * be running.
+ *
+ * None of that transfers to disk. The management server stages backups, caches
+ * modules, holds the celilo DB and writes the logs; it is the likeliest host in
+ * the fleet to fill, and it is the host that DID fill. A disk check that copied
+ * the probe's structure and inherited its local shortcut would skip the only
+ * machine the check exists to protect — reporting "all clear" about a
+ * filesystem it never looked at.
+ *
+ * So the local box reads `statfs` directly. There is no reachability question
+ * to answer about the machine running the code, only a usage one.
+ */
+
+import { execFile } from 'node:child_process';
+import { statfs } from 'node:fs/promises';
+import { promisify } from 'node:util';
+import type { DiskUsageResult } from './audit/disk-space';
+import { listMachines } from './machine-pool';
+import { LOCAL_MACHINE_IP } from './ssh-key-manager';
+
+const execFileAsync = promisify(execFile);
+
+/**
+ * Percent-used the way `df` reports it.
+ *
+ * Deliberately not `1 - bavail/blocks`. A filesystem reserves blocks for root,
+ * so free-to-root and free-to-everyone-else differ; `df` computes capacity
+ * against the space an ordinary process can actually use, and an operator
+ * comparing this alert to their own `df` output must see the same number.
+ */
+export function percentUsed(totalBlocks: number, freeBlocks: number, availBlocks: number): number {
+  const used = totalBlocks - freeBlocks;
+  const usable = used + availBlocks;
+  if (usable <= 0) return 0;
+  return Math.round((used / usable) * 100);
+}
+
+/** Parse the data row of `df -P /`. Returns null when the output is unusable. */
+export function parseDfOutput(
+  stdout: string,
+): { usedPercent: number; availableBytes: number } | null {
+  // -P guarantees one line per filesystem, so the row we want is the second.
+  const line = stdout.trim().split('\n')[1];
+  if (!line) return null;
+
+  // Filesystem 1024-blocks Used Available Capacity Mounted-on
+  const fields = line.trim().split(/\s+/);
+  if (fields.length < 5) return null;
+
+  const available = Number(fields[3]);
+  const percent = Number(fields[4]?.replace('%', ''));
+  if (!Number.isFinite(available) || !Number.isFinite(percent)) return null;
+
+  return { usedPercent: percent, availableBytes: available * 1024 };
+}
+
+async function probeLocal(hostname: string, ipAddress: string): Promise<DiskUsageResult> {
+  try {
+    const stats = await statfs('/');
+    return {
+      hostname,
+      ipAddress,
+      usedPercent: percentUsed(stats.blocks, stats.bfree, stats.bavail),
+      availableBytes: stats.bavail * stats.bsize,
+    };
+  } catch (err) {
+    return {
+      hostname,
+      ipAddress,
+      usedPercent: null,
+      message: err instanceof Error ? err.message : String(err),
+    };
+  }
+}
+
+async function probeRemote(
+  hostname: string,
+  ipAddress: string,
+  sshUser: string,
+): Promise<DiskUsageResult> {
+  try {
+    // Same bounding as machine-probe: BatchMode so a password prompt can never
+    // hang the probe, ConnectTimeout so an unresponsive host — the condition
+    // this check exists to notice — cannot wedge it.
+    const { stdout } = await execFileAsync(
+      'ssh',
+      [
+        '-o',
+        'BatchMode=yes',
+        '-o',
+        'ConnectTimeout=5',
+        '-o',
+        'StrictHostKeyChecking=no',
+        '-o',
+        'UserKnownHostsFile=/dev/null',
+        `${sshUser}@${ipAddress}`,
+        'df -P /',
+      ],
+      { timeout: 8000 },
+    );
+
+    const parsed = parseDfOutput(stdout);
+    if (!parsed) {
+      return {
+        hostname,
+        ipAddress,
+        usedPercent: null,
+        message: `unparseable df output: ${stdout}`,
+      };
+    }
+    return { hostname, ipAddress, ...parsed };
+  } catch (err) {
+    const e = err as { stderr?: string; message?: string };
+    return {
+      hostname,
+      ipAddress,
+      usedPercent: null,
+      message: e.stderr?.trim() || e.message || 'df probe failed',
+    };
+  }
+}
+
+export async function probeDiskUsage(): Promise<DiskUsageResult[]> {
+  const machines = await listMachines();
+  return Promise.all(
+    machines.map((m) =>
+      m.ipAddress === LOCAL_MACHINE_IP
+        ? probeLocal(m.hostname, m.ipAddress)
+        : probeRemote(m.hostname, m.ipAddress, m.sshUser),
+    ),
+  );
+}