@celilo/cli 0.14.2 → 0.15.0

package/src/cli/commands/module-operations.test.ts

@@ -0,0 +1,93 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+import { closeDb, getDb } from '../../db/client';
+import { runMigrations } from '../../db/migrate';
+import { moduleOperations } from '../../db/schema';
+import { OPERATION_TTL_MS } from '../../services/module-operations';
+import { handleModuleOperations } from './module-operations';
+
+describe('celilo module operations', () => {
+  let dir: string;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-ops-cmd-test-'));
+    process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
+    await runMigrations(process.env.CELILO_DB_PATH);
+  });
+
+  afterEach(() => {
+    closeDb();
+    process.env.CELILO_DB_PATH = undefined;
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  function insert(id: string, pid: number, ageMs: number): void {
+    getDb()
+      .insert(moduleOperations)
+      .values({
+        id,
+        moduleId: 'byoi',
+        operation: 'deploy',
+        status: 'in_progress',
+        pid,
+        startedAt: new Date(Date.now() - ageMs),
+      })
+      .run();
+  }
+
+  function statusOf(id: string): string | undefined {
+    return getDb().select().from(moduleOperations).where(eq(moduleOperations.id, id)).get()?.status;
+  }
+
+  it('reports nothing to clear when the lock is genuinely held', () => {
+    insert('live', process.pid, 60_000);
+
+    const result = handleModuleOperations(['clear'], {});
+
+    if (!result.success) throw new Error(`expected success, got: ${result.error}`);
+    expect(result.message).toContain('still look genuinely in flight');
+    expect(statusOf('live')).toBe('in_progress');
+  });
+
+  it('releases an expired row without touching a live one', () => {
+    insert('expired', process.pid, OPERATION_TTL_MS + 60_000);
+    insert('live', process.pid, 60_000);
+
+    const result = handleModuleOperations(['clear'], {});
+
+    expect(result.success).toBe(true);
+    expect(statusOf('expired')).toBe('failed');
+    expect(statusOf('live')).toBe('in_progress');
+  });
+
+  // The escape hatch's reason for existing: when liveness detection is wrong
+  // — a recycled pid reads as perfectly healthy — refusing to clear would
+  // recreate the outage the command exists to end.
+  it('--all releases a row whose process is still alive', () => {
+    insert('live', process.pid, 60_000);
+
+    const result = handleModuleOperations(['clear'], { all: true });
+
+    expect(result.success).toBe(true);
+    expect(statusOf('live')).toBe('failed');
+  });
+
+  it('lists without mutating anything', () => {
+    insert('expired', process.pid, OPERATION_TTL_MS + 60_000);
+
+    const result = handleModuleOperations([], {});
+
+    if (!result.success) throw new Error(`expected success, got: ${result.error}`);
+    expect(result.message).toContain('1 abandoned');
+    expect(statusOf('expired')).toBe('in_progress');
+  });
+
+  it('rejects an unknown action rather than silently listing', () => {
+    const result = handleModuleOperations(['nuke'], {});
+    if (result.success) throw new Error('expected an unknown action to fail');
+    expect(result.error).toContain('Unknown action');
+  });
+});

package/src/cli/commands/module-operations.ts

@@ -0,0 +1,134 @@
+/**
+ * `celilo module operations` — see and release the module-operation lock.
+ *
+ * Backup and restore refuse to run while another operation is in flight.
+ * When that refusal is wrong, the operator previously had no way to see
+ * the lock at all, let alone clear it: the error said "wait for it to
+ * complete", which for a suspended process is advice that can never come
+ * true. A `module deploy` Ctrl-Z'd on a lost terminal blocked every
+ * backup on the fleet for 20 days on exactly that advice.
+ *
+ * `clear` marks rows failed rather than deleting them — the history of
+ * what was abandoned, and when, is worth more than a tidy table.
+ */
+
+import { and, eq } from 'drizzle-orm';
+import { getDb } from '../../db/client';
+import { type ModuleOperation, moduleOperations } from '../../db/schema';
+import { OPERATION_TTL_MS, isPidRunnable } from '../../services/module-operations';
+import type { CommandResult } from '../types';
+
+/** Why a row is not holding the lock, or null when it still is. */
+function abandonedReason(row: ModuleOperation, now: number): string | null {
+  if (now - row.startedAt.getTime() > OPERATION_TTL_MS) return 'expired';
+  if (!isPidRunnable(row.pid)) return 'not running';
+  return null;
+}
+
+function formatAge(ms: number): string {
+  const minutes = Math.floor(ms / 60_000);
+  if (minutes < 60) return `${minutes}m`;
+  const hours = Math.floor(minutes / 60);
+  if (hours < 24) return `${hours}h`;
+  return `${Math.floor(hours / 24)}d`;
+}
+
+function inProgressRows(): ModuleOperation[] {
+  return getDb()
+    .select()
+    .from(moduleOperations)
+    .where(eq(moduleOperations.status, 'in_progress'))
+    .all();
+}
+
+function handleList(): CommandResult {
+  const now = Date.now();
+  const rows = inProgressRows();
+
+  if (rows.length === 0) {
+    console.log('\nNo module operations in progress.\n');
+    return { success: true, message: 'no operations in progress' };
+  }
+
+  console.log('\nModule operations in progress:\n');
+  let holding = 0;
+  for (const row of rows) {
+    const reason = abandonedReason(row, now);
+    if (!reason) holding++;
+    const age = formatAge(now - row.startedAt.getTime());
+    const status = reason ? `abandoned (${reason})` : 'HOLDING LOCK';
+    console.log(
+      `  ${row.operation.padEnd(9)} ${row.moduleId.padEnd(16)} pid ${String(row.pid).padEnd(8)} ${age.padStart(4)} ago   ${status}`,
+    );
+  }
+
+  const abandoned = rows.length - holding;
+  console.log('');
+  if (abandoned > 0) {
+    console.log(`${abandoned} abandoned row(s) — "celilo module operations clear" sweeps them.\n`);
+  }
+
+  return {
+    success: true,
+    message: `${rows.length} in progress (${holding} holding the lock, ${abandoned} abandoned)`,
+  };
+}
+
+/**
+ * Release abandoned rows. `--all` also releases rows whose process still
+ * looks alive.
+ *
+ * The permissive form exists because the pathological case is precisely
+ * the one where our liveness detection was wrong — a pid that has been
+ * recycled by an unrelated process reads as perfectly healthy. Refusing
+ * to clear it would recreate the outage this command exists to end. It
+ * is opt-in and names what it is overriding.
+ */
+function handleClear(flags: Record<string, boolean | string>): CommandResult {
+  const db = getDb();
+  const now = Date.now();
+  const force = flags.all === true;
+  const rows = inProgressRows();
+
+  const targets = force ? rows : rows.filter((row) => abandonedReason(row, now) !== null);
+
+  if (targets.length === 0) {
+    const held = rows.length;
+    if (held > 0) {
+      return {
+        success: true,
+        message: `Nothing to clear — ${held} operation(s) still look genuinely in flight. Use --all to release them anyway.`,
+      };
+    }
+    return { success: true, message: 'Nothing to clear — no operations in progress.' };
+  }
+
+  for (const row of targets) {
+    db.update(moduleOperations)
+      .set({
+        status: 'failed',
+        completedAt: new Date(),
+        errorMessage: 'abandoned — released by "celilo module operations clear"',
+      })
+      .where(and(eq(moduleOperations.id, row.id), eq(moduleOperations.status, 'in_progress')))
+      .run();
+    console.log(`  released ${row.operation} of ${row.moduleId} (pid ${row.pid})`);
+  }
+
+  return { success: true, message: `Released ${targets.length} operation(s)` };
+}
+
+export function handleModuleOperations(
+  args: string[],
+  flags: Record<string, boolean | string>,
+): CommandResult {
+  const action = args[0];
+
+  if (!action || action === 'list') return handleList();
+  if (action === 'clear') return handleClear(flags);
+
+  return {
+    success: false,
+    error: `Unknown action "${action}"\n\nUsage: celilo module operations [list|clear] [--all]`,
+  };
+}

package/src/cli/commands/module-upgrade.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test';
 import type { ModuleManifest } from '../../manifest/schema';
 import {
   type PollCandidate,
+  isPollInvocation,
   needsPreUpgradeBackup,
   pickAutoUpgrade,
   pickUpgradePolicy,
@@ -15,39 +16,50 @@ function manifest(hooks?: ModuleManifest['hooks']): ModuleManifest {
 const withBackupHook = manifest({ on_backup: { script: './scripts/backup.ts', timeout: 300000 } });
 const noBackupHook = manifest({ on_install: { script: './scripts/setup.ts', timeout: 180000 } });
 
-describe('pickUpgradePolicy (ISS-0138 — config override > manifest default > by-semver)', () => {
-  test('operator config wins over the manifest default', () => {
-    expect(pickUpgradePolicy('always-safe', 'always-fast')).toBe('always-safe');
+describe('pickUpgradePolicy (ISS-0138 — operator config, else by-semver)', () => {
+  test('operator config decides', () => {
+    expect(pickUpgradePolicy('always-safe')).toBe('always-safe');
+    expect(pickUpgradePolicy('always-fast')).toBe('always-fast');
   });
 
-  test('manifest default applies when there is no config override', () => {
-    expect(pickUpgradePolicy(undefined, 'always-fast')).toBe('always-fast');
+  test('defaults to by-semver when unset', () => {
+    expect(pickUpgradePolicy(undefined)).toBe('by-semver');
   });
 
-  test('defaults to by-semver when neither is set', () => {
-    expect(pickUpgradePolicy(undefined, undefined)).toBe('by-semver');
+  test('an unknown value falls back to by-semver (no crash on bad input)', () => {
+    expect(pickUpgradePolicy('bogus')).toBe('by-semver');
   });
+});
 
-  test('an unknown value falls back to by-semver (no crash on bad input)', () => {
-    expect(pickUpgradePolicy('bogus', undefined)).toBe('by-semver');
-    expect(pickUpgradePolicy(undefined, 'nonsense')).toBe('by-semver');
+describe('pickAutoUpgrade (ISS-0139 — opt-in via operator config, default off)', () => {
+  test('config value (string or boolean) decides', () => {
+    expect(pickAutoUpgrade('true')).toBe(true);
+    expect(pickAutoUpgrade('false')).toBe(false);
+    expect(pickAutoUpgrade(true)).toBe(true);
+    expect(pickAutoUpgrade(false)).toBe(false);
+  });
+
+  test('defaults to OFF (opt-in) when unset', () => {
+    expect(pickAutoUpgrade(undefined)).toBe(false);
   });
 });
 
-describe('pickAutoUpgrade (ISS-0139 — opt-in: config override > manifest default > off)', () => {
-  test('config override (string or boolean) wins', () => {
-    expect(pickAutoUpgrade('true', false)).toBe(true);
-    expect(pickAutoUpgrade('false', true)).toBe(false);
-    expect(pickAutoUpgrade(true, false)).toBe(true);
+// The dispatcher spawns a subprocess handler as `<handler> <event_id>`
+// (openspec/specs/event-bus/spec.md), so the registry-poll subscription's
+// handler arrives as `celilo module upgrade --poll 5517`. Without --poll the
+// event id landed in the module-name slot and every 15m tick died with
+// "Module not found: 5517" — the poll never ran on celilo-mgr for weeks.
+describe('isPollInvocation (the CD poll must survive the appended event id)', () => {
+  test('--poll wins over a positional (the dispatcher-appended event id)', () => {
+    expect(isPollInvocation(['5517'], { poll: true })).toBe(true);
   });
 
-  test('manifest default applies with no override', () => {
-    expect(pickAutoUpgrade(undefined, true)).toBe(true);
-    expect(pickAutoUpgrade(undefined, false)).toBe(false);
+  test('bare `module upgrade` is still the poll', () => {
+    expect(isPollInvocation([], {})).toBe(true);
   });
 
-  test('defaults to OFF (opt-in) when neither is set', () => {
-    expect(pickAutoUpgrade(undefined, undefined)).toBe(false);
+  test('a named module without --poll is a single-module upgrade', () => {
+    expect(isPollInvocation(['lunacycle'], {})).toBe(false);
   });
 });
 

package/src/cli/commands/module-upgrade.ts

@@ -37,33 +37,29 @@ import { classifyVersionChange, fetchAndUpdate } from './module-update';
 const VALID_POLICIES: readonly UpgradePolicy[] = ['by-semver', 'always-safe', 'always-fast'];
 
 /**
- * Pick the effective upgrade policy. Pure (Rule 10): operator config override
- * wins over the manifest default; an unknown/absent value falls back to
- * `by-semver`.
+ * Pick the effective upgrade policy. Pure (Rule 10): operator config decides;
+ * an unknown/absent value falls back to `by-semver`.
+ *
+ * Config-only by construction — the manifest schema is `.strict()` and declares
+ * no `upgrade_policy`, so a manifest that set one could never pass validation.
+ * (The old manifest-default branch read a key no valid manifest can carry —
+ * Rule 3.9 dead code, deleted rather than left as a corpse.)
  */
-export function pickUpgradePolicy(
-  fromConfig: string | undefined,
-  fromManifest: string | undefined,
-): UpgradePolicy {
-  const candidate = fromConfig ?? fromManifest;
-  return VALID_POLICIES.includes(candidate as UpgradePolicy)
-    ? (candidate as UpgradePolicy)
+export function pickUpgradePolicy(fromConfig: string | undefined): UpgradePolicy {
+  return VALID_POLICIES.includes(fromConfig as UpgradePolicy)
+    ? (fromConfig as UpgradePolicy)
     : 'by-semver';
 }
 
 /**
  * Resolve whether a module opts into auto-upgrade by the poll. Pure (Rule 10):
- * operator config override wins over the manifest default; the default is OFF
- * (opt-in — a production module isn't auto-upgraded unless chosen).
+ * operator config only (same `.strict()` reason as pickUpgradePolicy), default
+ * OFF — a production module isn't auto-upgraded unless the operator chose it:
+ * `celilo module config set <module> auto_upgrade true`.
  */
-export function pickAutoUpgrade(
-  fromConfig: string | boolean | undefined,
-  fromManifest: boolean | undefined,
-): boolean {
+export function pickAutoUpgrade(fromConfig: string | boolean | undefined): boolean {
   if (typeof fromConfig === 'boolean') return fromConfig;
-  if (fromConfig === 'true') return true;
-  if (fromConfig === 'false') return false;
-  return fromManifest ?? false;
+  return fromConfig === 'true';
 }
 
 /**
@@ -113,14 +109,11 @@ export function selectPollTargets(candidates: PollCandidate[]): PollTarget[] {
   return targets;
 }
 
-/** Resolve a module's auto_upgrade flag from config (override) + manifest (default). */
-function resolveAutoUpgrade(moduleId: string, manifest: ModuleManifest): boolean {
+/** Resolve a module's auto_upgrade opt-in from operator config. */
+function resolveAutoUpgrade(moduleId: string): boolean {
   const cfg = getModuleConfigValue(moduleId, 'auto_upgrade');
-  const fromConfig =
-    typeof cfg?.value === 'string' || typeof cfg?.value === 'boolean' ? cfg.value : undefined;
   return pickAutoUpgrade(
-    fromConfig,
-    (manifest as ModuleManifest & { auto_upgrade?: boolean }).auto_upgrade,
+    typeof cfg?.value === 'string' || typeof cfg?.value === 'boolean' ? cfg.value : undefined,
   );
 }
 
@@ -158,13 +151,11 @@ async function upgradeOneModule(
     (updatedRow?.manifestData as ModuleManifest | undefined) ??
     (mod.manifestData as ModuleManifest);
 
-  // Posture. Version delta is installed→target; the policy comes from the
-  // TARGET manifest (the version being installed declares its upgrade risk),
-  // with an operator config override still winning.
+  // Posture. Version delta is installed→target; the policy comes from operator
+  // config (`celilo module config set <m> upgrade_policy …`), else by-semver.
   const configPolicy = getModuleConfigValue(moduleId, 'upgrade_policy');
   const modulePolicy = pickUpgradePolicy(
     typeof configPolicy?.value === 'string' ? configPolicy.value : undefined,
-    (targetManifest as ModuleManifest & { upgrade_policy?: string }).upgrade_policy,
   );
   // Per-release deploy_posture override lives in the .netapp release metadata;
   // reading it requires fetching the package first. Deferred — the classifier
@@ -244,7 +235,7 @@ async function runRegistryPoll(
       moduleId: mod.id,
       installed: mod.version,
       latest: await latestRegistryVersion(client, mod.id),
-      autoUpgrade: resolveAutoUpgrade(mod.id, mod.manifestData as ModuleManifest),
+      autoUpgrade: resolveAutoUpgrade(mod.id),
     });
   }
 
@@ -278,6 +269,20 @@ async function runRegistryPoll(
   };
 }
 
+/**
+ * Pure (Rule 10.1): is this the CD poll rather than a single-module upgrade?
+ *
+ * `--poll` is what a bus subscription MUST use. The dispatcher spawns a
+ * subprocess handler as `<handler> <event_id>` (openspec/specs/event-bus/spec.md),
+ * so a bare `celilo module upgrade` handler arrives as `celilo module upgrade
+ * 5517` — the event id lands in the optional module-name slot and the poll dies
+ * with "Module not found: 5517" every tick. An explicit flag makes the poll
+ * invocation immune to the appended id instead of relying on argv position.
+ */
+export function isPollInvocation(args: string[], flags: Record<string, string | boolean>): boolean {
+  return Boolean(flags.poll) || !getArg(args, 0);
+}
+
 export async function handleModuleUpgrade(
   args: string[],
   flags: Record<string, string | boolean> = {},
@@ -285,8 +290,8 @@ export async function handleModuleUpgrade(
   const db = getDb();
   const moduleId = getArg(args, 0);
 
-  // No name → the CD poll over all auto_upgrade modules.
-  if (!moduleId) {
+  // `--poll` or no name → the CD poll over all auto_upgrade modules.
+  if (isPollInvocation(args, flags) || !moduleId) {
     return runRegistryPoll(db, flags);
   }
 

package/src/cli/commands/monitor.ts

@@ -11,7 +11,10 @@ import { getEventBusPath } from '../../config/paths';
 import { getDb } from '../../db/client';
 import type { MonitorKind } from '../../db/schema';
 import { parseIntervalMinutes } from '../../manifest/schema';
-import { runBuiltinCheckForMonitor } from '../../services/alerting/builtin-source';
+import {
+  isSchedulableBuiltin,
+  runBuiltinCheckForMonitor,
+} from '../../services/alerting/builtin-source';
 import { loadModuleCoverage } from '../../services/alerting/coverage-source';
 import { HEALTH_COVERAGE_CHECK } from '../../services/alerting/health-coverage';
 import {
@@ -21,6 +24,7 @@ import {
   listMonitors,
   setMonitorEnabled,
 } from '../../services/alerting/monitors';
+import { listPolicies } from '../../services/alerting/people';
 import { runOneMonitor } from '../../services/alerting/run-monitor';
 import { promoteReadyAlerts } from '../../services/alerting/store';
 import type { DriftCategory } from '../../services/audit/types';
@@ -37,7 +41,7 @@ function buildDeps() {
   return {
     runModuleCheck: (moduleId: string) =>
       runModuleHealthCheck(moduleId, db, { unattended: true, noInteractive: true }),
-    runBuiltinCheck: (category: DriftCategory) => runBuiltinCheckForMonitor(category),
+    runBuiltinCheck: (category: DriftCategory) => runBuiltinCheckForMonitor(category, db),
     loadModuleCoverage: () => loadModuleCoverage(db),
     now: () => new Date(),
     graceMs: DEFAULT_GRACE_MS,
@@ -53,15 +57,26 @@ function handleList(): CommandResult {
     return { success: true, message: 'No monitors configured' };
   }
 
+  // The POLICY column is the answer to "why did nothing page me". A monitor
+  // with no policy is one whose alerts reach nobody, and until this column
+  // existed there was no way to see that from the CLI at all — `assign`
+  // reported success and nothing anywhere reflected the result (#481).
+  const policies = new Map(listPolicies(getDb()).map((p) => [p.id, p.name]));
+  const policyOf = (id: string | null) =>
+    id ? (policies.get(id) ?? '(deleted policy)') : '— pages nobody';
+
   const width = Math.max(6, ...rows.map((r) => r.target.length));
+  const policyWidth = Math.max(6, ...rows.map((r) => policyOf(r.escalationPolicyId).length));
   console.log('');
-  console.log(`${'TARGET'.padEnd(width)}  ${'KIND'.padEnd(14)}  ${'EVERY'.padEnd(6)}  STATE`);
+  console.log(
+    `${'TARGET'.padEnd(width)}  ${'KIND'.padEnd(14)}  ${'EVERY'.padEnd(6)}  ${'POLICY'.padEnd(policyWidth)}  STATE`,
+  );
   for (const monitor of rows) {
     const state = monitor.enabled ? 'enabled' : 'disabled';
     const suffix = monitor.lastRunAt ? '' : '  (never run)';
     const every = `${monitor.intervalMinutes}m`;
     console.log(
-      `${monitor.target.padEnd(width)}  ${monitor.kind.padEnd(14)}  ${every.padEnd(6)}  ${state}${suffix}`,
+      `${monitor.target.padEnd(width)}  ${monitor.kind.padEnd(14)}  ${every.padEnd(6)}  ${policyOf(monitor.escalationPolicyId).padEnd(policyWidth)}  ${state}${suffix}`,
     );
   }
   console.log('');
@@ -89,9 +104,14 @@ function handleAdd(args: string[], flags: Record<string, boolean | string>): Com
   }
 
   // A target naming an audit category is a built-in check; anything else is a
-  // module's health_check hook.
+  // module's health_check hook. `isSchedulableBuiltin` is checked explicitly
+  // because not every category is snake_case — `backups` is one word, and the
+  // underscore heuristic alone would file it as a module hook against a module
+  // that does not exist.
   const kind: MonitorKind =
-    target === HEALTH_COVERAGE_CHECK || target.includes('_') ? 'builtin_check' : 'module_hook';
+    target === HEALTH_COVERAGE_CHECK || isSchedulableBuiltin(target) || target.includes('_')
+      ? 'builtin_check'
+      : 'module_hook';
 
   createMonitor(db, { kind, target, intervalMinutes });
 

package/src/cli/commands/system-audit.ts

@@ -20,12 +20,11 @@ import { existsSync, readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { promisify } from 'node:util';
-import { eq, isNotNull } from 'drizzle-orm';
+import { isNotNull } from 'drizzle-orm';
 import { testDigitalOceanConnection } from '../../api-clients/digitalocean';
 import { testProxmoxConnection } from '../../api-clients/proxmox';
 import { getDb } from '../../db/client';
 import {
-  backups,
   capabilitySecrets,
   moduleConfigs as moduleConfigsTbl,
   modules,
@@ -38,6 +37,7 @@ import { decryptSecret } from '../../secrets/encryption';
 import { getOrCreateMasterKey } from '../../secrets/master-key';
 import { runAudit } from '../../services/audit';
 import type { DriftFinding, SystemAuditReport } from '../../services/audit';
+import { loadBackupAuditInfo } from '../../services/audit/backup-source';
 import {
   type LatestCliVersionFetcher,
   fetchLatestCliVersion,
@@ -52,7 +52,7 @@ import type { TerraformPlanRunner } from '../../services/audit/terraform-plan';
 import { getServiceCredentials, listContainerServices } from '../../services/container-service';
 import { collectFirewallReach } from '../../services/firewall-reach';
 import { runAllHealthChecks } from '../../services/health-runner';
-import { listMachines } from '../../services/machine-pool';
+import { probeMachines } from '../../services/machine-probe';
 import { parseStoredConfigValue } from '../../services/module-config';
 import { buildTerraformEnvForModule } from '../../services/terraform-env';
 import { hasFlag } from '../parser';
@@ -154,28 +154,6 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
 
   const registryClient = new RegistryClient();
 
-  // Most recent successful backup per module.
-  // The `backups` table is added via inline ALTER statements in db/client.ts
-  // for upgraded databases; a freshly-initialized DB created from the
-  // drizzle journal alone may not have it yet. Treat absence as "no
-  // backups recorded" rather than crashing the audit.
-  const latestBackupByModule = new Map<string, number>();
-  try {
-    const successfulBackups = db
-      .select()
-      .from(backups)
-      .where(eq(backups.status, 'completed'))
-      .all();
-    for (const b of successfulBackups) {
-      if (!b.moduleId || !b.completedAt) continue;
-      const ts = b.completedAt.getTime();
-      const prev = latestBackupByModule.get(b.moduleId);
-      if (prev === undefined || ts > prev) latestBackupByModule.set(b.moduleId, ts);
-    }
-  } catch {
-    // backups table missing — leave map empty
-  }
-
   // Build per-module config map for module-configs check.
   const allConfigs = db
     .select()
@@ -196,12 +174,7 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
     configs: configsByModule.get(m.id) ?? {},
   }));
 
-  const installedBackupInfo = deployedModules.map((m) => ({
-    id: m.id,
-    state: m.state,
-    manifest: m.manifestData as ModuleManifest,
-    lastSuccessfulBackupAt: latestBackupByModule.get(m.id) ?? null,
-  }));
+  const installedBackupInfo = loadBackupAuditInfo(db);
 
   // Compose per-module TF_VAR_* env vars in parallel — each call hits
   // the secret store / DB so they're not free, but they're all
@@ -353,43 +326,11 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
     }),
   );
 
-  // Machines-reachable: SSH probe each pool machine in parallel.
-  // BatchMode=yes prevents password prompt hangs; ConnectTimeout=5
-  // bounds wait on unresponsive hosts. The audit's collapse logic
-  // turns "all unreachable" into a single host/network finding.
-  const allMachines = await listMachines();
-  const machineReachableResults: MachineReachableResult[] = await Promise.all(
-    allMachines.map(async (m): Promise<MachineReachableResult> => {
-      try {
-        await execFileAsync(
-          'ssh',
-          [
-            '-o',
-            'BatchMode=yes',
-            '-o',
-            'ConnectTimeout=5',
-            '-o',
-            'StrictHostKeyChecking=no',
-            '-o',
-            'UserKnownHostsFile=/dev/null',
-            `${m.sshUser}@${m.ipAddress}`,
-            'true',
-          ],
-          { timeout: 8000 },
-        );
-        return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
-      } catch (err) {
-        const e = err as { stderr?: string; message?: string };
-        return {
-          id: m.id,
-          hostname: m.hostname,
-          ipAddress: m.ipAddress,
-          reachable: false,
-          message: (e.stderr || e.message || 'unknown error').slice(0, 200),
-        };
-      }
-    }),
-  );
+  // Machines-reachable: probe each pool machine. Shared with the monitor sweep
+  // (services/machine-probe.ts) — this block used to be a second copy of the
+  // same SSH logic, and both copies reported the local management box as
+  // unreachable because celilo does not hold an SSH key for itself.
+  const machineReachableResults: MachineReachableResult[] = await probeMachines();
 
   const migrationsFolder = findMigrationsFolderSafe();
 

package/src/cli/completion.ts

@@ -185,6 +185,7 @@ export async function getCompletions(words: string[], current: number): Promise<
       'secret',
       'status',
       'where',
+      'operations',
       'terraform-unlock',
       'types',
       'validate',
@@ -198,6 +199,12 @@ export async function getCompletions(words: string[], current: number): Promise<
     return filterSuggestions(subcommands, args[2] || '');
   }
 
+  // Module operations subcommands (celilo module operations list/clear)
+  if (command === 'module' && args[1] === 'operations' && currentIndex === 2) {
+    const subcommands = ['list', 'clear'];
+    return filterSuggestions(subcommands, args[2] || '');
+  }
+
   // Module types subcommands (celilo module types generate/check)
   if (command === 'module' && args[1] === 'types' && currentIndex === 2) {
     const subcommands = ['generate', 'check'];
@@ -500,7 +507,17 @@ export async function getCompletions(words: string[], current: number): Promise<
 
   // Backup subcommands
   if (command === 'backup' && currentIndex === 1) {
-    const subcommands = ['create', 'list', 'restore', 'delete', 'prune', 'name', 'import', 'pull'];
+    const subcommands = [
+      'create',
+      'sweep',
+      'list',
+      'restore',
+      'delete',
+      'prune',
+      'name',
+      'import',
+      'pull',
+    ];
     return filterSuggestions(subcommands, args[1] || '');
   }