@celilo/cli 0.14.2 → 0.15.0

package/src/services/backup-create.ts

@@ -3,15 +3,7 @@
  * Orchestrates the backup workflow: create temp files, encrypt, upload to storage.
  */
 
-import {
-  copyFileSync,
-  existsSync,
-  mkdirSync,
-  readFileSync,
-  rmSync,
-  statSync,
-  writeFileSync,
-} from 'node:fs';
+import { copyFileSync, existsSync, mkdirSync, rmSync, statSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { eq } from 'drizzle-orm';
@@ -21,11 +13,13 @@ import { moduleConfigs, modules, secrets as secretsTable } from '../db/schema';
 import { invokeHook } from '../hooks/executor';
 import { createConsoleLogger } from '../hooks/logger';
 import type { ModuleManifest } from '../manifest/schema';
-import { decryptSecret, encryptSecret } from '../secrets/encryption';
+import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { shellEscape } from '../utils/shell';
+import { encryptFileToFile } from './backup-cipher';
 import { buildManifest } from './backup-manifest';
 import { completeBackup, createBackupRecord, failBackup, listBackups } from './backup-metadata';
+import type { BackupSchedule } from './backup-schedule';
 import {
   createStorageProvider,
   getBackupStorage,
@@ -55,7 +49,7 @@ export interface BackupCreateResult {
   error?: string;
 }
 
-export type BackupSchedule = 'hourly' | 'daily' | 'weekly' | 'monthly' | 'manual';
+export type { BackupSchedule } from './backup-schedule';
 
 /**
  * Resolve the target storage destination
@@ -155,15 +149,13 @@ export async function createSystemStateBackup(
     const { execSync } = await import('node:child_process');
     execSync(`tar -cf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)} .`);
 
-    // Encrypt the tar.
-    const tarData = readFileSync(tarPath);
+    // Encrypt the tar, streamed — see backup-cipher.ts. The plaintext is
+    // never held in memory, so a large fleet DB can't OOM the snapshot.
     const masterKey = await getOrCreateMasterKey();
-    const encrypted = encryptSecret(tarData.toString('base64'), masterKey);
-
-    // Write encrypted payload to temp file
     const encryptedPath = join(tempDir, 'system.enc');
-    writeFileSync(encryptedPath, JSON.stringify(encrypted));
+    await encryptFileToFile(tarPath, encryptedPath, masterKey);
 
+    const tarSize = statSync(tarPath).size;
     const encryptedSize = statSync(encryptedPath).size;
 
     // Upload to storage
@@ -173,7 +165,7 @@ export async function createSystemStateBackup(
     completeBackup(record.id, {
       sizeBytes: encryptedSize,
       metadata: {
-        originalSizeBytes: tarData.length,
+        originalSizeBytes: tarSize,
         dbPath,
         envelopeSchemaVersion: manifest.schemaVersion,
       },
@@ -411,14 +403,14 @@ export async function createModuleBackup(
     const { execSync } = await import('node:child_process');
     execSync(`tar -cf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)} .`);
 
-    // Encrypt the tar
-    const tarData = readFileSync(tarPath);
+    // Encrypt the tar, streamed — see backup-cipher.ts. Module artifacts run
+    // to hundreds of MB (forgejo's are ~774 MB); holding one in memory is
+    // what OOM-killed that backup.
     const masterKey = await getOrCreateMasterKey();
-    const encrypted = encryptSecret(tarData.toString('base64'), masterKey);
-
     const encryptedPath = join(tempDir, 'backup.tar.enc');
-    writeFileSync(encryptedPath, JSON.stringify(encrypted));
+    await encryptFileToFile(tarPath, encryptedPath, masterKey);
 
+    const tarSize = statSync(tarPath).size;
     const encryptedSize = statSync(encryptedPath).size;
 
     // Upload to storage
@@ -429,7 +421,7 @@ export async function createModuleBackup(
       sizeBytes: encryptedSize,
       metadata: {
         artifactCount: hookResult.outputs.artifact_count,
-        originalSizeBytes: tarData.length,
+        originalSizeBytes: tarSize,
         envelopeSchemaVersion: backupManifest.schemaVersion,
       },
       schemaVersion: dataSchemaVersion,
@@ -566,14 +558,12 @@ export async function importModuleBackup(
     const { execSync } = await import('node:child_process');
     execSync(`tar -cf ${shellEscape(tarPath)} -C ${shellEscape(artifactDir)} .`);
 
-    // Encrypt the tar
-    const tarData = readFileSync(tarPath);
+    // Encrypt the tar, streamed — see backup-cipher.ts.
     const masterKey = await getOrCreateMasterKey();
-    const encrypted = encryptSecret(tarData.toString('base64'), masterKey);
-
     const encryptedPath = join(tempDir, 'backup.tar.enc');
-    writeFileSync(encryptedPath, JSON.stringify(encrypted));
+    await encryptFileToFile(tarPath, encryptedPath, masterKey);
 
+    const tarSize = statSync(tarPath).size;
     const encryptedSize = statSync(encryptedPath).size;
 
     // Upload to storage
@@ -584,7 +574,7 @@ export async function importModuleBackup(
       sizeBytes: encryptedSize,
       metadata: {
         ...analyzedMetadata,
-        originalSizeBytes: tarData.length,
+        originalSizeBytes: tarSize,
       },
       schemaVersion: analyzedSchemaVersion,
     });

package/src/services/backup-envelope-roundtrip.test.ts

@@ -22,8 +22,8 @@ import { join } from 'node:path';
 import { closeDb, getDb } from '../db/client';
 import { runMigrations } from '../db/migrate';
 import { backups, systemConfig } from '../db/schema';
-import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
+import { decryptFileToFile, encryptFileToFile } from './backup-cipher';
 import { createSystemStateBackup } from './backup-create';
 import { MANIFEST_SCHEMA_VERSION, parseManifest } from './backup-manifest';
 import { restoreSystemStateBackup } from './backup-restore';
@@ -77,15 +77,11 @@ describe('backup envelope round-trip', () => {
     const artifactPath = join(storageDir, 'celilo-backups', result.storagePath as string);
     expect(existsSync(artifactPath)).toBe(true);
 
-    const encrypted = JSON.parse(readFileSync(artifactPath, 'utf-8'));
     const masterKey = await getOrCreateMasterKey();
-    const base64 = decryptSecret(encrypted, masterKey);
-    const tarBytes = Buffer.from(base64, 'base64');
-
     const extractDir = join(dir, 'extract');
     mkdirSync(extractDir, { recursive: true });
     const tarPath = join(dir, 'envelope.tar');
-    writeFileSync(tarPath, tarBytes);
+    await decryptFileToFile(artifactPath, tarPath, masterKey);
     execSync(`tar -xf '${tarPath}' -C '${extractDir}'`);
 
     // manifest.json present at root, valid, matches expectations.
@@ -129,14 +125,11 @@ describe('backup envelope round-trip', () => {
     // Create a valid backup, then poison the manifest by re-packing.
     const result = await createSystemStateBackup();
     const artifactPath = join(storageDir, 'celilo-backups', result.storagePath as string);
-    const encrypted = JSON.parse(readFileSync(artifactPath, 'utf-8'));
     const masterKey = await getOrCreateMasterKey();
-    const tarBytes = Buffer.from(decryptSecret(encrypted, masterKey), 'base64');
-
     const repackDir = join(dir, 'repack');
     mkdirSync(repackDir, { recursive: true });
     const tarPath = join(dir, 'orig.tar');
-    writeFileSync(tarPath, tarBytes);
+    await decryptFileToFile(artifactPath, tarPath, masterKey);
     execSync(`tar -xf '${tarPath}' -C '${repackDir}'`);
 
     // Rewrite manifest to claim kind='module'.
@@ -148,12 +141,7 @@ describe('backup envelope round-trip', () => {
     // Re-tar + re-encrypt + overwrite the storage entry.
     const repackedTarPath = join(dir, 'repacked.tar');
     execSync(`tar -cf '${repackedTarPath}' -C '${repackDir}' .`);
-    const { encryptSecret } = await import('../secrets/encryption');
-    const repackedEncrypted = encryptSecret(
-      readFileSync(repackedTarPath).toString('base64'),
-      masterKey,
-    );
-    writeFileSync(artifactPath, JSON.stringify(repackedEncrypted));
+    await encryptFileToFile(repackedTarPath, artifactPath, masterKey);
 
     const db = getDb();
     const backupRow = db.select().from(backups).all()[0];
@@ -165,14 +153,11 @@ describe('backup envelope round-trip', () => {
   it('system restore refuses an incompatible schemaVersion', async () => {
     const result = await createSystemStateBackup();
     const artifactPath = join(storageDir, 'celilo-backups', result.storagePath as string);
-    const encrypted = JSON.parse(readFileSync(artifactPath, 'utf-8'));
     const masterKey = await getOrCreateMasterKey();
-    const tarBytes = Buffer.from(decryptSecret(encrypted, masterKey), 'base64');
-
     const repackDir = join(dir, 'repack-schema');
     mkdirSync(repackDir, { recursive: true });
     const tarPath = join(dir, 'orig-schema.tar');
-    writeFileSync(tarPath, tarBytes);
+    await decryptFileToFile(artifactPath, tarPath, masterKey);
     execSync(`tar -xf '${tarPath}' -C '${repackDir}'`);
 
     // Bump schemaVersion to a different MAJOR version.
@@ -183,12 +168,7 @@ describe('backup envelope round-trip', () => {
 
     const repackedTarPath = join(dir, 'repacked-schema.tar');
     execSync(`tar -cf '${repackedTarPath}' -C '${repackDir}' .`);
-    const { encryptSecret } = await import('../secrets/encryption');
-    const repackedEncrypted = encryptSecret(
-      readFileSync(repackedTarPath).toString('base64'),
-      masterKey,
-    );
-    writeFileSync(artifactPath, JSON.stringify(repackedEncrypted));
+    await encryptFileToFile(repackedTarPath, artifactPath, masterKey);
 
     const db = getDb();
     const backupRow = db.select().from(backups).all()[0];

package/src/services/backup-restore.ts

@@ -5,7 +5,7 @@
  */
 
 import { execSync } from 'node:child_process';
-import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { eq } from 'drizzle-orm';
@@ -19,6 +19,7 @@ import type { ModuleManifest } from '../manifest/schema';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { shellEscape } from '../utils/shell';
+import { decryptFileToFile } from './backup-cipher';
 import { assertCompatibleSchema, parseManifest } from './backup-manifest';
 import { createStorageProvider } from './backup-storage';
 import { applyCrossModuleWriteRoot, moduleHasCrossModuleRead } from './cross-module-read';
@@ -61,17 +62,13 @@ export async function restoreSystemStateBackup(backup: Backup): Promise<RestoreR
     const encryptedPath = join(tempDir, 'system.enc');
     await provider.download(backup.storagePath, encryptedPath);
 
-    // Decrypt → tar
-    const encryptedData = JSON.parse(readFileSync(encryptedPath, 'utf-8'));
+    // Decrypt → tar, streamed. Reads both the current format and the legacy
+    // JSON envelope (see backup-cipher.ts).
     const masterKey = await getOrCreateMasterKey();
-    const base64Data = decryptSecret(encryptedData, masterKey);
-    const tarData = Buffer.from(base64Data, 'base64');
-
-    // Extract the envelope tar (manifest.json + celilo.db [+celilo.db-wal])
     const envelopeDir = join(tempDir, 'envelope');
     mkdirSync(envelopeDir, { recursive: true });
     const tarPath = join(tempDir, 'envelope.tar');
-    writeFileSync(tarPath, tarData);
+    await decryptFileToFile(encryptedPath, tarPath, masterKey);
     execSync(`tar -xf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)}`);
 
     // Read + validate manifest BEFORE touching the live DB. An
@@ -215,17 +212,14 @@ export async function restoreModuleBackup(
     const encryptedPath = join(tempDir, 'backup.tar.enc');
     await provider.download(backup.storagePath, encryptedPath);
 
-    // Decrypt → tar
-    const encryptedData = JSON.parse(readFileSync(encryptedPath, 'utf-8'));
-    const masterKey = await getOrCreateMasterKey();
-    const base64Data = decryptSecret(encryptedData, masterKey);
-    const tarData = Buffer.from(base64Data, 'base64');
-
-    // Write tar and extract into envelopeDir. The envelope contains:
+    // Decrypt → tar, streamed. Reads both the current format and the legacy
+    // JSON envelope (see backup-cipher.ts). Extract into envelopeDir; the
+    // envelope contains:
     //   manifest.json - validated below
     //   data/         - on_backup hook artifacts (passed to on_restore)
+    const masterKey = await getOrCreateMasterKey();
     const tarPath = join(tempDir, 'envelope.tar');
-    writeFileSync(tarPath, tarData);
+    await decryptFileToFile(encryptedPath, tarPath, masterKey);
     execSync(`tar -xf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)}`);
 
     // Read + validate envelope manifest BEFORE invoking the hook.

package/src/services/backup-schedule.ts

@@ -0,0 +1,35 @@
+/**
+ * What backup cadence a module actually has.
+ *
+ * One accessor, deliberately: the drift audit decides whether to ALERT
+ * that a backup is stale, and the backup sweep decides whether to RUN
+ * one. If those two read the manifest differently, a module can be
+ * alerted-on-but-never-backed-up — an alert no human action can clear.
+ *
+ * Absent means `daily`, not `manual`. Treating "the author didn't say"
+ * as "never check and never run" is what let celilo-mgmt go 55 days
+ * without a backup and forgejo and signal go without one entirely, all
+ * silently. Opting out is a decision worth writing down, so it takes an
+ * explicit `schedule: manual`.
+ */
+
+import type { ModuleManifest } from '../manifest/schema';
+
+export type BackupSchedule = 'hourly' | 'daily' | 'weekly' | 'monthly' | 'manual';
+
+/** Used when a manifest declares an `on_backup` hook but no cadence. */
+export const DEFAULT_BACKUP_SCHEDULE: BackupSchedule = 'daily';
+
+export function effectiveBackupSchedule(manifest: ModuleManifest): BackupSchedule {
+  const declared = manifest.backup?.schedule;
+  switch (declared) {
+    case 'hourly':
+    case 'daily':
+    case 'weekly':
+    case 'monthly':
+    case 'manual':
+      return declared;
+    default:
+      return DEFAULT_BACKUP_SCHEDULE;
+  }
+}

package/src/services/backup-sweep.test.ts

@@ -0,0 +1,148 @@
+import { describe, expect, test } from 'bun:test';
+import type { ModuleManifest } from '../manifest/schema';
+import type { BackupSchedule } from './backup-schedule';
+import {
+  BACKUP_SWEEP_PATTERN,
+  BACKUP_SWEEP_SUBSCRIBER,
+  type BackupSweepDeps,
+  type BackupSweepModule,
+  ensureBackupSweepSubscriber,
+  runBackupSweep,
+} from './backup-sweep';
+import { InFlightError } from './module-operations';
+
+function moduleWith(id: string, schedule?: BackupSchedule): BackupSweepModule {
+  const manifest = {
+    id,
+    hooks: { on_backup: { script: 'backup.ts' } },
+    ...(schedule ? { backup: { schedule } } : {}),
+  } as unknown as ModuleManifest;
+  return { id, manifest };
+}
+
+function deps(
+  modules: BackupSweepModule[],
+  overrides: Partial<BackupSweepDeps> = {},
+): BackupSweepDeps & { pruned: string[] } {
+  const pruned: string[] = [];
+  return {
+    pruned,
+    listEligible: () => modules,
+    isDue: () => true,
+    backup: async () => ({ success: true }),
+    prune: async ({ id }) => {
+      pruned.push(id);
+    },
+    ...overrides,
+  };
+}
+
+describe('runBackupSweep', () => {
+  test('backs up a module whose declared cadence is due', async () => {
+    const d = deps([moduleWith('authentik', 'daily')]);
+    const report = await runBackupSweep(d);
+
+    expect(report.backedUp).toEqual(['authentik']);
+    expect(report.failures).toEqual([]);
+    expect(d.pruned).toEqual(['authentik']);
+  });
+
+  test('skips a module that is not yet due', async () => {
+    const report = await runBackupSweep(
+      deps([moduleWith('authentik', 'daily')], { isDue: () => false }),
+    );
+
+    expect(report.backedUp).toEqual([]);
+    expect(report.skippedNotDue).toEqual(['authentik']);
+  });
+
+  test('never auto-backs-up an explicit manual schedule', async () => {
+    const report = await runBackupSweep(deps([moduleWith('scratch', 'manual')]));
+
+    expect(report.backedUp).toEqual([]);
+    expect(report.skippedManual).toEqual(['scratch']);
+  });
+
+  test('an undeclared schedule is backed up, not treated as manual', async () => {
+    // The regression this whole subsystem exists for: forgejo and signal
+    // declare no `backup:` block and had never been backed up.
+    const report = await runBackupSweep(deps([moduleWith('forgejo')]));
+
+    expect(report.backedUp).toEqual(['forgejo']);
+    expect(report.skippedManual).toEqual([]);
+  });
+
+  test('a held operation lock is a skip, not a failure, and stops the pass', async () => {
+    const d = deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], {
+      backup: async () => {
+        throw new InFlightError([]);
+      },
+    });
+    const report = await runBackupSweep(d);
+
+    expect(report.skippedLocked).toEqual(['authentik']);
+    expect(report.failures).toEqual([]);
+    expect(report.backedUp).toEqual([]);
+    expect(d.pruned).toEqual([]);
+  });
+
+  test('a failed backup is recorded and the remaining modules still run', async () => {
+    const report = await runBackupSweep(
+      deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], {
+        backup: async (moduleId) =>
+          moduleId === 'authentik' ? { success: false, error: 'hook exited 1' } : { success: true },
+      }),
+    );
+
+    expect(report.failures).toEqual([{ moduleId: 'authentik', error: 'hook exited 1' }]);
+    expect(report.backedUp).toEqual(['forgejo']);
+  });
+
+  test('a thrown storage error fails only that module', async () => {
+    const report = await runBackupSweep(
+      deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], {
+        backup: async (moduleId) => {
+          if (moduleId === 'authentik') throw new Error('Storage not verified');
+          return { success: true };
+        },
+      }),
+    );
+
+    expect(report.failures).toEqual([{ moduleId: 'authentik', error: 'Storage not verified' }]);
+    expect(report.backedUp).toEqual(['forgejo']);
+  });
+
+  test('does not prune when a backup failed', async () => {
+    const d = deps([moduleWith('authentik', 'daily')], {
+      backup: async () => ({ success: false, error: 'nope' }),
+    });
+    await runBackupSweep(d);
+
+    expect(d.pruned).toEqual([]);
+  });
+});
+
+describe('ensureBackupSweepSubscriber', () => {
+  test('registers the hourly sweep handler', () => {
+    const calls: Array<{
+      name: string;
+      pattern: string;
+      handler: string;
+      registeredBy?: string;
+    }> = [];
+    ensureBackupSweepSubscriber({
+      subscribe: (options) => calls.push(options),
+    });
+
+    expect(calls).toEqual([
+      {
+        name: BACKUP_SWEEP_SUBSCRIBER,
+        pattern: BACKUP_SWEEP_PATTERN,
+        handler: 'celilo backup sweep',
+        registeredBy: 'celilo-backup',
+      },
+    ]);
+    // 1h is the coarsest tick that can still serve an `hourly` cadence.
+    expect(BACKUP_SWEEP_PATTERN).toBe('timer.tick.1h');
+  });
+});

package/src/services/backup-sweep.ts

@@ -0,0 +1,124 @@
+/**
+ * The thing that actually runs backups on a schedule.
+ *
+ * A manifest's `backup.schedule` was decorative until this existed: the drift
+ * audit reported staleness and `celilo module backup` needed a human. This is
+ * the pass that makes a module declaring `daily` get backed up daily with
+ * nobody watching.
+ *
+ * No new scheduling mechanism. Registered as an ordinary bus subscriber whose
+ * handler is `celilo backup sweep` against `timer.tick.1h`, exactly like the
+ * alerting sweep (`services/alerting/monitors.ts`) — the timer already exists,
+ * already survives restarts, and already has retry and dedup. One hour is the
+ * coarsest tick that can still serve an `hourly` cadence; `1d` cannot.
+ *
+ * Dependencies are injected so the pass itself tests with no database, no
+ * storage, and no hook execution.
+ */
+
+import type { ModuleManifest } from '../manifest/schema';
+import { type BackupSchedule, effectiveBackupSchedule } from './backup-schedule';
+import { InFlightError } from './module-operations';
+
+export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep';
+export const BACKUP_SWEEP_PATTERN = 'timer.tick.1h';
+
+export interface SubscriberRegistrar {
+  subscribe(options: {
+    name: string;
+    pattern: string;
+    handler: string;
+    registeredBy?: string;
+  }): unknown;
+}
+
+/**
+ * Idempotent: `bus.subscribe` upserts by name, so this is safe to call on
+ * every module install and update.
+ */
+export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void {
+  bus.subscribe({
+    name: BACKUP_SWEEP_SUBSCRIBER,
+    pattern: BACKUP_SWEEP_PATTERN,
+    handler: 'celilo backup sweep',
+    registeredBy: 'celilo-backup',
+  });
+}
+
+export interface BackupSweepModule {
+  id: string;
+  manifest: ModuleManifest;
+}
+
+export interface BackupSweepDeps {
+  /** Installed modules that declare an `on_backup` hook. */
+  listEligible(): BackupSweepModule[];
+  isDue(moduleId: string, schedule: BackupSchedule): boolean;
+  backup(moduleId: string): Promise<{ success: boolean; error?: string }>;
+  /** Apply the module's declared retention. No-op when it declares none. */
+  prune(module: BackupSweepModule): Promise<void>;
+}
+
+export interface BackupSweepReport {
+  backedUp: string[];
+  /** Explicit `schedule: manual` — the author opted out. */
+  skippedManual: string[];
+  skippedNotDue: string[];
+  /** Another module operation held the lock. Not a failure; retried next tick. */
+  skippedLocked: string[];
+  failures: Array<{ moduleId: string; error: string }>;
+}
+
+export async function runBackupSweep(deps: BackupSweepDeps): Promise<BackupSweepReport> {
+  const report: BackupSweepReport = {
+    backedUp: [],
+    skippedManual: [],
+    skippedNotDue: [],
+    skippedLocked: [],
+    failures: [],
+  };
+
+  for (const module of deps.listEligible()) {
+    const schedule = effectiveBackupSchedule(module.manifest);
+    if (schedule === 'manual') {
+      report.skippedManual.push(module.id);
+      continue;
+    }
+    if (!deps.isDue(module.id, schedule)) {
+      report.skippedNotDue.push(module.id);
+      continue;
+    }
+
+    let result: { success: boolean; error?: string };
+    try {
+      result = await deps.backup(module.id);
+    } catch (error) {
+      // Refusing to run while a deploy/restore is in flight is correct, and a
+      // scheduled run must not bypass it. The lock is global, so once it is
+      // held every remaining module would refuse identically — stop the pass
+      // rather than collect the same refusal N times. The next tick retries.
+      if (error instanceof InFlightError) {
+        report.skippedLocked.push(module.id);
+        break;
+      }
+      // Everything else — an unverified storage destination, a missing default
+      // — throws before a backup row exists. Record it against the module and
+      // keep going; one module's failure must not cancel the rest.
+      report.failures.push({
+        moduleId: module.id,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      continue;
+    }
+
+    if (!result.success) {
+      report.failures.push({ moduleId: module.id, error: result.error ?? 'backup failed' });
+      continue;
+    }
+
+    report.backedUp.push(module.id);
+    await deps.prune(module);
+  }
+
+  return report;
+}

package/src/services/deploy-posture.ts

@@ -3,14 +3,27 @@
  * openspec/changes/build-bus-poll-cd/proposal.md).
  *
  * CD is CI-driven, so no operator is present to pass `--no-backup`. The posture
- * (fast = skip backup + extended verify; safe = backup + full verify) is
- * DERIVED, with this precedence:
+ * (fast = skip the pre-deploy backup; safe = back up first) is DERIVED, with
+ * this precedence:
  *   1. per-module `upgrade_policy: always-safe` — a hard floor (always safe).
  *   2. per-release `deploy_posture` stamped in the .netapp release metadata.
  *   3. per-module `upgrade_policy: always-fast`.
  *   4. default: the semver delta of installed → next (revision/patch = fast;
  *      minor/major = safe).
  *
+ * Posture gates the BACKUP ONLY. Both paths run the same post-deploy verify
+ * (module-upgrade.ts calls runModuleHealthCheck either way), and neither
+ * gates the deploy: verify is the last step, so a failing health check reports
+ * "upgraded but verify failed" on an upgrade that has already landed — there
+ * is no rollback and no auto-restore. This comment previously claimed safe
+ * posture ran a "full verify" against fast's "extended verify"; no such
+ * distinction has ever existed in the code, and believing it makes
+ * `auto_upgrade` sound far more guarded than it is.
+ *
+ * `upgrade_policy` (like `auto_upgrade`) is OPERATOR CONFIG, not a manifest
+ * key — `celilo module config set <module> upgrade_policy always-safe`. The
+ * manifest schema is `.strict()` and declares neither.
+ *
  * celilo reads version NUMBERS, never changesets — the default needs only the
  * versions. All functions here are pure (Rule 10).
  */

package/src/services/machine-probe.test.ts

@@ -0,0 +1,50 @@
+/**
+ * The local management box must never be reported unreachable.
+ *
+ * celilo runs there as the `celilo` user and deliberately holds no SSH key for
+ * itself, so probing `root@127.0.0.1` fails with `Permission denied (publickey)`
+ * on a completely healthy host. That produced a permanently firing
+ * `machines_reachable` alert against celilo-mgr — and that monitor is
+ * unsuppressible by design, so nothing could explain it away.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import { auditMachinesReachable } from './audit/machines-reachable';
+import { LOCAL_MACHINE_IP } from './ssh-key-manager';
+
+describe('local machine reachability', () => {
+  test('LOCAL_MACHINE_IP is the loopback address the machine pool records', () => {
+    // The probe's skip is keyed on this exact value, so pin it.
+    expect(LOCAL_MACHINE_IP).toBe('127.0.0.1');
+  });
+
+  test('a reachable local box produces no finding', async () => {
+    const findings = await auditMachinesReachable({
+      results: [
+        { id: 'mgr', hostname: 'celilo-mgr', ipAddress: LOCAL_MACHINE_IP, reachable: true },
+        { id: 'briq', hostname: 'briq', ipAddress: '192.168.0.254', reachable: true },
+      ],
+    });
+
+    expect(findings).toEqual([]);
+  });
+
+  test('an unreachable remote machine still produces a finding', async () => {
+    // The skip must not blunt the check for the machines it exists to watch.
+    const findings = await auditMachinesReachable({
+      results: [
+        { id: 'mgr', hostname: 'celilo-mgr', ipAddress: LOCAL_MACHINE_IP, reachable: true },
+        {
+          id: 'briq',
+          hostname: 'briq',
+          ipAddress: '192.168.0.254',
+          reachable: false,
+          message: 'connection refused',
+        },
+      ],
+    });
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0].message).toContain('briq');
+  });
+});