@celilo/cli 0.14.4 → 0.16.1

package/src/services/backup-restore.ts

@@ -5,7 +5,7 @@
  */
 
 import { execSync } from 'node:child_process';
-import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { copyFileSync, existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { eq } from 'drizzle-orm';
@@ -19,6 +19,7 @@ import type { ModuleManifest } from '../manifest/schema';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { shellEscape } from '../utils/shell';
+import { decryptFileToFile } from './backup-cipher';
 import { assertCompatibleSchema, parseManifest } from './backup-manifest';
 import { createStorageProvider } from './backup-storage';
 import { applyCrossModuleWriteRoot, moduleHasCrossModuleRead } from './cross-module-read';
@@ -61,17 +62,13 @@ export async function restoreSystemStateBackup(backup: Backup): Promise<RestoreR
     const encryptedPath = join(tempDir, 'system.enc');
     await provider.download(backup.storagePath, encryptedPath);
 
-    // Decrypt → tar
-    const encryptedData = JSON.parse(readFileSync(encryptedPath, 'utf-8'));
+    // Decrypt → tar, streamed. Reads both the current format and the legacy
+    // JSON envelope (see backup-cipher.ts).
     const masterKey = await getOrCreateMasterKey();
-    const base64Data = decryptSecret(encryptedData, masterKey);
-    const tarData = Buffer.from(base64Data, 'base64');
-
-    // Extract the envelope tar (manifest.json + celilo.db [+celilo.db-wal])
     const envelopeDir = join(tempDir, 'envelope');
     mkdirSync(envelopeDir, { recursive: true });
     const tarPath = join(tempDir, 'envelope.tar');
-    writeFileSync(tarPath, tarData);
+    await decryptFileToFile(encryptedPath, tarPath, masterKey);
     execSync(`tar -xf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)}`);
 
     // Read + validate manifest BEFORE touching the live DB. An
@@ -215,17 +212,14 @@ export async function restoreModuleBackup(
     const encryptedPath = join(tempDir, 'backup.tar.enc');
     await provider.download(backup.storagePath, encryptedPath);
 
-    // Decrypt → tar
-    const encryptedData = JSON.parse(readFileSync(encryptedPath, 'utf-8'));
-    const masterKey = await getOrCreateMasterKey();
-    const base64Data = decryptSecret(encryptedData, masterKey);
-    const tarData = Buffer.from(base64Data, 'base64');
-
-    // Write tar and extract into envelopeDir. The envelope contains:
+    // Decrypt → tar, streamed. Reads both the current format and the legacy
+    // JSON envelope (see backup-cipher.ts). Extract into envelopeDir; the
+    // envelope contains:
     //   manifest.json - validated below
     //   data/         - on_backup hook artifacts (passed to on_restore)
+    const masterKey = await getOrCreateMasterKey();
     const tarPath = join(tempDir, 'envelope.tar');
-    writeFileSync(tarPath, tarData);
+    await decryptFileToFile(encryptedPath, tarPath, masterKey);
     execSync(`tar -xf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)}`);
 
     // Read + validate envelope manifest BEFORE invoking the hook.

package/src/services/backup-schedule.ts

@@ -0,0 +1,35 @@
+/**
+ * What backup cadence a module actually has.
+ *
+ * One accessor, deliberately: the drift audit decides whether to ALERT
+ * that a backup is stale, and the backup sweep decides whether to RUN
+ * one. If those two read the manifest differently, a module can be
+ * alerted-on-but-never-backed-up — an alert no human action can clear.
+ *
+ * Absent means `daily`, not `manual`. Treating "the author didn't say"
+ * as "never check and never run" is what let celilo-mgmt go 55 days
+ * without a backup and forgejo and signal go without one entirely, all
+ * silently. Opting out is a decision worth writing down, so it takes an
+ * explicit `schedule: manual`.
+ */
+
+import type { ModuleManifest } from '../manifest/schema';
+
+export type BackupSchedule = 'hourly' | 'daily' | 'weekly' | 'monthly' | 'manual';
+
+/** Used when a manifest declares an `on_backup` hook but no cadence. */
+export const DEFAULT_BACKUP_SCHEDULE: BackupSchedule = 'daily';
+
+export function effectiveBackupSchedule(manifest: ModuleManifest): BackupSchedule {
+  const declared = manifest.backup?.schedule;
+  switch (declared) {
+    case 'hourly':
+    case 'daily':
+    case 'weekly':
+    case 'monthly':
+    case 'manual':
+      return declared;
+    default:
+      return DEFAULT_BACKUP_SCHEDULE;
+  }
+}

package/src/services/backup-sweep.test.ts

@@ -0,0 +1,148 @@
+import { describe, expect, test } from 'bun:test';
+import type { ModuleManifest } from '../manifest/schema';
+import type { BackupSchedule } from './backup-schedule';
+import {
+  BACKUP_SWEEP_PATTERN,
+  BACKUP_SWEEP_SUBSCRIBER,
+  type BackupSweepDeps,
+  type BackupSweepModule,
+  ensureBackupSweepSubscriber,
+  runBackupSweep,
+} from './backup-sweep';
+import { InFlightError } from './module-operations';
+
+function moduleWith(id: string, schedule?: BackupSchedule): BackupSweepModule {
+  const manifest = {
+    id,
+    hooks: { on_backup: { script: 'backup.ts' } },
+    ...(schedule ? { backup: { schedule } } : {}),
+  } as unknown as ModuleManifest;
+  return { id, manifest };
+}
+
+function deps(
+  modules: BackupSweepModule[],
+  overrides: Partial<BackupSweepDeps> = {},
+): BackupSweepDeps & { pruned: string[] } {
+  const pruned: string[] = [];
+  return {
+    pruned,
+    listEligible: () => modules,
+    isDue: () => true,
+    backup: async () => ({ success: true }),
+    prune: async ({ id }) => {
+      pruned.push(id);
+    },
+    ...overrides,
+  };
+}
+
+describe('runBackupSweep', () => {
+  test('backs up a module whose declared cadence is due', async () => {
+    const d = deps([moduleWith('authentik', 'daily')]);
+    const report = await runBackupSweep(d);
+
+    expect(report.backedUp).toEqual(['authentik']);
+    expect(report.failures).toEqual([]);
+    expect(d.pruned).toEqual(['authentik']);
+  });
+
+  test('skips a module that is not yet due', async () => {
+    const report = await runBackupSweep(
+      deps([moduleWith('authentik', 'daily')], { isDue: () => false }),
+    );
+
+    expect(report.backedUp).toEqual([]);
+    expect(report.skippedNotDue).toEqual(['authentik']);
+  });
+
+  test('never auto-backs-up an explicit manual schedule', async () => {
+    const report = await runBackupSweep(deps([moduleWith('scratch', 'manual')]));
+
+    expect(report.backedUp).toEqual([]);
+    expect(report.skippedManual).toEqual(['scratch']);
+  });
+
+  test('an undeclared schedule is backed up, not treated as manual', async () => {
+    // The regression this whole subsystem exists for: forgejo and signal
+    // declare no `backup:` block and had never been backed up.
+    const report = await runBackupSweep(deps([moduleWith('forgejo')]));
+
+    expect(report.backedUp).toEqual(['forgejo']);
+    expect(report.skippedManual).toEqual([]);
+  });
+
+  test('a held operation lock is a skip, not a failure, and stops the pass', async () => {
+    const d = deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], {
+      backup: async () => {
+        throw new InFlightError([]);
+      },
+    });
+    const report = await runBackupSweep(d);
+
+    expect(report.skippedLocked).toEqual(['authentik']);
+    expect(report.failures).toEqual([]);
+    expect(report.backedUp).toEqual([]);
+    expect(d.pruned).toEqual([]);
+  });
+
+  test('a failed backup is recorded and the remaining modules still run', async () => {
+    const report = await runBackupSweep(
+      deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], {
+        backup: async (moduleId) =>
+          moduleId === 'authentik' ? { success: false, error: 'hook exited 1' } : { success: true },
+      }),
+    );
+
+    expect(report.failures).toEqual([{ moduleId: 'authentik', error: 'hook exited 1' }]);
+    expect(report.backedUp).toEqual(['forgejo']);
+  });
+
+  test('a thrown storage error fails only that module', async () => {
+    const report = await runBackupSweep(
+      deps([moduleWith('authentik', 'daily'), moduleWith('forgejo', 'daily')], {
+        backup: async (moduleId) => {
+          if (moduleId === 'authentik') throw new Error('Storage not verified');
+          return { success: true };
+        },
+      }),
+    );
+
+    expect(report.failures).toEqual([{ moduleId: 'authentik', error: 'Storage not verified' }]);
+    expect(report.backedUp).toEqual(['forgejo']);
+  });
+
+  test('does not prune when a backup failed', async () => {
+    const d = deps([moduleWith('authentik', 'daily')], {
+      backup: async () => ({ success: false, error: 'nope' }),
+    });
+    await runBackupSweep(d);
+
+    expect(d.pruned).toEqual([]);
+  });
+});
+
+describe('ensureBackupSweepSubscriber', () => {
+  test('registers the hourly sweep handler', () => {
+    const calls: Array<{
+      name: string;
+      pattern: string;
+      handler: string;
+      registeredBy?: string;
+    }> = [];
+    ensureBackupSweepSubscriber({
+      subscribe: (options) => calls.push(options),
+    });
+
+    expect(calls).toEqual([
+      {
+        name: BACKUP_SWEEP_SUBSCRIBER,
+        pattern: BACKUP_SWEEP_PATTERN,
+        handler: 'celilo backup sweep',
+        registeredBy: 'celilo-backup',
+      },
+    ]);
+    // 1h is the coarsest tick that can still serve an `hourly` cadence.
+    expect(BACKUP_SWEEP_PATTERN).toBe('timer.tick.1h');
+  });
+});

package/src/services/backup-sweep.ts

@@ -0,0 +1,124 @@
+/**
+ * The thing that actually runs backups on a schedule.
+ *
+ * A manifest's `backup.schedule` was decorative until this existed: the drift
+ * audit reported staleness and `celilo module backup` needed a human. This is
+ * the pass that makes a module declaring `daily` get backed up daily with
+ * nobody watching.
+ *
+ * No new scheduling mechanism. Registered as an ordinary bus subscriber whose
+ * handler is `celilo backup sweep` against `timer.tick.1h`, exactly like the
+ * alerting sweep (`services/alerting/monitors.ts`) — the timer already exists,
+ * already survives restarts, and already has retry and dedup. One hour is the
+ * coarsest tick that can still serve an `hourly` cadence; `1d` cannot.
+ *
+ * Dependencies are injected so the pass itself tests with no database, no
+ * storage, and no hook execution.
+ */
+
+import type { ModuleManifest } from '../manifest/schema';
+import { type BackupSchedule, effectiveBackupSchedule } from './backup-schedule';
+import { InFlightError } from './module-operations';
+
+export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep';
+export const BACKUP_SWEEP_PATTERN = 'timer.tick.1h';
+
+export interface SubscriberRegistrar {
+  subscribe(options: {
+    name: string;
+    pattern: string;
+    handler: string;
+    registeredBy?: string;
+  }): unknown;
+}
+
+/**
+ * Idempotent: `bus.subscribe` upserts by name, so this is safe to call on
+ * every module install and update.
+ */
+export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void {
+  bus.subscribe({
+    name: BACKUP_SWEEP_SUBSCRIBER,
+    pattern: BACKUP_SWEEP_PATTERN,
+    handler: 'celilo backup sweep',
+    registeredBy: 'celilo-backup',
+  });
+}
+
+export interface BackupSweepModule {
+  id: string;
+  manifest: ModuleManifest;
+}
+
+export interface BackupSweepDeps {
+  /** Installed modules that declare an `on_backup` hook. */
+  listEligible(): BackupSweepModule[];
+  isDue(moduleId: string, schedule: BackupSchedule): boolean;
+  backup(moduleId: string): Promise<{ success: boolean; error?: string }>;
+  /** Apply the module's declared retention. No-op when it declares none. */
+  prune(module: BackupSweepModule): Promise<void>;
+}
+
+export interface BackupSweepReport {
+  backedUp: string[];
+  /** Explicit `schedule: manual` — the author opted out. */
+  skippedManual: string[];
+  skippedNotDue: string[];
+  /** Another module operation held the lock. Not a failure; retried next tick. */
+  skippedLocked: string[];
+  failures: Array<{ moduleId: string; error: string }>;
+}
+
+export async function runBackupSweep(deps: BackupSweepDeps): Promise<BackupSweepReport> {
+  const report: BackupSweepReport = {
+    backedUp: [],
+    skippedManual: [],
+    skippedNotDue: [],
+    skippedLocked: [],
+    failures: [],
+  };
+
+  for (const module of deps.listEligible()) {
+    const schedule = effectiveBackupSchedule(module.manifest);
+    if (schedule === 'manual') {
+      report.skippedManual.push(module.id);
+      continue;
+    }
+    if (!deps.isDue(module.id, schedule)) {
+      report.skippedNotDue.push(module.id);
+      continue;
+    }
+
+    let result: { success: boolean; error?: string };
+    try {
+      result = await deps.backup(module.id);
+    } catch (error) {
+      // Refusing to run while a deploy/restore is in flight is correct, and a
+      // scheduled run must not bypass it. The lock is global, so once it is
+      // held every remaining module would refuse identically — stop the pass
+      // rather than collect the same refusal N times. The next tick retries.
+      if (error instanceof InFlightError) {
+        report.skippedLocked.push(module.id);
+        break;
+      }
+      // Everything else — an unverified storage destination, a missing default
+      // — throws before a backup row exists. Record it against the module and
+      // keep going; one module's failure must not cancel the rest.
+      report.failures.push({
+        moduleId: module.id,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      continue;
+    }
+
+    if (!result.success) {
+      report.failures.push({ moduleId: module.id, error: result.error ?? 'backup failed' });
+      continue;
+    }
+
+    report.backedUp.push(module.id);
+    await deps.prune(module);
+  }
+
+  return report;
+}

package/src/services/deploy-posture.ts

@@ -3,14 +3,27 @@
  * openspec/changes/build-bus-poll-cd/proposal.md).
  *
  * CD is CI-driven, so no operator is present to pass `--no-backup`. The posture
- * (fast = skip backup + extended verify; safe = backup + full verify) is
- * DERIVED, with this precedence:
+ * (fast = skip the pre-deploy backup; safe = back up first) is DERIVED, with
+ * this precedence:
  *   1. per-module `upgrade_policy: always-safe` — a hard floor (always safe).
  *   2. per-release `deploy_posture` stamped in the .netapp release metadata.
  *   3. per-module `upgrade_policy: always-fast`.
  *   4. default: the semver delta of installed → next (revision/patch = fast;
  *      minor/major = safe).
  *
+ * Posture gates the BACKUP ONLY. Both paths run the same post-deploy verify
+ * (module-upgrade.ts calls runModuleHealthCheck either way), and neither
+ * gates the deploy: verify is the last step, so a failing health check reports
+ * "upgraded but verify failed" on an upgrade that has already landed — there
+ * is no rollback and no auto-restore. This comment previously claimed safe
+ * posture ran a "full verify" against fast's "extended verify"; no such
+ * distinction has ever existed in the code, and believing it makes
+ * `auto_upgrade` sound far more guarded than it is.
+ *
+ * `upgrade_policy` (like `auto_upgrade`) is OPERATOR CONFIG, not a manifest
+ * key — `celilo module config set <module> upgrade_policy always-safe`. The
+ * manifest schema is `.strict()` and declares neither.
+ *
  * celilo reads version NUMBERS, never changesets — the default needs only the
  * versions. All functions here are pure (Rule 10).
  */