@celilo/cli 0.22.0 → 0.24.0

package/src/services/backup-create.ts

@@ -17,8 +17,13 @@ import { getOrCreateMasterKey } from '../secrets/master-key';
 import { shellEscape } from '../utils/shell';
 import { encryptFileToFile } from './backup-cipher';
 import { buildManifest } from './backup-manifest';
-import { completeBackup, createBackupRecord, failBackup, listBackups } from './backup-metadata';
-import type { BackupSchedule } from './backup-schedule';
+import {
+  completeBackup,
+  createBackupRecord,
+  failBackup,
+  loadBackupHistory,
+} from './backup-metadata';
+import { isBackupDueFromHistory } from './backup-schedule';
 import { stagingDirFor } from './backup-staging';
 import {
   createStorageProvider,
@@ -26,8 +31,10 @@ import {
   getBackupStorageByStorageId,
   getDefaultBackupStorage,
 } from './backup-storage';
+import type { Cadence } from './cadence';
 import { materializeCrossModuleRoot, moduleHasCrossModuleRead } from './cross-module-read';
 import { getModuleSystems } from './deployed-systems';
+import { parseStoredConfigValue } from './module-config';
 import {
   completeOperation,
   failOperation,
@@ -49,8 +56,6 @@ export interface BackupCreateResult {
   error?: string;
 }
 
-export type { BackupSchedule } from './backup-schedule';
-
 /**
  * Resolve the target storage destination
  */
@@ -198,46 +203,47 @@ export async function createSystemStateBackup(
 }
 
 /**
- * Schedule intervals in milliseconds
- */
-const SCHEDULE_INTERVALS: Record<BackupSchedule, number> = {
-  hourly: 60 * 60 * 1000,
-  daily: 24 * 60 * 60 * 1000,
-  weekly: 7 * 24 * 60 * 60 * 1000,
-  monthly: 30 * 24 * 60 * 60 * 1000,
-  manual: Number.POSITIVE_INFINITY,
-};
-
-/**
- * Check if a module is due for backup based on its schedule
+ * Check if a module is due for backup based on its schedule.
+ *
+ * The policy lives in `isBackupDueFromHistory`; this reads the history it
+ * needs. Worth knowing what the previous version did, because its shape was
+ * the bug: it asked for `limit: 1` — the single most recent row, whatever its
+ * outcome — and then `.find()`-ed a *completed* one within that. Those two
+ * lines contradict each other. Once an attempt failed, the newest row was that
+ * failure, the find matched nothing, and "no successful backup exists" meant
+ * unconditionally due — at every hourly tick, forever, however the module's
+ * cadence read (celilo#685).
  */
-export function isBackupDue(moduleId: string, schedule: BackupSchedule): boolean {
-  if (schedule === 'manual') return false;
-
-  const existing = listBackups({ moduleId, limit: 1 });
-  const lastBackup = existing.find(
-    (b) => b.moduleId === moduleId && b.status === 'completed' && b.backupType === 'module_data',
-  );
-
-  if (!lastBackup) return true;
-
-  const elapsed = Date.now() - new Date(lastBackup.startedAt).getTime();
-  return elapsed >= SCHEDULE_INTERVALS[schedule];
+export function isBackupDue(moduleId: string, schedule: Cadence): boolean {
+  return isBackupDueFromHistory(schedule, loadBackupHistory(moduleId), Date.now());
 }
 
 /**
- * Find all installed modules that have an on_backup hook
+ * Find all installed modules that have an on_backup hook.
+ *
+ * Each carries its operator config alongside its manifest, because every
+ * caller then has to resolve a cadence or a retention policy out of the two
+ * together — and a caller that got the manifest alone would silently ignore
+ * the operator's override.
  */
 export function findBackupEligibleModules(): Array<{
   module: typeof modules.$inferSelect;
   manifest: ModuleManifest;
+  configs: Record<string, unknown>;
 }> {
   const db = getDb();
   const allModules = db.select().from(modules).all();
+  const configsByModule = new Map<string, Record<string, unknown>>();
+  for (const row of db.select().from(moduleConfigs).all()) {
+    const forModule = configsByModule.get(row.moduleId) ?? {};
+    forModule[row.key] = parseStoredConfigValue(row);
+    configsByModule.set(row.moduleId, forModule);
+  }
 
   const eligible: Array<{
     module: typeof modules.$inferSelect;
     manifest: ModuleManifest;
+    configs: Record<string, unknown>;
   }> = [];
 
   for (const mod of allModules) {
@@ -246,7 +252,7 @@ export function findBackupEligibleModules(): Array<{
     const manifest = mod.manifestData as unknown as ModuleManifest;
     if (!manifest.hooks?.on_backup) continue;
 
-    eligible.push({ module: mod, manifest });
+    eligible.push({ module: mod, manifest, configs: configsByModule.get(mod.id) ?? {} });
   }
 
   return eligible;

package/src/services/backup-metadata.ts

@@ -4,9 +4,10 @@
  */
 
 import { randomUUID } from 'node:crypto';
-import { desc, eq, like } from 'drizzle-orm';
+import { and, desc, eq, gt, isNotNull, like } from 'drizzle-orm';
 import { getDb } from '../db/client';
 import { type Backup, type BackupStatus, type BackupType, backups } from '../db/schema';
+import type { BackupHistory } from './backup-schedule';
 
 /** Short ID length used for display and lookup */
 export const SHORT_ID_LENGTH = 8;
@@ -156,6 +157,56 @@ export function listBackups(options?: {
   return db.select().from(backups).orderBy(desc(backups.startedAt)).limit(limit).all();
 }
 
+/**
+ * What a module's backup history says, for the due-ness decision.
+ *
+ * Only `module_data` rows count. A module's cadence is about its own data; a
+ * system backup that happens to name it must not make it look covered.
+ *
+ * Split from the decision itself so the policy stays pure and testable
+ * (Rule 2.3) — see `isBackupDueFromHistory`.
+ */
+export function loadBackupHistory(moduleId: string): BackupHistory {
+  const db = getDb();
+  const forThisModule = and(eq(backups.moduleId, moduleId), eq(backups.backupType, 'module_data'));
+
+  // Ordered and measured by `completedAt`, matching the freshness audit
+  // (services/audit/backup-source.ts). Measuring due-ness from `startedAt`
+  // while the audit measured from `completedAt` meant the two paths could
+  // disagree about a module's age by the duration of the backup itself —
+  // design.md D6. A `completed` row with no `completedAt` is not a backup
+  // either path will count.
+  const [lastSuccess] = db
+    .select()
+    .from(backups)
+    .where(and(forThisModule, eq(backups.status, 'completed'), isNotNull(backups.completedAt)))
+    .orderBy(desc(backups.completedAt))
+    .limit(1)
+    .all();
+
+  const [lastAttempt] = db
+    .select()
+    .from(backups)
+    .where(forThisModule)
+    .orderBy(desc(backups.startedAt))
+    .limit(1)
+    .all();
+
+  // Attempts since the last success — every row newer than it, or every row at
+  // all when there has never been one. An `in_progress` row is included: the
+  // decision only ever uses this to back OFF, so counting a live attempt is
+  // the safe direction.
+  const since = lastSuccess
+    ? and(forThisModule, gt(backups.startedAt, lastSuccess.startedAt))
+    : forThisModule;
+
+  return {
+    lastSuccessAt: lastSuccess?.completedAt ?? null,
+    lastAttemptAt: lastAttempt?.startedAt ?? null,
+    consecutiveFailures: db.select({ id: backups.id }).from(backups).where(since).all().length,
+  };
+}
+
 /**
  * Every backup record still claiming to be in progress, oldest first.
  *

package/src/services/backup-retention.test.ts

@@ -0,0 +1,123 @@
+/**
+ * Retention resolves per dimension, and an unset dimension is UNBOUNDED.
+ *
+ * The trap this file exists for: `backup.retention` is an optional block, so a
+ * manifest that omits it prunes nothing at all — its inner `count: 7` /
+ * `max_age_days: 30` defaults never apply. If setting one dimension let the
+ * other fall back to those defaults, an operator asking to keep 3 copies would
+ * silently arm a 30-day deletion on a module that had been keeping everything.
+ * Every other failure here is recoverable; that one destroys backups.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import type { Backup } from '../db/schema';
+import type { ModuleManifest } from '../manifest/schema';
+import {
+  BACKUP_RETENTION_COUNT_CONFIG_KEY,
+  BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY,
+  effectiveBackupRetention,
+  identifyExpiredBackups,
+  prunesNothing,
+} from './backup-retention';
+
+const DAY = 24 * 60 * 60 * 1000;
+const UNBOUNDED = Number.POSITIVE_INFINITY;
+
+function manifestWith(retention?: { count?: number; max_age_days?: number }): ModuleManifest {
+  return { backup: retention ? { retention } : undefined } as unknown as ModuleManifest;
+}
+
+/** Newest first, matching what `listCompletedBackupsForModule` returns. */
+function backupsAgedDays(...ages: number[]): Backup[] {
+  return ages.map(
+    (days, i) =>
+      ({
+        id: `backup-${i}`,
+        storagePath: `path-${i}`,
+        storageId: 'storage-1',
+        startedAt: new Date(Date.now() - days * DAY),
+      }) as unknown as Backup,
+  );
+}
+
+describe('effectiveBackupRetention', () => {
+  test('nothing declared and nothing set is unbounded in both dimensions', () => {
+    const policy = effectiveBackupRetention(manifestWith(), {});
+    expect(policy).toEqual({ count: UNBOUNDED, maxAgeDays: UNBOUNDED });
+    expect(prunesNothing(policy)).toBe(true);
+  });
+
+  // THE data-loss trap. Setting one dimension must not arm the other.
+  test('setting one dimension leaves the other unbounded when the manifest declares none', () => {
+    const policy = effectiveBackupRetention(manifestWith(), {
+      [BACKUP_RETENTION_COUNT_CONFIG_KEY]: 3,
+    });
+
+    expect(policy.count).toBe(3);
+    expect(policy.maxAgeDays).toBe(UNBOUNDED);
+    expect(prunesNothing(policy)).toBe(false);
+  });
+
+  test('an override bounds one dimension while the manifest bounds the other', () => {
+    const policy = effectiveBackupRetention(manifestWith({ count: 7, max_age_days: 30 }), {
+      [BACKUP_RETENTION_COUNT_CONFIG_KEY]: 3,
+    });
+
+    expect(policy).toEqual({ count: 3, maxAgeDays: 30 });
+  });
+
+  test('both dimensions can be overridden independently', () => {
+    const policy = effectiveBackupRetention(manifestWith({ count: 7, max_age_days: 30 }), {
+      [BACKUP_RETENTION_COUNT_CONFIG_KEY]: 2,
+      [BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY]: 90,
+    });
+
+    expect(policy).toEqual({ count: 2, maxAgeDays: 90 });
+  });
+
+  test("the manifest's suggestion applies when nobody has overridden", () => {
+    expect(effectiveBackupRetention(manifestWith({ count: 7, max_age_days: 30 }), {})).toEqual({
+      count: 7,
+      maxAgeDays: 30,
+    });
+  });
+
+  test('a non-positive or unparseable override falls back rather than deleting everything', () => {
+    // Validated at SET time, so this is hand-edited state. A `0` read as "keep
+    // zero copies" would delete every backup the module has.
+    for (const bad of [0, -1, 'lots', 2.5]) {
+      const policy = effectiveBackupRetention(manifestWith({ count: 7, max_age_days: 30 }), {
+        [BACKUP_RETENTION_COUNT_CONFIG_KEY]: bad,
+      });
+      expect(policy.count).toBe(7);
+    }
+  });
+});
+
+describe('identifyExpiredBackups with unbounded dimensions', () => {
+  test('an unbounded count never prunes on count', () => {
+    const expired = identifyExpiredBackups(backupsAgedDays(1, 2, 3, 4, 5), {
+      count: UNBOUNDED,
+      maxAgeDays: UNBOUNDED,
+    });
+    expect(expired).toEqual([]);
+  });
+
+  test('a bounded count prunes past it, and age alone prunes nothing', () => {
+    const expired = identifyExpiredBackups(backupsAgedDays(1, 2, 100, 200), {
+      count: 3,
+      maxAgeDays: UNBOUNDED,
+    });
+
+    expect(expired.map((b) => b.id)).toEqual(['backup-3']);
+  });
+
+  test('a bounded age prunes past it while every copy is kept on count', () => {
+    const expired = identifyExpiredBackups(backupsAgedDays(1, 2, 100), {
+      count: UNBOUNDED,
+      maxAgeDays: 30,
+    });
+
+    expect(expired.map((b) => b.id)).toEqual(['backup-2']);
+  });
+});

package/src/services/backup-retention.ts

@@ -1,19 +1,77 @@
 /**
- * Backup retention service.
- * Enforces retention policies defined in module manifests.
- * Policies are count-based (keep last N) and age-based (delete older than X days).
- * Whichever limit is hit first triggers deletion.
+ * Backup retention: how many copies to keep, and for how long.
+ *
+ * Policies are count-based (keep last N) and age-based (delete older than X
+ * days), and the two are INDEPENDENT — whichever limit is hit first triggers
+ * deletion. The manifest suggests; the operator's `backup_retention_count` and
+ * `backup_retention_max_age_days` overrides decide, each dimension resolved on
+ * its own.
+ *
+ * ⚠️ An unset dimension is UNBOUNDED, never a default bound. Today an absent
+ * `backup.retention` block means prune nothing at all — the block is optional,
+ * so its inner `count: 7` / `max_age_days: 30` defaults never apply. If setting
+ * one dimension made the other fall back to those defaults, an operator asking
+ * to keep 3 copies would silently also arm a 30-day deletion they never asked
+ * for, on a module that had been keeping everything. Deleting backups nobody
+ * asked to delete is the one failure here that cannot be undone.
  */
 
 import type { Backup } from '../db/schema';
+import type { ModuleManifest } from '../manifest/schema';
 import { deleteBackupRecord, listCompletedBackupsForModule } from './backup-metadata';
 import { createStorageProvider } from './backup-storage';
+import { configOverride } from './module-config';
+
+/** The `module_configs` keys an operator's retention policy is stored under. */
+export const BACKUP_RETENTION_COUNT_CONFIG_KEY = 'backup_retention_count';
+export const BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY = 'backup_retention_max_age_days';
 
 export interface RetentionPolicy {
+  /** Copies to keep. `Infinity` means unbounded — keep every copy. */
   count: number;
+  /** Days to keep. `Infinity` means unbounded — never delete on age. */
   maxAgeDays: number;
 }
 
+/** Neither dimension bounded: nothing is ever pruned, so the pass can be skipped. */
+export function prunesNothing(policy: RetentionPolicy): boolean {
+  return (
+    policy.count === Number.POSITIVE_INFINITY && policy.maxAgeDays === Number.POSITIVE_INFINITY
+  );
+}
+
+/**
+ * Takes the module's operator config as loaded from `module_configs`. An
+ * unparseable or non-positive override leaves that dimension to the manifest:
+ * values are validated at SET time, so a bad one here means hand-edited state,
+ * and keeping too much is the only safe direction to fail in.
+ */
+export function effectiveBackupRetention(
+  manifest: ModuleManifest,
+  configs: Record<string, unknown> | undefined,
+): RetentionPolicy {
+  const declared = manifest.backup?.retention;
+  return {
+    count: positiveIntegerOr(
+      configOverride(configs, BACKUP_RETENTION_COUNT_CONFIG_KEY),
+      declared?.count,
+    ),
+    maxAgeDays: positiveIntegerOr(
+      configOverride(configs, BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY),
+      declared?.max_age_days,
+    ),
+  };
+}
+
+/** Override, else the manifest's suggestion, else unbounded. Never a default bound. */
+function positiveIntegerOr(override: string | undefined, suggested: number | undefined): number {
+  if (override !== undefined) {
+    const parsed = Number(override);
+    if (Number.isInteger(parsed) && parsed > 0) return parsed;
+  }
+  return suggested ?? Number.POSITIVE_INFINITY;
+}
+
 export interface PruneResult {
   moduleId: string;
   deleted: number;
@@ -21,7 +79,10 @@ export interface PruneResult {
 }
 
 /**
- * Identify backups that should be pruned per the retention policy
+ * Identify backups that should be pruned per the retention policy.
+ *
+ * An unbounded dimension is `Infinity`, which needs no special case: no index
+ * reaches it and no age exceeds it.
  */
 export function identifyExpiredBackups(backupsList: Backup[], policy: RetentionPolicy): Backup[] {
   const now = Date.now();

package/src/services/backup-schedule.test.ts

@@ -0,0 +1,166 @@
+/**
+ * Backup cadence: what a manifest declares, and when the sweep acts on it.
+ *
+ * The due-ness tests are the interesting half. celilo#685 was a *daily* module
+ * attempted 24 times a day for a day, each attempt assembling ~1.9 GB before
+ * being OOM-killed, because due-ness was measured only from the last SUCCESS
+ * and a module that cannot succeed never advances that timestamp.
+ */
+
+import { describe, expect, test } from 'bun:test';
+import type { ModuleManifest } from '../manifest/schema';
+import {
+  DEFAULT_BACKUP_SCHEDULE,
+  MAX_RAPID_RETRIES,
+  effectiveBackupSchedule,
+  isBackupDueFromHistory,
+} from './backup-schedule';
+import { type Cadence, parseCadence } from './cadence';
+
+const HOUR = 60 * 60 * 1000;
+const DAY = 24 * HOUR;
+const NOW = Date.UTC(2026, 7, 13, 12, 0, 0);
+
+function ago(ms: number): Date {
+  return new Date(NOW - ms);
+}
+
+/** Parse a cadence in a test, failing loudly rather than passing null on (Rule 7.2). */
+function cadence(value: string): Cadence {
+  const parsed = parseCadence(value);
+  if (parsed === null) throw new Error(`test fixture is not a cadence: ${value}`);
+  return parsed;
+}
+
+function manifestWith(schedule?: string): ModuleManifest {
+  return { backup: schedule ? { schedule } : undefined } as unknown as ModuleManifest;
+}
+
+describe('effectiveBackupSchedule', () => {
+  test('an absent cadence from both sources means daily, not manual', () => {
+    expect(effectiveBackupSchedule(manifestWith(), undefined)).toEqual(cadence('daily'));
+    expect(DEFAULT_BACKUP_SCHEDULE).toEqual(cadence('daily'));
+  });
+
+  test("the manifest's suggestion is honoured when nobody has overridden", () => {
+    expect(effectiveBackupSchedule(manifestWith('manual'), undefined)).toBe('manual');
+    expect(effectiveBackupSchedule(manifestWith('weekly'), undefined)).toEqual(cadence('weekly'));
+  });
+
+  test('the operator override wins over the suggestion', () => {
+    expect(effectiveBackupSchedule(manifestWith('daily'), 'hourly')).toEqual(cadence('hourly'));
+    expect(effectiveBackupSchedule(manifestWith('daily'), '6h')).toEqual(cadence('6h'));
+  });
+
+  test('`manual` is reachable from either source', () => {
+    expect(effectiveBackupSchedule(manifestWith('daily'), 'manual')).toBe('manual');
+    expect(effectiveBackupSchedule(manifestWith('manual'), undefined)).toBe('manual');
+  });
+
+  test('an override applies to a module whose manifest suggests nothing', () => {
+    expect(effectiveBackupSchedule(manifestWith(), 'weekly')).toEqual(cadence('weekly'));
+  });
+
+  test('an unparseable override falls back to the manifest, never to manual', () => {
+    // Values are validated at SET time, so this is hand-edited state. Backing
+    // up MORE often than asked is the safe direction; silently never backing
+    // up is not.
+    expect(effectiveBackupSchedule(manifestWith('weekly'), 'dailyy')).toEqual(cadence('weekly'));
+  });
+});
+
+describe('isBackupDueFromHistory', () => {
+  test('manual never runs on a schedule', () => {
+    expect(
+      isBackupDueFromHistory(
+        'manual',
+        { lastSuccessAt: null, lastAttemptAt: null, consecutiveFailures: 0 },
+        NOW,
+      ),
+    ).toBe(false);
+  });
+
+  test('a module that has never been backed up is due', () => {
+    expect(
+      isBackupDueFromHistory(
+        cadence('daily'),
+        { lastSuccessAt: null, lastAttemptAt: null, consecutiveFailures: 0 },
+        NOW,
+      ),
+    ).toBe(true);
+  });
+
+  test('a fresh success is not due again until its interval has passed', () => {
+    const history = {
+      lastSuccessAt: ago(2 * HOUR),
+      lastAttemptAt: ago(2 * HOUR),
+      consecutiveFailures: 0,
+    };
+    expect(isBackupDueFromHistory(cadence('daily'), history, NOW)).toBe(false);
+    expect(isBackupDueFromHistory(cadence('hourly'), history, NOW)).toBe(true);
+  });
+
+  test('a single failure retries on the next tick', () => {
+    // Most failures are transient. Waiting a whole cadence period after one bad
+    // minute at the storage endpoint would cost more coverage than it saves.
+    expect(
+      isBackupDueFromHistory(
+        cadence('daily'),
+        {
+          lastSuccessAt: ago(2 * DAY),
+          lastAttemptAt: ago(5 * 60 * 1000),
+          consecutiveFailures: 1,
+        },
+        NOW,
+      ),
+    ).toBe(true);
+  });
+
+  test('a run of failures backs off to the module cadence', () => {
+    // The celilo#685 shape: daily module, no success in a week, failing every
+    // hour. Under the old rule this was due at every tick forever.
+    const forgejo = {
+      lastSuccessAt: ago(7 * DAY),
+      lastAttemptAt: ago(1 * HOUR),
+      consecutiveFailures: 20,
+    };
+    expect(isBackupDueFromHistory(cadence('daily'), forgejo, NOW)).toBe(false);
+
+    // ...and still runs once its own interval has elapsed. Backing off is not
+    // giving up.
+    expect(
+      isBackupDueFromHistory(cadence('daily'), { ...forgejo, lastAttemptAt: ago(25 * HOUR) }, NOW),
+    ).toBe(true);
+  });
+
+  test('the back-off boundary is MAX_RAPID_RETRIES', () => {
+    const justFailed = { lastSuccessAt: ago(7 * DAY), lastAttemptAt: ago(1 * HOUR) };
+    expect(
+      isBackupDueFromHistory(
+        cadence('daily'),
+        { ...justFailed, consecutiveFailures: MAX_RAPID_RETRIES - 1 },
+        NOW,
+      ),
+    ).toBe(true);
+    expect(
+      isBackupDueFromHistory(
+        cadence('daily'),
+        { ...justFailed, consecutiveFailures: MAX_RAPID_RETRIES },
+        NOW,
+      ),
+    ).toBe(false);
+  });
+
+  test('a never-succeeded module also backs off once it is clearly failing', () => {
+    // `signal` on celilo-mgr: no successful backup has ever existed. Without
+    // this branch "never succeeded" reads as "always due" and the doomed
+    // attempt runs every tick.
+    expect(
+      isBackupDueFromHistory(
+        cadence('daily'),
+        { lastSuccessAt: null, lastAttemptAt: ago(1 * HOUR), consecutiveFailures: 8 },
+        NOW,
+      ),
+    ).toBe(false);
+  });
+});