@celilo/cli 0.23.0 → 0.24.1

package/src/services/backup-schedule.ts

@@ -6,30 +6,120 @@
  * one. If those two read the manifest differently, a module can be
  * alerted-on-but-never-backed-up — an alert no human action can clear.
  *
- * Absent means `daily`, not `manual`. Treating "the author didn't say"
+ * The manifest is the author's SUGGESTION about a fleet they have never
+ * seen; the operator's `backup_schedule` override wins. Resolution
+ * happens HERE, at read time, so a corrected manifest still reaches
+ * every install that has not overridden it.
+ *
+ * Absent from both means `daily`, not `manual`. Treating "nobody said"
  * as "never check and never run" is what let celilo-mgmt go 55 days
  * without a backup and forgejo and signal go without one entirely, all
  * silently. Opting out is a decision worth writing down, so it takes an
- * explicit `schedule: manual`.
+ * explicit `manual`.
  */
 
 import type { ModuleManifest } from '../manifest/schema';
+import { type Cadence, cadenceMs, parseCadence } from './cadence';
 
-export type BackupSchedule = 'hourly' | 'daily' | 'weekly' | 'monthly' | 'manual';
+/** The `module_configs` key an operator's backup cadence is stored under. */
+export const BACKUP_SCHEDULE_CONFIG_KEY = 'backup_schedule';
 
-/** Used when a manifest declares an `on_backup` hook but no cadence. */
-export const DEFAULT_BACKUP_SCHEDULE: BackupSchedule = 'daily';
+/** Used when neither the operator nor the manifest says anything. */
+export const DEFAULT_BACKUP_SCHEDULE: Cadence = { minutes: 24 * 60 };
 
-export function effectiveBackupSchedule(manifest: ModuleManifest): BackupSchedule {
+/**
+ * `override` is the raw stored value of `backup_schedule`, or undefined when
+ * the operator has set none. An unparseable override falls back to the
+ * manifest rather than to `manual`: values are validated at SET time, so a bad
+ * one here means hand-edited state, and the safe direction is backing up more
+ * often than asked, never less.
+ */
+export function effectiveBackupSchedule(
+  manifest: ModuleManifest,
+  override: string | undefined,
+): Cadence {
+  if (override !== undefined) {
+    const chosen = parseCadence(override);
+    if (chosen !== null) return chosen;
+  }
   const declared = manifest.backup?.schedule;
-  switch (declared) {
-    case 'hourly':
-    case 'daily':
-    case 'weekly':
-    case 'monthly':
-    case 'manual':
-      return declared;
-    default:
-      return DEFAULT_BACKUP_SCHEDULE;
+  if (declared !== undefined) {
+    const suggested = parseCadence(declared);
+    if (suggested !== null) return suggested;
   }
+  return DEFAULT_BACKUP_SCHEDULE;
+}
+
+/**
+ * Failures in a row after which retrying stops being worth the resources.
+ *
+ * Under this many, a failure is assumed transient and the module stays due on
+ * the next tick, because most failures ARE transient — a storage endpoint
+ * having a bad minute should not cost a full cadence period of coverage.
+ *
+ * At or over it, the evidence says otherwise and the retry slows to the
+ * module's own cadence. Three is deliberately small: the useful information
+ * from a retry is almost entirely in the first one or two, and the cost of
+ * being wrong in this direction is bounded (one delayed backup) while the cost
+ * of the other direction is not.
+ */
+export const MAX_RAPID_RETRIES = 3;
+
+/** What the backup history says about one module, for the due-ness decision. */
+export interface BackupHistory {
+  /**
+   * When the last successful backup COMPLETED, or null if there has never
+   * been one. Completion, not start — the same instant the freshness audit
+   * measures from, so the run path and the alert path cannot disagree about
+   * how old a backup is (design.md D6).
+   */
+  lastSuccessAt: Date | null;
+  /** Last attempt of any outcome, or null if none has ever been made. */
+  lastAttemptAt: Date | null;
+  /** Attempts since the last success. */
+  consecutiveFailures: number;
+}
+
+/**
+ * Whether to start a backup for a module right now.
+ *
+ * Due-ness used to be "has it been an interval since the last SUCCESS", which
+ * is correct in the healthy case and degenerate in the failing one: a module
+ * that cannot back up never advances that timestamp, so it is due at every
+ * tick forever. The sweep runs hourly, so a *daily* module that started failing
+ * was re-picked 24 times a day. On celilo-mgr that meant 20+ consecutive
+ * forgejo attempts, each one assembling ~1.9 GB of staging and then dying, none
+ * of them ever going to succeed for a reason no retry could change (celilo#685).
+ *
+ * Retrying is still right — it just cannot be unconditional. So the failure
+ * count decides which clock applies: under `MAX_RAPID_RETRIES` the module stays
+ * due against its last success, and at or over it the interval is measured from
+ * the last ATTEMPT instead, which turns 24 doomed attempts a day into one.
+ *
+ * Backing off is not the same as going quiet. The `backups` drift monitor
+ * measures staleness from the last success and is unaffected by this, so a
+ * module that has slowed to one attempt a day still alerts as stale on exactly
+ * the schedule it would have before — see services/audit/backups.ts.
+ *
+ * Pure, and time is a parameter, so the policy tests without a database.
+ */
+export function isBackupDueFromHistory(
+  schedule: Cadence,
+  history: BackupHistory,
+  now: number,
+): boolean {
+  if (schedule === 'manual') return false;
+
+  const interval = cadenceMs(schedule);
+
+  if (history.consecutiveFailures >= MAX_RAPID_RETRIES) {
+    // A run of failures with no attempt recorded is not a state the sweep can
+    // produce, but "cannot prove an interval has passed" must not mean "start
+    // another 1.9 GB attempt".
+    if (!history.lastAttemptAt) return false;
+    return now - history.lastAttemptAt.getTime() >= interval;
+  }
+
+  if (!history.lastSuccessAt) return true;
+  return now - history.lastSuccessAt.getTime() >= interval;
 }

package/src/services/backup-staging.ts

@@ -95,9 +95,22 @@ export interface ReapedStaging {
  * drift check can read a module as recently backed up when every attempt in
  * fact died. Nine such rows were live on celilo-mgr while forgejo had no usable
  * backup at all.
+ *
+ * It names no cause because this pass genuinely cannot know one: it runs later,
+ * in a different process, and infers the death from a pid that is no longer
+ * there. The observer that DOES know is the dispatcher, which holds the exit
+ * code and translates it (`describeHandlerExit` in packages/event-bus) — a
+ * SIGKILL, say, and the OOM killer that most likely sent it.
+ *
+ * The two facts existing in two places is fine; the operator having no way to
+ * learn that is not. In celilo#685 this string was the whole of what an
+ * operator saw for twenty consecutive OOM kills, so it now points at the record
+ * that has the answer rather than terminating the trail.
  */
 export const ABANDONED_BACKUP_MESSAGE =
-  'abandoned — the backup process ended without recording an outcome';
+  'abandoned — the backup process ended without recording an outcome. ' +
+  'Only the process that ran it saw how it died, so the cause is recorded ' +
+  'against the event delivery rather than here: run `celilo system doctor`.';
 
 /**
  * Whether reclaiming this directory also means its record was lying about

package/src/services/backup-sweep.test.ts

@@ -1,6 +1,5 @@
 import { describe, expect, test } from 'bun:test';
 import type { ModuleManifest } from '../manifest/schema';
-import type { BackupSchedule } from './backup-schedule';
 import {
   BACKUP_SWEEP_MAX_ATTEMPTS,
   BACKUP_SWEEP_PATTERN,
@@ -13,13 +12,13 @@ import {
 } from './backup-sweep';
 import { InFlightError } from './module-operations';
 
-function moduleWith(id: string, schedule?: BackupSchedule): BackupSweepModule {
+function moduleWith(id: string, schedule?: string, scheduleOverride?: string): BackupSweepModule {
   const manifest = {
     id,
     hooks: { on_backup: { script: 'backup.ts' } },
     ...(schedule ? { backup: { schedule } } : {}),
   } as unknown as ModuleManifest;
-  return { id, manifest };
+  return { id, manifest, scheduleOverride };
 }
 
 function deps(
@@ -123,6 +122,26 @@ describe('runBackupSweep', () => {
     expect(report.skippedManual).toEqual(['scratch']);
   });
 
+  test("an operator's override decides the cadence, not the manifest", async () => {
+    const d = deps([moduleWith('caddy', 'daily', '6h')]);
+    const seen: Array<[string, unknown]> = [];
+    d.isDue = (moduleId, schedule) => {
+      seen.push([moduleId, schedule]);
+      return true;
+    };
+
+    await runBackupSweep(d);
+
+    expect(seen).toEqual([['caddy', { minutes: 360 }]]);
+  });
+
+  test('an override of manual stops a module the manifest wanted backed up', async () => {
+    const report = await runBackupSweep(deps([moduleWith('caddy', 'daily', 'manual')]));
+
+    expect(report.backedUp).toEqual([]);
+    expect(report.skippedManual).toEqual(['caddy']);
+  });
+
   test('an undeclared schedule is backed up, not treated as manual', async () => {
     // The regression this whole subsystem exists for: forgejo and signal
     // declare no `backup:` block and had never been backed up.

package/src/services/backup-sweep.ts

@@ -17,12 +17,16 @@
  */
 
 import type { ModuleManifest } from '../manifest/schema';
-import { type BackupSchedule, effectiveBackupSchedule } from './backup-schedule';
+import { effectiveBackupSchedule } from './backup-schedule';
 import type { ReapStagingReport, ResolveAbandonedReport } from './backup-staging';
+import { BACKUP_SWEEP_PATTERN, type Cadence } from './cadence';
 import { InFlightError } from './module-operations';
 
 export const BACKUP_SWEEP_SUBSCRIBER = 'celilo-backup-sweep';
-export const BACKUP_SWEEP_PATTERN = 'timer.tick.1h';
+// The tick itself lives in services/cadence.ts, next to the floor derived from
+// it — a sweep whose tick and whose finest servable cadence are stated in two
+// files is the pair that drifts.
+export { BACKUP_SWEEP_PATTERN };
 
 /**
  * How long the sweep may run before the dispatcher kills it.
@@ -99,12 +103,18 @@ export function ensureBackupSweepSubscriber(bus: SubscriberRegistrar): void {
 export interface BackupSweepModule {
   id: string;
   manifest: ModuleManifest;
+  /**
+   * The operator's `backup_schedule` override, or undefined when they have set
+   * none. Carried rather than pre-resolved so the cadence is resolved through
+   * the one shared accessor here, the same way the freshness audit resolves it.
+   */
+  scheduleOverride: string | undefined;
 }
 
 export interface BackupSweepDeps {
   /** Installed modules that declare an `on_backup` hook. */
   listEligible(): BackupSweepModule[];
-  isDue(moduleId: string, schedule: BackupSchedule): boolean;
+  isDue(moduleId: string, schedule: Cadence): boolean;
   backup(moduleId: string): Promise<{ success: boolean; error?: string }>;
   /** Apply the module's declared retention. No-op when it declares none. */
   prune(module: BackupSweepModule): Promise<void>;
@@ -126,7 +136,7 @@ export interface BackupSweepReport {
   /** Records corrected from a stale `in_progress`. */
   records: ResolveAbandonedReport;
   backedUp: string[];
-  /** Explicit `schedule: manual` — the author opted out. */
+  /** Effective cadence `manual` — the operator or the author opted out. */
   skippedManual: string[];
   skippedNotDue: string[];
   /** Another module operation held the lock. Not a failure; retried next tick. */
@@ -157,7 +167,7 @@ export async function runBackupSweep(deps: BackupSweepDeps): Promise<BackupSweep
   };
 
   for (const module of deps.listEligible()) {
-    const schedule = effectiveBackupSchedule(module.manifest);
+    const schedule = effectiveBackupSchedule(module.manifest, module.scheduleOverride);
     if (schedule === 'manual') {
       report.skippedManual.push(module.id);
       continue;

package/src/services/cadence.test.ts

@@ -0,0 +1,97 @@
+import { describe, expect, test } from 'bun:test';
+import {
+  ALERTING_SWEEP_PATTERN,
+  BACKUP_CADENCE_FLOOR_MINUTES,
+  BACKUP_SWEEP_PATTERN,
+  MONITOR_INTERVAL_FLOOR_MINUTES,
+  cadenceMs,
+  cadenceSchema,
+  formatCadence,
+  parseCadence,
+  tickIntervalMinutes,
+} from './cadence';
+
+describe('parseCadence', () => {
+  test('every named period equals its duration equivalent', () => {
+    expect(parseCadence('hourly')).toEqual(parseCadence('1h'));
+    expect(parseCadence('daily')).toEqual(parseCadence('24h'));
+    expect(parseCadence('weekly')).toEqual(parseCadence('7d'));
+    expect(parseCadence('monthly')).toEqual(parseCadence('30d'));
+  });
+
+  test('durations normalise to minutes', () => {
+    expect(parseCadence('90m')).toEqual({ minutes: 90 });
+    expect(parseCadence('6h')).toEqual({ minutes: 360 });
+    expect(parseCadence('3d')).toEqual({ minutes: 4320 });
+  });
+
+  test('manual is its own value, not a number', () => {
+    expect(parseCadence('manual')).toBe('manual');
+    expect(cadenceMs('manual')).toBe(Number.POSITIVE_INFINITY);
+  });
+
+  test('rejects malformed, zero and negative values', () => {
+    for (const bad of ['dailyy', '', '6', 'h6', '6w', '0h', '-1h', '1.5h', 'never']) {
+      expect(parseCadence(bad)).toBeNull();
+    }
+  });
+});
+
+describe('formatCadence', () => {
+  test('prefers the word an operator would have typed', () => {
+    expect(formatCadence({ minutes: 60 })).toBe('hourly');
+    expect(formatCadence({ minutes: 1440 })).toBe('daily');
+    expect(formatCadence('manual')).toBe('manual');
+  });
+
+  test('falls back to the coarsest exact duration', () => {
+    expect(formatCadence({ minutes: 360 })).toBe('6h');
+    expect(formatCadence({ minutes: 2880 })).toBe('2d');
+    expect(formatCadence({ minutes: 90 })).toBe('90m');
+  });
+});
+
+describe('the floor follows the sweep tick', () => {
+  test('each floor is the tick of the sweep that would serve it', () => {
+    expect(BACKUP_CADENCE_FLOOR_MINUTES).toBe(tickIntervalMinutes(BACKUP_SWEEP_PATTERN));
+    expect(MONITOR_INTERVAL_FLOOR_MINUTES).toBe(tickIntervalMinutes(ALERTING_SWEEP_PATTERN));
+  });
+
+  test('a finer tick admits a cadence the coarser one refused', () => {
+    const coarse = cadenceSchema({ floorMinutes: tickIntervalMinutes('timer.tick.1h') });
+    const fine = cadenceSchema({ floorMinutes: tickIntervalMinutes('timer.tick.5m') });
+    expect(coarse.safeParse('15m').success).toBe(false);
+    expect(fine.safeParse('15m').success).toBe(true);
+  });
+
+  test('a pattern that is not a timer tick is a programming error, not a floor of zero', () => {
+    expect(() => tickIntervalMinutes('module.deployed')).toThrow();
+  });
+});
+
+describe('cadenceSchema', () => {
+  const schema = cadenceSchema({ floorMinutes: 60 });
+
+  test('accepts words, durations at or above the floor, and manual', () => {
+    for (const good of ['hourly', 'daily', 'weekly', 'monthly', 'manual', '6h', '2d', '60m']) {
+      expect(schema.safeParse(good).success).toBe(true);
+    }
+  });
+
+  test('names the permitted forms when the value is unparseable', () => {
+    const result = schema.safeParse('dailyy');
+    expect(result.success).toBe(false);
+    if (!result.success) {
+      expect(result.error.issues[0].message).toContain('named period');
+      expect(result.error.issues[0].message).toContain('manual');
+    }
+  });
+
+  test('names the finest servable cadence when the value is below the floor', () => {
+    const result = schema.safeParse('5m');
+    expect(result.success).toBe(false);
+    if (!result.success) {
+      expect(result.error.issues[0].message).toContain('hourly');
+    }
+  });
+});

package/src/services/cadence.ts

@@ -0,0 +1,165 @@
+/**
+ * How often celilo does something to a module, in one type.
+ *
+ * celilo has three per-module cadences — how often to back a module up, how
+ * often to health-check it, and (in the same family) how long to keep the
+ * backups. They were three spellings of the same idea: an enum of four words
+ * for backups, a `15m`-style duration for health checks, nothing shared. One
+ * namespace holding two unrelated formats is not one concept, so an operator
+ * setting `6h` on a backup had no way to be right.
+ *
+ * A cadence is therefore a word OR a duration, both normalised to minutes, plus
+ * `manual` for "the operator opted out". Every existing manifest keeps working:
+ * the words are the same words.
+ *
+ * The FLOOR is derived from the tick of the sweep that would act on the
+ * cadence, never stated independently. A cadence finer than its sweep's tick
+ * cannot be served, and accepting one leaves the operator believing they
+ * configured something that silently never happens. Deriving it means a changed
+ * tick moves the floor with it rather than waiting for someone to remember.
+ *
+ * Pure: no database, no clock. See
+ * openspec/changes/operator-cadence-overrides/design.md D3, D4.
+ */
+
+import { z } from 'zod';
+
+/** Minutes between runs, or `manual` — the operator opted out entirely. */
+export type Cadence = { minutes: number } | 'manual';
+
+/** The words a cadence may be spelled with, and what each means in minutes. */
+const NAMED_PERIODS: Record<string, number> = {
+  hourly: 60,
+  daily: 24 * 60,
+  weekly: 7 * 24 * 60,
+  // 30 days, matching what the backup schedule has always meant by `monthly`.
+  // No calendar-month semantics are introduced here, and none are lost.
+  monthly: 30 * 24 * 60,
+};
+
+export const DURATION_PATTERN = /^(\d+)(m|h|d)$/;
+
+/**
+ * Everything a cadence may be spelled as, for the JSON Schema export.
+ *
+ * Editors validate `modules/*​/manifest.yml` against the exported JSON Schema,
+ * which cannot carry a Zod refinement — so well-formedness is duplicated as a
+ * regex the same way `health_check.interval` already does it. The floor is not
+ * expressible here and stays a refinement.
+ */
+export const CADENCE_PATTERN = /^(hourly|daily|weekly|monthly|manual|\d+(m|h|d))$/;
+
+/**
+ * Parse a duration string (`15m`, `1h`, `1d`) to whole minutes.
+ * Returns null when the string is not a well-formed duration.
+ */
+export function parseIntervalMinutes(value: string): number | null {
+  const match = DURATION_PATTERN.exec(value);
+  if (!match) return null;
+  const amount = Number.parseInt(match[1], 10);
+  if (!Number.isFinite(amount) || amount <= 0) return null;
+  const unit = match[2];
+  if (unit === 'm') return amount;
+  if (unit === 'h') return amount * 60;
+  return amount * 60 * 24;
+}
+
+/** Parse a cadence in any accepted spelling. Returns null when malformed. */
+export function parseCadence(value: string): Cadence | null {
+  if (value === 'manual') return 'manual';
+  const named = NAMED_PERIODS[value];
+  if (named !== undefined) return { minutes: named };
+  const minutes = parseIntervalMinutes(value);
+  return minutes === null ? null : { minutes };
+}
+
+/**
+ * Spell a cadence back the way an operator would write it, preferring the word
+ * when one exists — a resolved `1440` reads as `daily`, not `24h`.
+ */
+export function formatCadence(cadence: Cadence): string {
+  if (cadence === 'manual') return 'manual';
+  for (const [word, minutes] of Object.entries(NAMED_PERIODS)) {
+    if (minutes === cadence.minutes) return word;
+  }
+  if (cadence.minutes % (24 * 60) === 0) return `${cadence.minutes / (24 * 60)}d`;
+  if (cadence.minutes % 60 === 0) return `${cadence.minutes / 60}h`;
+  return `${cadence.minutes}m`;
+}
+
+/** Milliseconds between runs. `manual` is infinite — never due, never stale. */
+export function cadenceMs(cadence: Cadence): number {
+  return cadence === 'manual' ? Number.POSITIVE_INFINITY : cadence.minutes * 60_000;
+}
+
+/**
+ * Minutes between ticks of a bus timer pattern (`timer.tick.5m` → 5).
+ *
+ * The floors below are derived through this rather than written down, so
+ * changing which tick a sweep rides changes what it can serve in the same edit.
+ */
+export function tickIntervalMinutes(pattern: string): number {
+  const suffix = pattern.replace(/^timer\.tick\./, '');
+  const minutes = parseIntervalMinutes(suffix);
+  if (minutes === null) throw new Error(`Not a timer tick pattern: ${pattern}`);
+  return minutes;
+}
+
+/**
+ * The bus tick each sweep rides.
+ *
+ * They live here, with the floors that derive from them, rather than beside
+ * each sweep's subscriber registration — a floor stated in one file and a tick
+ * chosen in another is exactly the pair that drifts. The sweeps import their
+ * pattern from here.
+ */
+export const BACKUP_SWEEP_PATTERN = 'timer.tick.1h';
+export const ALERTING_SWEEP_PATTERN = 'timer.tick.5m';
+
+/** Finest backup cadence the hourly backup sweep can serve. */
+export const BACKUP_CADENCE_FLOOR_MINUTES = tickIntervalMinutes(BACKUP_SWEEP_PATTERN);
+
+/** Finest health-check cadence the five-minute alerting sweep can serve. */
+export const MONITOR_INTERVAL_FLOOR_MINUTES = tickIntervalMinutes(ALERTING_SWEEP_PATTERN);
+
+/** Human list of accepted spellings, naming the finest cadence this sweep serves. */
+export function describeCadenceForm(floorMinutes: number): string {
+  return `Allowed: a named period (hourly, daily, weekly, monthly), a duration like "6h", "90m" or "3d" no finer than ${formatCadence({ minutes: floorMinutes })}, or "manual" to opt out.`;
+}
+
+/**
+ * A cadence value that the named sweep can actually serve.
+ *
+ * Validates the string form (what an operator types and what a manifest
+ * carries); the caller parses it with `parseCadence` once it is known good.
+ *
+ * The well-formedness check is a `.regex` on the inner string rather than part
+ * of the refinement because only the regex survives the export to JSON Schema,
+ * and that export is what validates `modules/*​/manifest.yml` in an editor. The
+ * floor cannot be expressed in JSON Schema at all, so it stays a refinement.
+ */
+export function cadenceSchema({
+  floorMinutes,
+  description,
+}: {
+  floorMinutes: number;
+  description?: string;
+}): z.ZodType<string> {
+  const form = describeCadenceForm(floorMinutes);
+  let base = z.string().regex(CADENCE_PATTERN, { message: form });
+  if (description) base = base.describe(description);
+  return base.superRefine((value, ctx) => {
+    const cadence = parseCadence(value);
+    if (cadence === null) {
+      // `0h` matches the pattern and is not a cadence.
+      ctx.addIssue({ code: z.ZodIssueCode.custom, message: form });
+      return;
+    }
+    if (cadence !== 'manual' && cadence.minutes < floorMinutes) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: `"${value}" is finer than the sweep that would serve it can run (every ${formatCadence({ minutes: floorMinutes })}). ${form}`,
+      });
+    }
+  });
+}