@celilo/cli 0.22.0 → 0.24.0

package/src/services/alerting/monitors.ts

@@ -1,10 +1,20 @@
 /**
  * Monitor CRUD.
  *
- * A monitor binds a thing to check to a cadence, a severity, and an escalation
- * policy. Operators address them by their target (`caddy`,
- * `machines_reachable`) rather than by id — per CLAUDE.md, a UUID never
- * reaches the operator.
+ * A monitor binds a thing to check to a severity and an escalation policy.
+ * Operators address them by their target (`caddy`, `machines_reachable`) rather
+ * than by id — per CLAUDE.md, a UUID never reaches the operator.
+ *
+ * ⚠️ `intervalMinutes` and `enabled` are meaningful for `builtin_check` rows
+ * ONLY. A built-in check targets a fleet-level audit category with no module
+ * and no manifest, so nothing suggests its cadence and the row legitimately IS
+ * the config. A `module_hook` row targets a module whose manifest MAY suggest
+ * one, so its cadence and whether it is watched at all resolve at read time
+ * from `health_check_interval` — see [[services/alerting/health-cadence.ts]].
+ * Two columns whose meaning depends on `kind` is a smell, named here rather
+ * than discovered in review (design.md D8); the alternatives are a resolved
+ * value cached on the row, which rots, or splitting the table, which needs a
+ * synthetic monitor identity for `alerts.monitorId` and friends.
  */
 
 import { randomUUID } from 'node:crypto';
@@ -17,11 +27,12 @@ import {
   alerts,
   monitors,
 } from '../../db/schema';
-import { parseIntervalMinutes } from '../../manifest/schema';
+import { ALERTING_SWEEP_PATTERN, MONITOR_INTERVAL_FLOOR_MINUTES, parseCadence } from '../cadence';
 
 export interface CreateMonitorInput {
   kind: MonitorKind;
   target: string;
+  /** `builtin_check` only. Not consulted for `module_hook` — see the file header. */
   intervalMinutes: number;
   severity?: AlertSeverity;
   suppressible?: boolean;
@@ -69,29 +80,50 @@ export function createMonitor(db: DbClient, input: CreateMonitorInput): Monitor
 }
 
 /**
- * Ensure a monitor exists for a module that declares a suggested interval.
+ * Ensure a `module_hook` monitor row exists for a module that declares a
+ * `health_check` hook.
+ *
+ * The row is no longer where the cadence lives — it carries severity, the
+ * escalation policy and `lastRunAt`, and whether it runs at all is resolved
+ * from the module's effective cadence at sweep time. So it is created for any
+ * module with the hook, not only for one whose manifest happens to suggest an
+ * interval: an operator can now set a cadence on a module whose author never
+ * named one, and without a row there would be nothing to carry its last run.
  *
- * Called on deploy. Never overwrites an existing monitor: the manifest's
- * interval is the author's SUGGESTION and the operator's setting must survive
- * module upgrades, so a second call is a no-op rather than a reset.
+ * The stored `intervalMinutes` is the manifest's suggestion where there is one,
+ * and is NOT read back for this kind. It is written so that rolling back to a
+ * release which does read it behaves as it did before.
  *
- * Returns the monitor when one was created, null when one already existed or
- * the manifest declares no interval.
+ * Idempotent: a second call is a no-op, which is what keeps deploys from
+ * duplicating rows. It is no longer what protects an operator's setting — that
+ * lives in `module_configs` now and cannot be overwritten from here at all.
  */
 export function ensureMonitorForModule(
   db: DbClient,
   moduleId: string,
   suggestedInterval: string | undefined,
 ): Monitor | null {
-  if (!suggestedInterval) return null;
   if (findMonitor(db, 'module_hook', moduleId)) return null;
 
-  const intervalMinutes = parseIntervalMinutes(suggestedInterval);
-  if (intervalMinutes === null) return null;
+  const suggested = suggestedInterval ? parseCadence(suggestedInterval) : null;
+  const intervalMinutes =
+    suggested !== null && suggested !== 'manual'
+      ? suggested.minutes
+      : MONITOR_INTERVAL_FLOOR_MINUTES;
 
   return createMonitor(db, { kind: 'module_hook', target: moduleId, intervalMinutes });
 }
 
+/**
+ * Change a `builtin_check` monitor's cadence in place.
+ *
+ * In place, rather than remove-and-recreate, because the monitor id owns the
+ * alert history: recreating it would orphan every live alert it raised.
+ */
+export function updateMonitorInterval(db: DbClient, monitorId: string, minutes: number): void {
+  db.update(monitors).set({ intervalMinutes: minutes }).where(eq(monitors.id, monitorId)).run();
+}
+
 /**
  * Enable or disable a monitor.
  *
@@ -139,7 +171,10 @@ export function resolveMonitorAlerts(db: DbClient, monitorId: string, now: Date)
  * what switches on the sweep.
  */
 export const ALERTING_SWEEP_SUBSCRIBER = 'celilo-alerting-sweep';
-export const ALERTING_SWEEP_PATTERN = 'timer.tick.5m';
+// The tick itself lives in services/cadence.ts, next to the floor derived from
+// it — a sweep whose tick and whose finest servable cadence are stated in two
+// files is the pair that drifts.
+export { ALERTING_SWEEP_PATTERN };
 
 export interface SubscriberRegistrar {
   subscribe(options: {

package/src/services/alerting/suppression.test.ts

@@ -35,6 +35,7 @@ const TOPOLOGY: SuppressionTopology = {
 const noSuppression = {
   suppressible: true,
   modulesInDeployWindow: new Set<string>(),
+  pausedModules: new Set<string>(),
   topology: TOPOLOGY,
 };
 
@@ -185,6 +186,7 @@ describe('findSuppressor — guards', () => {
         firingKeys,
         suppressible: false,
         modulesInDeployWindow: new Set(),
+        pausedModules: new Set(),
         topology: TOPOLOGY,
       }),
     ).toBeNull();
@@ -197,6 +199,7 @@ describe('findSuppressor — guards', () => {
         firingKeys: new Set(),
         suppressible: true,
         modulesInDeployWindow: new Set(['forgejo']),
+        pausedModules: new Set(),
         topology: TOPOLOGY,
       }),
     ).toEqual({ kind: 'deploy_window', moduleId: 'forgejo' });
@@ -209,6 +212,7 @@ describe('findSuppressor — guards', () => {
         firingKeys: new Set(),
         suppressible: true,
         modulesInDeployWindow: new Set(['forgejo']),
+        pausedModules: new Set(),
         topology: TOPOLOGY,
       }),
     ).toBeNull();
@@ -221,6 +225,7 @@ describe('findSuppressor — guards', () => {
         firingKeys: new Set([machineAlertKey('iot')]),
         suppressible: true,
         modulesInDeployWindow: new Set(['homebridge']),
+        pausedModules: new Set(),
         topology: TOPOLOGY,
       }),
     ).toEqual({ kind: 'deploy_window', moduleId: 'homebridge' });

package/src/services/alerting/suppression.ts

@@ -106,12 +106,20 @@ export interface SuppressorLookup {
   suppressible: boolean;
   /** Modules currently inside a deploy window. */
   modulesInDeployWindow: ReadonlySet<string>;
+  /**
+   * Modules currently PAUSED. A pause is a deliberate quiescing, so its alerts
+   * are explained by the pause itself (openspec/changes/module-pause-lifecycle,
+   * task 2.3) — same mechanism as a deploy window, with the pause as the source
+   * instead of an ancestor alert.
+   */
+  pausedModules: ReadonlySet<string>;
   topology: SuppressionTopology;
 }
 
 export type Suppressor =
   | { kind: 'alert'; key: string }
-  | { kind: 'deploy_window'; moduleId: string };
+  | { kind: 'deploy_window'; moduleId: string }
+  | { kind: 'paused'; moduleId: string };
 
 /**
  * Find what is suppressing `key`, or null if it should be reported.
@@ -128,6 +136,15 @@ export function findSuppressor(lookup: SuppressorLookup): Suppressor | null {
 
   const parsed = parseAlertKey(lookup.key);
 
+  // A pause is checked before a deploy window because it is the longer-lived
+  // and more consequential explanation: a paused module may also be inside a
+  // deploy window (unpause redeploys), and "paused" is the fact the operator
+  // needs to see. Attributed to the pause rather than suppressed anonymously —
+  // silently dropping the alert is what turns a pause into an invisible outage.
+  if (parsed?.source === 'module' && lookup.pausedModules.has(parsed.moduleId)) {
+    return { kind: 'paused', moduleId: parsed.moduleId };
+  }
+
   // A deploy is the same mechanism with a window as the source instead of an
   // ancestor alert — which is why deploy auto-silencing is not a second feature.
   if (parsed?.source === 'module' && lookup.modulesInDeployWindow.has(parsed.moduleId)) {

package/src/services/alerting/sweep-runner.test.ts

@@ -4,9 +4,11 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { eq } from 'drizzle-orm';
 import type { DbClient } from '../../db/client';
-import { type Alert, type Monitor, alerts, monitors } from '../../db/schema';
+import { type Alert, type Monitor, alerts, modules, monitors } from '../../db/schema';
 import { setupTestDatabase } from '../../test-utils/setup-test-db';
 import type { HealthCheckResult } from '../health-runner';
+import { deleteModuleConfig, upsertModuleConfig } from '../module-config';
+import { HEALTH_CHECK_INTERVAL_CONFIG_KEY, reconcileModuleWatchState } from './health-cadence';
 import { moduleCheckAlertKey } from './keys';
 import type { MonitorRunDeps } from './run-monitor';
 import { type SuppressionTopology, machineAlertKey } from './suppression';
@@ -50,6 +52,7 @@ describe('runSweep', () => {
       monitorDeps: monitorDeps(result, now),
       loadTopology: () => TOPOLOGY,
       loadDeployWindowModules: () => new Set(),
+      loadPausedModules: () => new Set(),
       isSuppressible: () => true,
       // No routes configured: the sweep must still run everything else.
       notifyDepsFor: () => null,
@@ -66,6 +69,21 @@ describe('runSweep', () => {
     const dbPath = join(dir, 'celilo.db');
     process.env.CELILO_DB_PATH = dbPath;
     db = await setupTestDatabase(dbPath);
+    // The module row is load-bearing now: a `module_hook` monitor's cadence
+    // resolves from the module's manifest and config, not from its own row.
+    db.insert(modules)
+      .values({
+        id: MODULE,
+        name: MODULE,
+        sourcePath: dir,
+        version: '1.0.0',
+        state: 'INSTALLED',
+        manifestData: {
+          id: MODULE,
+          hooks: { health_check: { script: './h.ts', interval: '15m' } },
+        },
+      })
+      .run();
     db.insert(monitors)
       .values({
         id: 'mon-1',
@@ -116,11 +134,42 @@ describe('runSweep', () => {
     expect(report.monitorsRun).toBe(0);
   });
 
-  test('a disabled monitor is never run', async () => {
+  // `enabled` and `intervalMinutes` are builtin_check-only now (design.md D8).
+  // A future reader that reintroduces the dependency on the row fails here
+  // rather than silently regressing to write-time resolution.
+  test("a module monitor's stored enabled flag is not consulted", async () => {
     db.update(monitors).set({ enabled: false }).where(eq(monitors.id, 'mon-1')).run();
     const report = await runSweep(db, currentMonitors(), deps());
+    expect(report.monitorsRun).toBe(1);
+  });
+
+  test("a module monitor's stored interval is not consulted", async () => {
+    // The row says a full day; the manifest says 15m, which is what wins.
+    db.update(monitors).set({ intervalMinutes: 1440 }).where(eq(monitors.id, 'mon-1')).run();
+    await runSweep(db, currentMonitors(), deps());
+    const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
+    expect(report.monitorsRun).toBe(1);
+  });
+
+  test('a module whose effective cadence is manual is never run, and its alerts resolve', async () => {
+    await runSweep(db, currentMonitors(), deps());
+    expect(liveAlerts()).not.toEqual([]);
+
+    upsertModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual');
+    reconcileModuleWatchState(db, MODULE, later(20));
+
+    const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
     expect(report.monitorsRun).toBe(0);
-    expect(liveAlerts()).toEqual([]);
+    expect(liveAlerts().every((a) => a.state === 'resolved')).toBe(true);
+  });
+
+  test('unsetting the override resumes watching', async () => {
+    upsertModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual');
+    expect((await runSweep(db, currentMonitors(), deps())).monitorsRun).toBe(0);
+
+    deleteModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY);
+    const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
+    expect(report.monitorsRun).toBe(1);
   });
 
   test('recovery resolves the alert on a later sweep', async () => {

package/src/services/alerting/sweep-runner.ts

@@ -21,6 +21,8 @@
 
 import type { DbClient } from '../../db/client';
 import type { Alert, Monitor } from '../../db/schema';
+import { MONITOR_INTERVAL_FLOOR_MINUTES } from '../cadence';
+import { isScheduled, loadModuleHealthCadences } from './health-cadence';
 import type { NotifyDeps, NotifyOutcome } from './notifier';
 import { deliverDeferred, notifyAlert } from './notifier';
 import { type MonitorRunDeps, runOneMonitor } from './run-monitor';
@@ -43,6 +45,8 @@ export interface SweepDeps {
   loadTopology(): SuppressionTopology;
   /** Modules currently inside a deploy window. */
   loadDeployWindowModules(): Set<string>;
+  /** Ids of modules currently PAUSED — a pause explains its own module's alerts. */
+  loadPausedModules(): Set<string>;
   /** Whether the monitor owning an alert may be suppressed at all. */
   isSuppressible(alert: Alert): boolean;
   /** Compose the per-alert notification context. Null when nothing can page. */
@@ -122,14 +126,35 @@ export async function runSweep(
   };
 
   // 1. Run due monitors.
+  //
+  // A `module_hook` monitor's cadence and whether it is watched at all come
+  // from the module's effective health-check cadence, NOT from its row: the row
+  // was seeded once at first deploy and never reconsulted, so a corrected
+  // manifest could never reach an existing install (design.md D2/D8). A
+  // `builtin_check` has no module and no manifest, so its row is the config.
+  const cadences = loadModuleHealthCadences(db);
   const due = selectDueMonitors(
-    monitors.map((m) => ({
-      id: m.id,
-      intervalMinutes: m.intervalMinutes,
-      enabled: m.enabled,
-      lastRunAt: m.lastRunAt,
-      monitor: m,
-    })),
+    monitors.map((m) => {
+      if (m.kind !== 'module_hook') {
+        return {
+          id: m.id,
+          intervalMinutes: m.intervalMinutes,
+          enabled: m.enabled,
+          lastRunAt: m.lastRunAt,
+          monitor: m,
+        };
+      }
+      const cadence = cadences.get(m.target)?.cadence ?? null;
+      return {
+        id: m.id,
+        // Unscheduled monitors are filtered out by `enabled` below, so this
+        // value is never used to decide due-ness for them.
+        intervalMinutes: isScheduled(cadence) ? cadence.minutes : MONITOR_INTERVAL_FLOOR_MINUTES,
+        enabled: isScheduled(cadence),
+        lastRunAt: m.lastRunAt,
+        monitor: m,
+      };
+    }),
     deps.now(),
   );
 
@@ -156,6 +181,7 @@ export async function runSweep(
   );
   const topology = deps.loadTopology();
   const deployWindows = deps.loadDeployWindowModules();
+  const paused = deps.loadPausedModules();
 
   for (const alert of live) {
     const suppressor = findSuppressor({
@@ -163,13 +189,20 @@ export async function runSweep(
       firingKeys,
       suppressible: deps.isSuppressible(alert),
       modulesInDeployWindow: deployWindows,
+      pausedModules: paused,
       topology,
     });
     const wasSuppressed = alert.state === 'suppressed';
     if (suppressor && !wasSuppressed) {
       markSuppressed(db, alert.id, {
         alertId: suppressor.kind === 'alert' ? suppressor.key : undefined,
-        windowId: suppressor.kind === 'deploy_window' ? suppressor.moduleId : undefined,
+        // A pause is recorded on the same column as a deploy window: both are
+        // "a module-scoped condition explains this", and the operator reads the
+        // module id either way.
+        windowId:
+          suppressor.kind === 'deploy_window' || suppressor.kind === 'paused'
+            ? suppressor.moduleId
+            : undefined,
       });
       report.suppressed++;
     } else if (!suppressor && wasSuppressed) {

package/src/services/audit/backup-source.ts

@@ -9,8 +9,10 @@
 
 import { eq } from 'drizzle-orm';
 import type { DbClient } from '../../db/client';
-import { backups, modules } from '../../db/schema';
+import { backups, moduleConfigs, modules } from '../../db/schema';
 import type { ModuleManifest } from '../../manifest/schema';
+import { BACKUP_SCHEDULE_CONFIG_KEY } from '../backup-schedule';
+import { parseStoredConfigValue } from '../module-config';
 import type { InstalledModuleBackupInfo } from './backups';
 
 const DEPLOYED_STATES = ['INSTALLED', 'VERIFIED'];
@@ -40,6 +42,7 @@ function latestSuccessfulBackupByModule(db: DbClient): Map<string, number> {
 
 export function loadBackupAuditInfo(db: DbClient): InstalledModuleBackupInfo[] {
   const latest = latestSuccessfulBackupByModule(db);
+  const overrides = backupScheduleOverrides(db);
   return db
     .select()
     .from(modules)
@@ -49,6 +52,26 @@ export function loadBackupAuditInfo(db: DbClient): InstalledModuleBackupInfo[] {
       id: module.id,
       state: module.state,
       manifest: module.manifestData as ModuleManifest,
+      scheduleOverride: overrides.get(module.id),
       lastSuccessfulBackupAt: latest.get(module.id) ?? null,
     }));
 }
+
+/**
+ * Every operator backup-cadence override, by module.
+ *
+ * Read here rather than resolved here: the audit resolves override-against-
+ * manifest through the same accessor the backup sweep uses, so the two cannot
+ * disagree about what a module's cadence is.
+ */
+export function backupScheduleOverrides(db: DbClient): Map<string, string> {
+  const overrides = new Map<string, string>();
+  for (const row of db
+    .select()
+    .from(moduleConfigs)
+    .where(eq(moduleConfigs.key, BACKUP_SCHEDULE_CONFIG_KEY))
+    .all()) {
+    overrides.set(row.moduleId, String(parseStoredConfigValue(row)));
+  }
+  return overrides;
+}

package/src/services/audit/backups.test.ts

@@ -1,19 +1,19 @@
 import { describe, expect, test } from 'bun:test';
 import type { ModuleManifest } from '../../manifest/schema';
-import { type InstalledModuleBackupInfo, auditBackups } from './backups';
+import { parseCadence } from '../cadence';
+import { type InstalledModuleBackupInfo, auditBackups, backupStaleThresholdMs } from './backups';
 
 const NOW = new Date('2026-04-25T00:00:00Z').getTime();
 const HOUR = 60 * 60 * 1000;
 const ONE_DAY = 24 * HOUR;
 
-type Schedule = 'hourly' | 'daily' | 'weekly' | 'monthly' | 'manual';
-
 function makeModule(
   id: string,
   opts: {
     hasBackupHook: boolean;
     lastSuccessfulBackupAt: number | null;
-    schedule?: Schedule;
+    schedule?: string;
+    scheduleOverride?: string;
   },
 ): InstalledModuleBackupInfo {
   const manifest = {
@@ -27,9 +27,42 @@ function makeModule(
   // Default to INSTALLED — existing tests assert backup findings
   // fire, which is the deployed-module behavior. Tests for the
   // non-deployed-skip behavior override this explicitly.
-  return { id, state: 'INSTALLED', manifest, lastSuccessfulBackupAt: opts.lastSuccessfulBackupAt };
+  return {
+    id,
+    state: 'INSTALLED',
+    manifest,
+    scheduleOverride: opts.scheduleOverride,
+    lastSuccessfulBackupAt: opts.lastSuccessfulBackupAt,
+  };
 }
 
+describe('backupStaleThresholdMs', () => {
+  function cadence(value: string) {
+    const parsed = parseCadence(value);
+    if (parsed === null) throw new Error(`test fixture is not a cadence: ${value}`);
+    return parsed;
+  }
+
+  // The four values the change's release note promises. A table could not
+  // answer for `6h`, so these come out of `cadence + max(1h, cadence × 0.1)`;
+  // `weekly` is the only one that now alerts EARLIER than it used to (8d).
+  test('the four documented thresholds', () => {
+    expect(backupStaleThresholdMs(cadence('hourly'))).toBe(2 * HOUR);
+    expect(backupStaleThresholdMs(cadence('daily'))).toBe(26.4 * HOUR);
+    expect(backupStaleThresholdMs(cadence('weekly'))).toBe(7.7 * ONE_DAY);
+    expect(backupStaleThresholdMs(cadence('monthly'))).toBe(33 * ONE_DAY);
+  });
+
+  test('a custom duration has a defined threshold', () => {
+    expect(backupStaleThresholdMs(cadence('6h'))).toBe(7 * HOUR);
+    expect(backupStaleThresholdMs(cadence('90m'))).toBe(90 * 60_000 + HOUR);
+  });
+
+  test('manual has none — an opted-out module is never stale', () => {
+    expect(backupStaleThresholdMs('manual')).toBeNull();
+  });
+});
+
 describe('auditBackups', () => {
   test('skips modules without an on_backup hook', async () => {
     const result = await auditBackups({
@@ -39,7 +72,10 @@ describe('auditBackups', () => {
     expect(result).toEqual([]);
   });
 
-  test('reports missing backup regardless of schedule (still want at least one)', async () => {
+  // A module opted out of scheduled backups used to be reported as missing one
+  // forever: the never-backed-up check fired BEFORE the manual check, and the
+  // only remediation offered was the very thing the operator declined.
+  test('a module opted out is not reported as missing a backup', async () => {
     const result = await auditBackups({
       modules: [
         makeModule('lunacycle', {
@@ -51,14 +87,63 @@ describe('auditBackups', () => {
       now: () => NOW,
     });
 
+    expect(result).toEqual([]);
+  });
+
+  test('a module that has NOT opted out is still reported as missing a backup', async () => {
+    const result = await auditBackups({
+      modules: [
+        makeModule('authentik', {
+          hasBackupHook: true,
+          lastSuccessfulBackupAt: null,
+          schedule: 'daily',
+        }),
+      ],
+      now: () => NOW,
+    });
+
     expect(result).toHaveLength(1);
     expect(result[0]).toMatchObject({
       category: 'backups',
       severity: 'drift',
       code: 'backup_missing',
-      subject: 'lunacycle',
+      subject: 'authentik',
+    });
+  });
+
+  test("an operator's manual override silences a module the manifest wanted backed up", async () => {
+    const result = await auditBackups({
+      modules: [
+        makeModule('authentik', {
+          hasBackupHook: true,
+          lastSuccessfulBackupAt: null,
+          schedule: 'daily',
+          scheduleOverride: 'manual',
+        }),
+      ],
+      now: () => NOW,
+    });
+
+    expect(result).toEqual([]);
+  });
+
+  test('staleness is judged against the override, not the suggestion', async () => {
+    // Manifest says weekly (fresh at 2 days); the operator asked for hourly.
+    const result = await auditBackups({
+      modules: [
+        makeModule('caddy', {
+          hasBackupHook: true,
+          lastSuccessfulBackupAt: NOW - 2 * ONE_DAY,
+          schedule: 'weekly',
+          scheduleOverride: 'hourly',
+        }),
+      ],
+      now: () => NOW,
     });
-    expect(result[0].message).toContain('manual');
+
+    expect(result).toHaveLength(1);
+    expect(result[0].code).toBe('backup_stale');
+    expect(result[0].message).toContain('hourly');
   });
 
   test('manual schedule: never flags stale (user-driven cadence)', async () => {
@@ -93,12 +178,12 @@ describe('auditBackups', () => {
     expect(result[0].message).toContain('daily');
   });
 
-  test('daily schedule: 26h-old is stale', async () => {
+  test('daily schedule: 27h-old is stale', async () => {
     const result = await auditBackups({
       modules: [
         makeModule('authentik', {
           hasBackupHook: true,
-          lastSuccessfulBackupAt: NOW - 26 * HOUR,
+          lastSuccessfulBackupAt: NOW - 27 * HOUR,
           schedule: 'daily',
         }),
       ],