@celilo/cli 0.23.0 → 0.24.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/CELILO_CORE_MODULES.md +2 -2
  2. package/CELILO_SUBSYSTEMS.md +27 -7
  3. package/package.json +6 -5
  4. package/src/cli/commands/alerts-act.ts +1 -1
  5. package/src/cli/commands/backup-create.ts +26 -11
  6. package/src/cli/commands/backup-list.test.ts +83 -0
  7. package/src/cli/commands/backup-list.ts +67 -3
  8. package/src/cli/commands/backup-prune.ts +17 -17
  9. package/src/cli/commands/backup-sweep.ts +20 -8
  10. package/src/cli/commands/firewall-interface-list.test.ts +85 -0
  11. package/src/cli/commands/firewall-interface-list.ts +123 -0
  12. package/src/cli/commands/machine-add.ts +30 -2
  13. package/src/cli/commands/module-config.test.ts +70 -3
  14. package/src/cli/commands/module-config.ts +262 -28
  15. package/src/cli/commands/module-status.ts +155 -12
  16. package/src/cli/commands/monitor.ts +116 -19
  17. package/src/cli/commands/system-migrate.ts +14 -0
  18. package/src/cli/commands/system-update.ts +4 -1
  19. package/src/cli/completion.ts +35 -9
  20. package/src/cli/index.ts +59 -2
  21. package/src/cli/tui/audit-state.ts +2 -0
  22. package/src/hooks/capability-loader.ts +130 -4
  23. package/src/hooks/load-hook-config.test.ts +169 -1
  24. package/src/hooks/load-hook-config.ts +118 -20
  25. package/src/hooks/types.ts +2 -1
  26. package/src/manifest/contracts/v1.ts +16 -0
  27. package/src/manifest/schema.ts +40 -65
  28. package/src/services/alerting/builtin-monitors.test.ts +18 -10
  29. package/src/services/alerting/cadence-migration.test.ts +155 -0
  30. package/src/services/alerting/cadence-migration.ts +90 -0
  31. package/src/services/alerting/coverage-source.ts +8 -11
  32. package/src/services/alerting/deploy-hooks.test.ts +16 -7
  33. package/src/services/alerting/deploy-hooks.ts +11 -5
  34. package/src/services/alerting/health-cadence.test.ts +58 -0
  35. package/src/services/alerting/health-cadence.ts +128 -0
  36. package/src/services/alerting/health-coverage.ts +18 -8
  37. package/src/services/alerting/monitors.ts +50 -15
  38. package/src/services/alerting/sweep-runner.test.ts +51 -3
  39. package/src/services/alerting/sweep-runner.ts +30 -7
  40. package/src/services/audit/backup-source.ts +24 -1
  41. package/src/services/audit/backups.test.ts +95 -10
  42. package/src/services/audit/backups.ts +40 -37
  43. package/src/services/audit/interface-classification.test.ts +220 -0
  44. package/src/services/audit/interface-classification.ts +167 -0
  45. package/src/services/audit/types.ts +2 -1
  46. package/src/services/backup-age-agreement.test.ts +118 -0
  47. package/src/services/backup-create.ts +36 -30
  48. package/src/services/backup-metadata.ts +52 -1
  49. package/src/services/backup-retention.test.ts +123 -0
  50. package/src/services/backup-retention.ts +66 -5
  51. package/src/services/backup-schedule.test.ts +166 -0
  52. package/src/services/backup-schedule.ts +105 -15
  53. package/src/services/backup-staging.ts +14 -1
  54. package/src/services/backup-sweep.test.ts +22 -3
  55. package/src/services/backup-sweep.ts +15 -5
  56. package/src/services/cadence.test.ts +97 -0
  57. package/src/services/cadence.ts +165 -0
  58. package/src/services/config-provenance.test.ts +155 -0
  59. package/src/services/config-provenance.ts +104 -0
  60. package/src/services/machine-detector.ts +23 -1
  61. package/src/services/module-config.ts +33 -0
  62. package/src/services/storage-providers/s3.test.ts +96 -13
  63. package/src/services/storage-providers/s3.ts +48 -15
  64. package/src/services/zone-detector.test.ts +34 -3
  65. package/src/services/zone-detector.ts +33 -13
  66. package/src/variables/context.ts +69 -15
  67. package/src/variables/declarative-derivation.test.ts +53 -0
  68. package/src/variables/declarative-derivation.ts +13 -2
@@ -0,0 +1,128 @@
1
+ /**
2
+ * How often celilo health-checks a module.
3
+ *
4
+ * The same shape as the backup cadence, and for the same reason. The manifest's
5
+ * `hooks.health_check.interval` is the author's SUGGESTION; the operator's
6
+ * `health_check_interval` override decides; both resolve HERE, at read time.
7
+ *
8
+ * What this replaces is worth stating, because it failed silently in both
9
+ * directions. The cadence used to be seeded onto the `monitors` row at first
10
+ * deploy and never reconsulted (`monitors.ts` returned early if a row existed),
11
+ * so an author who corrected a bad interval never reached an existing install,
12
+ * with nothing an operator could read to discover it. And since nothing ever
13
+ * wrote `monitors.intervalMinutes` after that insert, an operator's only way to
14
+ * re-cadence a module was raw SQL against `celilo.db`.
15
+ *
16
+ * `manual` means the operator has stopped watching the module. That is a
17
+ * decision, not a gap: it raises no health-coverage finding, because a finding
18
+ * asking for the action they just declined is one no action can clear.
19
+ */
20
+
21
+ import { eq } from 'drizzle-orm';
22
+ import type { DbClient } from '../../db/client';
23
+ import { moduleConfigs, modules } from '../../db/schema';
24
+ import type { ModuleManifest } from '../../manifest/schema';
25
+ import { type Cadence, parseCadence } from '../cadence';
26
+ import { getModuleConfigValue, parseStoredConfigValue } from '../module-config';
27
+ import { findMonitor, resolveMonitorAlerts } from './monitors';
28
+
29
+ /** The `module_configs` key an operator's health-check cadence is stored under. */
30
+ export const HEALTH_CHECK_INTERVAL_CONFIG_KEY = 'health_check_interval';
31
+
32
+ /**
33
+ * `null` means NOBODY has said how often — the module is unscheduled and that
34
+ * is a coverage gap. It is distinct from `'manual'`, which is an operator
35
+ * saying not to watch it. There is no default: an interval celilo invented
36
+ * would run someone's health hook on a cadence no one chose.
37
+ *
38
+ * An unparseable override falls back to the manifest, the same direction the
39
+ * backup cadence fails in — values are validated at SET time, so a bad one here
40
+ * means hand-edited state, and continuing to watch beats going quiet.
41
+ */
42
+ export function effectiveHealthCheckCadence(
43
+ manifest: ModuleManifest,
44
+ override: string | undefined,
45
+ ): Cadence | null {
46
+ if (override !== undefined) {
47
+ const chosen = parseCadence(override);
48
+ if (chosen !== null) return chosen;
49
+ }
50
+ const suggested = manifest.hooks?.health_check?.interval;
51
+ if (suggested !== undefined) {
52
+ const parsed = parseCadence(suggested);
53
+ if (parsed !== null) return parsed;
54
+ }
55
+ return null;
56
+ }
57
+
58
+ /** Whether a resolved cadence schedules anything at all. */
59
+ export function isScheduled(cadence: Cadence | null): cadence is { minutes: number } {
60
+ return cadence !== null && cadence !== 'manual';
61
+ }
62
+
63
+ export interface ModuleHealthCadence {
64
+ moduleId: string;
65
+ manifest: ModuleManifest;
66
+ state: string;
67
+ cadence: Cadence | null;
68
+ }
69
+
70
+ /**
71
+ * Every module's effective health-check cadence, in one pass.
72
+ *
73
+ * Read here rather than resolved per call site (Rule 2.3): the alerting sweep,
74
+ * the coverage check and `monitor list` all need the same answer, and three
75
+ * queries that could disagree is the shape this change exists to remove.
76
+ */
77
+ export function loadModuleHealthCadences(db: DbClient): Map<string, ModuleHealthCadence> {
78
+ const overrides = new Map<string, string>();
79
+ for (const row of db
80
+ .select()
81
+ .from(moduleConfigs)
82
+ .where(eq(moduleConfigs.key, HEALTH_CHECK_INTERVAL_CONFIG_KEY))
83
+ .all()) {
84
+ overrides.set(row.moduleId, String(parseStoredConfigValue(row)));
85
+ }
86
+
87
+ const cadences = new Map<string, ModuleHealthCadence>();
88
+ for (const module of db.select().from(modules).all()) {
89
+ const manifest = module.manifestData as ModuleManifest;
90
+ cadences.set(module.id, {
91
+ moduleId: module.id,
92
+ manifest,
93
+ state: module.state,
94
+ cadence: effectiveHealthCheckCadence(manifest, overrides.get(module.id)),
95
+ });
96
+ }
97
+ return cadences;
98
+ }
99
+
100
+ /**
101
+ * Bring a module's watch state into line with its effective cadence.
102
+ *
103
+ * Only one direction needs doing: a module that has just become `manual` may
104
+ * still own live alerts from its last scheduled runs, and nothing will ever
105
+ * report on them again — they would sit firing forever with no operator action
106
+ * able to clear them. This resolves them, the way disabling a monitor always
107
+ * has.
108
+ *
109
+ * Idempotent, and safe to call when nothing changed. Called wherever a
110
+ * `health_check_interval` override is written: `module config set`/`unset`, and
111
+ * the migrate step.
112
+ */
113
+ export function reconcileModuleWatchState(db: DbClient, moduleId: string, now: Date): number {
114
+ const monitor = findMonitor(db, 'module_hook', moduleId);
115
+ if (!monitor) return 0;
116
+
117
+ const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
118
+ if (!module) return 0;
119
+
120
+ const override = getModuleConfigValue(moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, db);
121
+ const cadence = effectiveHealthCheckCadence(
122
+ module.manifestData as ModuleManifest,
123
+ override === null ? undefined : String(override.value),
124
+ );
125
+ if (isScheduled(cadence)) return 0;
126
+
127
+ return resolveMonitorAlerts(db, monitor.id, now);
128
+ }
@@ -10,7 +10,13 @@
10
10
  *
11
11
  * Two ways that happens, and both are worth saying out loud:
12
12
  * - the module ships no `health_check` hook, so it can never be verified;
13
- * - it has one, but no enabled monitor schedules it.
13
+ * - it has one, but nothing gives it a cadence, so nothing schedules it.
14
+ *
15
+ * An effective cadence of `manual` raises NOTHING. The check exists to catch
16
+ * blindness nobody chose; `manual` is a recorded decision, visible in `module
17
+ * status` and removable with one command. A finding raised by an operator's
18
+ * explicit opt-out asks for the action they just declined, so no action of
19
+ * theirs could ever clear it.
14
20
  *
15
21
  * Deliberately cheap: this reads the module roster against the monitor set and
16
22
  * contacts nothing. It is safe to run on every sweep.
@@ -19,6 +25,8 @@
19
25
  */
20
26
 
21
27
  import type { ModuleState } from '../../db/schema';
28
+ import type { Cadence } from '../cadence';
29
+ import { isScheduled } from './health-cadence';
22
30
  import { type FailingKey, builtinAlertKey } from './keys';
23
31
 
24
32
  export const HEALTH_COVERAGE_CHECK = 'health_coverage';
@@ -35,8 +43,11 @@ export interface ModuleCoverageInput {
35
43
  state: ModuleState;
36
44
  /** Whether the manifest declares a `health_check` hook at all. */
37
45
  hasHealthCheckHook: boolean;
38
- /** Whether an ENABLED monitor schedules that hook. */
39
- hasEnabledMonitor: boolean;
46
+ /**
47
+ * The module's effective health-check cadence: `null` when nobody has named
48
+ * one (a gap), `'manual'` when the operator opted out (a decision).
49
+ */
50
+ cadence: Cadence | null;
40
51
  }
41
52
 
42
53
  /**
@@ -65,15 +76,14 @@ export function healthCoverageFailingKeys(modules: ModuleCoverageInput[]): Faili
65
76
  continue;
66
77
  }
67
78
 
68
- if (!module.hasEnabledMonitor) {
79
+ if (module.cadence === 'manual') continue;
80
+
81
+ if (!isScheduled(module.cadence)) {
69
82
  failing.push({
70
83
  key,
71
84
  severity: 'warning',
72
85
  message: `${module.id} has a health_check hook but nothing schedules it`,
73
- details:
74
- 'The module can be verified on demand but is not being watched.\n' +
75
- 'Create a monitor for it, or set hooks.health_check.interval in the\n' +
76
- 'module manifest so one is created on deploy.',
86
+ details: `The module can be verified on demand but is not being watched.\nGive it a cadence: celilo module config set ${module.id} health_check_interval 15m\nor set hooks.health_check.interval in the module manifest.`,
77
87
  });
78
88
  }
79
89
  }
@@ -1,10 +1,20 @@
1
1
  /**
2
2
  * Monitor CRUD.
3
3
  *
4
- * A monitor binds a thing to check to a cadence, a severity, and an escalation
5
- * policy. Operators address them by their target (`caddy`,
6
- * `machines_reachable`) rather than by id — per CLAUDE.md, a UUID never
7
- * reaches the operator.
4
+ * A monitor binds a thing to check to a severity and an escalation policy.
5
+ * Operators address them by their target (`caddy`, `machines_reachable`) rather
6
+ * than by id — per CLAUDE.md, a UUID never reaches the operator.
7
+ *
8
+ * ⚠️ `intervalMinutes` and `enabled` are meaningful for `builtin_check` rows
9
+ * ONLY. A built-in check targets a fleet-level audit category with no module
10
+ * and no manifest, so nothing suggests its cadence and the row legitimately IS
11
+ * the config. A `module_hook` row targets a module whose manifest MAY suggest
12
+ * one, so its cadence and whether it is watched at all resolve at read time
13
+ * from `health_check_interval` — see [[services/alerting/health-cadence.ts]].
14
+ * Two columns whose meaning depends on `kind` is a smell, named here rather
15
+ * than discovered in review (design.md D8); the alternatives are a resolved
16
+ * value cached on the row, which rots, or splitting the table, which needs a
17
+ * synthetic monitor identity for `alerts.monitorId` and friends.
8
18
  */
9
19
 
10
20
  import { randomUUID } from 'node:crypto';
@@ -17,11 +27,12 @@ import {
17
27
  alerts,
18
28
  monitors,
19
29
  } from '../../db/schema';
20
- import { parseIntervalMinutes } from '../../manifest/schema';
30
+ import { ALERTING_SWEEP_PATTERN, MONITOR_INTERVAL_FLOOR_MINUTES, parseCadence } from '../cadence';
21
31
 
22
32
  export interface CreateMonitorInput {
23
33
  kind: MonitorKind;
24
34
  target: string;
35
+ /** `builtin_check` only. Not consulted for `module_hook` — see the file header. */
25
36
  intervalMinutes: number;
26
37
  severity?: AlertSeverity;
27
38
  suppressible?: boolean;
@@ -69,29 +80,50 @@ export function createMonitor(db: DbClient, input: CreateMonitorInput): Monitor
69
80
  }
70
81
 
71
82
  /**
72
- * Ensure a monitor exists for a module that declares a suggested interval.
83
+ * Ensure a `module_hook` monitor row exists for a module that declares a
84
+ * `health_check` hook.
85
+ *
86
+ * The row is no longer where the cadence lives — it carries severity, the
87
+ * escalation policy and `lastRunAt`, and whether it runs at all is resolved
88
+ * from the module's effective cadence at sweep time. So it is created for any
89
+ * module with the hook, not only for one whose manifest happens to suggest an
90
+ * interval: an operator can now set a cadence on a module whose author never
91
+ * named one, and without a row there would be nothing to carry its last run.
73
92
  *
74
- * Called on deploy. Never overwrites an existing monitor: the manifest's
75
- * interval is the author's SUGGESTION and the operator's setting must survive
76
- * module upgrades, so a second call is a no-op rather than a reset.
93
+ * The stored `intervalMinutes` is the manifest's suggestion where there is one,
94
+ * and is NOT read back for this kind. It is written so that rolling back to a
95
+ * release which does read it behaves as it did before.
77
96
  *
78
- * Returns the monitor when one was created, null when one already existed or
79
- * the manifest declares no interval.
97
+ * Idempotent: a second call is a no-op, which is what keeps deploys from
98
+ * duplicating rows. It is no longer what protects an operator's setting — that
99
+ * lives in `module_configs` now and cannot be overwritten from here at all.
80
100
  */
81
101
  export function ensureMonitorForModule(
82
102
  db: DbClient,
83
103
  moduleId: string,
84
104
  suggestedInterval: string | undefined,
85
105
  ): Monitor | null {
86
- if (!suggestedInterval) return null;
87
106
  if (findMonitor(db, 'module_hook', moduleId)) return null;
88
107
 
89
- const intervalMinutes = parseIntervalMinutes(suggestedInterval);
90
- if (intervalMinutes === null) return null;
108
+ const suggested = suggestedInterval ? parseCadence(suggestedInterval) : null;
109
+ const intervalMinutes =
110
+ suggested !== null && suggested !== 'manual'
111
+ ? suggested.minutes
112
+ : MONITOR_INTERVAL_FLOOR_MINUTES;
91
113
 
92
114
  return createMonitor(db, { kind: 'module_hook', target: moduleId, intervalMinutes });
93
115
  }
94
116
 
117
+ /**
118
+ * Change a `builtin_check` monitor's cadence in place.
119
+ *
120
+ * In place, rather than remove-and-recreate, because the monitor id owns the
121
+ * alert history: recreating it would orphan every live alert it raised.
122
+ */
123
+ export function updateMonitorInterval(db: DbClient, monitorId: string, minutes: number): void {
124
+ db.update(monitors).set({ intervalMinutes: minutes }).where(eq(monitors.id, monitorId)).run();
125
+ }
126
+
95
127
  /**
96
128
  * Enable or disable a monitor.
97
129
  *
@@ -139,7 +171,10 @@ export function resolveMonitorAlerts(db: DbClient, monitorId: string, now: Date)
139
171
  * what switches on the sweep.
140
172
  */
141
173
  export const ALERTING_SWEEP_SUBSCRIBER = 'celilo-alerting-sweep';
142
- export const ALERTING_SWEEP_PATTERN = 'timer.tick.5m';
174
+ // The tick itself lives in services/cadence.ts, next to the floor derived from
175
+ // it — a sweep whose tick and whose finest servable cadence are stated in two
176
+ // files is the pair that drifts.
177
+ export { ALERTING_SWEEP_PATTERN };
143
178
 
144
179
  export interface SubscriberRegistrar {
145
180
  subscribe(options: {
@@ -4,9 +4,11 @@ import { tmpdir } from 'node:os';
4
4
  import { join } from 'node:path';
5
5
  import { eq } from 'drizzle-orm';
6
6
  import type { DbClient } from '../../db/client';
7
- import { type Alert, type Monitor, alerts, monitors } from '../../db/schema';
7
+ import { type Alert, type Monitor, alerts, modules, monitors } from '../../db/schema';
8
8
  import { setupTestDatabase } from '../../test-utils/setup-test-db';
9
9
  import type { HealthCheckResult } from '../health-runner';
10
+ import { deleteModuleConfig, upsertModuleConfig } from '../module-config';
11
+ import { HEALTH_CHECK_INTERVAL_CONFIG_KEY, reconcileModuleWatchState } from './health-cadence';
10
12
  import { moduleCheckAlertKey } from './keys';
11
13
  import type { MonitorRunDeps } from './run-monitor';
12
14
  import { type SuppressionTopology, machineAlertKey } from './suppression';
@@ -67,6 +69,21 @@ describe('runSweep', () => {
67
69
  const dbPath = join(dir, 'celilo.db');
68
70
  process.env.CELILO_DB_PATH = dbPath;
69
71
  db = await setupTestDatabase(dbPath);
72
+ // The module row is load-bearing now: a `module_hook` monitor's cadence
73
+ // resolves from the module's manifest and config, not from its own row.
74
+ db.insert(modules)
75
+ .values({
76
+ id: MODULE,
77
+ name: MODULE,
78
+ sourcePath: dir,
79
+ version: '1.0.0',
80
+ state: 'INSTALLED',
81
+ manifestData: {
82
+ id: MODULE,
83
+ hooks: { health_check: { script: './h.ts', interval: '15m' } },
84
+ },
85
+ })
86
+ .run();
70
87
  db.insert(monitors)
71
88
  .values({
72
89
  id: 'mon-1',
@@ -117,11 +134,42 @@ describe('runSweep', () => {
117
134
  expect(report.monitorsRun).toBe(0);
118
135
  });
119
136
 
120
- test('a disabled monitor is never run', async () => {
137
+ // `enabled` and `intervalMinutes` are builtin_check-only now (design.md D8).
138
+ // A future reader that reintroduces the dependency on the row fails here
139
+ // rather than silently regressing to write-time resolution.
140
+ test("a module monitor's stored enabled flag is not consulted", async () => {
121
141
  db.update(monitors).set({ enabled: false }).where(eq(monitors.id, 'mon-1')).run();
122
142
  const report = await runSweep(db, currentMonitors(), deps());
143
+ expect(report.monitorsRun).toBe(1);
144
+ });
145
+
146
+ test("a module monitor's stored interval is not consulted", async () => {
147
+ // The row says a full day; the manifest says 15m, which is what wins.
148
+ db.update(monitors).set({ intervalMinutes: 1440 }).where(eq(monitors.id, 'mon-1')).run();
149
+ await runSweep(db, currentMonitors(), deps());
150
+ const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
151
+ expect(report.monitorsRun).toBe(1);
152
+ });
153
+
154
+ test('a module whose effective cadence is manual is never run, and its alerts resolve', async () => {
155
+ await runSweep(db, currentMonitors(), deps());
156
+ expect(liveAlerts()).not.toEqual([]);
157
+
158
+ upsertModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual');
159
+ reconcileModuleWatchState(db, MODULE, later(20));
160
+
161
+ const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
123
162
  expect(report.monitorsRun).toBe(0);
124
- expect(liveAlerts()).toEqual([]);
163
+ expect(liveAlerts().every((a) => a.state === 'resolved')).toBe(true);
164
+ });
165
+
166
+ test('unsetting the override resumes watching', async () => {
167
+ upsertModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual');
168
+ expect((await runSweep(db, currentMonitors(), deps())).monitorsRun).toBe(0);
169
+
170
+ deleteModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY);
171
+ const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
172
+ expect(report.monitorsRun).toBe(1);
125
173
  });
126
174
 
127
175
  test('recovery resolves the alert on a later sweep', async () => {
@@ -21,6 +21,8 @@
21
21
 
22
22
  import type { DbClient } from '../../db/client';
23
23
  import type { Alert, Monitor } from '../../db/schema';
24
+ import { MONITOR_INTERVAL_FLOOR_MINUTES } from '../cadence';
25
+ import { isScheduled, loadModuleHealthCadences } from './health-cadence';
24
26
  import type { NotifyDeps, NotifyOutcome } from './notifier';
25
27
  import { deliverDeferred, notifyAlert } from './notifier';
26
28
  import { type MonitorRunDeps, runOneMonitor } from './run-monitor';
@@ -124,14 +126,35 @@ export async function runSweep(
124
126
  };
125
127
 
126
128
  // 1. Run due monitors.
129
+ //
130
+ // A `module_hook` monitor's cadence and whether it is watched at all come
131
+ // from the module's effective health-check cadence, NOT from its row: the row
132
+ // was seeded once at first deploy and never reconsulted, so a corrected
133
+ // manifest could never reach an existing install (design.md D2/D8). A
134
+ // `builtin_check` has no module and no manifest, so its row is the config.
135
+ const cadences = loadModuleHealthCadences(db);
127
136
  const due = selectDueMonitors(
128
- monitors.map((m) => ({
129
- id: m.id,
130
- intervalMinutes: m.intervalMinutes,
131
- enabled: m.enabled,
132
- lastRunAt: m.lastRunAt,
133
- monitor: m,
134
- })),
137
+ monitors.map((m) => {
138
+ if (m.kind !== 'module_hook') {
139
+ return {
140
+ id: m.id,
141
+ intervalMinutes: m.intervalMinutes,
142
+ enabled: m.enabled,
143
+ lastRunAt: m.lastRunAt,
144
+ monitor: m,
145
+ };
146
+ }
147
+ const cadence = cadences.get(m.target)?.cadence ?? null;
148
+ return {
149
+ id: m.id,
150
+ // Unscheduled monitors are filtered out by `enabled` below, so this
151
+ // value is never used to decide due-ness for them.
152
+ intervalMinutes: isScheduled(cadence) ? cadence.minutes : MONITOR_INTERVAL_FLOOR_MINUTES,
153
+ enabled: isScheduled(cadence),
154
+ lastRunAt: m.lastRunAt,
155
+ monitor: m,
156
+ };
157
+ }),
135
158
  deps.now(),
136
159
  );
137
160
 
@@ -9,8 +9,10 @@
9
9
 
10
10
  import { eq } from 'drizzle-orm';
11
11
  import type { DbClient } from '../../db/client';
12
- import { backups, modules } from '../../db/schema';
12
+ import { backups, moduleConfigs, modules } from '../../db/schema';
13
13
  import type { ModuleManifest } from '../../manifest/schema';
14
+ import { BACKUP_SCHEDULE_CONFIG_KEY } from '../backup-schedule';
15
+ import { parseStoredConfigValue } from '../module-config';
14
16
  import type { InstalledModuleBackupInfo } from './backups';
15
17
 
16
18
  const DEPLOYED_STATES = ['INSTALLED', 'VERIFIED'];
@@ -40,6 +42,7 @@ function latestSuccessfulBackupByModule(db: DbClient): Map<string, number> {
40
42
 
41
43
  export function loadBackupAuditInfo(db: DbClient): InstalledModuleBackupInfo[] {
42
44
  const latest = latestSuccessfulBackupByModule(db);
45
+ const overrides = backupScheduleOverrides(db);
43
46
  return db
44
47
  .select()
45
48
  .from(modules)
@@ -49,6 +52,26 @@ export function loadBackupAuditInfo(db: DbClient): InstalledModuleBackupInfo[] {
49
52
  id: module.id,
50
53
  state: module.state,
51
54
  manifest: module.manifestData as ModuleManifest,
55
+ scheduleOverride: overrides.get(module.id),
52
56
  lastSuccessfulBackupAt: latest.get(module.id) ?? null,
53
57
  }));
54
58
  }
59
+
60
+ /**
61
+ * Every operator backup-cadence override, by module.
62
+ *
63
+ * Read here rather than resolved here: the audit resolves override-against-
64
+ * manifest through the same accessor the backup sweep uses, so the two cannot
65
+ * disagree about what a module's cadence is.
66
+ */
67
+ export function backupScheduleOverrides(db: DbClient): Map<string, string> {
68
+ const overrides = new Map<string, string>();
69
+ for (const row of db
70
+ .select()
71
+ .from(moduleConfigs)
72
+ .where(eq(moduleConfigs.key, BACKUP_SCHEDULE_CONFIG_KEY))
73
+ .all()) {
74
+ overrides.set(row.moduleId, String(parseStoredConfigValue(row)));
75
+ }
76
+ return overrides;
77
+ }
@@ -1,19 +1,19 @@
1
1
  import { describe, expect, test } from 'bun:test';
2
2
  import type { ModuleManifest } from '../../manifest/schema';
3
- import { type InstalledModuleBackupInfo, auditBackups } from './backups';
3
+ import { parseCadence } from '../cadence';
4
+ import { type InstalledModuleBackupInfo, auditBackups, backupStaleThresholdMs } from './backups';
4
5
 
5
6
  const NOW = new Date('2026-04-25T00:00:00Z').getTime();
6
7
  const HOUR = 60 * 60 * 1000;
7
8
  const ONE_DAY = 24 * HOUR;
8
9
 
9
- type Schedule = 'hourly' | 'daily' | 'weekly' | 'monthly' | 'manual';
10
-
11
10
  function makeModule(
12
11
  id: string,
13
12
  opts: {
14
13
  hasBackupHook: boolean;
15
14
  lastSuccessfulBackupAt: number | null;
16
- schedule?: Schedule;
15
+ schedule?: string;
16
+ scheduleOverride?: string;
17
17
  },
18
18
  ): InstalledModuleBackupInfo {
19
19
  const manifest = {
@@ -27,9 +27,42 @@ function makeModule(
27
27
  // Default to INSTALLED — existing tests assert backup findings
28
28
  // fire, which is the deployed-module behavior. Tests for the
29
29
  // non-deployed-skip behavior override this explicitly.
30
- return { id, state: 'INSTALLED', manifest, lastSuccessfulBackupAt: opts.lastSuccessfulBackupAt };
30
+ return {
31
+ id,
32
+ state: 'INSTALLED',
33
+ manifest,
34
+ scheduleOverride: opts.scheduleOverride,
35
+ lastSuccessfulBackupAt: opts.lastSuccessfulBackupAt,
36
+ };
31
37
  }
32
38
 
39
+ describe('backupStaleThresholdMs', () => {
40
+ function cadence(value: string) {
41
+ const parsed = parseCadence(value);
42
+ if (parsed === null) throw new Error(`test fixture is not a cadence: ${value}`);
43
+ return parsed;
44
+ }
45
+
46
+ // The four values the change's release note promises. A table could not
47
+ // answer for `6h`, so these come out of `cadence + max(1h, cadence × 0.1)`;
48
+ // `weekly` is the only one that now alerts EARLIER than it used to (8d).
49
+ test('the four documented thresholds', () => {
50
+ expect(backupStaleThresholdMs(cadence('hourly'))).toBe(2 * HOUR);
51
+ expect(backupStaleThresholdMs(cadence('daily'))).toBe(26.4 * HOUR);
52
+ expect(backupStaleThresholdMs(cadence('weekly'))).toBe(7.7 * ONE_DAY);
53
+ expect(backupStaleThresholdMs(cadence('monthly'))).toBe(33 * ONE_DAY);
54
+ });
55
+
56
+ test('a custom duration has a defined threshold', () => {
57
+ expect(backupStaleThresholdMs(cadence('6h'))).toBe(7 * HOUR);
58
+ expect(backupStaleThresholdMs(cadence('90m'))).toBe(90 * 60_000 + HOUR);
59
+ });
60
+
61
+ test('manual has none — an opted-out module is never stale', () => {
62
+ expect(backupStaleThresholdMs('manual')).toBeNull();
63
+ });
64
+ });
65
+
33
66
  describe('auditBackups', () => {
34
67
  test('skips modules without an on_backup hook', async () => {
35
68
  const result = await auditBackups({
@@ -39,7 +72,10 @@ describe('auditBackups', () => {
39
72
  expect(result).toEqual([]);
40
73
  });
41
74
 
42
- test('reports missing backup regardless of schedule (still want at least one)', async () => {
75
+ // A module opted out of scheduled backups used to be reported as missing one
76
+ // forever: the never-backed-up check fired BEFORE the manual check, and the
77
+ // only remediation offered was the very thing the operator declined.
78
+ test('a module opted out is not reported as missing a backup', async () => {
43
79
  const result = await auditBackups({
44
80
  modules: [
45
81
  makeModule('lunacycle', {
@@ -51,14 +87,63 @@ describe('auditBackups', () => {
51
87
  now: () => NOW,
52
88
  });
53
89
 
90
+ expect(result).toEqual([]);
91
+ });
92
+
93
+ test('a module that has NOT opted out is still reported as missing a backup', async () => {
94
+ const result = await auditBackups({
95
+ modules: [
96
+ makeModule('authentik', {
97
+ hasBackupHook: true,
98
+ lastSuccessfulBackupAt: null,
99
+ schedule: 'daily',
100
+ }),
101
+ ],
102
+ now: () => NOW,
103
+ });
104
+
54
105
  expect(result).toHaveLength(1);
55
106
  expect(result[0]).toMatchObject({
56
107
  category: 'backups',
57
108
  severity: 'drift',
58
109
  code: 'backup_missing',
59
- subject: 'lunacycle',
110
+ subject: 'authentik',
111
+ });
112
+ });
113
+
114
+ test("an operator's manual override silences a module the manifest wanted backed up", async () => {
115
+ const result = await auditBackups({
116
+ modules: [
117
+ makeModule('authentik', {
118
+ hasBackupHook: true,
119
+ lastSuccessfulBackupAt: null,
120
+ schedule: 'daily',
121
+ scheduleOverride: 'manual',
122
+ }),
123
+ ],
124
+ now: () => NOW,
125
+ });
126
+
127
+ expect(result).toEqual([]);
128
+ });
129
+
130
+ test('staleness is judged against the override, not the suggestion', async () => {
131
+ // Manifest says weekly (fresh at 2 days); the operator asked for hourly.
132
+ const result = await auditBackups({
133
+ modules: [
134
+ makeModule('caddy', {
135
+ hasBackupHook: true,
136
+ lastSuccessfulBackupAt: NOW - 2 * ONE_DAY,
137
+ schedule: 'weekly',
138
+ scheduleOverride: 'hourly',
139
+ }),
140
+ ],
141
+ now: () => NOW,
60
142
  });
61
- expect(result[0].message).toContain('manual');
143
+
144
+ expect(result).toHaveLength(1);
145
+ expect(result[0].code).toBe('backup_stale');
146
+ expect(result[0].message).toContain('hourly');
62
147
  });
63
148
 
64
149
  test('manual schedule: never flags stale (user-driven cadence)', async () => {
@@ -93,12 +178,12 @@ describe('auditBackups', () => {
93
178
  expect(result[0].message).toContain('daily');
94
179
  });
95
180
 
96
- test('daily schedule: 26h-old is stale', async () => {
181
+ test('daily schedule: 27h-old is stale', async () => {
97
182
  const result = await auditBackups({
98
183
  modules: [
99
184
  makeModule('authentik', {
100
185
  hasBackupHook: true,
101
- lastSuccessfulBackupAt: NOW - 26 * HOUR,
186
+ lastSuccessfulBackupAt: NOW - 27 * HOUR,
102
187
  schedule: 'daily',
103
188
  }),
104
189
  ],