@celilo/cli 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CELILO_CORE_MODULES.md +2 -2
  2. package/CELILO_SUBSYSTEMS.md +27 -7
  3. package/package.json +6 -5
  4. package/src/cli/commands/alerts-act.ts +1 -1
  5. package/src/cli/commands/backup-create.ts +26 -11
  6. package/src/cli/commands/backup-list.test.ts +83 -0
  7. package/src/cli/commands/backup-list.ts +67 -3
  8. package/src/cli/commands/backup-prune.ts +17 -17
  9. package/src/cli/commands/backup-sweep.ts +20 -8
  10. package/src/cli/commands/firewall-interface-list.test.ts +85 -0
  11. package/src/cli/commands/firewall-interface-list.ts +123 -0
  12. package/src/cli/commands/machine-add.ts +30 -2
  13. package/src/cli/commands/module-config.test.ts +64 -2
  14. package/src/cli/commands/module-config.ts +159 -8
  15. package/src/cli/commands/module-status.ts +124 -0
  16. package/src/cli/commands/monitor.ts +116 -19
  17. package/src/cli/commands/system-migrate.ts +14 -0
  18. package/src/cli/commands/system-update.ts +4 -1
  19. package/src/cli/completion.ts +35 -9
  20. package/src/cli/index.ts +59 -2
  21. package/src/cli/tui/audit-state.ts +2 -0
  22. package/src/hooks/capability-loader.ts +130 -4
  23. package/src/hooks/types.ts +2 -1
  24. package/src/manifest/contracts/v1.ts +16 -0
  25. package/src/manifest/schema.ts +40 -65
  26. package/src/services/alerting/builtin-monitors.test.ts +18 -10
  27. package/src/services/alerting/cadence-migration.test.ts +155 -0
  28. package/src/services/alerting/cadence-migration.ts +90 -0
  29. package/src/services/alerting/coverage-source.ts +8 -11
  30. package/src/services/alerting/deploy-hooks.test.ts +16 -7
  31. package/src/services/alerting/deploy-hooks.ts +11 -5
  32. package/src/services/alerting/health-cadence.test.ts +58 -0
  33. package/src/services/alerting/health-cadence.ts +128 -0
  34. package/src/services/alerting/health-coverage.ts +18 -8
  35. package/src/services/alerting/monitors.ts +50 -15
  36. package/src/services/alerting/sweep-runner.test.ts +51 -3
  37. package/src/services/alerting/sweep-runner.ts +30 -7
  38. package/src/services/audit/backup-source.ts +24 -1
  39. package/src/services/audit/backups.test.ts +95 -10
  40. package/src/services/audit/backups.ts +40 -37
  41. package/src/services/audit/interface-classification.test.ts +220 -0
  42. package/src/services/audit/interface-classification.ts +167 -0
  43. package/src/services/audit/types.ts +2 -1
  44. package/src/services/backup-age-agreement.test.ts +118 -0
  45. package/src/services/backup-create.ts +36 -30
  46. package/src/services/backup-metadata.ts +52 -1
  47. package/src/services/backup-retention.test.ts +123 -0
  48. package/src/services/backup-retention.ts +66 -5
  49. package/src/services/backup-schedule.test.ts +166 -0
  50. package/src/services/backup-schedule.ts +105 -15
  51. package/src/services/backup-staging.ts +14 -1
  52. package/src/services/backup-sweep.test.ts +22 -3
  53. package/src/services/backup-sweep.ts +15 -5
  54. package/src/services/cadence.test.ts +97 -0
  55. package/src/services/cadence.ts +165 -0
  56. package/src/services/machine-detector.ts +23 -1
  57. package/src/services/module-config.ts +33 -0
  58. package/src/services/storage-providers/s3.test.ts +96 -13
  59. package/src/services/storage-providers/s3.ts +48 -15
  60. package/src/services/zone-detector.test.ts +34 -3
  61. package/src/services/zone-detector.ts +33 -13
@@ -6,21 +6,18 @@
6
6
  * lives here. See Rule 2.3.
7
7
  */
8
8
 
9
- import { eq } from 'drizzle-orm';
10
9
  import type { DbClient } from '../../db/client';
11
- import { modules, monitors } from '../../db/schema';
10
+ import { modules } from '../../db/schema';
12
11
  import type { ModuleManifest } from '../../manifest/schema';
12
+ import { loadModuleHealthCadences } from './health-cadence';
13
13
  import type { ModuleCoverageInput } from './health-coverage';
14
14
 
15
15
  export function loadModuleCoverage(db: DbClient): ModuleCoverageInput[] {
16
- const monitored = new Set(
17
- db
18
- .select({ target: monitors.target })
19
- .from(monitors)
20
- .where(eq(monitors.enabled, true))
21
- .all()
22
- .map((row) => row.target),
23
- );
16
+ // Coverage now asks the same question the sweep asks — "what is this module's
17
+ // effective cadence" — rather than "does an enabled monitor row exist". The
18
+ // row's `enabled` column is `builtin_check`-only, so reading it here would
19
+ // have reported a module as watched that the sweep had stopped scheduling.
20
+ const cadences = loadModuleHealthCadences(db);
24
21
 
25
22
  return db
26
23
  .select({ id: modules.id, state: modules.state, manifestData: modules.manifestData })
@@ -32,7 +29,7 @@ export function loadModuleCoverage(db: DbClient): ModuleCoverageInput[] {
32
29
  id: module.id,
33
30
  state: module.state,
34
31
  hasHealthCheckHook: Boolean(manifest.hooks?.health_check),
35
- hasEnabledMonitor: monitored.has(module.id),
32
+ cadence: cadences.get(module.id)?.cadence ?? null,
36
33
  };
37
34
  });
38
35
  }
@@ -101,21 +101,30 @@ describe('deploy hooks', () => {
101
101
  expect(monitor.intervalMinutes).toBe(15);
102
102
  });
103
103
 
104
- test('a module without an interval gets nothing', () => {
104
+ // The row carries severity, escalation policy and lastRunAt; WHETHER it
105
+ // runs comes from the module's effective cadence. So a module with the hook
106
+ // and no suggested interval still gets a row — otherwise an operator who
107
+ // sets `health_check_interval` on it would have nothing to schedule against.
108
+ test('a module with the hook but no interval still gets a row to carry its state', () => {
105
109
  addModule('caddy', { hooks: { health_check: { script: 'x' } } });
106
- expect(ensureMonitorOnDeploy(db, 'caddy')).toBe(false);
110
+ expect(ensureMonitorOnDeploy(db, 'caddy')).toBe(true);
111
+ expect(db.select().from(monitors).all()).toHaveLength(1);
112
+ });
113
+
114
+ test('a module with no health_check hook at all gets nothing', () => {
115
+ addModule('signal', { hooks: {} });
116
+ expect(ensureMonitorOnDeploy(db, 'signal')).toBe(false);
107
117
  expect(db.select().from(monitors).all()).toEqual([]);
108
118
  });
109
119
 
110
- // The manifest interval is a SUGGESTION. An operator's setting has to
111
- // survive module upgrades.
112
- test('an existing monitor is never overwritten', () => {
120
+ // Not because the row protects an operator's setting any more — that lives
121
+ // in module_configs now — but because a redeploy must not duplicate rows.
122
+ test('a second deploy does not create a second row', () => {
113
123
  addModule('caddy', { hooks: { health_check: { script: 'x', interval: '15m' } } });
114
124
  ensureMonitorOnDeploy(db, 'caddy');
115
- db.update(monitors).set({ intervalMinutes: 60 }).run();
116
125
 
117
126
  expect(ensureMonitorOnDeploy(db, 'caddy')).toBe(false);
118
- expect(db.select().from(monitors).all()[0].intervalMinutes).toBe(60);
127
+ expect(db.select().from(monitors).all()).toHaveLength(1);
119
128
  });
120
129
 
121
130
  test('an unknown module does not throw', () => {
@@ -92,10 +92,14 @@ export function modulesInDeployWindow(db: DbClient): Set<string> {
92
92
  }
93
93
 
94
94
  /**
95
- * Create a monitor for a freshly deployed module that asks for one.
95
+ * Create the monitor row for a freshly deployed module that declares a
96
+ * `health_check` hook.
96
97
  *
97
- * Never overwrites: the manifest's interval is the author's suggestion, and an
98
- * operator's setting must survive module upgrades.
98
+ * A row for any such module, not only one whose manifest suggests an interval:
99
+ * the row carries severity, escalation policy and `lastRunAt`, while whether it
100
+ * runs comes from the module's effective cadence. Without a row, an operator
101
+ * setting `health_check_interval` on a module whose author named none would
102
+ * have nothing to schedule against.
99
103
  */
100
104
  export function ensureMonitorOnDeploy(db: DbClient, moduleId: string): boolean {
101
105
  try {
@@ -103,8 +107,10 @@ export function ensureMonitorOnDeploy(db: DbClient, moduleId: string): boolean {
103
107
  if (!module) return false;
104
108
 
105
109
  const manifest = module.manifestData as ModuleManifest;
106
- const interval = manifest.hooks?.health_check?.interval;
107
- return ensureMonitorForModule(db, moduleId, interval) !== null;
110
+ const healthCheck = manifest.hooks?.health_check;
111
+ if (!healthCheck) return false;
112
+
113
+ return ensureMonitorForModule(db, moduleId, healthCheck.interval) !== null;
108
114
  } catch {
109
115
  return false;
110
116
  }
@@ -0,0 +1,58 @@
1
+ import { describe, expect, test } from 'bun:test';
2
+ import type { ModuleManifest } from '../../manifest/schema';
3
+ import { effectiveHealthCheckCadence, isScheduled } from './health-cadence';
4
+
5
+ function manifestWith(interval?: string): ModuleManifest {
6
+ return {
7
+ hooks: { health_check: { script: './health.ts', ...(interval ? { interval } : {}) } },
8
+ } as unknown as ModuleManifest;
9
+ }
10
+
11
+ describe('effectiveHealthCheckCadence', () => {
12
+ test("the manifest's suggestion applies when nobody has overridden", () => {
13
+ expect(effectiveHealthCheckCadence(manifestWith('15m'), undefined)).toEqual({ minutes: 15 });
14
+ });
15
+
16
+ test('the operator override wins', () => {
17
+ expect(effectiveHealthCheckCadence(manifestWith('15m'), '1h')).toEqual({ minutes: 60 });
18
+ });
19
+
20
+ // The failure this whole change exists for: the cadence used to be seeded onto
21
+ // the monitor row at first deploy and never reconsulted, so an author who
22
+ // corrected a bad interval never reached an existing install.
23
+ test('a corrected suggestion reaches an un-overridden module', () => {
24
+ const before = effectiveHealthCheckCadence(manifestWith('1h'), undefined);
25
+ const afterUpgrade = effectiveHealthCheckCadence(manifestWith('15m'), undefined);
26
+
27
+ expect(before).toEqual({ minutes: 60 });
28
+ expect(afterUpgrade).toEqual({ minutes: 15 });
29
+ });
30
+
31
+ test('a corrected suggestion does NOT disturb an overridden module', () => {
32
+ expect(effectiveHealthCheckCadence(manifestWith('15m'), '1h')).toEqual({ minutes: 60 });
33
+ });
34
+
35
+ test('`manual` stops it being scheduled, and is not the same as unset', () => {
36
+ expect(effectiveHealthCheckCadence(manifestWith('15m'), 'manual')).toBe('manual');
37
+ expect(isScheduled('manual')).toBe(false);
38
+ expect(isScheduled(null)).toBe(false);
39
+ expect(isScheduled({ minutes: 15 })).toBe(true);
40
+ });
41
+
42
+ test('unsetting the override resumes the manifest cadence', () => {
43
+ // `undefined` is what the accessor sees once the row is deleted.
44
+ expect(effectiveHealthCheckCadence(manifestWith('15m'), undefined)).toEqual({ minutes: 15 });
45
+ });
46
+
47
+ test('nobody naming a cadence is null — a gap, not an opt-out', () => {
48
+ expect(effectiveHealthCheckCadence(manifestWith(), undefined)).toBeNull();
49
+ });
50
+
51
+ test('an operator can set a cadence a module never suggested', () => {
52
+ expect(effectiveHealthCheckCadence(manifestWith(), 'daily')).toEqual({ minutes: 1440 });
53
+ });
54
+
55
+ test('an unparseable override falls back to the manifest rather than going quiet', () => {
56
+ expect(effectiveHealthCheckCadence(manifestWith('15m'), 'fifteen')).toEqual({ minutes: 15 });
57
+ });
58
+ });
@@ -0,0 +1,128 @@
1
+ /**
2
+ * How often celilo health-checks a module.
3
+ *
4
+ * The same shape as the backup cadence, and for the same reason. The manifest's
5
+ * `hooks.health_check.interval` is the author's SUGGESTION; the operator's
6
+ * `health_check_interval` override decides; both resolve HERE, at read time.
7
+ *
8
+ * What this replaces is worth stating, because it failed silently in both
9
+ * directions. The cadence used to be seeded onto the `monitors` row at first
10
+ * deploy and never reconsulted (`monitors.ts` returned early if a row existed),
11
+ * so an author who corrected a bad interval never reached an existing install,
12
+ * with nothing an operator could read to discover it. And since nothing ever
13
+ * wrote `monitors.intervalMinutes` after that insert, an operator's only way to
14
+ * re-cadence a module was raw SQL against `celilo.db`.
15
+ *
16
+ * `manual` means the operator has stopped watching the module. That is a
17
+ * decision, not a gap: it raises no health-coverage finding, because a finding
18
+ * asking for the action they just declined is one no action can clear.
19
+ */
20
+
21
+ import { eq } from 'drizzle-orm';
22
+ import type { DbClient } from '../../db/client';
23
+ import { moduleConfigs, modules } from '../../db/schema';
24
+ import type { ModuleManifest } from '../../manifest/schema';
25
+ import { type Cadence, parseCadence } from '../cadence';
26
+ import { getModuleConfigValue, parseStoredConfigValue } from '../module-config';
27
+ import { findMonitor, resolveMonitorAlerts } from './monitors';
28
+
29
+ /** The `module_configs` key an operator's health-check cadence is stored under. */
30
+ export const HEALTH_CHECK_INTERVAL_CONFIG_KEY = 'health_check_interval';
31
+
32
+ /**
33
+ * `null` means NOBODY has said how often — the module is unscheduled and that
34
+ * is a coverage gap. It is distinct from `'manual'`, which is an operator
35
+ * saying not to watch it. There is no default: an interval celilo invented
36
+ * would run someone's health hook on a cadence no one chose.
37
+ *
38
+ * An unparseable override falls back to the manifest, the same direction the
39
+ * backup cadence fails in — values are validated at SET time, so a bad one here
40
+ * means hand-edited state, and continuing to watch beats going quiet.
41
+ */
42
+ export function effectiveHealthCheckCadence(
43
+ manifest: ModuleManifest,
44
+ override: string | undefined,
45
+ ): Cadence | null {
46
+ if (override !== undefined) {
47
+ const chosen = parseCadence(override);
48
+ if (chosen !== null) return chosen;
49
+ }
50
+ const suggested = manifest.hooks?.health_check?.interval;
51
+ if (suggested !== undefined) {
52
+ const parsed = parseCadence(suggested);
53
+ if (parsed !== null) return parsed;
54
+ }
55
+ return null;
56
+ }
57
+
58
+ /** Whether a resolved cadence schedules anything at all. */
59
+ export function isScheduled(cadence: Cadence | null): cadence is { minutes: number } {
60
+ return cadence !== null && cadence !== 'manual';
61
+ }
62
+
63
+ export interface ModuleHealthCadence {
64
+ moduleId: string;
65
+ manifest: ModuleManifest;
66
+ state: string;
67
+ cadence: Cadence | null;
68
+ }
69
+
70
+ /**
71
+ * Every module's effective health-check cadence, in one pass.
72
+ *
73
+ * Read here rather than resolved per call site (Rule 2.3): the alerting sweep,
74
+ * the coverage check and `monitor list` all need the same answer, and three
75
+ * queries that could disagree is the shape this change exists to remove.
76
+ */
77
+ export function loadModuleHealthCadences(db: DbClient): Map<string, ModuleHealthCadence> {
78
+ const overrides = new Map<string, string>();
79
+ for (const row of db
80
+ .select()
81
+ .from(moduleConfigs)
82
+ .where(eq(moduleConfigs.key, HEALTH_CHECK_INTERVAL_CONFIG_KEY))
83
+ .all()) {
84
+ overrides.set(row.moduleId, String(parseStoredConfigValue(row)));
85
+ }
86
+
87
+ const cadences = new Map<string, ModuleHealthCadence>();
88
+ for (const module of db.select().from(modules).all()) {
89
+ const manifest = module.manifestData as ModuleManifest;
90
+ cadences.set(module.id, {
91
+ moduleId: module.id,
92
+ manifest,
93
+ state: module.state,
94
+ cadence: effectiveHealthCheckCadence(manifest, overrides.get(module.id)),
95
+ });
96
+ }
97
+ return cadences;
98
+ }
99
+
100
+ /**
101
+ * Bring a module's watch state into line with its effective cadence.
102
+ *
103
+ * Only one direction needs doing: a module that has just become `manual` may
104
+ * still own live alerts from its last scheduled runs, and nothing will ever
105
+ * report on them again — they would sit firing forever with no operator action
106
+ * able to clear them. This resolves them, the way disabling a monitor always
107
+ * has.
108
+ *
109
+ * Idempotent, and safe to call when nothing changed. Called wherever a
110
+ * `health_check_interval` override is written: `module config set`/`unset`, and
111
+ * the migrate step.
112
+ */
113
+ export function reconcileModuleWatchState(db: DbClient, moduleId: string, now: Date): number {
114
+ const monitor = findMonitor(db, 'module_hook', moduleId);
115
+ if (!monitor) return 0;
116
+
117
+ const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
118
+ if (!module) return 0;
119
+
120
+ const override = getModuleConfigValue(moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, db);
121
+ const cadence = effectiveHealthCheckCadence(
122
+ module.manifestData as ModuleManifest,
123
+ override === null ? undefined : String(override.value),
124
+ );
125
+ if (isScheduled(cadence)) return 0;
126
+
127
+ return resolveMonitorAlerts(db, monitor.id, now);
128
+ }
@@ -10,7 +10,13 @@
10
10
  *
11
11
  * Two ways that happens, and both are worth saying out loud:
12
12
  * - the module ships no `health_check` hook, so it can never be verified;
13
- * - it has one, but no enabled monitor schedules it.
13
+ * - it has one, but nothing gives it a cadence, so nothing schedules it.
14
+ *
15
+ * An effective cadence of `manual` raises NOTHING. The check exists to catch
16
+ * blindness nobody chose; `manual` is a recorded decision, visible in `module
17
+ * status` and removable with one command. A finding raised by an operator's
18
+ * explicit opt-out asks for the action they just declined, so no action of
19
+ * theirs could ever clear it.
14
20
  *
15
21
  * Deliberately cheap: this reads the module roster against the monitor set and
16
22
  * contacts nothing. It is safe to run on every sweep.
@@ -19,6 +25,8 @@
19
25
  */
20
26
 
21
27
  import type { ModuleState } from '../../db/schema';
28
+ import type { Cadence } from '../cadence';
29
+ import { isScheduled } from './health-cadence';
22
30
  import { type FailingKey, builtinAlertKey } from './keys';
23
31
 
24
32
  export const HEALTH_COVERAGE_CHECK = 'health_coverage';
@@ -35,8 +43,11 @@ export interface ModuleCoverageInput {
35
43
  state: ModuleState;
36
44
  /** Whether the manifest declares a `health_check` hook at all. */
37
45
  hasHealthCheckHook: boolean;
38
- /** Whether an ENABLED monitor schedules that hook. */
39
- hasEnabledMonitor: boolean;
46
+ /**
47
+ * The module's effective health-check cadence: `null` when nobody has named
48
+ * one (a gap), `'manual'` when the operator opted out (a decision).
49
+ */
50
+ cadence: Cadence | null;
40
51
  }
41
52
 
42
53
  /**
@@ -65,15 +76,14 @@ export function healthCoverageFailingKeys(modules: ModuleCoverageInput[]): Faili
65
76
  continue;
66
77
  }
67
78
 
68
- if (!module.hasEnabledMonitor) {
79
+ if (module.cadence === 'manual') continue;
80
+
81
+ if (!isScheduled(module.cadence)) {
69
82
  failing.push({
70
83
  key,
71
84
  severity: 'warning',
72
85
  message: `${module.id} has a health_check hook but nothing schedules it`,
73
- details:
74
- 'The module can be verified on demand but is not being watched.\n' +
75
- 'Create a monitor for it, or set hooks.health_check.interval in the\n' +
76
- 'module manifest so one is created on deploy.',
86
+ details: `The module can be verified on demand but is not being watched.\nGive it a cadence: celilo module config set ${module.id} health_check_interval 15m\nor set hooks.health_check.interval in the module manifest.`,
77
87
  });
78
88
  }
79
89
  }
@@ -1,10 +1,20 @@
1
1
  /**
2
2
  * Monitor CRUD.
3
3
  *
4
- * A monitor binds a thing to check to a cadence, a severity, and an escalation
5
- * policy. Operators address them by their target (`caddy`,
6
- * `machines_reachable`) rather than by id — per CLAUDE.md, a UUID never
7
- * reaches the operator.
4
+ * A monitor binds a thing to check to a severity and an escalation policy.
5
+ * Operators address them by their target (`caddy`, `machines_reachable`) rather
6
+ * than by id — per CLAUDE.md, a UUID never reaches the operator.
7
+ *
8
+ * ⚠️ `intervalMinutes` and `enabled` are meaningful for `builtin_check` rows
9
+ * ONLY. A built-in check targets a fleet-level audit category with no module
10
+ * and no manifest, so nothing suggests its cadence and the row legitimately IS
11
+ * the config. A `module_hook` row targets a module whose manifest MAY suggest
12
+ * one, so its cadence and whether it is watched at all resolve at read time
13
+ * from `health_check_interval` — see [[services/alerting/health-cadence.ts]].
14
+ * Two columns whose meaning depends on `kind` is a smell, named here rather
15
+ * than discovered in review (design.md D8); the alternatives are a resolved
16
+ * value cached on the row, which rots, or splitting the table, which needs a
17
+ * synthetic monitor identity for `alerts.monitorId` and friends.
8
18
  */
9
19
 
10
20
  import { randomUUID } from 'node:crypto';
@@ -17,11 +27,12 @@ import {
17
27
  alerts,
18
28
  monitors,
19
29
  } from '../../db/schema';
20
- import { parseIntervalMinutes } from '../../manifest/schema';
30
+ import { ALERTING_SWEEP_PATTERN, MONITOR_INTERVAL_FLOOR_MINUTES, parseCadence } from '../cadence';
21
31
 
22
32
  export interface CreateMonitorInput {
23
33
  kind: MonitorKind;
24
34
  target: string;
35
+ /** `builtin_check` only. Not consulted for `module_hook` — see the file header. */
25
36
  intervalMinutes: number;
26
37
  severity?: AlertSeverity;
27
38
  suppressible?: boolean;
@@ -69,29 +80,50 @@ export function createMonitor(db: DbClient, input: CreateMonitorInput): Monitor
69
80
  }
70
81
 
71
82
  /**
72
- * Ensure a monitor exists for a module that declares a suggested interval.
83
+ * Ensure a `module_hook` monitor row exists for a module that declares a
84
+ * `health_check` hook.
85
+ *
86
+ * The row is no longer where the cadence lives — it carries severity, the
87
+ * escalation policy and `lastRunAt`, and whether it runs at all is resolved
88
+ * from the module's effective cadence at sweep time. So it is created for any
89
+ * module with the hook, not only for one whose manifest happens to suggest an
90
+ * interval: an operator can now set a cadence on a module whose author never
91
+ * named one, and without a row there would be nothing to carry its last run.
73
92
  *
74
- * Called on deploy. Never overwrites an existing monitor: the manifest's
75
- * interval is the author's SUGGESTION and the operator's setting must survive
76
- * module upgrades, so a second call is a no-op rather than a reset.
93
+ * The stored `intervalMinutes` is the manifest's suggestion where there is one,
94
+ * and is NOT read back for this kind. It is written so that rolling back to a
95
+ * release which does read it behaves as it did before.
77
96
  *
78
- * Returns the monitor when one was created, null when one already existed or
79
- * the manifest declares no interval.
97
+ * Idempotent: a second call is a no-op, which is what keeps deploys from
98
+ * duplicating rows. It is no longer what protects an operator's setting — that
99
+ * lives in `module_configs` now and cannot be overwritten from here at all.
80
100
  */
81
101
  export function ensureMonitorForModule(
82
102
  db: DbClient,
83
103
  moduleId: string,
84
104
  suggestedInterval: string | undefined,
85
105
  ): Monitor | null {
86
- if (!suggestedInterval) return null;
87
106
  if (findMonitor(db, 'module_hook', moduleId)) return null;
88
107
 
89
- const intervalMinutes = parseIntervalMinutes(suggestedInterval);
90
- if (intervalMinutes === null) return null;
108
+ const suggested = suggestedInterval ? parseCadence(suggestedInterval) : null;
109
+ const intervalMinutes =
110
+ suggested !== null && suggested !== 'manual'
111
+ ? suggested.minutes
112
+ : MONITOR_INTERVAL_FLOOR_MINUTES;
91
113
 
92
114
  return createMonitor(db, { kind: 'module_hook', target: moduleId, intervalMinutes });
93
115
  }
94
116
 
117
+ /**
118
+ * Change a `builtin_check` monitor's cadence in place.
119
+ *
120
+ * In place, rather than remove-and-recreate, because the monitor id owns the
121
+ * alert history: recreating it would orphan every live alert it raised.
122
+ */
123
+ export function updateMonitorInterval(db: DbClient, monitorId: string, minutes: number): void {
124
+ db.update(monitors).set({ intervalMinutes: minutes }).where(eq(monitors.id, monitorId)).run();
125
+ }
126
+
95
127
  /**
96
128
  * Enable or disable a monitor.
97
129
  *
@@ -139,7 +171,10 @@ export function resolveMonitorAlerts(db: DbClient, monitorId: string, now: Date)
139
171
  * what switches on the sweep.
140
172
  */
141
173
  export const ALERTING_SWEEP_SUBSCRIBER = 'celilo-alerting-sweep';
142
- export const ALERTING_SWEEP_PATTERN = 'timer.tick.5m';
174
+ // The tick itself lives in services/cadence.ts, next to the floor derived from
175
+ // it — a sweep whose tick and whose finest servable cadence are stated in two
176
+ // files is the pair that drifts.
177
+ export { ALERTING_SWEEP_PATTERN };
143
178
 
144
179
  export interface SubscriberRegistrar {
145
180
  subscribe(options: {
@@ -4,9 +4,11 @@ import { tmpdir } from 'node:os';
4
4
  import { join } from 'node:path';
5
5
  import { eq } from 'drizzle-orm';
6
6
  import type { DbClient } from '../../db/client';
7
- import { type Alert, type Monitor, alerts, monitors } from '../../db/schema';
7
+ import { type Alert, type Monitor, alerts, modules, monitors } from '../../db/schema';
8
8
  import { setupTestDatabase } from '../../test-utils/setup-test-db';
9
9
  import type { HealthCheckResult } from '../health-runner';
10
+ import { deleteModuleConfig, upsertModuleConfig } from '../module-config';
11
+ import { HEALTH_CHECK_INTERVAL_CONFIG_KEY, reconcileModuleWatchState } from './health-cadence';
10
12
  import { moduleCheckAlertKey } from './keys';
11
13
  import type { MonitorRunDeps } from './run-monitor';
12
14
  import { type SuppressionTopology, machineAlertKey } from './suppression';
@@ -67,6 +69,21 @@ describe('runSweep', () => {
67
69
  const dbPath = join(dir, 'celilo.db');
68
70
  process.env.CELILO_DB_PATH = dbPath;
69
71
  db = await setupTestDatabase(dbPath);
72
+ // The module row is load-bearing now: a `module_hook` monitor's cadence
73
+ // resolves from the module's manifest and config, not from its own row.
74
+ db.insert(modules)
75
+ .values({
76
+ id: MODULE,
77
+ name: MODULE,
78
+ sourcePath: dir,
79
+ version: '1.0.0',
80
+ state: 'INSTALLED',
81
+ manifestData: {
82
+ id: MODULE,
83
+ hooks: { health_check: { script: './h.ts', interval: '15m' } },
84
+ },
85
+ })
86
+ .run();
70
87
  db.insert(monitors)
71
88
  .values({
72
89
  id: 'mon-1',
@@ -117,11 +134,42 @@ describe('runSweep', () => {
117
134
  expect(report.monitorsRun).toBe(0);
118
135
  });
119
136
 
120
- test('a disabled monitor is never run', async () => {
137
+ // `enabled` and `intervalMinutes` are builtin_check-only now (design.md D8).
138
+ // A future reader that reintroduces the dependency on the row fails here
139
+ // rather than silently regressing to write-time resolution.
140
+ test("a module monitor's stored enabled flag is not consulted", async () => {
121
141
  db.update(monitors).set({ enabled: false }).where(eq(monitors.id, 'mon-1')).run();
122
142
  const report = await runSweep(db, currentMonitors(), deps());
143
+ expect(report.monitorsRun).toBe(1);
144
+ });
145
+
146
+ test("a module monitor's stored interval is not consulted", async () => {
147
+ // The row says a full day; the manifest says 15m, which is what wins.
148
+ db.update(monitors).set({ intervalMinutes: 1440 }).where(eq(monitors.id, 'mon-1')).run();
149
+ await runSweep(db, currentMonitors(), deps());
150
+ const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
151
+ expect(report.monitorsRun).toBe(1);
152
+ });
153
+
154
+ test('a module whose effective cadence is manual is never run, and its alerts resolve', async () => {
155
+ await runSweep(db, currentMonitors(), deps());
156
+ expect(liveAlerts()).not.toEqual([]);
157
+
158
+ upsertModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual');
159
+ reconcileModuleWatchState(db, MODULE, later(20));
160
+
161
+ const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
123
162
  expect(report.monitorsRun).toBe(0);
124
- expect(liveAlerts()).toEqual([]);
163
+ expect(liveAlerts().every((a) => a.state === 'resolved')).toBe(true);
164
+ });
165
+
166
+ test('unsetting the override resumes watching', async () => {
167
+ upsertModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual');
168
+ expect((await runSweep(db, currentMonitors(), deps())).monitorsRun).toBe(0);
169
+
170
+ deleteModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY);
171
+ const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
172
+ expect(report.monitorsRun).toBe(1);
125
173
  });
126
174
 
127
175
  test('recovery resolves the alert on a later sweep', async () => {
@@ -21,6 +21,8 @@
21
21
 
22
22
  import type { DbClient } from '../../db/client';
23
23
  import type { Alert, Monitor } from '../../db/schema';
24
+ import { MONITOR_INTERVAL_FLOOR_MINUTES } from '../cadence';
25
+ import { isScheduled, loadModuleHealthCadences } from './health-cadence';
24
26
  import type { NotifyDeps, NotifyOutcome } from './notifier';
25
27
  import { deliverDeferred, notifyAlert } from './notifier';
26
28
  import { type MonitorRunDeps, runOneMonitor } from './run-monitor';
@@ -124,14 +126,35 @@ export async function runSweep(
124
126
  };
125
127
 
126
128
  // 1. Run due monitors.
129
+ //
130
+ // A `module_hook` monitor's cadence and whether it is watched at all come
131
+ // from the module's effective health-check cadence, NOT from its row: the row
132
+ // was seeded once at first deploy and never reconsulted, so a corrected
133
+ // manifest could never reach an existing install (design.md D2/D8). A
134
+ // `builtin_check` has no module and no manifest, so its row is the config.
135
+ const cadences = loadModuleHealthCadences(db);
127
136
  const due = selectDueMonitors(
128
- monitors.map((m) => ({
129
- id: m.id,
130
- intervalMinutes: m.intervalMinutes,
131
- enabled: m.enabled,
132
- lastRunAt: m.lastRunAt,
133
- monitor: m,
134
- })),
137
+ monitors.map((m) => {
138
+ if (m.kind !== 'module_hook') {
139
+ return {
140
+ id: m.id,
141
+ intervalMinutes: m.intervalMinutes,
142
+ enabled: m.enabled,
143
+ lastRunAt: m.lastRunAt,
144
+ monitor: m,
145
+ };
146
+ }
147
+ const cadence = cadences.get(m.target)?.cadence ?? null;
148
+ return {
149
+ id: m.id,
150
+ // Unscheduled monitors are filtered out by `enabled` below, so this
151
+ // value is never used to decide due-ness for them.
152
+ intervalMinutes: isScheduled(cadence) ? cadence.minutes : MONITOR_INTERVAL_FLOOR_MINUTES,
153
+ enabled: isScheduled(cadence),
154
+ lastRunAt: m.lastRunAt,
155
+ monitor: m,
156
+ };
157
+ }),
135
158
  deps.now(),
136
159
  );
137
160
 
@@ -9,8 +9,10 @@
9
9
 
10
10
  import { eq } from 'drizzle-orm';
11
11
  import type { DbClient } from '../../db/client';
12
- import { backups, modules } from '../../db/schema';
12
+ import { backups, moduleConfigs, modules } from '../../db/schema';
13
13
  import type { ModuleManifest } from '../../manifest/schema';
14
+ import { BACKUP_SCHEDULE_CONFIG_KEY } from '../backup-schedule';
15
+ import { parseStoredConfigValue } from '../module-config';
14
16
  import type { InstalledModuleBackupInfo } from './backups';
15
17
 
16
18
  const DEPLOYED_STATES = ['INSTALLED', 'VERIFIED'];
@@ -40,6 +42,7 @@ function latestSuccessfulBackupByModule(db: DbClient): Map<string, number> {
40
42
 
41
43
  export function loadBackupAuditInfo(db: DbClient): InstalledModuleBackupInfo[] {
42
44
  const latest = latestSuccessfulBackupByModule(db);
45
+ const overrides = backupScheduleOverrides(db);
43
46
  return db
44
47
  .select()
45
48
  .from(modules)
@@ -49,6 +52,26 @@ export function loadBackupAuditInfo(db: DbClient): InstalledModuleBackupInfo[] {
49
52
  id: module.id,
50
53
  state: module.state,
51
54
  manifest: module.manifestData as ModuleManifest,
55
+ scheduleOverride: overrides.get(module.id),
52
56
  lastSuccessfulBackupAt: latest.get(module.id) ?? null,
53
57
  }));
54
58
  }
59
+
60
+ /**
61
+ * Every operator backup-cadence override, by module.
62
+ *
63
+ * Read here rather than resolved here: the audit resolves override-against-
64
+ * manifest through the same accessor the backup sweep uses, so the two cannot
65
+ * disagree about what a module's cadence is.
66
+ */
67
+ export function backupScheduleOverrides(db: DbClient): Map<string, string> {
68
+ const overrides = new Map<string, string>();
69
+ for (const row of db
70
+ .select()
71
+ .from(moduleConfigs)
72
+ .where(eq(moduleConfigs.key, BACKUP_SCHEDULE_CONFIG_KEY))
73
+ .all()) {
74
+ overrides.set(row.moduleId, String(parseStoredConfigValue(row)));
75
+ }
76
+ return overrides;
77
+ }