@celilo/cli 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +27 -7
- package/package.json +6 -5
- package/src/cli/commands/alerts-act.ts +1 -1
- package/src/cli/commands/backup-create.ts +26 -11
- package/src/cli/commands/backup-list.test.ts +83 -0
- package/src/cli/commands/backup-list.ts +67 -3
- package/src/cli/commands/backup-prune.ts +17 -17
- package/src/cli/commands/backup-sweep.ts +20 -8
- package/src/cli/commands/firewall-interface-list.test.ts +85 -0
- package/src/cli/commands/firewall-interface-list.ts +123 -0
- package/src/cli/commands/machine-add.ts +30 -2
- package/src/cli/commands/module-config.test.ts +64 -2
- package/src/cli/commands/module-config.ts +159 -8
- package/src/cli/commands/module-status.ts +124 -0
- package/src/cli/commands/monitor.ts +116 -19
- package/src/cli/commands/system-migrate.ts +14 -0
- package/src/cli/commands/system-update.ts +4 -1
- package/src/cli/completion.ts +35 -9
- package/src/cli/index.ts +59 -2
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/hooks/capability-loader.ts +130 -4
- package/src/hooks/types.ts +2 -1
- package/src/manifest/contracts/v1.ts +16 -0
- package/src/manifest/schema.ts +40 -65
- package/src/services/alerting/builtin-monitors.test.ts +18 -10
- package/src/services/alerting/cadence-migration.test.ts +155 -0
- package/src/services/alerting/cadence-migration.ts +90 -0
- package/src/services/alerting/coverage-source.ts +8 -11
- package/src/services/alerting/deploy-hooks.test.ts +16 -7
- package/src/services/alerting/deploy-hooks.ts +11 -5
- package/src/services/alerting/health-cadence.test.ts +58 -0
- package/src/services/alerting/health-cadence.ts +128 -0
- package/src/services/alerting/health-coverage.ts +18 -8
- package/src/services/alerting/monitors.ts +50 -15
- package/src/services/alerting/sweep-runner.test.ts +51 -3
- package/src/services/alerting/sweep-runner.ts +30 -7
- package/src/services/audit/backup-source.ts +24 -1
- package/src/services/audit/backups.test.ts +95 -10
- package/src/services/audit/backups.ts +40 -37
- package/src/services/audit/interface-classification.test.ts +220 -0
- package/src/services/audit/interface-classification.ts +167 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/backup-age-agreement.test.ts +118 -0
- package/src/services/backup-create.ts +36 -30
- package/src/services/backup-metadata.ts +52 -1
- package/src/services/backup-retention.test.ts +123 -0
- package/src/services/backup-retention.ts +66 -5
- package/src/services/backup-schedule.test.ts +166 -0
- package/src/services/backup-schedule.ts +105 -15
- package/src/services/backup-staging.ts +14 -1
- package/src/services/backup-sweep.test.ts +22 -3
- package/src/services/backup-sweep.ts +15 -5
- package/src/services/cadence.test.ts +97 -0
- package/src/services/cadence.ts +165 -0
- package/src/services/machine-detector.ts +23 -1
- package/src/services/module-config.ts +33 -0
- package/src/services/storage-providers/s3.test.ts +96 -13
- package/src/services/storage-providers/s3.ts +48 -15
- package/src/services/zone-detector.test.ts +34 -3
- package/src/services/zone-detector.ts +33 -13
|
@@ -6,21 +6,18 @@
|
|
|
6
6
|
* lives here. See Rule 2.3.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import { eq } from 'drizzle-orm';
|
|
10
9
|
import type { DbClient } from '../../db/client';
|
|
11
|
-
import { modules
|
|
10
|
+
import { modules } from '../../db/schema';
|
|
12
11
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
12
|
+
import { loadModuleHealthCadences } from './health-cadence';
|
|
13
13
|
import type { ModuleCoverageInput } from './health-coverage';
|
|
14
14
|
|
|
15
15
|
export function loadModuleCoverage(db: DbClient): ModuleCoverageInput[] {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
.all()
|
|
22
|
-
.map((row) => row.target),
|
|
23
|
-
);
|
|
16
|
+
// Coverage now asks the same question the sweep asks — "what is this module's
|
|
17
|
+
// effective cadence" — rather than "does an enabled monitor row exist". The
|
|
18
|
+
// row's `enabled` column is `builtin_check`-only, so reading it here would
|
|
19
|
+
// have reported a module as watched that the sweep had stopped scheduling.
|
|
20
|
+
const cadences = loadModuleHealthCadences(db);
|
|
24
21
|
|
|
25
22
|
return db
|
|
26
23
|
.select({ id: modules.id, state: modules.state, manifestData: modules.manifestData })
|
|
@@ -32,7 +29,7 @@ export function loadModuleCoverage(db: DbClient): ModuleCoverageInput[] {
|
|
|
32
29
|
id: module.id,
|
|
33
30
|
state: module.state,
|
|
34
31
|
hasHealthCheckHook: Boolean(manifest.hooks?.health_check),
|
|
35
|
-
|
|
32
|
+
cadence: cadences.get(module.id)?.cadence ?? null,
|
|
36
33
|
};
|
|
37
34
|
});
|
|
38
35
|
}
|
|
@@ -101,21 +101,30 @@ describe('deploy hooks', () => {
|
|
|
101
101
|
expect(monitor.intervalMinutes).toBe(15);
|
|
102
102
|
});
|
|
103
103
|
|
|
104
|
-
|
|
104
|
+
// The row carries severity, escalation policy and lastRunAt; WHETHER it
|
|
105
|
+
// runs comes from the module's effective cadence. So a module with the hook
|
|
106
|
+
// and no suggested interval still gets a row — otherwise an operator who
|
|
107
|
+
// sets `health_check_interval` on it would have nothing to schedule against.
|
|
108
|
+
test('a module with the hook but no interval still gets a row to carry its state', () => {
|
|
105
109
|
addModule('caddy', { hooks: { health_check: { script: 'x' } } });
|
|
106
|
-
expect(ensureMonitorOnDeploy(db, 'caddy')).toBe(
|
|
110
|
+
expect(ensureMonitorOnDeploy(db, 'caddy')).toBe(true);
|
|
111
|
+
expect(db.select().from(monitors).all()).toHaveLength(1);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test('a module with no health_check hook at all gets nothing', () => {
|
|
115
|
+
addModule('signal', { hooks: {} });
|
|
116
|
+
expect(ensureMonitorOnDeploy(db, 'signal')).toBe(false);
|
|
107
117
|
expect(db.select().from(monitors).all()).toEqual([]);
|
|
108
118
|
});
|
|
109
119
|
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
test('
|
|
120
|
+
// Not because the row protects an operator's setting any more — that lives
|
|
121
|
+
// in module_configs now — but because a redeploy must not duplicate rows.
|
|
122
|
+
test('a second deploy does not create a second row', () => {
|
|
113
123
|
addModule('caddy', { hooks: { health_check: { script: 'x', interval: '15m' } } });
|
|
114
124
|
ensureMonitorOnDeploy(db, 'caddy');
|
|
115
|
-
db.update(monitors).set({ intervalMinutes: 60 }).run();
|
|
116
125
|
|
|
117
126
|
expect(ensureMonitorOnDeploy(db, 'caddy')).toBe(false);
|
|
118
|
-
expect(db.select().from(monitors).all()
|
|
127
|
+
expect(db.select().from(monitors).all()).toHaveLength(1);
|
|
119
128
|
});
|
|
120
129
|
|
|
121
130
|
test('an unknown module does not throw', () => {
|
|
@@ -92,10 +92,14 @@ export function modulesInDeployWindow(db: DbClient): Set<string> {
|
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
/**
|
|
95
|
-
* Create
|
|
95
|
+
* Create the monitor row for a freshly deployed module that declares a
|
|
96
|
+
* `health_check` hook.
|
|
96
97
|
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
98
|
+
* A row for any such module, not only one whose manifest suggests an interval:
|
|
99
|
+
* the row carries severity, escalation policy and `lastRunAt`, while whether it
|
|
100
|
+
* runs comes from the module's effective cadence. Without a row, an operator
|
|
101
|
+
* setting `health_check_interval` on a module whose author named none would
|
|
102
|
+
* have nothing to schedule against.
|
|
99
103
|
*/
|
|
100
104
|
export function ensureMonitorOnDeploy(db: DbClient, moduleId: string): boolean {
|
|
101
105
|
try {
|
|
@@ -103,8 +107,10 @@ export function ensureMonitorOnDeploy(db: DbClient, moduleId: string): boolean {
|
|
|
103
107
|
if (!module) return false;
|
|
104
108
|
|
|
105
109
|
const manifest = module.manifestData as ModuleManifest;
|
|
106
|
-
const
|
|
107
|
-
|
|
110
|
+
const healthCheck = manifest.hooks?.health_check;
|
|
111
|
+
if (!healthCheck) return false;
|
|
112
|
+
|
|
113
|
+
return ensureMonitorForModule(db, moduleId, healthCheck.interval) !== null;
|
|
108
114
|
} catch {
|
|
109
115
|
return false;
|
|
110
116
|
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import type { ModuleManifest } from '../../manifest/schema';
|
|
3
|
+
import { effectiveHealthCheckCadence, isScheduled } from './health-cadence';
|
|
4
|
+
|
|
5
|
+
function manifestWith(interval?: string): ModuleManifest {
|
|
6
|
+
return {
|
|
7
|
+
hooks: { health_check: { script: './health.ts', ...(interval ? { interval } : {}) } },
|
|
8
|
+
} as unknown as ModuleManifest;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
describe('effectiveHealthCheckCadence', () => {
|
|
12
|
+
test("the manifest's suggestion applies when nobody has overridden", () => {
|
|
13
|
+
expect(effectiveHealthCheckCadence(manifestWith('15m'), undefined)).toEqual({ minutes: 15 });
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
test('the operator override wins', () => {
|
|
17
|
+
expect(effectiveHealthCheckCadence(manifestWith('15m'), '1h')).toEqual({ minutes: 60 });
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
// The failure this whole change exists for: the cadence used to be seeded onto
|
|
21
|
+
// the monitor row at first deploy and never reconsulted, so an author who
|
|
22
|
+
// corrected a bad interval never reached an existing install.
|
|
23
|
+
test('a corrected suggestion reaches an un-overridden module', () => {
|
|
24
|
+
const before = effectiveHealthCheckCadence(manifestWith('1h'), undefined);
|
|
25
|
+
const afterUpgrade = effectiveHealthCheckCadence(manifestWith('15m'), undefined);
|
|
26
|
+
|
|
27
|
+
expect(before).toEqual({ minutes: 60 });
|
|
28
|
+
expect(afterUpgrade).toEqual({ minutes: 15 });
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test('a corrected suggestion does NOT disturb an overridden module', () => {
|
|
32
|
+
expect(effectiveHealthCheckCadence(manifestWith('15m'), '1h')).toEqual({ minutes: 60 });
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test('`manual` stops it being scheduled, and is not the same as unset', () => {
|
|
36
|
+
expect(effectiveHealthCheckCadence(manifestWith('15m'), 'manual')).toBe('manual');
|
|
37
|
+
expect(isScheduled('manual')).toBe(false);
|
|
38
|
+
expect(isScheduled(null)).toBe(false);
|
|
39
|
+
expect(isScheduled({ minutes: 15 })).toBe(true);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test('unsetting the override resumes the manifest cadence', () => {
|
|
43
|
+
// `undefined` is what the accessor sees once the row is deleted.
|
|
44
|
+
expect(effectiveHealthCheckCadence(manifestWith('15m'), undefined)).toEqual({ minutes: 15 });
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test('nobody naming a cadence is null — a gap, not an opt-out', () => {
|
|
48
|
+
expect(effectiveHealthCheckCadence(manifestWith(), undefined)).toBeNull();
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test('an operator can set a cadence a module never suggested', () => {
|
|
52
|
+
expect(effectiveHealthCheckCadence(manifestWith(), 'daily')).toEqual({ minutes: 1440 });
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test('an unparseable override falls back to the manifest rather than going quiet', () => {
|
|
56
|
+
expect(effectiveHealthCheckCadence(manifestWith('15m'), 'fifteen')).toEqual({ minutes: 15 });
|
|
57
|
+
});
|
|
58
|
+
});
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How often celilo health-checks a module.
|
|
3
|
+
*
|
|
4
|
+
* The same shape as the backup cadence, and for the same reason. The manifest's
|
|
5
|
+
* `hooks.health_check.interval` is the author's SUGGESTION; the operator's
|
|
6
|
+
* `health_check_interval` override decides; both resolve HERE, at read time.
|
|
7
|
+
*
|
|
8
|
+
* What this replaces is worth stating, because it failed silently in both
|
|
9
|
+
* directions. The cadence used to be seeded onto the `monitors` row at first
|
|
10
|
+
* deploy and never reconsulted (`monitors.ts` returned early if a row existed),
|
|
11
|
+
* so an author who corrected a bad interval never reached an existing install,
|
|
12
|
+
* with nothing an operator could read to discover it. And since nothing ever
|
|
13
|
+
* wrote `monitors.intervalMinutes` after that insert, an operator's only way to
|
|
14
|
+
* re-cadence a module was raw SQL against `celilo.db`.
|
|
15
|
+
*
|
|
16
|
+
* `manual` means the operator has stopped watching the module. That is a
|
|
17
|
+
* decision, not a gap: it raises no health-coverage finding, because a finding
|
|
18
|
+
* asking for the action they just declined is one no action can clear.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { eq } from 'drizzle-orm';
|
|
22
|
+
import type { DbClient } from '../../db/client';
|
|
23
|
+
import { moduleConfigs, modules } from '../../db/schema';
|
|
24
|
+
import type { ModuleManifest } from '../../manifest/schema';
|
|
25
|
+
import { type Cadence, parseCadence } from '../cadence';
|
|
26
|
+
import { getModuleConfigValue, parseStoredConfigValue } from '../module-config';
|
|
27
|
+
import { findMonitor, resolveMonitorAlerts } from './monitors';
|
|
28
|
+
|
|
29
|
+
/** The `module_configs` key an operator's health-check cadence is stored under. */
|
|
30
|
+
export const HEALTH_CHECK_INTERVAL_CONFIG_KEY = 'health_check_interval';
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* `null` means NOBODY has said how often — the module is unscheduled and that
|
|
34
|
+
* is a coverage gap. It is distinct from `'manual'`, which is an operator
|
|
35
|
+
* saying not to watch it. There is no default: an interval celilo invented
|
|
36
|
+
* would run someone's health hook on a cadence no one chose.
|
|
37
|
+
*
|
|
38
|
+
* An unparseable override falls back to the manifest, the same direction the
|
|
39
|
+
* backup cadence fails in — values are validated at SET time, so a bad one here
|
|
40
|
+
* means hand-edited state, and continuing to watch beats going quiet.
|
|
41
|
+
*/
|
|
42
|
+
export function effectiveHealthCheckCadence(
|
|
43
|
+
manifest: ModuleManifest,
|
|
44
|
+
override: string | undefined,
|
|
45
|
+
): Cadence | null {
|
|
46
|
+
if (override !== undefined) {
|
|
47
|
+
const chosen = parseCadence(override);
|
|
48
|
+
if (chosen !== null) return chosen;
|
|
49
|
+
}
|
|
50
|
+
const suggested = manifest.hooks?.health_check?.interval;
|
|
51
|
+
if (suggested !== undefined) {
|
|
52
|
+
const parsed = parseCadence(suggested);
|
|
53
|
+
if (parsed !== null) return parsed;
|
|
54
|
+
}
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Whether a resolved cadence schedules anything at all. */
|
|
59
|
+
export function isScheduled(cadence: Cadence | null): cadence is { minutes: number } {
|
|
60
|
+
return cadence !== null && cadence !== 'manual';
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface ModuleHealthCadence {
|
|
64
|
+
moduleId: string;
|
|
65
|
+
manifest: ModuleManifest;
|
|
66
|
+
state: string;
|
|
67
|
+
cadence: Cadence | null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Every module's effective health-check cadence, in one pass.
|
|
72
|
+
*
|
|
73
|
+
* Read here rather than resolved per call site (Rule 2.3): the alerting sweep,
|
|
74
|
+
* the coverage check and `monitor list` all need the same answer, and three
|
|
75
|
+
* queries that could disagree is the shape this change exists to remove.
|
|
76
|
+
*/
|
|
77
|
+
export function loadModuleHealthCadences(db: DbClient): Map<string, ModuleHealthCadence> {
|
|
78
|
+
const overrides = new Map<string, string>();
|
|
79
|
+
for (const row of db
|
|
80
|
+
.select()
|
|
81
|
+
.from(moduleConfigs)
|
|
82
|
+
.where(eq(moduleConfigs.key, HEALTH_CHECK_INTERVAL_CONFIG_KEY))
|
|
83
|
+
.all()) {
|
|
84
|
+
overrides.set(row.moduleId, String(parseStoredConfigValue(row)));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const cadences = new Map<string, ModuleHealthCadence>();
|
|
88
|
+
for (const module of db.select().from(modules).all()) {
|
|
89
|
+
const manifest = module.manifestData as ModuleManifest;
|
|
90
|
+
cadences.set(module.id, {
|
|
91
|
+
moduleId: module.id,
|
|
92
|
+
manifest,
|
|
93
|
+
state: module.state,
|
|
94
|
+
cadence: effectiveHealthCheckCadence(manifest, overrides.get(module.id)),
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
return cadences;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Bring a module's watch state into line with its effective cadence.
|
|
102
|
+
*
|
|
103
|
+
* Only one direction needs doing: a module that has just become `manual` may
|
|
104
|
+
* still own live alerts from its last scheduled runs, and nothing will ever
|
|
105
|
+
* report on them again — they would sit firing forever with no operator action
|
|
106
|
+
* able to clear them. This resolves them, the way disabling a monitor always
|
|
107
|
+
* has.
|
|
108
|
+
*
|
|
109
|
+
* Idempotent, and safe to call when nothing changed. Called wherever a
|
|
110
|
+
* `health_check_interval` override is written: `module config set`/`unset`, and
|
|
111
|
+
* the migrate step.
|
|
112
|
+
*/
|
|
113
|
+
export function reconcileModuleWatchState(db: DbClient, moduleId: string, now: Date): number {
|
|
114
|
+
const monitor = findMonitor(db, 'module_hook', moduleId);
|
|
115
|
+
if (!monitor) return 0;
|
|
116
|
+
|
|
117
|
+
const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
|
|
118
|
+
if (!module) return 0;
|
|
119
|
+
|
|
120
|
+
const override = getModuleConfigValue(moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, db);
|
|
121
|
+
const cadence = effectiveHealthCheckCadence(
|
|
122
|
+
module.manifestData as ModuleManifest,
|
|
123
|
+
override === null ? undefined : String(override.value),
|
|
124
|
+
);
|
|
125
|
+
if (isScheduled(cadence)) return 0;
|
|
126
|
+
|
|
127
|
+
return resolveMonitorAlerts(db, monitor.id, now);
|
|
128
|
+
}
|
|
@@ -10,7 +10,13 @@
|
|
|
10
10
|
*
|
|
11
11
|
* Two ways that happens, and both are worth saying out loud:
|
|
12
12
|
* - the module ships no `health_check` hook, so it can never be verified;
|
|
13
|
-
* - it has one, but
|
|
13
|
+
* - it has one, but nothing gives it a cadence, so nothing schedules it.
|
|
14
|
+
*
|
|
15
|
+
* An effective cadence of `manual` raises NOTHING. The check exists to catch
|
|
16
|
+
* blindness nobody chose; `manual` is a recorded decision, visible in `module
|
|
17
|
+
* status` and removable with one command. A finding raised by an operator's
|
|
18
|
+
* explicit opt-out asks for the action they just declined, so no action of
|
|
19
|
+
* theirs could ever clear it.
|
|
14
20
|
*
|
|
15
21
|
* Deliberately cheap: this reads the module roster against the monitor set and
|
|
16
22
|
* contacts nothing. It is safe to run on every sweep.
|
|
@@ -19,6 +25,8 @@
|
|
|
19
25
|
*/
|
|
20
26
|
|
|
21
27
|
import type { ModuleState } from '../../db/schema';
|
|
28
|
+
import type { Cadence } from '../cadence';
|
|
29
|
+
import { isScheduled } from './health-cadence';
|
|
22
30
|
import { type FailingKey, builtinAlertKey } from './keys';
|
|
23
31
|
|
|
24
32
|
export const HEALTH_COVERAGE_CHECK = 'health_coverage';
|
|
@@ -35,8 +43,11 @@ export interface ModuleCoverageInput {
|
|
|
35
43
|
state: ModuleState;
|
|
36
44
|
/** Whether the manifest declares a `health_check` hook at all. */
|
|
37
45
|
hasHealthCheckHook: boolean;
|
|
38
|
-
/**
|
|
39
|
-
|
|
46
|
+
/**
|
|
47
|
+
* The module's effective health-check cadence: `null` when nobody has named
|
|
48
|
+
* one (a gap), `'manual'` when the operator opted out (a decision).
|
|
49
|
+
*/
|
|
50
|
+
cadence: Cadence | null;
|
|
40
51
|
}
|
|
41
52
|
|
|
42
53
|
/**
|
|
@@ -65,15 +76,14 @@ export function healthCoverageFailingKeys(modules: ModuleCoverageInput[]): Faili
|
|
|
65
76
|
continue;
|
|
66
77
|
}
|
|
67
78
|
|
|
68
|
-
if (
|
|
79
|
+
if (module.cadence === 'manual') continue;
|
|
80
|
+
|
|
81
|
+
if (!isScheduled(module.cadence)) {
|
|
69
82
|
failing.push({
|
|
70
83
|
key,
|
|
71
84
|
severity: 'warning',
|
|
72
85
|
message: `${module.id} has a health_check hook but nothing schedules it`,
|
|
73
|
-
details:
|
|
74
|
-
'The module can be verified on demand but is not being watched.\n' +
|
|
75
|
-
'Create a monitor for it, or set hooks.health_check.interval in the\n' +
|
|
76
|
-
'module manifest so one is created on deploy.',
|
|
86
|
+
details: `The module can be verified on demand but is not being watched.\nGive it a cadence: celilo module config set ${module.id} health_check_interval 15m\nor set hooks.health_check.interval in the module manifest.`,
|
|
77
87
|
});
|
|
78
88
|
}
|
|
79
89
|
}
|
|
@@ -1,10 +1,20 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Monitor CRUD.
|
|
3
3
|
*
|
|
4
|
-
* A monitor binds a thing to check to a
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* A monitor binds a thing to check to a severity and an escalation policy.
|
|
5
|
+
* Operators address them by their target (`caddy`, `machines_reachable`) rather
|
|
6
|
+
* than by id — per CLAUDE.md, a UUID never reaches the operator.
|
|
7
|
+
*
|
|
8
|
+
* ⚠️ `intervalMinutes` and `enabled` are meaningful for `builtin_check` rows
|
|
9
|
+
* ONLY. A built-in check targets a fleet-level audit category with no module
|
|
10
|
+
* and no manifest, so nothing suggests its cadence and the row legitimately IS
|
|
11
|
+
* the config. A `module_hook` row targets a module whose manifest MAY suggest
|
|
12
|
+
* one, so its cadence and whether it is watched at all resolve at read time
|
|
13
|
+
* from `health_check_interval` — see [[services/alerting/health-cadence.ts]].
|
|
14
|
+
* Two columns whose meaning depends on `kind` is a smell, named here rather
|
|
15
|
+
* than discovered in review (design.md D8); the alternatives are a resolved
|
|
16
|
+
* value cached on the row, which rots, or splitting the table, which needs a
|
|
17
|
+
* synthetic monitor identity for `alerts.monitorId` and friends.
|
|
8
18
|
*/
|
|
9
19
|
|
|
10
20
|
import { randomUUID } from 'node:crypto';
|
|
@@ -17,11 +27,12 @@ import {
|
|
|
17
27
|
alerts,
|
|
18
28
|
monitors,
|
|
19
29
|
} from '../../db/schema';
|
|
20
|
-
import {
|
|
30
|
+
import { ALERTING_SWEEP_PATTERN, MONITOR_INTERVAL_FLOOR_MINUTES, parseCadence } from '../cadence';
|
|
21
31
|
|
|
22
32
|
export interface CreateMonitorInput {
|
|
23
33
|
kind: MonitorKind;
|
|
24
34
|
target: string;
|
|
35
|
+
/** `builtin_check` only. Not consulted for `module_hook` — see the file header. */
|
|
25
36
|
intervalMinutes: number;
|
|
26
37
|
severity?: AlertSeverity;
|
|
27
38
|
suppressible?: boolean;
|
|
@@ -69,29 +80,50 @@ export function createMonitor(db: DbClient, input: CreateMonitorInput): Monitor
|
|
|
69
80
|
}
|
|
70
81
|
|
|
71
82
|
/**
|
|
72
|
-
* Ensure a monitor exists for a module that declares a
|
|
83
|
+
* Ensure a `module_hook` monitor row exists for a module that declares a
|
|
84
|
+
* `health_check` hook.
|
|
85
|
+
*
|
|
86
|
+
* The row is no longer where the cadence lives — it carries severity, the
|
|
87
|
+
* escalation policy and `lastRunAt`, and whether it runs at all is resolved
|
|
88
|
+
* from the module's effective cadence at sweep time. So it is created for any
|
|
89
|
+
* module with the hook, not only for one whose manifest happens to suggest an
|
|
90
|
+
* interval: an operator can now set a cadence on a module whose author never
|
|
91
|
+
* named one, and without a row there would be nothing to carry its last run.
|
|
73
92
|
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
93
|
+
* The stored `intervalMinutes` is the manifest's suggestion where there is one,
|
|
94
|
+
* and is NOT read back for this kind. It is written so that rolling back to a
|
|
95
|
+
* release which does read it behaves as it did before.
|
|
77
96
|
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
97
|
+
* Idempotent: a second call is a no-op, which is what keeps deploys from
|
|
98
|
+
* duplicating rows. It is no longer what protects an operator's setting — that
|
|
99
|
+
* lives in `module_configs` now and cannot be overwritten from here at all.
|
|
80
100
|
*/
|
|
81
101
|
export function ensureMonitorForModule(
|
|
82
102
|
db: DbClient,
|
|
83
103
|
moduleId: string,
|
|
84
104
|
suggestedInterval: string | undefined,
|
|
85
105
|
): Monitor | null {
|
|
86
|
-
if (!suggestedInterval) return null;
|
|
87
106
|
if (findMonitor(db, 'module_hook', moduleId)) return null;
|
|
88
107
|
|
|
89
|
-
const
|
|
90
|
-
|
|
108
|
+
const suggested = suggestedInterval ? parseCadence(suggestedInterval) : null;
|
|
109
|
+
const intervalMinutes =
|
|
110
|
+
suggested !== null && suggested !== 'manual'
|
|
111
|
+
? suggested.minutes
|
|
112
|
+
: MONITOR_INTERVAL_FLOOR_MINUTES;
|
|
91
113
|
|
|
92
114
|
return createMonitor(db, { kind: 'module_hook', target: moduleId, intervalMinutes });
|
|
93
115
|
}
|
|
94
116
|
|
|
117
|
+
/**
|
|
118
|
+
* Change a `builtin_check` monitor's cadence in place.
|
|
119
|
+
*
|
|
120
|
+
* In place, rather than remove-and-recreate, because the monitor id owns the
|
|
121
|
+
* alert history: recreating it would orphan every live alert it raised.
|
|
122
|
+
*/
|
|
123
|
+
export function updateMonitorInterval(db: DbClient, monitorId: string, minutes: number): void {
|
|
124
|
+
db.update(monitors).set({ intervalMinutes: minutes }).where(eq(monitors.id, monitorId)).run();
|
|
125
|
+
}
|
|
126
|
+
|
|
95
127
|
/**
|
|
96
128
|
* Enable or disable a monitor.
|
|
97
129
|
*
|
|
@@ -139,7 +171,10 @@ export function resolveMonitorAlerts(db: DbClient, monitorId: string, now: Date)
|
|
|
139
171
|
* what switches on the sweep.
|
|
140
172
|
*/
|
|
141
173
|
export const ALERTING_SWEEP_SUBSCRIBER = 'celilo-alerting-sweep';
|
|
142
|
-
|
|
174
|
+
// The tick itself lives in services/cadence.ts, next to the floor derived from
|
|
175
|
+
// it — a sweep whose tick and whose finest servable cadence are stated in two
|
|
176
|
+
// files is the pair that drifts.
|
|
177
|
+
export { ALERTING_SWEEP_PATTERN };
|
|
143
178
|
|
|
144
179
|
export interface SubscriberRegistrar {
|
|
145
180
|
subscribe(options: {
|
|
@@ -4,9 +4,11 @@ import { tmpdir } from 'node:os';
|
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { eq } from 'drizzle-orm';
|
|
6
6
|
import type { DbClient } from '../../db/client';
|
|
7
|
-
import { type Alert, type Monitor, alerts, monitors } from '../../db/schema';
|
|
7
|
+
import { type Alert, type Monitor, alerts, modules, monitors } from '../../db/schema';
|
|
8
8
|
import { setupTestDatabase } from '../../test-utils/setup-test-db';
|
|
9
9
|
import type { HealthCheckResult } from '../health-runner';
|
|
10
|
+
import { deleteModuleConfig, upsertModuleConfig } from '../module-config';
|
|
11
|
+
import { HEALTH_CHECK_INTERVAL_CONFIG_KEY, reconcileModuleWatchState } from './health-cadence';
|
|
10
12
|
import { moduleCheckAlertKey } from './keys';
|
|
11
13
|
import type { MonitorRunDeps } from './run-monitor';
|
|
12
14
|
import { type SuppressionTopology, machineAlertKey } from './suppression';
|
|
@@ -67,6 +69,21 @@ describe('runSweep', () => {
|
|
|
67
69
|
const dbPath = join(dir, 'celilo.db');
|
|
68
70
|
process.env.CELILO_DB_PATH = dbPath;
|
|
69
71
|
db = await setupTestDatabase(dbPath);
|
|
72
|
+
// The module row is load-bearing now: a `module_hook` monitor's cadence
|
|
73
|
+
// resolves from the module's manifest and config, not from its own row.
|
|
74
|
+
db.insert(modules)
|
|
75
|
+
.values({
|
|
76
|
+
id: MODULE,
|
|
77
|
+
name: MODULE,
|
|
78
|
+
sourcePath: dir,
|
|
79
|
+
version: '1.0.0',
|
|
80
|
+
state: 'INSTALLED',
|
|
81
|
+
manifestData: {
|
|
82
|
+
id: MODULE,
|
|
83
|
+
hooks: { health_check: { script: './h.ts', interval: '15m' } },
|
|
84
|
+
},
|
|
85
|
+
})
|
|
86
|
+
.run();
|
|
70
87
|
db.insert(monitors)
|
|
71
88
|
.values({
|
|
72
89
|
id: 'mon-1',
|
|
@@ -117,11 +134,42 @@ describe('runSweep', () => {
|
|
|
117
134
|
expect(report.monitorsRun).toBe(0);
|
|
118
135
|
});
|
|
119
136
|
|
|
120
|
-
|
|
137
|
+
// `enabled` and `intervalMinutes` are builtin_check-only now (design.md D8).
|
|
138
|
+
// A future reader that reintroduces the dependency on the row fails here
|
|
139
|
+
// rather than silently regressing to write-time resolution.
|
|
140
|
+
test("a module monitor's stored enabled flag is not consulted", async () => {
|
|
121
141
|
db.update(monitors).set({ enabled: false }).where(eq(monitors.id, 'mon-1')).run();
|
|
122
142
|
const report = await runSweep(db, currentMonitors(), deps());
|
|
143
|
+
expect(report.monitorsRun).toBe(1);
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test("a module monitor's stored interval is not consulted", async () => {
|
|
147
|
+
// The row says a full day; the manifest says 15m, which is what wins.
|
|
148
|
+
db.update(monitors).set({ intervalMinutes: 1440 }).where(eq(monitors.id, 'mon-1')).run();
|
|
149
|
+
await runSweep(db, currentMonitors(), deps());
|
|
150
|
+
const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
|
|
151
|
+
expect(report.monitorsRun).toBe(1);
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
test('a module whose effective cadence is manual is never run, and its alerts resolve', async () => {
|
|
155
|
+
await runSweep(db, currentMonitors(), deps());
|
|
156
|
+
expect(liveAlerts()).not.toEqual([]);
|
|
157
|
+
|
|
158
|
+
upsertModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual');
|
|
159
|
+
reconcileModuleWatchState(db, MODULE, later(20));
|
|
160
|
+
|
|
161
|
+
const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
|
|
123
162
|
expect(report.monitorsRun).toBe(0);
|
|
124
|
-
expect(liveAlerts()).
|
|
163
|
+
expect(liveAlerts().every((a) => a.state === 'resolved')).toBe(true);
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
test('unsetting the override resumes watching', async () => {
|
|
167
|
+
upsertModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual');
|
|
168
|
+
expect((await runSweep(db, currentMonitors(), deps())).monitorsRun).toBe(0);
|
|
169
|
+
|
|
170
|
+
deleteModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY);
|
|
171
|
+
const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
|
|
172
|
+
expect(report.monitorsRun).toBe(1);
|
|
125
173
|
});
|
|
126
174
|
|
|
127
175
|
test('recovery resolves the alert on a later sweep', async () => {
|
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
|
|
22
22
|
import type { DbClient } from '../../db/client';
|
|
23
23
|
import type { Alert, Monitor } from '../../db/schema';
|
|
24
|
+
import { MONITOR_INTERVAL_FLOOR_MINUTES } from '../cadence';
|
|
25
|
+
import { isScheduled, loadModuleHealthCadences } from './health-cadence';
|
|
24
26
|
import type { NotifyDeps, NotifyOutcome } from './notifier';
|
|
25
27
|
import { deliverDeferred, notifyAlert } from './notifier';
|
|
26
28
|
import { type MonitorRunDeps, runOneMonitor } from './run-monitor';
|
|
@@ -124,14 +126,35 @@ export async function runSweep(
|
|
|
124
126
|
};
|
|
125
127
|
|
|
126
128
|
// 1. Run due monitors.
|
|
129
|
+
//
|
|
130
|
+
// A `module_hook` monitor's cadence and whether it is watched at all come
|
|
131
|
+
// from the module's effective health-check cadence, NOT from its row: the row
|
|
132
|
+
// was seeded once at first deploy and never reconsulted, so a corrected
|
|
133
|
+
// manifest could never reach an existing install (design.md D2/D8). A
|
|
134
|
+
// `builtin_check` has no module and no manifest, so its row is the config.
|
|
135
|
+
const cadences = loadModuleHealthCadences(db);
|
|
127
136
|
const due = selectDueMonitors(
|
|
128
|
-
monitors.map((m) =>
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
137
|
+
monitors.map((m) => {
|
|
138
|
+
if (m.kind !== 'module_hook') {
|
|
139
|
+
return {
|
|
140
|
+
id: m.id,
|
|
141
|
+
intervalMinutes: m.intervalMinutes,
|
|
142
|
+
enabled: m.enabled,
|
|
143
|
+
lastRunAt: m.lastRunAt,
|
|
144
|
+
monitor: m,
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
const cadence = cadences.get(m.target)?.cadence ?? null;
|
|
148
|
+
return {
|
|
149
|
+
id: m.id,
|
|
150
|
+
// Unscheduled monitors are filtered out by `enabled` below, so this
|
|
151
|
+
// value is never used to decide due-ness for them.
|
|
152
|
+
intervalMinutes: isScheduled(cadence) ? cadence.minutes : MONITOR_INTERVAL_FLOOR_MINUTES,
|
|
153
|
+
enabled: isScheduled(cadence),
|
|
154
|
+
lastRunAt: m.lastRunAt,
|
|
155
|
+
monitor: m,
|
|
156
|
+
};
|
|
157
|
+
}),
|
|
135
158
|
deps.now(),
|
|
136
159
|
);
|
|
137
160
|
|
|
@@ -9,8 +9,10 @@
|
|
|
9
9
|
|
|
10
10
|
import { eq } from 'drizzle-orm';
|
|
11
11
|
import type { DbClient } from '../../db/client';
|
|
12
|
-
import { backups, modules } from '../../db/schema';
|
|
12
|
+
import { backups, moduleConfigs, modules } from '../../db/schema';
|
|
13
13
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
14
|
+
import { BACKUP_SCHEDULE_CONFIG_KEY } from '../backup-schedule';
|
|
15
|
+
import { parseStoredConfigValue } from '../module-config';
|
|
14
16
|
import type { InstalledModuleBackupInfo } from './backups';
|
|
15
17
|
|
|
16
18
|
const DEPLOYED_STATES = ['INSTALLED', 'VERIFIED'];
|
|
@@ -40,6 +42,7 @@ function latestSuccessfulBackupByModule(db: DbClient): Map<string, number> {
|
|
|
40
42
|
|
|
41
43
|
export function loadBackupAuditInfo(db: DbClient): InstalledModuleBackupInfo[] {
|
|
42
44
|
const latest = latestSuccessfulBackupByModule(db);
|
|
45
|
+
const overrides = backupScheduleOverrides(db);
|
|
43
46
|
return db
|
|
44
47
|
.select()
|
|
45
48
|
.from(modules)
|
|
@@ -49,6 +52,26 @@ export function loadBackupAuditInfo(db: DbClient): InstalledModuleBackupInfo[] {
|
|
|
49
52
|
id: module.id,
|
|
50
53
|
state: module.state,
|
|
51
54
|
manifest: module.manifestData as ModuleManifest,
|
|
55
|
+
scheduleOverride: overrides.get(module.id),
|
|
52
56
|
lastSuccessfulBackupAt: latest.get(module.id) ?? null,
|
|
53
57
|
}));
|
|
54
58
|
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Every operator backup-cadence override, by module.
|
|
62
|
+
*
|
|
63
|
+
* Read here rather than resolved here: the audit resolves override-against-
|
|
64
|
+
* manifest through the same accessor the backup sweep uses, so the two cannot
|
|
65
|
+
* disagree about what a module's cadence is.
|
|
66
|
+
*/
|
|
67
|
+
export function backupScheduleOverrides(db: DbClient): Map<string, string> {
|
|
68
|
+
const overrides = new Map<string, string>();
|
|
69
|
+
for (const row of db
|
|
70
|
+
.select()
|
|
71
|
+
.from(moduleConfigs)
|
|
72
|
+
.where(eq(moduleConfigs.key, BACKUP_SCHEDULE_CONFIG_KEY))
|
|
73
|
+
.all()) {
|
|
74
|
+
overrides.set(row.moduleId, String(parseStoredConfigValue(row)));
|
|
75
|
+
}
|
|
76
|
+
return overrides;
|
|
77
|
+
}
|