@celilo/cli 0.23.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +27 -7
- package/package.json +6 -5
- package/src/cli/commands/alerts-act.ts +1 -1
- package/src/cli/commands/backup-create.ts +26 -11
- package/src/cli/commands/backup-list.test.ts +83 -0
- package/src/cli/commands/backup-list.ts +67 -3
- package/src/cli/commands/backup-prune.ts +17 -17
- package/src/cli/commands/backup-sweep.ts +20 -8
- package/src/cli/commands/firewall-interface-list.test.ts +85 -0
- package/src/cli/commands/firewall-interface-list.ts +123 -0
- package/src/cli/commands/machine-add.ts +30 -2
- package/src/cli/commands/module-config.test.ts +70 -3
- package/src/cli/commands/module-config.ts +262 -28
- package/src/cli/commands/module-status.ts +155 -12
- package/src/cli/commands/monitor.ts +116 -19
- package/src/cli/commands/system-migrate.ts +14 -0
- package/src/cli/commands/system-update.ts +4 -1
- package/src/cli/completion.ts +35 -9
- package/src/cli/index.ts +59 -2
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/hooks/capability-loader.ts +130 -4
- package/src/hooks/load-hook-config.test.ts +169 -1
- package/src/hooks/load-hook-config.ts +118 -20
- package/src/hooks/types.ts +2 -1
- package/src/manifest/contracts/v1.ts +16 -0
- package/src/manifest/schema.ts +40 -65
- package/src/services/alerting/builtin-monitors.test.ts +18 -10
- package/src/services/alerting/cadence-migration.test.ts +155 -0
- package/src/services/alerting/cadence-migration.ts +90 -0
- package/src/services/alerting/coverage-source.ts +8 -11
- package/src/services/alerting/deploy-hooks.test.ts +16 -7
- package/src/services/alerting/deploy-hooks.ts +11 -5
- package/src/services/alerting/health-cadence.test.ts +58 -0
- package/src/services/alerting/health-cadence.ts +128 -0
- package/src/services/alerting/health-coverage.ts +18 -8
- package/src/services/alerting/monitors.ts +50 -15
- package/src/services/alerting/sweep-runner.test.ts +51 -3
- package/src/services/alerting/sweep-runner.ts +30 -7
- package/src/services/audit/backup-source.ts +24 -1
- package/src/services/audit/backups.test.ts +95 -10
- package/src/services/audit/backups.ts +40 -37
- package/src/services/audit/interface-classification.test.ts +220 -0
- package/src/services/audit/interface-classification.ts +167 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/backup-age-agreement.test.ts +118 -0
- package/src/services/backup-create.ts +36 -30
- package/src/services/backup-metadata.ts +52 -1
- package/src/services/backup-retention.test.ts +123 -0
- package/src/services/backup-retention.ts +66 -5
- package/src/services/backup-schedule.test.ts +166 -0
- package/src/services/backup-schedule.ts +105 -15
- package/src/services/backup-staging.ts +14 -1
- package/src/services/backup-sweep.test.ts +22 -3
- package/src/services/backup-sweep.ts +15 -5
- package/src/services/cadence.test.ts +97 -0
- package/src/services/cadence.ts +165 -0
- package/src/services/config-provenance.test.ts +155 -0
- package/src/services/config-provenance.ts +104 -0
- package/src/services/machine-detector.ts +23 -1
- package/src/services/module-config.ts +33 -0
- package/src/services/storage-providers/s3.test.ts +96 -13
- package/src/services/storage-providers/s3.ts +48 -15
- package/src/services/zone-detector.test.ts +34 -3
- package/src/services/zone-detector.ts +33 -13
- package/src/variables/context.ts +69 -15
- package/src/variables/declarative-derivation.test.ts +53 -0
- package/src/variables/declarative-derivation.ts +13 -2
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* How often celilo health-checks a module.
|
|
3
|
+
*
|
|
4
|
+
* The same shape as the backup cadence, and for the same reason. The manifest's
|
|
5
|
+
* `hooks.health_check.interval` is the author's SUGGESTION; the operator's
|
|
6
|
+
* `health_check_interval` override decides; both resolve HERE, at read time.
|
|
7
|
+
*
|
|
8
|
+
* What this replaces is worth stating, because it failed silently in both
|
|
9
|
+
* directions. The cadence used to be seeded onto the `monitors` row at first
|
|
10
|
+
* deploy and never reconsulted (`monitors.ts` returned early if a row existed),
|
|
11
|
+
* so an author who corrected a bad interval never reached an existing install,
|
|
12
|
+
* with nothing an operator could read to discover it. And since nothing ever
|
|
13
|
+
* wrote `monitors.intervalMinutes` after that insert, an operator's only way to
|
|
14
|
+
* re-cadence a module was raw SQL against `celilo.db`.
|
|
15
|
+
*
|
|
16
|
+
* `manual` means the operator has stopped watching the module. That is a
|
|
17
|
+
* decision, not a gap: it raises no health-coverage finding, because a finding
|
|
18
|
+
* asking for the action they just declined is one no action can clear.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { eq } from 'drizzle-orm';
|
|
22
|
+
import type { DbClient } from '../../db/client';
|
|
23
|
+
import { moduleConfigs, modules } from '../../db/schema';
|
|
24
|
+
import type { ModuleManifest } from '../../manifest/schema';
|
|
25
|
+
import { type Cadence, parseCadence } from '../cadence';
|
|
26
|
+
import { getModuleConfigValue, parseStoredConfigValue } from '../module-config';
|
|
27
|
+
import { findMonitor, resolveMonitorAlerts } from './monitors';
|
|
28
|
+
|
|
29
|
+
/** The `module_configs` key an operator's health-check cadence is stored under. */
|
|
30
|
+
export const HEALTH_CHECK_INTERVAL_CONFIG_KEY = 'health_check_interval';
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* `null` means NOBODY has said how often — the module is unscheduled and that
|
|
34
|
+
* is a coverage gap. It is distinct from `'manual'`, which is an operator
|
|
35
|
+
* saying not to watch it. There is no default: an interval celilo invented
|
|
36
|
+
* would run someone's health hook on a cadence no one chose.
|
|
37
|
+
*
|
|
38
|
+
* An unparseable override falls back to the manifest, the same direction the
|
|
39
|
+
* backup cadence fails in — values are validated at SET time, so a bad one here
|
|
40
|
+
* means hand-edited state, and continuing to watch beats going quiet.
|
|
41
|
+
*/
|
|
42
|
+
export function effectiveHealthCheckCadence(
|
|
43
|
+
manifest: ModuleManifest,
|
|
44
|
+
override: string | undefined,
|
|
45
|
+
): Cadence | null {
|
|
46
|
+
if (override !== undefined) {
|
|
47
|
+
const chosen = parseCadence(override);
|
|
48
|
+
if (chosen !== null) return chosen;
|
|
49
|
+
}
|
|
50
|
+
const suggested = manifest.hooks?.health_check?.interval;
|
|
51
|
+
if (suggested !== undefined) {
|
|
52
|
+
const parsed = parseCadence(suggested);
|
|
53
|
+
if (parsed !== null) return parsed;
|
|
54
|
+
}
|
|
55
|
+
return null;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Whether a resolved cadence schedules anything at all. */
|
|
59
|
+
export function isScheduled(cadence: Cadence | null): cadence is { minutes: number } {
|
|
60
|
+
return cadence !== null && cadence !== 'manual';
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export interface ModuleHealthCadence {
|
|
64
|
+
moduleId: string;
|
|
65
|
+
manifest: ModuleManifest;
|
|
66
|
+
state: string;
|
|
67
|
+
cadence: Cadence | null;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Every module's effective health-check cadence, in one pass.
|
|
72
|
+
*
|
|
73
|
+
* Read here rather than resolved per call site (Rule 2.3): the alerting sweep,
|
|
74
|
+
* the coverage check and `monitor list` all need the same answer, and three
|
|
75
|
+
* queries that could disagree is the shape this change exists to remove.
|
|
76
|
+
*/
|
|
77
|
+
export function loadModuleHealthCadences(db: DbClient): Map<string, ModuleHealthCadence> {
|
|
78
|
+
const overrides = new Map<string, string>();
|
|
79
|
+
for (const row of db
|
|
80
|
+
.select()
|
|
81
|
+
.from(moduleConfigs)
|
|
82
|
+
.where(eq(moduleConfigs.key, HEALTH_CHECK_INTERVAL_CONFIG_KEY))
|
|
83
|
+
.all()) {
|
|
84
|
+
overrides.set(row.moduleId, String(parseStoredConfigValue(row)));
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const cadences = new Map<string, ModuleHealthCadence>();
|
|
88
|
+
for (const module of db.select().from(modules).all()) {
|
|
89
|
+
const manifest = module.manifestData as ModuleManifest;
|
|
90
|
+
cadences.set(module.id, {
|
|
91
|
+
moduleId: module.id,
|
|
92
|
+
manifest,
|
|
93
|
+
state: module.state,
|
|
94
|
+
cadence: effectiveHealthCheckCadence(manifest, overrides.get(module.id)),
|
|
95
|
+
});
|
|
96
|
+
}
|
|
97
|
+
return cadences;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Bring a module's watch state into line with its effective cadence.
|
|
102
|
+
*
|
|
103
|
+
* Only one direction needs doing: a module that has just become `manual` may
|
|
104
|
+
* still own live alerts from its last scheduled runs, and nothing will ever
|
|
105
|
+
* report on them again — they would sit firing forever with no operator action
|
|
106
|
+
* able to clear them. This resolves them, the way disabling a monitor always
|
|
107
|
+
* has.
|
|
108
|
+
*
|
|
109
|
+
* Idempotent, and safe to call when nothing changed. Called wherever a
|
|
110
|
+
* `health_check_interval` override is written: `module config set`/`unset`, and
|
|
111
|
+
* the migrate step.
|
|
112
|
+
*/
|
|
113
|
+
export function reconcileModuleWatchState(db: DbClient, moduleId: string, now: Date): number {
|
|
114
|
+
const monitor = findMonitor(db, 'module_hook', moduleId);
|
|
115
|
+
if (!monitor) return 0;
|
|
116
|
+
|
|
117
|
+
const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
|
|
118
|
+
if (!module) return 0;
|
|
119
|
+
|
|
120
|
+
const override = getModuleConfigValue(moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, db);
|
|
121
|
+
const cadence = effectiveHealthCheckCadence(
|
|
122
|
+
module.manifestData as ModuleManifest,
|
|
123
|
+
override === null ? undefined : String(override.value),
|
|
124
|
+
);
|
|
125
|
+
if (isScheduled(cadence)) return 0;
|
|
126
|
+
|
|
127
|
+
return resolveMonitorAlerts(db, monitor.id, now);
|
|
128
|
+
}
|
|
@@ -10,7 +10,13 @@
|
|
|
10
10
|
*
|
|
11
11
|
* Two ways that happens, and both are worth saying out loud:
|
|
12
12
|
* - the module ships no `health_check` hook, so it can never be verified;
|
|
13
|
-
* - it has one, but
|
|
13
|
+
* - it has one, but nothing gives it a cadence, so nothing schedules it.
|
|
14
|
+
*
|
|
15
|
+
* An effective cadence of `manual` raises NOTHING. The check exists to catch
|
|
16
|
+
* blindness nobody chose; `manual` is a recorded decision, visible in `module
|
|
17
|
+
* status` and removable with one command. A finding raised by an operator's
|
|
18
|
+
* explicit opt-out asks for the action they just declined, so no action of
|
|
19
|
+
* theirs could ever clear it.
|
|
14
20
|
*
|
|
15
21
|
* Deliberately cheap: this reads the module roster against the monitor set and
|
|
16
22
|
* contacts nothing. It is safe to run on every sweep.
|
|
@@ -19,6 +25,8 @@
|
|
|
19
25
|
*/
|
|
20
26
|
|
|
21
27
|
import type { ModuleState } from '../../db/schema';
|
|
28
|
+
import type { Cadence } from '../cadence';
|
|
29
|
+
import { isScheduled } from './health-cadence';
|
|
22
30
|
import { type FailingKey, builtinAlertKey } from './keys';
|
|
23
31
|
|
|
24
32
|
export const HEALTH_COVERAGE_CHECK = 'health_coverage';
|
|
@@ -35,8 +43,11 @@ export interface ModuleCoverageInput {
|
|
|
35
43
|
state: ModuleState;
|
|
36
44
|
/** Whether the manifest declares a `health_check` hook at all. */
|
|
37
45
|
hasHealthCheckHook: boolean;
|
|
38
|
-
/**
|
|
39
|
-
|
|
46
|
+
/**
|
|
47
|
+
* The module's effective health-check cadence: `null` when nobody has named
|
|
48
|
+
* one (a gap), `'manual'` when the operator opted out (a decision).
|
|
49
|
+
*/
|
|
50
|
+
cadence: Cadence | null;
|
|
40
51
|
}
|
|
41
52
|
|
|
42
53
|
/**
|
|
@@ -65,15 +76,14 @@ export function healthCoverageFailingKeys(modules: ModuleCoverageInput[]): Faili
|
|
|
65
76
|
continue;
|
|
66
77
|
}
|
|
67
78
|
|
|
68
|
-
if (
|
|
79
|
+
if (module.cadence === 'manual') continue;
|
|
80
|
+
|
|
81
|
+
if (!isScheduled(module.cadence)) {
|
|
69
82
|
failing.push({
|
|
70
83
|
key,
|
|
71
84
|
severity: 'warning',
|
|
72
85
|
message: `${module.id} has a health_check hook but nothing schedules it`,
|
|
73
|
-
details:
|
|
74
|
-
'The module can be verified on demand but is not being watched.\n' +
|
|
75
|
-
'Create a monitor for it, or set hooks.health_check.interval in the\n' +
|
|
76
|
-
'module manifest so one is created on deploy.',
|
|
86
|
+
details: `The module can be verified on demand but is not being watched.\nGive it a cadence: celilo module config set ${module.id} health_check_interval 15m\nor set hooks.health_check.interval in the module manifest.`,
|
|
77
87
|
});
|
|
78
88
|
}
|
|
79
89
|
}
|
|
@@ -1,10 +1,20 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Monitor CRUD.
|
|
3
3
|
*
|
|
4
|
-
* A monitor binds a thing to check to a
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* A monitor binds a thing to check to a severity and an escalation policy.
|
|
5
|
+
* Operators address them by their target (`caddy`, `machines_reachable`) rather
|
|
6
|
+
* than by id — per CLAUDE.md, a UUID never reaches the operator.
|
|
7
|
+
*
|
|
8
|
+
* ⚠️ `intervalMinutes` and `enabled` are meaningful for `builtin_check` rows
|
|
9
|
+
* ONLY. A built-in check targets a fleet-level audit category with no module
|
|
10
|
+
* and no manifest, so nothing suggests its cadence and the row legitimately IS
|
|
11
|
+
* the config. A `module_hook` row targets a module whose manifest MAY suggest
|
|
12
|
+
* one, so its cadence and whether it is watched at all resolve at read time
|
|
13
|
+
* from `health_check_interval` — see [[services/alerting/health-cadence.ts]].
|
|
14
|
+
* Two columns whose meaning depends on `kind` is a smell, named here rather
|
|
15
|
+
* than discovered in review (design.md D8); the alternatives are a resolved
|
|
16
|
+
* value cached on the row, which rots, or splitting the table, which needs a
|
|
17
|
+
* synthetic monitor identity for `alerts.monitorId` and friends.
|
|
8
18
|
*/
|
|
9
19
|
|
|
10
20
|
import { randomUUID } from 'node:crypto';
|
|
@@ -17,11 +27,12 @@ import {
|
|
|
17
27
|
alerts,
|
|
18
28
|
monitors,
|
|
19
29
|
} from '../../db/schema';
|
|
20
|
-
import {
|
|
30
|
+
import { ALERTING_SWEEP_PATTERN, MONITOR_INTERVAL_FLOOR_MINUTES, parseCadence } from '../cadence';
|
|
21
31
|
|
|
22
32
|
export interface CreateMonitorInput {
|
|
23
33
|
kind: MonitorKind;
|
|
24
34
|
target: string;
|
|
35
|
+
/** `builtin_check` only. Not consulted for `module_hook` — see the file header. */
|
|
25
36
|
intervalMinutes: number;
|
|
26
37
|
severity?: AlertSeverity;
|
|
27
38
|
suppressible?: boolean;
|
|
@@ -69,29 +80,50 @@ export function createMonitor(db: DbClient, input: CreateMonitorInput): Monitor
|
|
|
69
80
|
}
|
|
70
81
|
|
|
71
82
|
/**
|
|
72
|
-
* Ensure a monitor exists for a module that declares a
|
|
83
|
+
* Ensure a `module_hook` monitor row exists for a module that declares a
|
|
84
|
+
* `health_check` hook.
|
|
85
|
+
*
|
|
86
|
+
* The row is no longer where the cadence lives — it carries severity, the
|
|
87
|
+
* escalation policy and `lastRunAt`, and whether it runs at all is resolved
|
|
88
|
+
* from the module's effective cadence at sweep time. So it is created for any
|
|
89
|
+
* module with the hook, not only for one whose manifest happens to suggest an
|
|
90
|
+
* interval: an operator can now set a cadence on a module whose author never
|
|
91
|
+
* named one, and without a row there would be nothing to carry its last run.
|
|
73
92
|
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
93
|
+
* The stored `intervalMinutes` is the manifest's suggestion where there is one,
|
|
94
|
+
* and is NOT read back for this kind. It is written so that rolling back to a
|
|
95
|
+
* release which does read it behaves as it did before.
|
|
77
96
|
*
|
|
78
|
-
*
|
|
79
|
-
*
|
|
97
|
+
* Idempotent: a second call is a no-op, which is what keeps deploys from
|
|
98
|
+
* duplicating rows. It is no longer what protects an operator's setting — that
|
|
99
|
+
* lives in `module_configs` now and cannot be overwritten from here at all.
|
|
80
100
|
*/
|
|
81
101
|
export function ensureMonitorForModule(
|
|
82
102
|
db: DbClient,
|
|
83
103
|
moduleId: string,
|
|
84
104
|
suggestedInterval: string | undefined,
|
|
85
105
|
): Monitor | null {
|
|
86
|
-
if (!suggestedInterval) return null;
|
|
87
106
|
if (findMonitor(db, 'module_hook', moduleId)) return null;
|
|
88
107
|
|
|
89
|
-
const
|
|
90
|
-
|
|
108
|
+
const suggested = suggestedInterval ? parseCadence(suggestedInterval) : null;
|
|
109
|
+
const intervalMinutes =
|
|
110
|
+
suggested !== null && suggested !== 'manual'
|
|
111
|
+
? suggested.minutes
|
|
112
|
+
: MONITOR_INTERVAL_FLOOR_MINUTES;
|
|
91
113
|
|
|
92
114
|
return createMonitor(db, { kind: 'module_hook', target: moduleId, intervalMinutes });
|
|
93
115
|
}
|
|
94
116
|
|
|
117
|
+
/**
|
|
118
|
+
* Change a `builtin_check` monitor's cadence in place.
|
|
119
|
+
*
|
|
120
|
+
* In place, rather than remove-and-recreate, because the monitor id owns the
|
|
121
|
+
* alert history: recreating it would orphan every live alert it raised.
|
|
122
|
+
*/
|
|
123
|
+
export function updateMonitorInterval(db: DbClient, monitorId: string, minutes: number): void {
|
|
124
|
+
db.update(monitors).set({ intervalMinutes: minutes }).where(eq(monitors.id, monitorId)).run();
|
|
125
|
+
}
|
|
126
|
+
|
|
95
127
|
/**
|
|
96
128
|
* Enable or disable a monitor.
|
|
97
129
|
*
|
|
@@ -139,7 +171,10 @@ export function resolveMonitorAlerts(db: DbClient, monitorId: string, now: Date)
|
|
|
139
171
|
* what switches on the sweep.
|
|
140
172
|
*/
|
|
141
173
|
export const ALERTING_SWEEP_SUBSCRIBER = 'celilo-alerting-sweep';
|
|
142
|
-
|
|
174
|
+
// The tick itself lives in services/cadence.ts, next to the floor derived from
|
|
175
|
+
// it — a sweep whose tick and whose finest servable cadence are stated in two
|
|
176
|
+
// files is the pair that drifts.
|
|
177
|
+
export { ALERTING_SWEEP_PATTERN };
|
|
143
178
|
|
|
144
179
|
export interface SubscriberRegistrar {
|
|
145
180
|
subscribe(options: {
|
|
@@ -4,9 +4,11 @@ import { tmpdir } from 'node:os';
|
|
|
4
4
|
import { join } from 'node:path';
|
|
5
5
|
import { eq } from 'drizzle-orm';
|
|
6
6
|
import type { DbClient } from '../../db/client';
|
|
7
|
-
import { type Alert, type Monitor, alerts, monitors } from '../../db/schema';
|
|
7
|
+
import { type Alert, type Monitor, alerts, modules, monitors } from '../../db/schema';
|
|
8
8
|
import { setupTestDatabase } from '../../test-utils/setup-test-db';
|
|
9
9
|
import type { HealthCheckResult } from '../health-runner';
|
|
10
|
+
import { deleteModuleConfig, upsertModuleConfig } from '../module-config';
|
|
11
|
+
import { HEALTH_CHECK_INTERVAL_CONFIG_KEY, reconcileModuleWatchState } from './health-cadence';
|
|
10
12
|
import { moduleCheckAlertKey } from './keys';
|
|
11
13
|
import type { MonitorRunDeps } from './run-monitor';
|
|
12
14
|
import { type SuppressionTopology, machineAlertKey } from './suppression';
|
|
@@ -67,6 +69,21 @@ describe('runSweep', () => {
|
|
|
67
69
|
const dbPath = join(dir, 'celilo.db');
|
|
68
70
|
process.env.CELILO_DB_PATH = dbPath;
|
|
69
71
|
db = await setupTestDatabase(dbPath);
|
|
72
|
+
// The module row is load-bearing now: a `module_hook` monitor's cadence
|
|
73
|
+
// resolves from the module's manifest and config, not from its own row.
|
|
74
|
+
db.insert(modules)
|
|
75
|
+
.values({
|
|
76
|
+
id: MODULE,
|
|
77
|
+
name: MODULE,
|
|
78
|
+
sourcePath: dir,
|
|
79
|
+
version: '1.0.0',
|
|
80
|
+
state: 'INSTALLED',
|
|
81
|
+
manifestData: {
|
|
82
|
+
id: MODULE,
|
|
83
|
+
hooks: { health_check: { script: './h.ts', interval: '15m' } },
|
|
84
|
+
},
|
|
85
|
+
})
|
|
86
|
+
.run();
|
|
70
87
|
db.insert(monitors)
|
|
71
88
|
.values({
|
|
72
89
|
id: 'mon-1',
|
|
@@ -117,11 +134,42 @@ describe('runSweep', () => {
|
|
|
117
134
|
expect(report.monitorsRun).toBe(0);
|
|
118
135
|
});
|
|
119
136
|
|
|
120
|
-
|
|
137
|
+
// `enabled` and `intervalMinutes` are builtin_check-only now (design.md D8).
|
|
138
|
+
// A future reader that reintroduces the dependency on the row fails here
|
|
139
|
+
// rather than silently regressing to write-time resolution.
|
|
140
|
+
test("a module monitor's stored enabled flag is not consulted", async () => {
|
|
121
141
|
db.update(monitors).set({ enabled: false }).where(eq(monitors.id, 'mon-1')).run();
|
|
122
142
|
const report = await runSweep(db, currentMonitors(), deps());
|
|
143
|
+
expect(report.monitorsRun).toBe(1);
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
test("a module monitor's stored interval is not consulted", async () => {
|
|
147
|
+
// The row says a full day; the manifest says 15m, which is what wins.
|
|
148
|
+
db.update(monitors).set({ intervalMinutes: 1440 }).where(eq(monitors.id, 'mon-1')).run();
|
|
149
|
+
await runSweep(db, currentMonitors(), deps());
|
|
150
|
+
const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
|
|
151
|
+
expect(report.monitorsRun).toBe(1);
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
test('a module whose effective cadence is manual is never run, and its alerts resolve', async () => {
|
|
155
|
+
await runSweep(db, currentMonitors(), deps());
|
|
156
|
+
expect(liveAlerts()).not.toEqual([]);
|
|
157
|
+
|
|
158
|
+
upsertModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual');
|
|
159
|
+
reconcileModuleWatchState(db, MODULE, later(20));
|
|
160
|
+
|
|
161
|
+
const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
|
|
123
162
|
expect(report.monitorsRun).toBe(0);
|
|
124
|
-
expect(liveAlerts()).
|
|
163
|
+
expect(liveAlerts().every((a) => a.state === 'resolved')).toBe(true);
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
test('unsetting the override resumes watching', async () => {
|
|
167
|
+
upsertModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual');
|
|
168
|
+
expect((await runSweep(db, currentMonitors(), deps())).monitorsRun).toBe(0);
|
|
169
|
+
|
|
170
|
+
deleteModuleConfig(db, MODULE, HEALTH_CHECK_INTERVAL_CONFIG_KEY);
|
|
171
|
+
const report = await runSweep(db, currentMonitors(), deps({}, failing, later(20)));
|
|
172
|
+
expect(report.monitorsRun).toBe(1);
|
|
125
173
|
});
|
|
126
174
|
|
|
127
175
|
test('recovery resolves the alert on a later sweep', async () => {
|
|
@@ -21,6 +21,8 @@
|
|
|
21
21
|
|
|
22
22
|
import type { DbClient } from '../../db/client';
|
|
23
23
|
import type { Alert, Monitor } from '../../db/schema';
|
|
24
|
+
import { MONITOR_INTERVAL_FLOOR_MINUTES } from '../cadence';
|
|
25
|
+
import { isScheduled, loadModuleHealthCadences } from './health-cadence';
|
|
24
26
|
import type { NotifyDeps, NotifyOutcome } from './notifier';
|
|
25
27
|
import { deliverDeferred, notifyAlert } from './notifier';
|
|
26
28
|
import { type MonitorRunDeps, runOneMonitor } from './run-monitor';
|
|
@@ -124,14 +126,35 @@ export async function runSweep(
|
|
|
124
126
|
};
|
|
125
127
|
|
|
126
128
|
// 1. Run due monitors.
|
|
129
|
+
//
|
|
130
|
+
// A `module_hook` monitor's cadence and whether it is watched at all come
|
|
131
|
+
// from the module's effective health-check cadence, NOT from its row: the row
|
|
132
|
+
// was seeded once at first deploy and never reconsulted, so a corrected
|
|
133
|
+
// manifest could never reach an existing install (design.md D2/D8). A
|
|
134
|
+
// `builtin_check` has no module and no manifest, so its row is the config.
|
|
135
|
+
const cadences = loadModuleHealthCadences(db);
|
|
127
136
|
const due = selectDueMonitors(
|
|
128
|
-
monitors.map((m) =>
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
137
|
+
monitors.map((m) => {
|
|
138
|
+
if (m.kind !== 'module_hook') {
|
|
139
|
+
return {
|
|
140
|
+
id: m.id,
|
|
141
|
+
intervalMinutes: m.intervalMinutes,
|
|
142
|
+
enabled: m.enabled,
|
|
143
|
+
lastRunAt: m.lastRunAt,
|
|
144
|
+
monitor: m,
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
const cadence = cadences.get(m.target)?.cadence ?? null;
|
|
148
|
+
return {
|
|
149
|
+
id: m.id,
|
|
150
|
+
// Unscheduled monitors are filtered out by `enabled` below, so this
|
|
151
|
+
// value is never used to decide due-ness for them.
|
|
152
|
+
intervalMinutes: isScheduled(cadence) ? cadence.minutes : MONITOR_INTERVAL_FLOOR_MINUTES,
|
|
153
|
+
enabled: isScheduled(cadence),
|
|
154
|
+
lastRunAt: m.lastRunAt,
|
|
155
|
+
monitor: m,
|
|
156
|
+
};
|
|
157
|
+
}),
|
|
135
158
|
deps.now(),
|
|
136
159
|
);
|
|
137
160
|
|
|
@@ -9,8 +9,10 @@
|
|
|
9
9
|
|
|
10
10
|
import { eq } from 'drizzle-orm';
|
|
11
11
|
import type { DbClient } from '../../db/client';
|
|
12
|
-
import { backups, modules } from '../../db/schema';
|
|
12
|
+
import { backups, moduleConfigs, modules } from '../../db/schema';
|
|
13
13
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
14
|
+
import { BACKUP_SCHEDULE_CONFIG_KEY } from '../backup-schedule';
|
|
15
|
+
import { parseStoredConfigValue } from '../module-config';
|
|
14
16
|
import type { InstalledModuleBackupInfo } from './backups';
|
|
15
17
|
|
|
16
18
|
const DEPLOYED_STATES = ['INSTALLED', 'VERIFIED'];
|
|
@@ -40,6 +42,7 @@ function latestSuccessfulBackupByModule(db: DbClient): Map<string, number> {
|
|
|
40
42
|
|
|
41
43
|
export function loadBackupAuditInfo(db: DbClient): InstalledModuleBackupInfo[] {
|
|
42
44
|
const latest = latestSuccessfulBackupByModule(db);
|
|
45
|
+
const overrides = backupScheduleOverrides(db);
|
|
43
46
|
return db
|
|
44
47
|
.select()
|
|
45
48
|
.from(modules)
|
|
@@ -49,6 +52,26 @@ export function loadBackupAuditInfo(db: DbClient): InstalledModuleBackupInfo[] {
|
|
|
49
52
|
id: module.id,
|
|
50
53
|
state: module.state,
|
|
51
54
|
manifest: module.manifestData as ModuleManifest,
|
|
55
|
+
scheduleOverride: overrides.get(module.id),
|
|
52
56
|
lastSuccessfulBackupAt: latest.get(module.id) ?? null,
|
|
53
57
|
}));
|
|
54
58
|
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Every operator backup-cadence override, by module.
|
|
62
|
+
*
|
|
63
|
+
* Read here rather than resolved here: the audit resolves override-against-
|
|
64
|
+
* manifest through the same accessor the backup sweep uses, so the two cannot
|
|
65
|
+
* disagree about what a module's cadence is.
|
|
66
|
+
*/
|
|
67
|
+
export function backupScheduleOverrides(db: DbClient): Map<string, string> {
|
|
68
|
+
const overrides = new Map<string, string>();
|
|
69
|
+
for (const row of db
|
|
70
|
+
.select()
|
|
71
|
+
.from(moduleConfigs)
|
|
72
|
+
.where(eq(moduleConfigs.key, BACKUP_SCHEDULE_CONFIG_KEY))
|
|
73
|
+
.all()) {
|
|
74
|
+
overrides.set(row.moduleId, String(parseStoredConfigValue(row)));
|
|
75
|
+
}
|
|
76
|
+
return overrides;
|
|
77
|
+
}
|
|
@@ -1,19 +1,19 @@
|
|
|
1
1
|
import { describe, expect, test } from 'bun:test';
|
|
2
2
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
3
|
-
import {
|
|
3
|
+
import { parseCadence } from '../cadence';
|
|
4
|
+
import { type InstalledModuleBackupInfo, auditBackups, backupStaleThresholdMs } from './backups';
|
|
4
5
|
|
|
5
6
|
const NOW = new Date('2026-04-25T00:00:00Z').getTime();
|
|
6
7
|
const HOUR = 60 * 60 * 1000;
|
|
7
8
|
const ONE_DAY = 24 * HOUR;
|
|
8
9
|
|
|
9
|
-
type Schedule = 'hourly' | 'daily' | 'weekly' | 'monthly' | 'manual';
|
|
10
|
-
|
|
11
10
|
function makeModule(
|
|
12
11
|
id: string,
|
|
13
12
|
opts: {
|
|
14
13
|
hasBackupHook: boolean;
|
|
15
14
|
lastSuccessfulBackupAt: number | null;
|
|
16
|
-
schedule?:
|
|
15
|
+
schedule?: string;
|
|
16
|
+
scheduleOverride?: string;
|
|
17
17
|
},
|
|
18
18
|
): InstalledModuleBackupInfo {
|
|
19
19
|
const manifest = {
|
|
@@ -27,9 +27,42 @@ function makeModule(
|
|
|
27
27
|
// Default to INSTALLED — existing tests assert backup findings
|
|
28
28
|
// fire, which is the deployed-module behavior. Tests for the
|
|
29
29
|
// non-deployed-skip behavior override this explicitly.
|
|
30
|
-
return {
|
|
30
|
+
return {
|
|
31
|
+
id,
|
|
32
|
+
state: 'INSTALLED',
|
|
33
|
+
manifest,
|
|
34
|
+
scheduleOverride: opts.scheduleOverride,
|
|
35
|
+
lastSuccessfulBackupAt: opts.lastSuccessfulBackupAt,
|
|
36
|
+
};
|
|
31
37
|
}
|
|
32
38
|
|
|
39
|
+
describe('backupStaleThresholdMs', () => {
|
|
40
|
+
function cadence(value: string) {
|
|
41
|
+
const parsed = parseCadence(value);
|
|
42
|
+
if (parsed === null) throw new Error(`test fixture is not a cadence: ${value}`);
|
|
43
|
+
return parsed;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// The four values the change's release note promises. A table could not
|
|
47
|
+
// answer for `6h`, so these come out of `cadence + max(1h, cadence × 0.1)`;
|
|
48
|
+
// `weekly` is the only one that now alerts EARLIER than it used to (8d).
|
|
49
|
+
test('the four documented thresholds', () => {
|
|
50
|
+
expect(backupStaleThresholdMs(cadence('hourly'))).toBe(2 * HOUR);
|
|
51
|
+
expect(backupStaleThresholdMs(cadence('daily'))).toBe(26.4 * HOUR);
|
|
52
|
+
expect(backupStaleThresholdMs(cadence('weekly'))).toBe(7.7 * ONE_DAY);
|
|
53
|
+
expect(backupStaleThresholdMs(cadence('monthly'))).toBe(33 * ONE_DAY);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test('a custom duration has a defined threshold', () => {
|
|
57
|
+
expect(backupStaleThresholdMs(cadence('6h'))).toBe(7 * HOUR);
|
|
58
|
+
expect(backupStaleThresholdMs(cadence('90m'))).toBe(90 * 60_000 + HOUR);
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test('manual has none — an opted-out module is never stale', () => {
|
|
62
|
+
expect(backupStaleThresholdMs('manual')).toBeNull();
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
|
|
33
66
|
describe('auditBackups', () => {
|
|
34
67
|
test('skips modules without an on_backup hook', async () => {
|
|
35
68
|
const result = await auditBackups({
|
|
@@ -39,7 +72,10 @@ describe('auditBackups', () => {
|
|
|
39
72
|
expect(result).toEqual([]);
|
|
40
73
|
});
|
|
41
74
|
|
|
42
|
-
|
|
75
|
+
// A module opted out of scheduled backups used to be reported as missing one
|
|
76
|
+
// forever: the never-backed-up check fired BEFORE the manual check, and the
|
|
77
|
+
// only remediation offered was the very thing the operator declined.
|
|
78
|
+
test('a module opted out is not reported as missing a backup', async () => {
|
|
43
79
|
const result = await auditBackups({
|
|
44
80
|
modules: [
|
|
45
81
|
makeModule('lunacycle', {
|
|
@@ -51,14 +87,63 @@ describe('auditBackups', () => {
|
|
|
51
87
|
now: () => NOW,
|
|
52
88
|
});
|
|
53
89
|
|
|
90
|
+
expect(result).toEqual([]);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test('a module that has NOT opted out is still reported as missing a backup', async () => {
|
|
94
|
+
const result = await auditBackups({
|
|
95
|
+
modules: [
|
|
96
|
+
makeModule('authentik', {
|
|
97
|
+
hasBackupHook: true,
|
|
98
|
+
lastSuccessfulBackupAt: null,
|
|
99
|
+
schedule: 'daily',
|
|
100
|
+
}),
|
|
101
|
+
],
|
|
102
|
+
now: () => NOW,
|
|
103
|
+
});
|
|
104
|
+
|
|
54
105
|
expect(result).toHaveLength(1);
|
|
55
106
|
expect(result[0]).toMatchObject({
|
|
56
107
|
category: 'backups',
|
|
57
108
|
severity: 'drift',
|
|
58
109
|
code: 'backup_missing',
|
|
59
|
-
subject: '
|
|
110
|
+
subject: 'authentik',
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("an operator's manual override silences a module the manifest wanted backed up", async () => {
|
|
115
|
+
const result = await auditBackups({
|
|
116
|
+
modules: [
|
|
117
|
+
makeModule('authentik', {
|
|
118
|
+
hasBackupHook: true,
|
|
119
|
+
lastSuccessfulBackupAt: null,
|
|
120
|
+
schedule: 'daily',
|
|
121
|
+
scheduleOverride: 'manual',
|
|
122
|
+
}),
|
|
123
|
+
],
|
|
124
|
+
now: () => NOW,
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
expect(result).toEqual([]);
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
test('staleness is judged against the override, not the suggestion', async () => {
|
|
131
|
+
// Manifest says weekly (fresh at 2 days); the operator asked for hourly.
|
|
132
|
+
const result = await auditBackups({
|
|
133
|
+
modules: [
|
|
134
|
+
makeModule('caddy', {
|
|
135
|
+
hasBackupHook: true,
|
|
136
|
+
lastSuccessfulBackupAt: NOW - 2 * ONE_DAY,
|
|
137
|
+
schedule: 'weekly',
|
|
138
|
+
scheduleOverride: 'hourly',
|
|
139
|
+
}),
|
|
140
|
+
],
|
|
141
|
+
now: () => NOW,
|
|
60
142
|
});
|
|
61
|
-
|
|
143
|
+
|
|
144
|
+
expect(result).toHaveLength(1);
|
|
145
|
+
expect(result[0].code).toBe('backup_stale');
|
|
146
|
+
expect(result[0].message).toContain('hourly');
|
|
62
147
|
});
|
|
63
148
|
|
|
64
149
|
test('manual schedule: never flags stale (user-driven cadence)', async () => {
|
|
@@ -93,12 +178,12 @@ describe('auditBackups', () => {
|
|
|
93
178
|
expect(result[0].message).toContain('daily');
|
|
94
179
|
});
|
|
95
180
|
|
|
96
|
-
test('daily schedule:
|
|
181
|
+
test('daily schedule: 27h-old is stale', async () => {
|
|
97
182
|
const result = await auditBackups({
|
|
98
183
|
modules: [
|
|
99
184
|
makeModule('authentik', {
|
|
100
185
|
hasBackupHook: true,
|
|
101
|
-
lastSuccessfulBackupAt: NOW -
|
|
186
|
+
lastSuccessfulBackupAt: NOW - 27 * HOUR,
|
|
102
187
|
schedule: 'daily',
|
|
103
188
|
}),
|
|
104
189
|
],
|