@celilo/cli 0.13.2 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/CELILO_CORE_MODULES.md +3 -0
  2. package/CELILO_SUBSYSTEMS.md +71 -2
  3. package/docs/ALERTING.md +298 -0
  4. package/docs/INDEX.md +103 -0
  5. package/drizzle/0016_trusted_sources.sql +10 -0
  6. package/drizzle/0017_alerting.sql +127 -0
  7. package/drizzle/meta/_journal.json +15 -1
  8. package/package.json +3 -2
  9. package/schemas/system_config.json +9 -0
  10. package/src/ansible/inventory.ts +2 -2
  11. package/src/cli/commands/alerts-act.ts +107 -0
  12. package/src/cli/commands/alerts-list.ts +62 -0
  13. package/src/cli/commands/alerts-poll.ts +129 -0
  14. package/src/cli/commands/alerts-sweep.ts +156 -0
  15. package/src/cli/commands/module-list.ts +50 -3
  16. package/src/cli/commands/monitor.ts +178 -0
  17. package/src/cli/commands/notify-config.ts +453 -0
  18. package/src/cli/commands/system-audit.ts +2 -0
  19. package/src/cli/commands/system-update.ts +1 -0
  20. package/src/cli/completion.ts +26 -0
  21. package/src/cli/generate-zsh-completion.ts +2 -0
  22. package/src/cli/index.ts +58 -0
  23. package/src/cli/tui/audit-state.ts +2 -0
  24. package/src/db/schema.ts +358 -0
  25. package/src/hooks/capability-loader.ts +158 -46
  26. package/src/hooks/capability-map-coverage.test.ts +101 -0
  27. package/src/manifest/schema.ts +60 -1
  28. package/src/services/alerting/ack.test.ts +212 -0
  29. package/src/services/alerting/ack.ts +119 -0
  30. package/src/services/alerting/builtin-monitors.test.ts +132 -0
  31. package/src/services/alerting/builtin-monitors.ts +84 -0
  32. package/src/services/alerting/builtin-source.ts +82 -0
  33. package/src/services/alerting/coverage-source.ts +38 -0
  34. package/src/services/alerting/deferral.test.ts +161 -0
  35. package/src/services/alerting/delivery-loop.test.ts +396 -0
  36. package/src/services/alerting/deploy-hooks.test.ts +125 -0
  37. package/src/services/alerting/deploy-hooks.ts +111 -0
  38. package/src/services/alerting/escalation.test.ts +207 -0
  39. package/src/services/alerting/escalation.ts +151 -0
  40. package/src/services/alerting/format.test.ts +193 -0
  41. package/src/services/alerting/format.ts +150 -0
  42. package/src/services/alerting/health-coverage.ts +81 -0
  43. package/src/services/alerting/inbound-poller.test.ts +298 -0
  44. package/src/services/alerting/inbound-poller.ts +236 -0
  45. package/src/services/alerting/inbound.test.ts +201 -0
  46. package/src/services/alerting/inbound.ts +112 -0
  47. package/src/services/alerting/interview-responder.test.ts +169 -0
  48. package/src/services/alerting/interview-responder.ts +158 -0
  49. package/src/services/alerting/keys.test.ts +155 -0
  50. package/src/services/alerting/keys.ts +190 -0
  51. package/src/services/alerting/monitors.ts +185 -0
  52. package/src/services/alerting/notification-responder.test.ts +290 -0
  53. package/src/services/alerting/notification-responder.ts +260 -0
  54. package/src/services/alerting/notifier.ts +219 -0
  55. package/src/services/alerting/people.ts +178 -0
  56. package/src/services/alerting/quiet-hours.test.ts +140 -0
  57. package/src/services/alerting/quiet-hours.ts +99 -0
  58. package/src/services/alerting/reconcile.test.ts +190 -0
  59. package/src/services/alerting/reconcile.ts +166 -0
  60. package/src/services/alerting/run-monitor.test.ts +185 -0
  61. package/src/services/alerting/run-monitor.ts +177 -0
  62. package/src/services/alerting/store.test.ts +222 -0
  63. package/src/services/alerting/store.ts +289 -0
  64. package/src/services/alerting/suppression.test.ts +228 -0
  65. package/src/services/alerting/suppression.ts +142 -0
  66. package/src/services/alerting/sweep-runner.test.ts +229 -0
  67. package/src/services/alerting/sweep-runner.ts +204 -0
  68. package/src/services/alerting/sweep.test.ts +61 -0
  69. package/src/services/alerting/sweep.ts +41 -0
  70. package/src/services/alerting/tokens.test.ts +152 -0
  71. package/src/services/alerting/tokens.ts +119 -0
  72. package/src/services/alerting/transport-loader.ts +48 -0
  73. package/src/services/aspect-runner.ts +2 -2
  74. package/src/services/audit/index.test.ts +1 -0
  75. package/src/services/audit/index.ts +3 -0
  76. package/src/services/audit/trusted-sources.test.ts +137 -0
  77. package/src/services/audit/trusted-sources.ts +124 -0
  78. package/src/services/audit/types.ts +2 -1
  79. package/src/services/firewall-reach.ts +83 -0
  80. package/src/services/health-runner.test.ts +50 -0
  81. package/src/services/health-runner.ts +116 -82
  82. package/src/services/module-deploy.ts +32 -3
  83. package/src/services/ssh-key-manager.test.ts +14 -0
  84. package/src/services/ssh-key-manager.ts +12 -0
  85. package/src/services/system-config-validator.test.ts +31 -1
  86. package/src/services/trusted-sources.test.ts +221 -0
  87. package/src/services/trusted-sources.ts +159 -0
  88. package/src/services/update/orchestrator.test.ts +1 -0
  89. package/src/templates/generator.ts +6 -29
@@ -0,0 +1,212 @@
1
+ import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
2
+ import { mkdtempSync, rmSync } from 'node:fs';
3
+ import { tmpdir } from 'node:os';
4
+ import { join } from 'node:path';
5
+ import type { DbClient } from '../../db/client';
6
+ import { type Route, alerts, modules, monitors } from '../../db/schema';
7
+ import { setupTestDatabase } from '../../test-utils/setup-test-db';
8
+ import { acknowledgeAlert, findLiveAlertByKey, resolveAlertManually, silenceAlert } from './ack';
9
+ import { moduleCheckAlertKey } from './keys';
10
+ import { createPerson, createRoute } from './people';
11
+ import { mintDelivery } from './tokens';
12
+
13
+ const NOW = new Date('2026-07-28T03:00:00Z');
14
+ const LATER = new Date('2026-07-28T04:00:00Z');
15
+ const KEY = moduleCheckAlertKey('caddy', 'disk-space');
16
+
17
+ describe('ack / silence / resolve', () => {
18
+ let dir: string;
19
+ let db: DbClient;
20
+ let peterRoute: Route;
21
+ let wifeRoute: Route;
22
+ let peterId: string;
23
+ let wifeId: string;
24
+
25
+ beforeEach(async () => {
26
+ dir = mkdtempSync(join(tmpdir(), 'ack-'));
27
+ const dbPath = join(dir, 'celilo.db');
28
+ process.env.CELILO_DB_PATH = dbPath;
29
+ db = await setupTestDatabase(dbPath);
30
+
31
+ db.insert(modules)
32
+ .values({
33
+ id: 'signal',
34
+ name: 'Signal',
35
+ version: '0.1.0',
36
+ manifestData: {},
37
+ sourcePath: '/tmp/signal',
38
+ })
39
+ .run();
40
+ db.insert(monitors)
41
+ .values({
42
+ id: 'mon-1',
43
+ kind: 'module_hook',
44
+ target: 'caddy',
45
+ intervalMinutes: 15,
46
+ severity: 'critical',
47
+ })
48
+ .run();
49
+ db.insert(alerts)
50
+ .values({
51
+ id: 'alert-1',
52
+ key: KEY,
53
+ activeKey: KEY,
54
+ monitorId: 'mon-1',
55
+ state: 'firing',
56
+ severity: 'critical',
57
+ graceUntil: NOW,
58
+ message: '/var 94% used',
59
+ })
60
+ .run();
61
+
62
+ const peter = createPerson(db, { name: 'peter', timezone: 'UTC' });
63
+ const wife = createPerson(db, { name: 'wife', timezone: 'UTC' });
64
+ peterId = peter.id;
65
+ wifeId = wife.id;
66
+ peterRoute = createRoute(db, {
67
+ personId: peter.id,
68
+ transportModuleId: 'signal',
69
+ address: '+1555001',
70
+ canAck: true,
71
+ });
72
+ wifeRoute = createRoute(db, {
73
+ personId: wife.id,
74
+ transportModuleId: 'signal',
75
+ address: '+1555002',
76
+ canAck: true,
77
+ });
78
+ });
79
+
80
+ afterEach(() => {
81
+ db.$client.close();
82
+ process.env.CELILO_DB_PATH = undefined;
83
+ try {
84
+ rmSync(dir, { recursive: true, force: true });
85
+ } catch {
86
+ /* ignore */
87
+ }
88
+ });
89
+
90
+ const page = (routeId: string) =>
91
+ mintDelivery(db, {
92
+ kind: 'alert',
93
+ targetId: 'alert-1',
94
+ routeId,
95
+ now: NOW,
96
+ ttlMs: 86_400_000,
97
+ });
98
+
99
+ describe('acknowledge', () => {
100
+ // Ack stops escalation. It does NOT resolve — the problem is still
101
+ // happening, someone is just dealing with it.
102
+ test('sets acked state and records who and when, leaving it unresolved', () => {
103
+ const result = acknowledgeAlert(db, 'alert-1', peterId, LATER);
104
+ expect(result?.alert.state).toBe('acked');
105
+ expect(result?.alert.ackedBy).toBe(peterId);
106
+ expect(result?.alert.ackedAt).toEqual(LATER);
107
+ expect(result?.alert.resolvedAt).toBeNull();
108
+ expect(result?.alert.activeKey).toBe(KEY);
109
+ });
110
+
111
+ // Without this the primary keeps believing they must act, which is the
112
+ // duplicated effort the escalation chain exists to avoid.
113
+ test("a secondary's ack is broadcast to everyone else who was paged", () => {
114
+ page(peterRoute.id);
115
+ page(wifeRoute.id);
116
+
117
+ const result = acknowledgeAlert(db, 'alert-1', wifeId, LATER);
118
+ expect(result?.broadcastTo).toEqual([peterRoute.id]);
119
+ });
120
+
121
+ test('the acknowledger is not told about their own ack', () => {
122
+ page(peterRoute.id);
123
+ const result = acknowledgeAlert(db, 'alert-1', peterId, LATER);
124
+ expect(result?.broadcastTo).toEqual([]);
125
+ });
126
+
127
+ test('nobody paged means nobody to broadcast to', () => {
128
+ const result = acknowledgeAlert(db, 'alert-1', peterId, LATER);
129
+ expect(result?.broadcastTo).toEqual([]);
130
+ });
131
+
132
+ test('duplicate deliveries to one route broadcast once', () => {
133
+ page(peterRoute.id);
134
+ page(peterRoute.id);
135
+ const result = acknowledgeAlert(db, 'alert-1', wifeId, LATER);
136
+ expect(result?.broadcastTo).toEqual([peterRoute.id]);
137
+ });
138
+
139
+ // A reply arriving just after recovery is a normal race, not an error.
140
+ test('acking an already-resolved alert is a no-op, not a failure', () => {
141
+ resolveAlertManually(db, 'alert-1', LATER);
142
+ const result = acknowledgeAlert(db, 'alert-1', peterId, LATER);
143
+ expect(result).not.toBeNull();
144
+ expect(result?.alert.state).toBe('resolved');
145
+ });
146
+
147
+ test('an unknown alert returns null rather than throwing', () => {
148
+ expect(acknowledgeAlert(db, 'nope', peterId, LATER)).toBeNull();
149
+ });
150
+ });
151
+
152
+ describe('silence', () => {
153
+ // Silence is not ack: nobody owns the problem, they just do not want to
154
+ // hear about it yet.
155
+ test('records an expiry without acknowledging', () => {
156
+ const alert = silenceAlert(db, 'alert-1', LATER);
157
+ expect(alert?.silencedUntil).toEqual(LATER);
158
+ expect(alert?.state).toBe('firing');
159
+ expect(alert?.ackedBy).toBeNull();
160
+ });
161
+ });
162
+
163
+ describe('manual resolve', () => {
164
+ test('clears activeKey so the alert leaves the live set', () => {
165
+ const alert = resolveAlertManually(db, 'alert-1', LATER);
166
+ expect(alert?.state).toBe('resolved');
167
+ expect(alert?.activeKey).toBeNull();
168
+ expect(alert?.resolvedAt).toEqual(LATER);
169
+ });
170
+
171
+ // Manual resolve cannot hide a real problem: the monitor is still the
172
+ // authority, and a key that is still failing simply fires again.
173
+ test('a still-failing key can re-fire afterwards', () => {
174
+ resolveAlertManually(db, 'alert-1', LATER);
175
+ expect(findLiveAlertByKey(db, KEY)).toBeUndefined();
176
+
177
+ db.insert(alerts)
178
+ .values({
179
+ id: 'alert-2',
180
+ key: KEY,
181
+ activeKey: KEY,
182
+ monitorId: 'mon-1',
183
+ state: 'pending',
184
+ severity: 'critical',
185
+ graceUntil: LATER,
186
+ message: '/var 96% used',
187
+ })
188
+ .run();
189
+ expect(findLiveAlertByKey(db, KEY)?.id).toBe('alert-2');
190
+ });
191
+ });
192
+
193
+ describe('findLiveAlertByKey', () => {
194
+ test('finds a live alert', () => {
195
+ expect(findLiveAlertByKey(db, KEY)?.id).toBe('alert-1');
196
+ });
197
+
198
+ test('ignores resolved alerts', () => {
199
+ resolveAlertManually(db, 'alert-1', LATER);
200
+ expect(findLiveAlertByKey(db, KEY)).toBeUndefined();
201
+ });
202
+
203
+ test('an unknown key finds nothing', () => {
204
+ expect(findLiveAlertByKey(db, 'module:nope')).toBeUndefined();
205
+ });
206
+
207
+ test('an acked alert is still live — it is not resolved', () => {
208
+ acknowledgeAlert(db, 'alert-1', peterId, LATER);
209
+ expect(findLiveAlertByKey(db, KEY)?.id).toBe('alert-1');
210
+ });
211
+ });
212
+ });
@@ -0,0 +1,119 @@
1
+ /**
2
+ * Acknowledgement, silence, and manual resolution.
3
+ *
4
+ * Three operations that look similar and mean entirely different things
5
+ * (design S5). Collapsing any two is how an alerting system stops being
6
+ * trusted:
7
+ *
8
+ * ack a person has this. Escalation stops; the alert is still FIRING,
9
+ * because the problem is still happening.
10
+ * silence deliberate, expiring, audited. "I know, stop telling me until X."
11
+ * resolve the operator asserts the condition is gone. The next successful
12
+ * monitor run is still the authority — if it disagrees, the alert
13
+ * comes straight back, which is correct.
14
+ */
15
+
16
+ import { and, eq, isNull, ne } from 'drizzle-orm';
17
+ import type { DbClient } from '../../db/client';
18
+ import { type Alert, alerts, notificationDeliveries, routes } from '../../db/schema';
19
+
20
+ export interface AckResult {
21
+ alert: Alert;
22
+ /** Routes that were paged and should be told someone has it. */
23
+ broadcastTo: string[];
24
+ }
25
+
26
+ /**
27
+ * Acknowledge an alert.
28
+ *
29
+ * Returns every OTHER route that was paged, so the caller can tell them who
30
+ * took it. Without that, a secondary who acks leaves the primary still
31
+ * believing they need to act — the exact duplicated-effort the escalation
32
+ * chain exists to avoid.
33
+ */
34
+ export function acknowledgeAlert(
35
+ db: DbClient,
36
+ alertId: string,
37
+ personId: string,
38
+ now: Date,
39
+ ): AckResult | null {
40
+ const alert = db.select().from(alerts).where(eq(alerts.id, alertId)).get();
41
+ if (!alert) return null;
42
+
43
+ // Acking a resolved alert is a no-op rather than an error: a reply that
44
+ // arrives just after recovery is a normal race, not operator error.
45
+ if (alert.state === 'resolved') return { alert, broadcastTo: [] };
46
+
47
+ db.update(alerts)
48
+ .set({ state: 'acked', ackedBy: personId, ackedAt: now })
49
+ .where(eq(alerts.id, alertId))
50
+ .run();
51
+
52
+ const paged = db
53
+ .select({ routeId: notificationDeliveries.routeId })
54
+ .from(notificationDeliveries)
55
+ .where(
56
+ and(eq(notificationDeliveries.kind, 'alert'), eq(notificationDeliveries.targetId, alertId)),
57
+ )
58
+ .all();
59
+
60
+ const acknowledgerRoutes = new Set(
61
+ db
62
+ .select({ id: routes.id })
63
+ .from(routes)
64
+ .where(eq(routes.personId, personId))
65
+ .all()
66
+ .map((r) => r.id),
67
+ );
68
+
69
+ // Don't tell the acknowledger about their own ack.
70
+ const broadcastTo = [...new Set(paged.map((p) => p.routeId))].filter(
71
+ (routeId) => !acknowledgerRoutes.has(routeId),
72
+ );
73
+
74
+ const updated = db.select().from(alerts).where(eq(alerts.id, alertId)).get() as Alert;
75
+ return { alert: updated, broadcastTo };
76
+ }
77
+
78
+ /**
79
+ * Silence an alert until a given instant.
80
+ *
81
+ * Distinct from ack: silence says "stop telling me", ack says "I have this".
82
+ * An alert can be silenced without anyone owning it, which is exactly the
83
+ * state worth being able to see later.
84
+ */
85
+ export function silenceAlert(db: DbClient, alertId: string, until: Date): Alert | null {
86
+ const alert = db.select().from(alerts).where(eq(alerts.id, alertId)).get();
87
+ if (!alert) return null;
88
+
89
+ db.update(alerts).set({ silencedUntil: until }).where(eq(alerts.id, alertId)).run();
90
+ return db.select().from(alerts).where(eq(alerts.id, alertId)).get() as Alert;
91
+ }
92
+
93
+ /**
94
+ * Manually resolve an alert.
95
+ *
96
+ * The operator's assertion, not the monitor's. If the condition is still
97
+ * failing, the next successful run re-creates the alert — which is correct,
98
+ * and is why this is safe to offer: it cannot be used to permanently hide a
99
+ * real problem, only to clear one the operator knows is finished.
100
+ */
101
+ export function resolveAlertManually(db: DbClient, alertId: string, now: Date): Alert | null {
102
+ const alert = db.select().from(alerts).where(eq(alerts.id, alertId)).get();
103
+ if (!alert) return null;
104
+
105
+ db.update(alerts)
106
+ .set({ state: 'resolved', activeKey: null, resolvedAt: now })
107
+ .where(eq(alerts.id, alertId))
108
+ .run();
109
+ return db.select().from(alerts).where(eq(alerts.id, alertId)).get() as Alert;
110
+ }
111
+
112
+ /** Live alerts whose key matches, for resolving an operator-typed key. */
113
+ export function findLiveAlertByKey(db: DbClient, key: string): Alert | undefined {
114
+ return db
115
+ .select()
116
+ .from(alerts)
117
+ .where(and(eq(alerts.key, key), isNull(alerts.resolvedAt), ne(alerts.state, 'resolved')))
118
+ .get();
119
+ }
@@ -0,0 +1,132 @@
1
+ import { describe, expect, test } from 'bun:test';
2
+ import type { DriftFinding } from '../audit/types';
3
+ import {
4
+ failingKeysFromFindings,
5
+ severityForDriftSeverity,
6
+ targetKindForCategory,
7
+ } from './builtin-monitors';
8
+ import { healthCoverageFailingKeys } from './health-coverage';
9
+
10
+ function finding(over: Partial<DriftFinding> = {}): DriftFinding {
11
+ return {
12
+ category: 'machines_reachable',
13
+ severity: 'drift',
14
+ code: 'machine_unreachable',
15
+ message: 'iot unreachable',
16
+ subject: 'iot',
17
+ ...over,
18
+ };
19
+ }
20
+
21
+ describe('targetKindForCategory', () => {
22
+ test('machine-scoped category', () => {
23
+ expect(targetKindForCategory('machines_reachable')).toBe('machine');
24
+ });
25
+
26
+ test('module-scoped category', () => {
27
+ expect(targetKindForCategory('backups')).toBe('module');
28
+ });
29
+
30
+ // Whole-system categories have no narrower subject to suppress against.
31
+ test('unmapped category falls back to system', () => {
32
+ expect(targetKindForCategory('cli_version')).toBe('system');
33
+ expect(targetKindForCategory('schema')).toBe('system');
34
+ });
35
+ });
36
+
37
+ describe('severityForDriftSeverity', () => {
38
+ test('todo never pages, whatever the monitor says', () => {
39
+ expect(severityForDriftSeverity('todo', 'critical')).toBe('warning');
40
+ });
41
+
42
+ test('drift and blocked take the monitor severity', () => {
43
+ expect(severityForDriftSeverity('drift', 'critical')).toBe('critical');
44
+ expect(severityForDriftSeverity('blocked', 'critical')).toBe('critical');
45
+ });
46
+ });
47
+
48
+ describe('failingKeysFromFindings', () => {
49
+ test('projects findings into builtin keys', () => {
50
+ const keys = failingKeysFromFindings('machines_reachable', [finding()], 'critical');
51
+ expect(keys).toEqual([
52
+ {
53
+ key: 'builtin:machines_reachable/machine:iot',
54
+ severity: 'critical',
55
+ message: 'iot unreachable',
56
+ details: undefined,
57
+ },
58
+ ]);
59
+ });
60
+
61
+ // A monitor owns exactly one category. Resolution is set-difference over the
62
+ // keys a monitor reports, so letting a stray category through would make some
63
+ // OTHER monitor's alerts resolve at random.
64
+ test('ignores findings from other categories', () => {
65
+ const mixed = [finding(), finding({ category: 'backups', subject: 'forgejo' })];
66
+ const keys = failingKeysFromFindings('machines_reachable', mixed, 'critical');
67
+ expect(keys).toHaveLength(1);
68
+ expect(keys[0].key).toBe('builtin:machines_reachable/machine:iot');
69
+ });
70
+
71
+ test('no findings produces an empty failing set', () => {
72
+ expect(failingKeysFromFindings('machines_reachable', [], 'critical')).toEqual([]);
73
+ });
74
+ });
75
+
76
+ describe('healthCoverageFailingKeys', () => {
77
+ const base = { hasHealthCheckHook: true, hasEnabledMonitor: true } as const;
78
+
79
+ test('a monitored module produces no finding', () => {
80
+ expect(healthCoverageFailingKeys([{ id: 'caddy', state: 'VERIFIED', ...base }])).toEqual([]);
81
+ });
82
+
83
+ test('a module with no health_check hook is surfaced', () => {
84
+ const keys = healthCoverageFailingKeys([
85
+ { id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false, hasEnabledMonitor: false },
86
+ ]);
87
+ expect(keys).toHaveLength(1);
88
+ expect(keys[0].key).toBe('builtin:health_coverage/module:signal');
89
+ expect(keys[0].message).toContain('no health_check hook');
90
+ });
91
+
92
+ test('a module with a hook but no monitor is surfaced differently', () => {
93
+ const keys = healthCoverageFailingKeys([
94
+ { id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: true, hasEnabledMonitor: false },
95
+ ]);
96
+ expect(keys).toHaveLength(1);
97
+ expect(keys[0].message).toContain('nothing schedules it');
98
+ });
99
+
100
+ // A coverage gap is real but is not an outage — it must never page.
101
+ test('findings are warning severity', () => {
102
+ const keys = healthCoverageFailingKeys([
103
+ { id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: false, hasEnabledMonitor: false },
104
+ ]);
105
+ expect(keys[0].severity).toBe('warning');
106
+ });
107
+
108
+ // Not-yet-deployed modules have nothing to observe; an absent monitor there
109
+ // is expected, not a gap.
110
+ test.each(['IMPORTED', 'VALIDATED', 'CONFIGURED', 'DEPLOYING', 'ERROR'] as const)(
111
+ 'ignores a module in state %s',
112
+ (state) => {
113
+ expect(
114
+ healthCoverageFailingKeys([
115
+ { id: 'x', state, hasHealthCheckHook: false, hasEnabledMonitor: false },
116
+ ]),
117
+ ).toEqual([]);
118
+ },
119
+ );
120
+
121
+ test('reports only the modules that are actually uncovered', () => {
122
+ const keys = healthCoverageFailingKeys([
123
+ { id: 'caddy', state: 'VERIFIED', ...base },
124
+ { id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false, hasEnabledMonitor: false },
125
+ { id: 'forgejo', state: 'INSTALLED', hasHealthCheckHook: true, hasEnabledMonitor: false },
126
+ ]);
127
+ expect(keys.map((k) => k.key)).toEqual([
128
+ 'builtin:health_coverage/module:signal',
129
+ 'builtin:health_coverage/module:forgejo',
130
+ ]);
131
+ });
132
+ });
@@ -0,0 +1,84 @@
1
+ /**
2
+ * Built-in monitors — scheduling celilo's own audit checks.
3
+ *
4
+ * `celilo system audit` already implements fourteen checks that know how to
5
+ * find a broken fleet; until now nothing ran them on a schedule or told anyone
6
+ * what they found. A `builtin_check` monitor runs ONE of those categories and
7
+ * projects its findings into alert keys, so the same machinery that carries a
8
+ * module's failing health check carries "machine iot is unreachable".
9
+ *
10
+ * See openspec/changes/add-alerting/design.md (Gap 6 → D4).
11
+ */
12
+
13
+ import type { AlertSeverity } from '../../db/schema';
14
+ import type { DriftCategory, DriftFinding, DriftSeverity } from '../audit/types';
15
+ import { type FailingKey, builtinAlertKey } from './keys';
16
+
17
+ /**
18
+ * What kind of entity a category's findings are about.
19
+ *
20
+ * This is not cosmetic: suppression resolves a key's ancestors through the
21
+ * deployment topology, and it can only do that if it knows whether
22
+ * `builtin:…/x:iot` names a machine or a module. Categories absent from the map
23
+ * are about the management system as a whole and have no narrower subject.
24
+ */
25
+ const TARGET_KIND_BY_CATEGORY: Partial<Record<DriftCategory, string>> = {
26
+ machines_reachable: 'machine',
27
+ services_reachable: 'service',
28
+ services_credentials: 'service',
29
+ backups: 'module',
30
+ health: 'module',
31
+ module_versions: 'module',
32
+ module_configs: 'module',
33
+ undeployed_modules: 'module',
34
+ unconfigured_modules: 'module',
35
+ };
36
+
37
+ const SYSTEM_TARGET_KIND = 'system';
38
+
39
+ export function targetKindForCategory(category: DriftCategory): string {
40
+ return TARGET_KIND_BY_CATEGORY[category] ?? SYSTEM_TARGET_KIND;
41
+ }
42
+
43
+ /**
44
+ * Map a drift severity to the severity its alert carries.
45
+ *
46
+ * `todo` findings are next-step reminders ("you imported this and haven't
47
+ * deployed it"), not divergence — they are recorded but never page, mirroring
48
+ * how a `warn` health-check item behaves. `drift` and `blocked` take the
49
+ * monitor's configured severity.
50
+ */
51
+ export function severityForDriftSeverity(
52
+ driftSeverity: DriftSeverity,
53
+ monitorSeverity: AlertSeverity,
54
+ ): AlertSeverity {
55
+ return driftSeverity === 'todo' ? 'warning' : monitorSeverity;
56
+ }
57
+
58
+ /**
59
+ * Project the findings of ONE audit category into the complete set of currently
60
+ * failing keys for that monitor.
61
+ *
62
+ * Findings from other categories are ignored rather than silently folded in: a
63
+ * `builtin_check` monitor owns exactly one category, and resolution works by
64
+ * set difference over the keys a monitor reports. Letting a stray category's
65
+ * findings through would make another monitor's alerts resolve at random.
66
+ */
67
+ export function failingKeysFromFindings(
68
+ category: DriftCategory,
69
+ findings: DriftFinding[],
70
+ monitorSeverity: AlertSeverity,
71
+ ): FailingKey[] {
72
+ const kind = targetKindForCategory(category);
73
+ const failing: FailingKey[] = [];
74
+ for (const finding of findings) {
75
+ if (finding.category !== category) continue;
76
+ failing.push({
77
+ key: builtinAlertKey(category, kind, finding.subject),
78
+ severity: severityForDriftSeverity(finding.severity, monitorSeverity),
79
+ message: finding.message,
80
+ details: finding.details,
81
+ });
82
+ }
83
+ return failing;
84
+ }
@@ -0,0 +1,82 @@
1
+ /**
2
+ * Running a built-in check for a monitor.
3
+ *
4
+ * Deliberately NOT `runAudit`. That runner computes all fourteen categories and
5
+ * needs the whole world injected — proxmox connections, a registry client,
6
+ * secret decryption, a terraform binary — which is the wrong cost and the wrong
7
+ * failure surface for a check that runs every few minutes. A monitor owns one
8
+ * category, so it calls that category directly.
9
+ *
10
+ * Only the categories the MVP schedules are wired. An unwired one fails loudly
11
+ * rather than silently returning no findings, which the reconciler would read
12
+ * as "nothing is wrong".
13
+ */
14
+
15
+ import { execFile } from 'node:child_process';
16
+ import { promisify } from 'node:util';
17
+ import { auditMachinesReachable } from '../audit/machines-reachable';
18
+ import type { MachineReachableResult } from '../audit/machines-reachable';
19
+ import type { DriftCategory, DriftFinding } from '../audit/types';
20
+ import { listMachines } from '../machine-pool';
21
+
22
+ const execFileAsync = promisify(execFile);
23
+
24
+ /** Categories a monitor can currently schedule. */
25
+ export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = ['machines_reachable'];
26
+
27
+ export function isSchedulableBuiltin(category: string): category is DriftCategory {
28
+ return (SCHEDULABLE_BUILTIN_CHECKS as readonly string[]).includes(category);
29
+ }
30
+
31
+ /**
32
+ * SSH-probe every pool machine.
33
+ *
34
+ * `BatchMode=yes` prevents a password prompt from hanging the probe forever,
35
+ * and `ConnectTimeout` bounds the wait on an unresponsive host — the exact
36
+ * condition this check exists to detect must not be the one that wedges it.
37
+ */
38
+ async function probeMachines(): Promise<MachineReachableResult[]> {
39
+ const machines = await listMachines();
40
+ return Promise.all(
41
+ machines.map(async (m): Promise<MachineReachableResult> => {
42
+ try {
43
+ await execFileAsync(
44
+ 'ssh',
45
+ [
46
+ '-o',
47
+ 'BatchMode=yes',
48
+ '-o',
49
+ 'ConnectTimeout=5',
50
+ '-o',
51
+ 'StrictHostKeyChecking=no',
52
+ '-o',
53
+ 'UserKnownHostsFile=/dev/null',
54
+ `${m.sshUser}@${m.ipAddress}`,
55
+ 'true',
56
+ ],
57
+ { timeout: 8000 },
58
+ );
59
+ return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
60
+ } catch (err) {
61
+ const e = err as { stderr?: string; message?: string };
62
+ return {
63
+ id: m.id,
64
+ hostname: m.hostname,
65
+ ipAddress: m.ipAddress,
66
+ reachable: false,
67
+ message: e.stderr?.trim() || e.message || 'SSH probe failed',
68
+ };
69
+ }
70
+ }),
71
+ );
72
+ }
73
+
74
+ export async function runBuiltinCheckForMonitor(category: DriftCategory): Promise<DriftFinding[]> {
75
+ if (category === 'machines_reachable') {
76
+ return auditMachinesReachable({ results: await probeMachines() });
77
+ }
78
+
79
+ throw new Error(
80
+ `Built-in check "${category}" is not schedulable yet. Schedulable: ${SCHEDULABLE_BUILTIN_CHECKS.join(', ')}.`,
81
+ );
82
+ }
@@ -0,0 +1,38 @@
1
+ /**
2
+ * Reading the module roster for the health-coverage check.
3
+ *
4
+ * Split out from `health-coverage.ts` so that the decision — which modules
5
+ * count as unobserved — stays pure and testable, while the query that feeds it
6
+ * lives here. See Rule 2.3.
7
+ */
8
+
9
+ import { eq } from 'drizzle-orm';
10
+ import type { DbClient } from '../../db/client';
11
+ import { modules, monitors } from '../../db/schema';
12
+ import type { ModuleManifest } from '../../manifest/schema';
13
+ import type { ModuleCoverageInput } from './health-coverage';
14
+
15
+ export function loadModuleCoverage(db: DbClient): ModuleCoverageInput[] {
16
+ const monitored = new Set(
17
+ db
18
+ .select({ target: monitors.target })
19
+ .from(monitors)
20
+ .where(eq(monitors.enabled, true))
21
+ .all()
22
+ .map((row) => row.target),
23
+ );
24
+
25
+ return db
26
+ .select({ id: modules.id, state: modules.state, manifestData: modules.manifestData })
27
+ .from(modules)
28
+ .all()
29
+ .map((module) => {
30
+ const manifest = module.manifestData as ModuleManifest;
31
+ return {
32
+ id: module.id,
33
+ state: module.state,
34
+ hasHealthCheckHook: Boolean(manifest.hooks?.health_check),
35
+ hasEnabledMonitor: monitored.has(module.id),
36
+ };
37
+ });
38
+ }