@celilo/cli 0.14.4 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +1 -1
- package/CELILO_SUBSYSTEMS.md +18 -2
- package/drizzle/0018_drop_alert_policy_snapshot.sql +46 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +3 -3
- package/src/cli/commands/alerts-list.ts +10 -0
- package/src/cli/commands/alerts-poll.ts +12 -6
- package/src/cli/commands/alerts-sweep.ts +22 -81
- package/src/cli/commands/backup-sweep.ts +65 -0
- package/src/cli/commands/module-operations.test.ts +93 -0
- package/src/cli/commands/module-operations.ts +134 -0
- package/src/cli/commands/module-upgrade.test.ts +32 -20
- package/src/cli/commands/module-upgrade.ts +37 -32
- package/src/cli/commands/monitor.ts +26 -6
- package/src/cli/commands/system-audit.ts +3 -30
- package/src/cli/completion.ts +18 -1
- package/src/cli/index.ts +11 -0
- package/src/db/schema.ts +5 -3
- package/src/manifest/schema.ts +4 -1
- package/src/module/packaging/build.ts +4 -0
- package/src/services/alerting/builtin-source.ts +17 -2
- package/src/services/alerting/delivery-loop.test.ts +5 -1
- package/src/services/alerting/format.test.ts +0 -1
- package/src/services/alerting/inbound-poller.test.ts +44 -8
- package/src/services/alerting/inbound-poller.ts +65 -28
- package/src/services/alerting/notify-deps.ts +113 -0
- package/src/services/alerting/run-monitor.ts +0 -1
- package/src/services/alerting/store.test.ts +1 -1
- package/src/services/alerting/store.ts +0 -2
- package/src/services/alerting/sweep-runner.test.ts +11 -2
- package/src/services/alerting/sweep-runner.ts +14 -7
- package/src/services/audit/backup-source.ts +54 -0
- package/src/services/audit/backups.test.ts +7 -2
- package/src/services/audit/backups.ts +10 -18
- package/src/services/backup-cipher.test.ts +188 -0
- package/src/services/backup-cipher.ts +178 -0
- package/src/services/backup-create.ts +20 -30
- package/src/services/backup-envelope-roundtrip.test.ts +6 -26
- package/src/services/backup-restore.ts +10 -16
- package/src/services/backup-schedule.ts +35 -0
- package/src/services/backup-sweep.test.ts +148 -0
- package/src/services/backup-sweep.ts +124 -0
- package/src/services/deploy-posture.ts +15 -2
- package/src/services/module-operations.test.ts +67 -6
- package/src/services/module-operations.ts +69 -19
- package/src/services/module-subscriptions.test.ts +33 -2
- package/src/services/module-subscriptions.ts +10 -1
- package/src/services/module-validator/typescript-build.test.ts +20 -1
- package/src/services/module-validator/typescript-build.ts +9 -5
- package/src/services/restore-from-file.ts +6 -21
- package/src/templates/generator.test.ts +88 -0
- package/src/templates/generator.ts +119 -16
|
@@ -27,7 +27,7 @@ describe('pollInbound', () => {
|
|
|
27
27
|
|
|
28
28
|
function deps(messages: InboundMessage[], over: Partial<InboundPollDeps> = {}): InboundPollDeps {
|
|
29
29
|
return {
|
|
30
|
-
receiveFrom: async () => ({ messages, cursor: 'cursor-1' }),
|
|
30
|
+
receiveFrom: async () => ({ status: 'received', messages, cursor: 'cursor-1' }),
|
|
31
31
|
readCursor: (t) => cursors.get(t) ?? null,
|
|
32
32
|
writeCursor: (t, c) => {
|
|
33
33
|
cursors.set(t, c);
|
|
@@ -190,7 +190,7 @@ describe('pollInbound', () => {
|
|
|
190
190
|
const delivery = page(peterRoute.id);
|
|
191
191
|
const report = await pollInbound(db, deps([inbound(WIFE, delivery.token)]));
|
|
192
192
|
|
|
193
|
-
expect(report.
|
|
193
|
+
expect(report.unheard).toEqual([{ senderAddress: WIFE, reason: 'wrong_sender' }]);
|
|
194
194
|
expect(report.acked).toBe(0);
|
|
195
195
|
expect(alertState()).toBe('firing');
|
|
196
196
|
});
|
|
@@ -200,7 +200,7 @@ describe('pollInbound', () => {
|
|
|
200
200
|
const delivery = page(peterRoute.id);
|
|
201
201
|
const report = await pollInbound(db, deps([inbound(STRANGER, delivery.token)]));
|
|
202
202
|
|
|
203
|
-
expect(report.
|
|
203
|
+
expect(report.unheard).toEqual([{ senderAddress: STRANGER, reason: 'unknown_sender' }]);
|
|
204
204
|
expect(alertState()).toBe('firing');
|
|
205
205
|
});
|
|
206
206
|
|
|
@@ -228,7 +228,7 @@ describe('pollInbound', () => {
|
|
|
228
228
|
expect(first.acked).toBe(1);
|
|
229
229
|
// The token is spent, so the replay is refused rather than acking again.
|
|
230
230
|
expect(second.acked).toBe(0);
|
|
231
|
-
expect(second.
|
|
231
|
+
expect(second.unheard).toEqual([{ senderAddress: PETER, reason: 'unknown_token' }]);
|
|
232
232
|
expect(alertState()).toBe('acked');
|
|
233
233
|
});
|
|
234
234
|
|
|
@@ -262,31 +262,67 @@ describe('pollInbound', () => {
|
|
|
262
262
|
});
|
|
263
263
|
|
|
264
264
|
const report = await pollInbound(db, deps([inbound(PETER, 'ack')]));
|
|
265
|
-
expect(report.
|
|
265
|
+
expect(report.unheard).toEqual([{ senderAddress: PETER, reason: 'ambiguous' }]);
|
|
266
266
|
expect(alertState()).toBe('firing');
|
|
267
267
|
});
|
|
268
268
|
|
|
269
269
|
// One dead transport must not stop the others being read.
|
|
270
270
|
test('an unreachable transport is skipped without throwing', async () => {
|
|
271
|
-
const report = await pollInbound(
|
|
271
|
+
const report = await pollInbound(
|
|
272
|
+
db,
|
|
273
|
+
deps([], { receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
|
|
274
|
+
);
|
|
272
275
|
expect(report.transportsPolled).toBe(1);
|
|
273
276
|
expect(report.messagesRead).toBe(0);
|
|
274
277
|
});
|
|
275
278
|
|
|
279
|
+
/**
|
|
280
|
+
* The regression that cost a week. signal-cli's daemon refused celilo's read
|
|
281
|
+
* call outright ("Receive command cannot be used if messages are already
|
|
282
|
+
* being received") and the poller reported the same "0 message(s)" it
|
|
283
|
+
* reports when nobody has replied. The reason must reach the report, or the
|
|
284
|
+
* two are indistinguishable from outside.
|
|
285
|
+
*/
|
|
286
|
+
test('a transport that REFUSES the read says so instead of looking quiet', async () => {
|
|
287
|
+
const refusal = 'Receive command cannot be used if messages are already being received.';
|
|
288
|
+
const report = await pollInbound(
|
|
289
|
+
db,
|
|
290
|
+
deps([], { receiveFrom: async () => ({ status: 'failed', error: refusal }) }),
|
|
291
|
+
);
|
|
292
|
+
|
|
293
|
+
expect(report.failures).toEqual([{ transportModuleId: 'signal', error: refusal }]);
|
|
294
|
+
// ...and a genuinely quiet transport must NOT look like a failure.
|
|
295
|
+
const quiet = await pollInbound(db, deps([]));
|
|
296
|
+
expect(quiet.failures).toEqual([]);
|
|
297
|
+
expect(quiet.messagesRead).toBe(0);
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
test('a unidirectional transport is not reported as a failure', async () => {
|
|
301
|
+
const report = await pollInbound(
|
|
302
|
+
db,
|
|
303
|
+
deps([], { receiveFrom: async () => ({ status: 'unidirectional' }) }),
|
|
304
|
+
);
|
|
305
|
+
expect(report.failures).toEqual([]);
|
|
306
|
+
expect(report.transportsPolled).toBe(1);
|
|
307
|
+
});
|
|
308
|
+
|
|
276
309
|
test('the cursor is persisted after a successful poll', async () => {
|
|
277
310
|
await pollInbound(db, deps([]));
|
|
278
311
|
expect(cursors.get('signal')).toBe('cursor-1');
|
|
279
312
|
});
|
|
280
313
|
|
|
281
314
|
test('the cursor is NOT advanced when the transport could not be reached', async () => {
|
|
282
|
-
await pollInbound(
|
|
315
|
+
await pollInbound(
|
|
316
|
+
db,
|
|
317
|
+
deps([], { receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
|
|
318
|
+
);
|
|
283
319
|
expect(cursors.get('signal')).toBeUndefined();
|
|
284
320
|
});
|
|
285
321
|
|
|
286
322
|
test('gibberish is rejected without touching the alert', async () => {
|
|
287
323
|
page(peterRoute.id);
|
|
288
324
|
const report = await pollInbound(db, deps([inbound(PETER, 'what is going on')]));
|
|
289
|
-
expect(report.
|
|
325
|
+
expect(report.unheard).toEqual([{ senderAddress: PETER, reason: 'unrecognised' }]);
|
|
290
326
|
expect(alertState()).toBe('firing');
|
|
291
327
|
});
|
|
292
328
|
|
|
@@ -25,12 +25,35 @@ import type { NotificationTransport } from './notifier';
|
|
|
25
25
|
import { composeAckBroadcastBody } from './notifier';
|
|
26
26
|
import { consumeDelivery } from './tokens';
|
|
27
27
|
|
|
28
|
+
/**
|
|
29
|
+
* What one attempt to read a transport produced.
|
|
30
|
+
*
|
|
31
|
+
* `unidirectional` and `failed` used to collapse into a bare null, and that
|
|
32
|
+
* cost a week: a transport whose read call was being REFUSED reported the same
|
|
33
|
+
* "0 message(s)" as a transport nobody had replied on. An operator could not
|
|
34
|
+
* tell "my reply never arrived" from "celilo cannot read this transport at
|
|
35
|
+
* all", and neither could anyone debugging it.
|
|
36
|
+
*/
|
|
37
|
+
export type TransportReceive =
|
|
38
|
+
| { status: 'received'; messages: InboundMessage[]; cursor: string | null }
|
|
39
|
+
| { status: 'unidirectional' }
|
|
40
|
+
| { status: 'failed'; error: string };
|
|
41
|
+
|
|
42
|
+
/** A transport that could not be read, and why. */
|
|
43
|
+
export interface TransportFailure {
|
|
44
|
+
transportModuleId: string;
|
|
45
|
+
error: string;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** A message that was read but not acted on, and why. */
|
|
49
|
+
export interface UnheardMessage {
|
|
50
|
+
senderAddress: string;
|
|
51
|
+
reason: 'unknown_sender' | 'unknown_token' | 'wrong_sender' | 'ambiguous' | 'unrecognised';
|
|
52
|
+
}
|
|
53
|
+
|
|
28
54
|
export interface InboundPollDeps {
|
|
29
|
-
/** Receive from one transport module
|
|
30
|
-
receiveFrom(
|
|
31
|
-
transportModuleId: string,
|
|
32
|
-
cursor: string | null,
|
|
33
|
-
): Promise<{ messages: InboundMessage[]; cursor: string | null } | null>;
|
|
55
|
+
/** Receive from one transport module. */
|
|
56
|
+
receiveFrom(transportModuleId: string, cursor: string | null): Promise<TransportReceive>;
|
|
34
57
|
/** Persisted receive cursor per transport. */
|
|
35
58
|
readCursor(transportModuleId: string): string | null;
|
|
36
59
|
writeCursor(transportModuleId: string, cursor: string | null): void;
|
|
@@ -53,12 +76,14 @@ export interface InboundPollReport {
|
|
|
53
76
|
transportsPolled: number;
|
|
54
77
|
messagesRead: number;
|
|
55
78
|
acked: number;
|
|
56
|
-
ignored: number;
|
|
57
|
-
rejected: number;
|
|
58
79
|
/** Routes told that someone else took the alert. */
|
|
59
80
|
broadcast: number;
|
|
60
81
|
/** Deploy questions answered from a phone. */
|
|
61
82
|
answered: number;
|
|
83
|
+
/** Transports that could not be read at all. Never silent — see above. */
|
|
84
|
+
failures: TransportFailure[];
|
|
85
|
+
/** Messages read and then discarded, with the reason each was discarded. */
|
|
86
|
+
unheard: UnheardMessage[];
|
|
62
87
|
}
|
|
63
88
|
|
|
64
89
|
/** Transports that have at least one route pointing at them. */
|
|
@@ -89,19 +114,24 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
|
|
|
89
114
|
transportsPolled: 0,
|
|
90
115
|
messagesRead: 0,
|
|
91
116
|
acked: 0,
|
|
92
|
-
ignored: 0,
|
|
93
|
-
rejected: 0,
|
|
94
117
|
broadcast: 0,
|
|
95
118
|
answered: 0,
|
|
119
|
+
failures: [],
|
|
120
|
+
unheard: [],
|
|
96
121
|
};
|
|
97
122
|
|
|
98
123
|
for (const transportId of transportsWithRoutes(db)) {
|
|
99
124
|
const received = await deps.receiveFrom(transportId, deps.readCursor(transportId));
|
|
100
125
|
report.transportsPolled++;
|
|
101
|
-
//
|
|
102
|
-
//
|
|
103
|
-
// the
|
|
104
|
-
|
|
126
|
+
// One dead transport must not stop the others being read — but it is
|
|
127
|
+
// RECORDED rather than skipped in silence, because "cannot read" and
|
|
128
|
+
// "nothing to read" are the two things an operator most needs to tell
|
|
129
|
+
// apart at 3am.
|
|
130
|
+
if (received.status === 'failed') {
|
|
131
|
+
report.failures.push({ transportModuleId: transportId, error: received.error });
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
if (received.status === 'unidirectional') continue;
|
|
105
135
|
|
|
106
136
|
for (const message of received.messages) {
|
|
107
137
|
report.messagesRead++;
|
|
@@ -113,12 +143,8 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
|
|
|
113
143
|
outstandingForRoute: (routeId) => outstandingAlertDeliveries(db, routeId, deps.now()),
|
|
114
144
|
});
|
|
115
145
|
|
|
116
|
-
if (outcome.action === 'ignored') {
|
|
117
|
-
report.
|
|
118
|
-
continue;
|
|
119
|
-
}
|
|
120
|
-
if (outcome.action === 'rejected') {
|
|
121
|
-
report.rejected++;
|
|
146
|
+
if (outcome.action === 'ignored' || outcome.action === 'rejected') {
|
|
147
|
+
report.unheard.push({ senderAddress: message.senderAddress, reason: outcome.reason });
|
|
122
148
|
continue;
|
|
123
149
|
}
|
|
124
150
|
|
|
@@ -127,7 +153,7 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
|
|
|
127
153
|
if (outcome.delivery.kind === 'interview') {
|
|
128
154
|
const value = parseInterviewAnswer(message.body, outcome.delivery.token);
|
|
129
155
|
if (!value || !deps.answerInterview) {
|
|
130
|
-
report.
|
|
156
|
+
report.unheard.push({ senderAddress: message.senderAddress, reason: 'unrecognised' });
|
|
131
157
|
continue;
|
|
132
158
|
}
|
|
133
159
|
consumeDelivery(db, outcome.delivery.id, deps.now());
|
|
@@ -213,12 +239,23 @@ function outstandingAlertDeliveries(db: DbClient, routeId: string, now: Date) {
|
|
|
213
239
|
*
|
|
214
240
|
* A transport with no `receive` is unidirectional — that is not an error, it
|
|
215
241
|
* just means replies cannot arrive, which the route's `can_ack` already
|
|
216
|
-
* records.
|
|
242
|
+
* records. Anything else going wrong IS an error and is returned as one.
|
|
243
|
+
*
|
|
244
|
+
* This used to be a bare `catch {}` returning null, on the reasoning that the
|
|
245
|
+
* transport's own health check would report an unreachable daemon. That was
|
|
246
|
+
* wrong twice over: a health check that only proves the daemon answers cannot
|
|
247
|
+
* see a read call being REFUSED by a daemon that is otherwise perfectly
|
|
248
|
+
* healthy, and the swallowed error was the only place the reason existed. The
|
|
249
|
+
* live signal-cli case was exactly that shape — `receive` refused with
|
|
250
|
+
* "Receive command cannot be used if messages are already being received."
|
|
251
|
+
* while every health check passed and every poll reported zero messages.
|
|
217
252
|
*/
|
|
218
253
|
export function makeReceiver(db: DbClient) {
|
|
219
|
-
return async (transportModuleId: string, cursor: string | null) => {
|
|
254
|
+
return async (transportModuleId: string, cursor: string | null): Promise<TransportReceive> => {
|
|
220
255
|
const module = db.select().from(modules).where(eq(modules.id, transportModuleId)).get();
|
|
221
|
-
if (!module)
|
|
256
|
+
if (!module) {
|
|
257
|
+
return { status: 'failed', error: `no module '${transportModuleId}' is installed` };
|
|
258
|
+
}
|
|
222
259
|
|
|
223
260
|
try {
|
|
224
261
|
const { logger } = createCapturingLogger();
|
|
@@ -226,11 +263,11 @@ export function makeReceiver(db: DbClient) {
|
|
|
226
263
|
const notification = (capabilities as Record<string, unknown>).notification as
|
|
227
264
|
| NotificationCapability
|
|
228
265
|
| undefined;
|
|
229
|
-
if (!notification?.receive) return
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
return
|
|
266
|
+
if (!notification?.receive) return { status: 'unidirectional' };
|
|
267
|
+
const result = await notification.receive(cursor);
|
|
268
|
+
return { status: 'received', messages: result.messages, cursor: result.cursor };
|
|
269
|
+
} catch (error) {
|
|
270
|
+
return { status: 'failed', error: error instanceof Error ? error.message : String(error) };
|
|
234
271
|
}
|
|
235
272
|
};
|
|
236
273
|
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolving an alert to the people it should page.
|
|
3
|
+
*
|
|
4
|
+
* The escalation policy is a property of the MONITOR, and it is read here on
|
|
5
|
+
* every sweep rather than copied onto the alert when the alert is created.
|
|
6
|
+
* That ordering is the whole point: an operator only reaches for
|
|
7
|
+
* `escalation-policy assign` once something is already firing and reaching
|
|
8
|
+
* nobody, so a policy that only applied to alerts created afterwards would be
|
|
9
|
+
* a no-op for the one alert they were trying to fix (#481).
|
|
10
|
+
*
|
|
11
|
+
* The cost of resolving late is that a policy edited mid-escalation changes
|
|
12
|
+
* where the remaining steps go. That is the behaviour an operator asks for
|
|
13
|
+
* when they edit it — the alert's own progress through the chain
|
|
14
|
+
* (`escalationStep`, `nextEscalationAt`) is what stays on the alert.
|
|
15
|
+
*
|
|
16
|
+
* Lives here rather than in the sweep command so the sweep's tests can drive
|
|
17
|
+
* the real resolution instead of a fake, which is how the snapshot bug
|
|
18
|
+
* survived: every existing test stubbed `notifyDepsFor`.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { eq } from 'drizzle-orm';
|
|
22
|
+
import type { DbClient } from '../../db/client';
|
|
23
|
+
import { type Alert, type EscalationPolicy, escalationPolicies, monitors } from '../../db/schema';
|
|
24
|
+
import type { NotifyDeps } from './notifier';
|
|
25
|
+
import { listPeople, listPolicySteps, listRoutes } from './people';
|
|
26
|
+
import { mintDelivery } from './tokens';
|
|
27
|
+
import { loadNotificationTransport } from './transport-loader';
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* How long a reply token stays valid. A day is long enough that someone who
|
|
31
|
+
* sees a page overnight can still act on it in the morning, and short enough
|
|
32
|
+
* that a token found later is useless.
|
|
33
|
+
*/
|
|
34
|
+
const TOKEN_TTL_MS = 24 * 60 * 60_000;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The escalation policy an alert resolves to right now, or null.
|
|
38
|
+
*
|
|
39
|
+
* The single answer to "who would this page", shared by the sweep and by every
|
|
40
|
+
* command that has to show it. Two readers computing it separately is how a
|
|
41
|
+
* CLI ends up confidently reporting a policy the sweep does not use.
|
|
42
|
+
*/
|
|
43
|
+
export function policyForAlert(
|
|
44
|
+
db: DbClient,
|
|
45
|
+
alert: Pick<Alert, 'monitorId'>,
|
|
46
|
+
): EscalationPolicy | null {
|
|
47
|
+
const monitor = db.select().from(monitors).where(eq(monitors.id, alert.monitorId)).get();
|
|
48
|
+
if (!monitor?.escalationPolicyId) return null;
|
|
49
|
+
return (
|
|
50
|
+
db
|
|
51
|
+
.select()
|
|
52
|
+
.from(escalationPolicies)
|
|
53
|
+
.where(eq(escalationPolicies.id, monitor.escalationPolicyId))
|
|
54
|
+
.get() ?? null
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Assemble everything needed to page for one alert, or null when nobody can be.
|
|
60
|
+
*
|
|
61
|
+
* Returning null rather than an empty policy is deliberate: "no escalation
|
|
62
|
+
* policy assigned" and "policy exists but nobody is eligible" are different
|
|
63
|
+
* situations, and only the second is worth an escalation decision.
|
|
64
|
+
*/
|
|
65
|
+
export function buildNotifyDeps(db: DbClient, alert: Alert, now: Date): NotifyDeps | null {
|
|
66
|
+
const policy = policyForAlert(db, alert);
|
|
67
|
+
if (!policy) return null;
|
|
68
|
+
|
|
69
|
+
const steps = listPolicySteps(db, policy.id).map((step) => ({
|
|
70
|
+
stepIndex: step.stepIndex,
|
|
71
|
+
routeId: step.routeId,
|
|
72
|
+
delayMinutes: step.delayMinutes,
|
|
73
|
+
}));
|
|
74
|
+
if (steps.length === 0) return null;
|
|
75
|
+
|
|
76
|
+
const routeRows = listRoutes(db);
|
|
77
|
+
const people = listPeople(db);
|
|
78
|
+
|
|
79
|
+
return {
|
|
80
|
+
steps,
|
|
81
|
+
routes: new Map(
|
|
82
|
+
routeRows.map((r) => [
|
|
83
|
+
r.id,
|
|
84
|
+
{ id: r.id, severityFloor: r.severityFloor, enabled: r.enabled },
|
|
85
|
+
]),
|
|
86
|
+
),
|
|
87
|
+
routeDetails: new Map(routeRows.map((r) => [r.id, r])),
|
|
88
|
+
quietHoursByPerson: new Map(
|
|
89
|
+
people
|
|
90
|
+
.filter((p) => p.quietHoursStart && p.quietHoursEnd)
|
|
91
|
+
.map((p) => [
|
|
92
|
+
p.id,
|
|
93
|
+
{
|
|
94
|
+
personId: p.id,
|
|
95
|
+
start: p.quietHoursStart,
|
|
96
|
+
end: p.quietHoursEnd,
|
|
97
|
+
timezone: p.timezone,
|
|
98
|
+
},
|
|
99
|
+
]),
|
|
100
|
+
),
|
|
101
|
+
bypassQuietHours: policy.bypassQuietHours,
|
|
102
|
+
transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
|
|
103
|
+
mintToken: (alertId, routeId) =>
|
|
104
|
+
mintDelivery(db, {
|
|
105
|
+
kind: 'alert',
|
|
106
|
+
targetId: alertId,
|
|
107
|
+
routeId,
|
|
108
|
+
now,
|
|
109
|
+
ttlMs: TOKEN_TTL_MS,
|
|
110
|
+
}).token,
|
|
111
|
+
now,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
@@ -27,7 +27,7 @@ describe('alert store', () => {
|
|
|
27
27
|
let dir: string;
|
|
28
28
|
let db: DbClient;
|
|
29
29
|
|
|
30
|
-
const context = { monitorId: MONITOR,
|
|
30
|
+
const context = { monitorId: MONITOR, now: NOW };
|
|
31
31
|
|
|
32
32
|
const create = (key: string): ReconcileAction => ({
|
|
33
33
|
type: 'create',
|
|
@@ -28,7 +28,6 @@ export function loadLiveAlerts(db: DbClient, monitorId: string): LiveAlert[] {
|
|
|
28
28
|
|
|
29
29
|
export interface ApplyContext {
|
|
30
30
|
monitorId: string;
|
|
31
|
-
escalationPolicyId: string | null;
|
|
32
31
|
now: Date;
|
|
33
32
|
}
|
|
34
33
|
|
|
@@ -66,7 +65,6 @@ export function applyReconcileActions(
|
|
|
66
65
|
lastSeenAt: context.now,
|
|
67
66
|
graceUntil: action.graceUntil,
|
|
68
67
|
escalationStep: 0,
|
|
69
|
-
escalationPolicyId: context.escalationPolicyId,
|
|
70
68
|
message: action.message,
|
|
71
69
|
details: action.details,
|
|
72
70
|
})
|
|
@@ -238,7 +238,16 @@ describe('runSweep', () => {
|
|
|
238
238
|
|
|
239
239
|
expect(liveAlerts()).toHaveLength(1);
|
|
240
240
|
expect(report.notified).toBe(0);
|
|
241
|
-
expect(report.noPolicy).
|
|
241
|
+
expect(report.noPolicy).toEqual([{ alertKey: PORT_CHECK, monitor: MODULE }]);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
// A count told the operator something was unroutable but not WHICH thing,
|
|
245
|
+
// so the remedy printed next to it could not be aimed anywhere (#481).
|
|
246
|
+
test('the unroutable alert is named, along with the monitor to assign to', async () => {
|
|
247
|
+
const report = await runSweep(db, currentMonitors(), deps());
|
|
248
|
+
|
|
249
|
+
expect(report.noPolicy[0].alertKey).toBe(PORT_CHECK);
|
|
250
|
+
expect(report.noPolicy[0].monitor).toBe(MODULE);
|
|
242
251
|
});
|
|
243
252
|
|
|
244
253
|
test('escalation declining to notify records WHICH reason', async () => {
|
|
@@ -270,7 +279,7 @@ describe('runSweep', () => {
|
|
|
270
279
|
);
|
|
271
280
|
|
|
272
281
|
expect(report.notified).toBe(0);
|
|
273
|
-
expect(report.noPolicy).
|
|
282
|
+
expect(report.noPolicy).toEqual([]);
|
|
274
283
|
expect(report.skipped.within_grace).toBe(1);
|
|
275
284
|
});
|
|
276
285
|
|
|
@@ -63,13 +63,16 @@ export interface SweepReport {
|
|
|
63
63
|
failed: number;
|
|
64
64
|
/**
|
|
65
65
|
* Live alerts nobody is configured to be told about — no escalation policy on
|
|
66
|
-
* the monitor, so `notifyDepsFor` returns null.
|
|
66
|
+
* the monitor, so `notifyDepsFor` returns null. Each carries the monitor that
|
|
67
|
+
* owns it, which is the thing an operator has to assign a policy TO.
|
|
67
68
|
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
69
|
+
* Identified rather than merely counted: `1 no-policy` told an operator that
|
|
70
|
+
* something was unreachable but not WHICH thing, so the remedy the sweep
|
|
71
|
+
* printed alongside it could not be aimed at anything. Assigning the policy to
|
|
72
|
+
* all eighteen monitors then changed nothing observable (#481). A bare count
|
|
73
|
+
* is only half a step better than the silence it replaced.
|
|
71
74
|
*/
|
|
72
|
-
noPolicy:
|
|
75
|
+
noPolicy: { alertKey: string; monitor: string }[];
|
|
73
76
|
/**
|
|
74
77
|
* Deliveries escalation declined, keyed by its reason (`within_grace`,
|
|
75
78
|
* `no_eligible_route`, …).
|
|
@@ -113,7 +116,7 @@ export async function runSweep(
|
|
|
113
116
|
deferred: 0,
|
|
114
117
|
deferredDelivered: 0,
|
|
115
118
|
failed: 0,
|
|
116
|
-
noPolicy:
|
|
119
|
+
noPolicy: [],
|
|
117
120
|
skipped: {},
|
|
118
121
|
failures: [],
|
|
119
122
|
};
|
|
@@ -206,10 +209,14 @@ export async function runSweep(
|
|
|
206
209
|
}
|
|
207
210
|
|
|
208
211
|
// 5. Notify. Re-read: the steps above changed state under us.
|
|
212
|
+
const monitorTargets = new Map(monitors.map((m) => [m.id, m.target]));
|
|
209
213
|
for (const alert of loadAllLiveAlerts(db)) {
|
|
210
214
|
const notifyDeps = deps.notifyDepsFor(alert);
|
|
211
215
|
if (!notifyDeps) {
|
|
212
|
-
report.noPolicy
|
|
216
|
+
report.noPolicy.push({
|
|
217
|
+
alertKey: alert.key,
|
|
218
|
+
monitor: monitorTargets.get(alert.monitorId) ?? '(unknown monitor)',
|
|
219
|
+
});
|
|
213
220
|
continue;
|
|
214
221
|
}
|
|
215
222
|
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Reading the roster the backup-freshness check runs against.
|
|
3
|
+
*
|
|
4
|
+
* Split out from the check itself so the decision — is this backup too
|
|
5
|
+
* old — stays pure, while the queries that feed it live here (Rule 2.3).
|
|
6
|
+
* Shared by `celilo system audit` and by the scheduled `backups` monitor,
|
|
7
|
+
* so both judge the same fleet from the same data.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { eq } from 'drizzle-orm';
|
|
11
|
+
import type { DbClient } from '../../db/client';
|
|
12
|
+
import { backups, modules } from '../../db/schema';
|
|
13
|
+
import type { ModuleManifest } from '../../manifest/schema';
|
|
14
|
+
import type { InstalledModuleBackupInfo } from './backups';
|
|
15
|
+
|
|
16
|
+
const DEPLOYED_STATES = ['INSTALLED', 'VERIFIED'];
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Most recent COMPLETED backup per module, in epoch ms.
|
|
20
|
+
*
|
|
21
|
+
* The `backups` table is added via inline ALTER statements in
|
|
22
|
+
* db/client.ts for upgraded databases, so a freshly-initialized DB built
|
|
23
|
+
* from the drizzle journal alone may not have it yet. Absence means "no
|
|
24
|
+
* backups recorded", never a crashed audit.
|
|
25
|
+
*/
|
|
26
|
+
function latestSuccessfulBackupByModule(db: DbClient): Map<string, number> {
|
|
27
|
+
const latest = new Map<string, number>();
|
|
28
|
+
try {
|
|
29
|
+
for (const backup of db.select().from(backups).where(eq(backups.status, 'completed')).all()) {
|
|
30
|
+
if (!backup.moduleId || !backup.completedAt) continue;
|
|
31
|
+
const at = backup.completedAt.getTime();
|
|
32
|
+
const previous = latest.get(backup.moduleId);
|
|
33
|
+
if (previous === undefined || at > previous) latest.set(backup.moduleId, at);
|
|
34
|
+
}
|
|
35
|
+
} catch {
|
|
36
|
+
// Table missing — leave the map empty.
|
|
37
|
+
}
|
|
38
|
+
return latest;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function loadBackupAuditInfo(db: DbClient): InstalledModuleBackupInfo[] {
|
|
42
|
+
const latest = latestSuccessfulBackupByModule(db);
|
|
43
|
+
return db
|
|
44
|
+
.select()
|
|
45
|
+
.from(modules)
|
|
46
|
+
.all()
|
|
47
|
+
.filter((module) => DEPLOYED_STATES.includes(module.state))
|
|
48
|
+
.map((module) => ({
|
|
49
|
+
id: module.id,
|
|
50
|
+
state: module.state,
|
|
51
|
+
manifest: module.manifestData as ModuleManifest,
|
|
52
|
+
lastSuccessfulBackupAt: latest.get(module.id) ?? null,
|
|
53
|
+
}));
|
|
54
|
+
}
|
|
@@ -75,7 +75,10 @@ describe('auditBackups', () => {
|
|
|
75
75
|
expect(result).toEqual([]);
|
|
76
76
|
});
|
|
77
77
|
|
|
78
|
-
|
|
78
|
+
// Silence-by-default is the bug: an undeclared cadence is how celilo-mgmt
|
|
79
|
+
// went 55 days without a backup and nobody was told. Opting out takes an
|
|
80
|
+
// explicit `manual` — see services/backup-schedule.ts.
|
|
81
|
+
test('no schedule declared → daily, so a year-old backup is stale', async () => {
|
|
79
82
|
const result = await auditBackups({
|
|
80
83
|
modules: [
|
|
81
84
|
makeModule('lunacycle', {
|
|
@@ -85,7 +88,9 @@ describe('auditBackups', () => {
|
|
|
85
88
|
],
|
|
86
89
|
now: () => NOW,
|
|
87
90
|
});
|
|
88
|
-
expect(result).
|
|
91
|
+
expect(result).toHaveLength(1);
|
|
92
|
+
expect(result[0]).toMatchObject({ code: 'backup_stale', subject: 'lunacycle' });
|
|
93
|
+
expect(result[0].message).toContain('daily');
|
|
89
94
|
});
|
|
90
95
|
|
|
91
96
|
test('daily schedule: 26h-old is stale', async () => {
|
|
@@ -8,14 +8,17 @@
|
|
|
8
8
|
* scheduled run doesn't flag drift on every audit.
|
|
9
9
|
*
|
|
10
10
|
* Modules without an `on_backup` hook are skipped — there's nothing
|
|
11
|
-
* to back up. Modules whose schedule is `manual`
|
|
12
|
-
* staleness check (the
|
|
13
|
-
* `backup_missing` finding if no backup has ever been recorded.
|
|
11
|
+
* to back up. Modules whose schedule is explicitly `manual` skip the
|
|
12
|
+
* staleness check (the operator decides cadence) but still get a
|
|
13
|
+
* `backup_missing` finding if no backup has ever been recorded. An
|
|
14
|
+
* unset schedule is `daily`, not `manual` — see
|
|
15
|
+
* [[services/backup-schedule.ts]] for why that default matters.
|
|
14
16
|
*
|
|
15
17
|
* Time is injected so tests can pin "now" deterministically.
|
|
16
18
|
*/
|
|
17
19
|
|
|
18
20
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
21
|
+
import { effectiveBackupSchedule } from '../backup-schedule';
|
|
19
22
|
import type { DriftFinding } from './types';
|
|
20
23
|
|
|
21
24
|
export interface InstalledModuleBackupInfo {
|
|
@@ -50,10 +53,7 @@ const DAY = 24 * HOUR;
|
|
|
50
53
|
* - daily → 25h (24h + 1h grace)
|
|
51
54
|
* - weekly → 8d (7d + 1d grace)
|
|
52
55
|
* - monthly → 32d (~30d + 2d grace)
|
|
53
|
-
* - manual → null (no staleness check;
|
|
54
|
-
*
|
|
55
|
-
* `undefined` (no `backup:` block in manifest) is treated as
|
|
56
|
-
* `manual` — author opted out of declaring a cadence.
|
|
56
|
+
* - manual → null (no staleness check; operator-driven cadence)
|
|
57
57
|
*/
|
|
58
58
|
const SCHEDULE_THRESHOLDS = {
|
|
59
59
|
hourly: 2 * HOUR,
|
|
@@ -63,21 +63,13 @@ const SCHEDULE_THRESHOLDS = {
|
|
|
63
63
|
manual: null,
|
|
64
64
|
} as const;
|
|
65
65
|
|
|
66
|
-
type ScheduleKey = keyof typeof SCHEDULE_THRESHOLDS;
|
|
67
|
-
|
|
68
66
|
function moduleHasBackupHook(manifest: ModuleManifest): boolean {
|
|
69
67
|
return Boolean(manifest.hooks?.on_backup);
|
|
70
68
|
}
|
|
71
69
|
|
|
72
|
-
function scheduleFor(manifest: ModuleManifest): ScheduleKey {
|
|
73
|
-
const s = manifest.backup?.schedule;
|
|
74
|
-
if (s === 'hourly' || s === 'daily' || s === 'weekly' || s === 'monthly') return s;
|
|
75
|
-
return 'manual';
|
|
76
|
-
}
|
|
77
|
-
|
|
78
70
|
function thresholdFor(manifest: ModuleManifest, override: number | undefined): number | null {
|
|
79
71
|
if (override !== undefined) return override;
|
|
80
|
-
return SCHEDULE_THRESHOLDS[
|
|
72
|
+
return SCHEDULE_THRESHOLDS[effectiveBackupSchedule(manifest)];
|
|
81
73
|
}
|
|
82
74
|
|
|
83
75
|
function formatAge(ms: number): string {
|
|
@@ -109,7 +101,7 @@ export async function auditBackups(deps: BackupsAuditDeps): Promise<DriftFinding
|
|
|
109
101
|
category: 'backups',
|
|
110
102
|
severity: 'drift',
|
|
111
103
|
code: 'backup_missing',
|
|
112
|
-
message: `${m.id}: no successful backup recorded (schedule: ${
|
|
104
|
+
message: `${m.id}: no successful backup recorded (schedule: ${effectiveBackupSchedule(m.manifest)})`,
|
|
113
105
|
remediation: `celilo backup create ${m.id} --force`,
|
|
114
106
|
actionable: true,
|
|
115
107
|
subject: m.id,
|
|
@@ -126,7 +118,7 @@ export async function auditBackups(deps: BackupsAuditDeps): Promise<DriftFinding
|
|
|
126
118
|
category: 'backups',
|
|
127
119
|
severity: 'drift',
|
|
128
120
|
code: 'backup_stale',
|
|
129
|
-
message: `${m.id}: last successful backup is ${formatAge(age)} old (schedule: ${
|
|
121
|
+
message: `${m.id}: last successful backup is ${formatAge(age)} old (schedule: ${effectiveBackupSchedule(m.manifest)}, threshold: ${formatAge(threshold)})`,
|
|
130
122
|
remediation: `celilo backup create ${m.id} --force`,
|
|
131
123
|
actionable: true,
|
|
132
124
|
subject: m.id,
|