@celilo/cli 0.14.4 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CELILO_CORE_MODULES.md +1 -1
  2. package/CELILO_SUBSYSTEMS.md +18 -2
  3. package/drizzle/0018_drop_alert_policy_snapshot.sql +46 -0
  4. package/drizzle/meta/_journal.json +8 -1
  5. package/package.json +3 -3
  6. package/src/cli/commands/alerts-list.ts +10 -0
  7. package/src/cli/commands/alerts-poll.ts +12 -6
  8. package/src/cli/commands/alerts-sweep.ts +22 -81
  9. package/src/cli/commands/backup-sweep.ts +65 -0
  10. package/src/cli/commands/module-operations.test.ts +93 -0
  11. package/src/cli/commands/module-operations.ts +134 -0
  12. package/src/cli/commands/module-upgrade.test.ts +32 -20
  13. package/src/cli/commands/module-upgrade.ts +37 -32
  14. package/src/cli/commands/monitor.ts +26 -6
  15. package/src/cli/commands/system-audit.ts +3 -30
  16. package/src/cli/completion.ts +18 -1
  17. package/src/cli/index.ts +11 -0
  18. package/src/db/schema.ts +5 -3
  19. package/src/manifest/schema.ts +4 -1
  20. package/src/module/packaging/build.ts +4 -0
  21. package/src/services/alerting/builtin-source.ts +17 -2
  22. package/src/services/alerting/delivery-loop.test.ts +5 -1
  23. package/src/services/alerting/format.test.ts +0 -1
  24. package/src/services/alerting/inbound-poller.test.ts +44 -8
  25. package/src/services/alerting/inbound-poller.ts +65 -28
  26. package/src/services/alerting/notify-deps.ts +113 -0
  27. package/src/services/alerting/run-monitor.ts +0 -1
  28. package/src/services/alerting/store.test.ts +1 -1
  29. package/src/services/alerting/store.ts +0 -2
  30. package/src/services/alerting/sweep-runner.test.ts +11 -2
  31. package/src/services/alerting/sweep-runner.ts +14 -7
  32. package/src/services/audit/backup-source.ts +54 -0
  33. package/src/services/audit/backups.test.ts +7 -2
  34. package/src/services/audit/backups.ts +10 -18
  35. package/src/services/backup-cipher.test.ts +188 -0
  36. package/src/services/backup-cipher.ts +178 -0
  37. package/src/services/backup-create.ts +20 -30
  38. package/src/services/backup-envelope-roundtrip.test.ts +6 -26
  39. package/src/services/backup-restore.ts +10 -16
  40. package/src/services/backup-schedule.ts +35 -0
  41. package/src/services/backup-sweep.test.ts +148 -0
  42. package/src/services/backup-sweep.ts +124 -0
  43. package/src/services/deploy-posture.ts +15 -2
  44. package/src/services/module-operations.test.ts +67 -6
  45. package/src/services/module-operations.ts +69 -19
  46. package/src/services/module-subscriptions.test.ts +33 -2
  47. package/src/services/module-subscriptions.ts +10 -1
  48. package/src/services/module-validator/typescript-build.test.ts +20 -1
  49. package/src/services/module-validator/typescript-build.ts +9 -5
  50. package/src/services/restore-from-file.ts +6 -21
  51. package/src/templates/generator.test.ts +88 -0
  52. package/src/templates/generator.ts +119 -16
@@ -34,7 +34,6 @@ function alert(over: Partial<Alert> = {}): Alert {
34
34
  silencedUntil: null,
35
35
  escalationStep: 0,
36
36
  nextEscalationAt: null,
37
- escalationPolicyId: null,
38
37
  message: '/var 94% used',
39
38
  details: null,
40
39
  resolvedAt: null,
@@ -27,7 +27,7 @@ describe('pollInbound', () => {
27
27
 
28
28
  function deps(messages: InboundMessage[], over: Partial<InboundPollDeps> = {}): InboundPollDeps {
29
29
  return {
30
- receiveFrom: async () => ({ messages, cursor: 'cursor-1' }),
30
+ receiveFrom: async () => ({ status: 'received', messages, cursor: 'cursor-1' }),
31
31
  readCursor: (t) => cursors.get(t) ?? null,
32
32
  writeCursor: (t, c) => {
33
33
  cursors.set(t, c);
@@ -190,7 +190,7 @@ describe('pollInbound', () => {
190
190
  const delivery = page(peterRoute.id);
191
191
  const report = await pollInbound(db, deps([inbound(WIFE, delivery.token)]));
192
192
 
193
- expect(report.rejected).toBe(1);
193
+ expect(report.unheard).toEqual([{ senderAddress: WIFE, reason: 'wrong_sender' }]);
194
194
  expect(report.acked).toBe(0);
195
195
  expect(alertState()).toBe('firing');
196
196
  });
@@ -200,7 +200,7 @@ describe('pollInbound', () => {
200
200
  const delivery = page(peterRoute.id);
201
201
  const report = await pollInbound(db, deps([inbound(STRANGER, delivery.token)]));
202
202
 
203
- expect(report.ignored).toBe(1);
203
+ expect(report.unheard).toEqual([{ senderAddress: STRANGER, reason: 'unknown_sender' }]);
204
204
  expect(alertState()).toBe('firing');
205
205
  });
206
206
 
@@ -228,7 +228,7 @@ describe('pollInbound', () => {
228
228
  expect(first.acked).toBe(1);
229
229
  // The token is spent, so the replay is refused rather than acking again.
230
230
  expect(second.acked).toBe(0);
231
- expect(second.rejected).toBe(1);
231
+ expect(second.unheard).toEqual([{ senderAddress: PETER, reason: 'unknown_token' }]);
232
232
  expect(alertState()).toBe('acked');
233
233
  });
234
234
 
@@ -262,31 +262,67 @@ describe('pollInbound', () => {
262
262
  });
263
263
 
264
264
  const report = await pollInbound(db, deps([inbound(PETER, 'ack')]));
265
- expect(report.rejected).toBe(1);
265
+ expect(report.unheard).toEqual([{ senderAddress: PETER, reason: 'ambiguous' }]);
266
266
  expect(alertState()).toBe('firing');
267
267
  });
268
268
 
269
269
  // One dead transport must not stop the others being read.
270
270
  test('an unreachable transport is skipped without throwing', async () => {
271
- const report = await pollInbound(db, deps([], { receiveFrom: async () => null }));
271
+ const report = await pollInbound(
272
+ db,
273
+ deps([], { receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
274
+ );
272
275
  expect(report.transportsPolled).toBe(1);
273
276
  expect(report.messagesRead).toBe(0);
274
277
  });
275
278
 
279
+ /**
280
+ * The regression that cost a week. signal-cli's daemon refused celilo's read
281
+ * call outright ("Receive command cannot be used if messages are already
282
+ * being received") and the poller reported the same "0 message(s)" it
283
+ * reports when nobody has replied. The reason must reach the report, or the
284
+ * two are indistinguishable from outside.
285
+ */
286
+ test('a transport that REFUSES the read says so instead of looking quiet', async () => {
287
+ const refusal = 'Receive command cannot be used if messages are already being received.';
288
+ const report = await pollInbound(
289
+ db,
290
+ deps([], { receiveFrom: async () => ({ status: 'failed', error: refusal }) }),
291
+ );
292
+
293
+ expect(report.failures).toEqual([{ transportModuleId: 'signal', error: refusal }]);
294
+ // ...and a genuinely quiet transport must NOT look like a failure.
295
+ const quiet = await pollInbound(db, deps([]));
296
+ expect(quiet.failures).toEqual([]);
297
+ expect(quiet.messagesRead).toBe(0);
298
+ });
299
+
300
+ test('a unidirectional transport is not reported as a failure', async () => {
301
+ const report = await pollInbound(
302
+ db,
303
+ deps([], { receiveFrom: async () => ({ status: 'unidirectional' }) }),
304
+ );
305
+ expect(report.failures).toEqual([]);
306
+ expect(report.transportsPolled).toBe(1);
307
+ });
308
+
276
309
  test('the cursor is persisted after a successful poll', async () => {
277
310
  await pollInbound(db, deps([]));
278
311
  expect(cursors.get('signal')).toBe('cursor-1');
279
312
  });
280
313
 
281
314
  test('the cursor is NOT advanced when the transport could not be reached', async () => {
282
- await pollInbound(db, deps([], { receiveFrom: async () => null }));
315
+ await pollInbound(
316
+ db,
317
+ deps([], { receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
318
+ );
283
319
  expect(cursors.get('signal')).toBeUndefined();
284
320
  });
285
321
 
286
322
  test('gibberish is rejected without touching the alert', async () => {
287
323
  page(peterRoute.id);
288
324
  const report = await pollInbound(db, deps([inbound(PETER, 'what is going on')]));
289
- expect(report.rejected).toBe(1);
325
+ expect(report.unheard).toEqual([{ senderAddress: PETER, reason: 'unrecognised' }]);
290
326
  expect(alertState()).toBe('firing');
291
327
  });
292
328
 
@@ -25,12 +25,35 @@ import type { NotificationTransport } from './notifier';
25
25
  import { composeAckBroadcastBody } from './notifier';
26
26
  import { consumeDelivery } from './tokens';
27
27
 
28
+ /**
29
+ * What one attempt to read a transport produced.
30
+ *
31
+ * `unidirectional` and `failed` used to collapse into a bare null, and that
32
+ * cost a week: a transport whose read call was being REFUSED reported the same
33
+ * "0 message(s)" as a transport nobody had replied on. An operator could not
34
+ * tell "my reply never arrived" from "celilo cannot read this transport at
35
+ * all", and neither could anyone debugging it.
36
+ */
37
+ export type TransportReceive =
38
+ | { status: 'received'; messages: InboundMessage[]; cursor: string | null }
39
+ | { status: 'unidirectional' }
40
+ | { status: 'failed'; error: string };
41
+
42
+ /** A transport that could not be read, and why. */
43
+ export interface TransportFailure {
44
+ transportModuleId: string;
45
+ error: string;
46
+ }
47
+
48
+ /** A message that was read but not acted on, and why. */
49
+ export interface UnheardMessage {
50
+ senderAddress: string;
51
+ reason: 'unknown_sender' | 'unknown_token' | 'wrong_sender' | 'ambiguous' | 'unrecognised';
52
+ }
53
+
28
54
  export interface InboundPollDeps {
29
- /** Receive from one transport module, or null when it cannot be reached. */
30
- receiveFrom(
31
- transportModuleId: string,
32
- cursor: string | null,
33
- ): Promise<{ messages: InboundMessage[]; cursor: string | null } | null>;
55
+ /** Receive from one transport module. */
56
+ receiveFrom(transportModuleId: string, cursor: string | null): Promise<TransportReceive>;
34
57
  /** Persisted receive cursor per transport. */
35
58
  readCursor(transportModuleId: string): string | null;
36
59
  writeCursor(transportModuleId: string, cursor: string | null): void;
@@ -53,12 +76,14 @@ export interface InboundPollReport {
53
76
  transportsPolled: number;
54
77
  messagesRead: number;
55
78
  acked: number;
56
- ignored: number;
57
- rejected: number;
58
79
  /** Routes told that someone else took the alert. */
59
80
  broadcast: number;
60
81
  /** Deploy questions answered from a phone. */
61
82
  answered: number;
83
+ /** Transports that could not be read at all. Never silent — see above. */
84
+ failures: TransportFailure[];
85
+ /** Messages read and then discarded, with the reason each was discarded. */
86
+ unheard: UnheardMessage[];
62
87
  }
63
88
 
64
89
  /** Transports that have at least one route pointing at them. */
@@ -89,19 +114,24 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
89
114
  transportsPolled: 0,
90
115
  messagesRead: 0,
91
116
  acked: 0,
92
- ignored: 0,
93
- rejected: 0,
94
117
  broadcast: 0,
95
118
  answered: 0,
119
+ failures: [],
120
+ unheard: [],
96
121
  };
97
122
 
98
123
  for (const transportId of transportsWithRoutes(db)) {
99
124
  const received = await deps.receiveFrom(transportId, deps.readCursor(transportId));
100
125
  report.transportsPolled++;
101
- // A transport that cannot be reached is not an error here its own
102
- // health check is what reports that, and one dead transport must not stop
103
- // the others from being read.
104
- if (!received) continue;
126
+ // One dead transport must not stop the others being read but it is
127
+ // RECORDED rather than skipped in silence, because "cannot read" and
128
+ // "nothing to read" are the two things an operator most needs to tell
129
+ // apart at 3am.
130
+ if (received.status === 'failed') {
131
+ report.failures.push({ transportModuleId: transportId, error: received.error });
132
+ continue;
133
+ }
134
+ if (received.status === 'unidirectional') continue;
105
135
 
106
136
  for (const message of received.messages) {
107
137
  report.messagesRead++;
@@ -113,12 +143,8 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
113
143
  outstandingForRoute: (routeId) => outstandingAlertDeliveries(db, routeId, deps.now()),
114
144
  });
115
145
 
116
- if (outcome.action === 'ignored') {
117
- report.ignored++;
118
- continue;
119
- }
120
- if (outcome.action === 'rejected') {
121
- report.rejected++;
146
+ if (outcome.action === 'ignored' || outcome.action === 'rejected') {
147
+ report.unheard.push({ senderAddress: message.senderAddress, reason: outcome.reason });
122
148
  continue;
123
149
  }
124
150
 
@@ -127,7 +153,7 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
127
153
  if (outcome.delivery.kind === 'interview') {
128
154
  const value = parseInterviewAnswer(message.body, outcome.delivery.token);
129
155
  if (!value || !deps.answerInterview) {
130
- report.rejected++;
156
+ report.unheard.push({ senderAddress: message.senderAddress, reason: 'unrecognised' });
131
157
  continue;
132
158
  }
133
159
  consumeDelivery(db, outcome.delivery.id, deps.now());
@@ -213,12 +239,23 @@ function outstandingAlertDeliveries(db: DbClient, routeId: string, now: Date) {
213
239
  *
214
240
  * A transport with no `receive` is unidirectional — that is not an error, it
215
241
  * just means replies cannot arrive, which the route's `can_ack` already
216
- * records.
242
+ * records. Anything else going wrong IS an error and is returned as one.
243
+ *
244
+ * This used to be a bare `catch {}` returning null, on the reasoning that the
245
+ * transport's own health check would report an unreachable daemon. That was
246
+ * wrong twice over: a health check that only proves the daemon answers cannot
247
+ * see a read call being REFUSED by a daemon that is otherwise perfectly
248
+ * healthy, and the swallowed error was the only place the reason existed. The
249
+ * live signal-cli case was exactly that shape — `receive` refused with
250
+ * "Receive command cannot be used if messages are already being received."
251
+ * while every health check passed and every poll reported zero messages.
217
252
  */
218
253
  export function makeReceiver(db: DbClient) {
219
- return async (transportModuleId: string, cursor: string | null) => {
254
+ return async (transportModuleId: string, cursor: string | null): Promise<TransportReceive> => {
220
255
  const module = db.select().from(modules).where(eq(modules.id, transportModuleId)).get();
221
- if (!module) return null;
256
+ if (!module) {
257
+ return { status: 'failed', error: `no module '${transportModuleId}' is installed` };
258
+ }
222
259
 
223
260
  try {
224
261
  const { logger } = createCapturingLogger();
@@ -226,11 +263,11 @@ export function makeReceiver(db: DbClient) {
226
263
  const notification = (capabilities as Record<string, unknown>).notification as
227
264
  | NotificationCapability
228
265
  | undefined;
229
- if (!notification?.receive) return null;
230
- return await notification.receive(cursor);
231
- } catch {
232
- // Unreachable transport: the transport's own health check reports it.
233
- return null;
266
+ if (!notification?.receive) return { status: 'unidirectional' };
267
+ const result = await notification.receive(cursor);
268
+ return { status: 'received', messages: result.messages, cursor: result.cursor };
269
+ } catch (error) {
270
+ return { status: 'failed', error: error instanceof Error ? error.message : String(error) };
234
271
  }
235
272
  };
236
273
  }
@@ -0,0 +1,113 @@
1
+ /**
2
+ * Resolving an alert to the people it should page.
3
+ *
4
+ * The escalation policy is a property of the MONITOR, and it is read here on
5
+ * every sweep rather than copied onto the alert when the alert is created.
6
+ * That ordering is the whole point: an operator only reaches for
7
+ * `escalation-policy assign` once something is already firing and reaching
8
+ * nobody, so a policy that only applied to alerts created afterwards would be
9
+ * a no-op for the one alert they were trying to fix (#481).
10
+ *
11
+ * The cost of resolving late is that a policy edited mid-escalation changes
12
+ * where the remaining steps go. That is the behaviour an operator asks for
13
+ * when they edit it — the alert's own progress through the chain
14
+ * (`escalationStep`, `nextEscalationAt`) is what stays on the alert.
15
+ *
16
+ * Lives here rather than in the sweep command so the sweep's tests can drive
17
+ * the real resolution instead of a fake, which is how the snapshot bug
18
+ * survived: every existing test stubbed `notifyDepsFor`.
19
+ */
20
+
21
+ import { eq } from 'drizzle-orm';
22
+ import type { DbClient } from '../../db/client';
23
+ import { type Alert, type EscalationPolicy, escalationPolicies, monitors } from '../../db/schema';
24
+ import type { NotifyDeps } from './notifier';
25
+ import { listPeople, listPolicySteps, listRoutes } from './people';
26
+ import { mintDelivery } from './tokens';
27
+ import { loadNotificationTransport } from './transport-loader';
28
+
29
+ /**
30
+ * How long a reply token stays valid. A day is long enough that someone who
31
+ * sees a page overnight can still act on it in the morning, and short enough
32
+ * that a token found later is useless.
33
+ */
34
+ const TOKEN_TTL_MS = 24 * 60 * 60_000;
35
+
36
+ /**
37
+ * The escalation policy an alert resolves to right now, or null.
38
+ *
39
+ * The single answer to "who would this page", shared by the sweep and by every
40
+ * command that has to show it. Two readers computing it separately is how a
41
+ * CLI ends up confidently reporting a policy the sweep does not use.
42
+ */
43
+ export function policyForAlert(
44
+ db: DbClient,
45
+ alert: Pick<Alert, 'monitorId'>,
46
+ ): EscalationPolicy | null {
47
+ const monitor = db.select().from(monitors).where(eq(monitors.id, alert.monitorId)).get();
48
+ if (!monitor?.escalationPolicyId) return null;
49
+ return (
50
+ db
51
+ .select()
52
+ .from(escalationPolicies)
53
+ .where(eq(escalationPolicies.id, monitor.escalationPolicyId))
54
+ .get() ?? null
55
+ );
56
+ }
57
+
58
+ /**
59
+ * Assemble everything needed to page for one alert, or null when nobody can be.
60
+ *
61
+ * Returning null rather than an empty policy is deliberate: "no escalation
62
+ * policy assigned" and "policy exists but nobody is eligible" are different
63
+ * situations, and only the second is worth an escalation decision.
64
+ */
65
+ export function buildNotifyDeps(db: DbClient, alert: Alert, now: Date): NotifyDeps | null {
66
+ const policy = policyForAlert(db, alert);
67
+ if (!policy) return null;
68
+
69
+ const steps = listPolicySteps(db, policy.id).map((step) => ({
70
+ stepIndex: step.stepIndex,
71
+ routeId: step.routeId,
72
+ delayMinutes: step.delayMinutes,
73
+ }));
74
+ if (steps.length === 0) return null;
75
+
76
+ const routeRows = listRoutes(db);
77
+ const people = listPeople(db);
78
+
79
+ return {
80
+ steps,
81
+ routes: new Map(
82
+ routeRows.map((r) => [
83
+ r.id,
84
+ { id: r.id, severityFloor: r.severityFloor, enabled: r.enabled },
85
+ ]),
86
+ ),
87
+ routeDetails: new Map(routeRows.map((r) => [r.id, r])),
88
+ quietHoursByPerson: new Map(
89
+ people
90
+ .filter((p) => p.quietHoursStart && p.quietHoursEnd)
91
+ .map((p) => [
92
+ p.id,
93
+ {
94
+ personId: p.id,
95
+ start: p.quietHoursStart,
96
+ end: p.quietHoursEnd,
97
+ timezone: p.timezone,
98
+ },
99
+ ]),
100
+ ),
101
+ bypassQuietHours: policy.bypassQuietHours,
102
+ transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
103
+ mintToken: (alertId, routeId) =>
104
+ mintDelivery(db, {
105
+ kind: 'alert',
106
+ targetId: alertId,
107
+ routeId,
108
+ now,
109
+ ttlMs: TOKEN_TTL_MS,
110
+ }).token,
111
+ now,
112
+ };
113
+ }
@@ -151,7 +151,6 @@ export async function runOneMonitor(
151
151
 
152
152
  const { createdIds, resolvedIds } = applyReconcileActions(db, actions, {
153
153
  monitorId: monitor.id,
154
- escalationPolicyId: monitor.escalationPolicyId,
155
154
  now,
156
155
  });
157
156
 
@@ -27,7 +27,7 @@ describe('alert store', () => {
27
27
  let dir: string;
28
28
  let db: DbClient;
29
29
 
30
- const context = { monitorId: MONITOR, escalationPolicyId: null, now: NOW };
30
+ const context = { monitorId: MONITOR, now: NOW };
31
31
 
32
32
  const create = (key: string): ReconcileAction => ({
33
33
  type: 'create',
@@ -28,7 +28,6 @@ export function loadLiveAlerts(db: DbClient, monitorId: string): LiveAlert[] {
28
28
 
29
29
  export interface ApplyContext {
30
30
  monitorId: string;
31
- escalationPolicyId: string | null;
32
31
  now: Date;
33
32
  }
34
33
 
@@ -66,7 +65,6 @@ export function applyReconcileActions(
66
65
  lastSeenAt: context.now,
67
66
  graceUntil: action.graceUntil,
68
67
  escalationStep: 0,
69
- escalationPolicyId: context.escalationPolicyId,
70
68
  message: action.message,
71
69
  details: action.details,
72
70
  })
@@ -238,7 +238,16 @@ describe('runSweep', () => {
238
238
 
239
239
  expect(liveAlerts()).toHaveLength(1);
240
240
  expect(report.notified).toBe(0);
241
- expect(report.noPolicy).toBe(1);
241
+ expect(report.noPolicy).toEqual([{ alertKey: PORT_CHECK, monitor: MODULE }]);
242
+ });
243
+
244
+ // A count told the operator something was unroutable but not WHICH thing,
245
+ // so the remedy printed next to it could not be aimed anywhere (#481).
246
+ test('the unroutable alert is named, along with the monitor to assign to', async () => {
247
+ const report = await runSweep(db, currentMonitors(), deps());
248
+
249
+ expect(report.noPolicy[0].alertKey).toBe(PORT_CHECK);
250
+ expect(report.noPolicy[0].monitor).toBe(MODULE);
242
251
  });
243
252
 
244
253
  test('escalation declining to notify records WHICH reason', async () => {
@@ -270,7 +279,7 @@ describe('runSweep', () => {
270
279
  );
271
280
 
272
281
  expect(report.notified).toBe(0);
273
- expect(report.noPolicy).toBe(0);
282
+ expect(report.noPolicy).toEqual([]);
274
283
  expect(report.skipped.within_grace).toBe(1);
275
284
  });
276
285
 
@@ -63,13 +63,16 @@ export interface SweepReport {
63
63
  failed: number;
64
64
  /**
65
65
  * Live alerts nobody is configured to be told about — no escalation policy on
66
- * the monitor, so `notifyDepsFor` returns null.
66
+ * the monitor, so `notifyDepsFor` returns null. Each carries the monitor that
67
+ * owns it, which is the thing an operator has to assign a policy TO.
67
68
  *
68
- * Counted rather than skipped in silence: "nothing needed sending" and "an
69
- * alert is firing and no policy points at anyone" are opposite situations that
70
- * previously rendered identically as `0 notified`.
69
+ * Identified rather than merely counted: `1 no-policy` told an operator that
70
+ * something was unreachable but not WHICH thing, so the remedy the sweep
71
+ * printed alongside it could not be aimed at anything. Assigning the policy to
72
+ * all eighteen monitors then changed nothing observable (#481). A bare count
73
+ * is only half a step better than the silence it replaced.
71
74
  */
72
- noPolicy: number;
75
+ noPolicy: { alertKey: string; monitor: string }[];
73
76
  /**
74
77
  * Deliveries escalation declined, keyed by its reason (`within_grace`,
75
78
  * `no_eligible_route`, …).
@@ -113,7 +116,7 @@ export async function runSweep(
113
116
  deferred: 0,
114
117
  deferredDelivered: 0,
115
118
  failed: 0,
116
- noPolicy: 0,
119
+ noPolicy: [],
117
120
  skipped: {},
118
121
  failures: [],
119
122
  };
@@ -206,10 +209,14 @@ export async function runSweep(
206
209
  }
207
210
 
208
211
  // 5. Notify. Re-read: the steps above changed state under us.
212
+ const monitorTargets = new Map(monitors.map((m) => [m.id, m.target]));
209
213
  for (const alert of loadAllLiveAlerts(db)) {
210
214
  const notifyDeps = deps.notifyDepsFor(alert);
211
215
  if (!notifyDeps) {
212
- report.noPolicy++;
216
+ report.noPolicy.push({
217
+ alertKey: alert.key,
218
+ monitor: monitorTargets.get(alert.monitorId) ?? '(unknown monitor)',
219
+ });
213
220
  continue;
214
221
  }
215
222
 
@@ -0,0 +1,54 @@
1
+ /**
2
+ * Reading the roster the backup-freshness check runs against.
3
+ *
4
+ * Split out from the check itself so the decision — is this backup too
5
+ * old — stays pure, while the queries that feed it live here (Rule 2.3).
6
+ * Shared by `celilo system audit` and by the scheduled `backups` monitor,
7
+ * so both judge the same fleet from the same data.
8
+ */
9
+
10
+ import { eq } from 'drizzle-orm';
11
+ import type { DbClient } from '../../db/client';
12
+ import { backups, modules } from '../../db/schema';
13
+ import type { ModuleManifest } from '../../manifest/schema';
14
+ import type { InstalledModuleBackupInfo } from './backups';
15
+
16
+ const DEPLOYED_STATES = ['INSTALLED', 'VERIFIED'];
17
+
18
+ /**
19
+ * Most recent COMPLETED backup per module, in epoch ms.
20
+ *
21
+ * The `backups` table is added via inline ALTER statements in
22
+ * db/client.ts for upgraded databases, so a freshly-initialized DB built
23
+ * from the drizzle journal alone may not have it yet. Absence means "no
24
+ * backups recorded", never a crashed audit.
25
+ */
26
+ function latestSuccessfulBackupByModule(db: DbClient): Map<string, number> {
27
+ const latest = new Map<string, number>();
28
+ try {
29
+ for (const backup of db.select().from(backups).where(eq(backups.status, 'completed')).all()) {
30
+ if (!backup.moduleId || !backup.completedAt) continue;
31
+ const at = backup.completedAt.getTime();
32
+ const previous = latest.get(backup.moduleId);
33
+ if (previous === undefined || at > previous) latest.set(backup.moduleId, at);
34
+ }
35
+ } catch {
36
+ // Table missing — leave the map empty.
37
+ }
38
+ return latest;
39
+ }
40
+
41
+ export function loadBackupAuditInfo(db: DbClient): InstalledModuleBackupInfo[] {
42
+ const latest = latestSuccessfulBackupByModule(db);
43
+ return db
44
+ .select()
45
+ .from(modules)
46
+ .all()
47
+ .filter((module) => DEPLOYED_STATES.includes(module.state))
48
+ .map((module) => ({
49
+ id: module.id,
50
+ state: module.state,
51
+ manifest: module.manifestData as ModuleManifest,
52
+ lastSuccessfulBackupAt: latest.get(module.id) ?? null,
53
+ }));
54
+ }
@@ -75,7 +75,10 @@ describe('auditBackups', () => {
75
75
  expect(result).toEqual([]);
76
76
  });
77
77
 
78
- test('no schedule declared treated as manual (no stale flag)', async () => {
78
+ // Silence-by-default is the bug: an undeclared cadence is how celilo-mgmt
79
+ // went 55 days without a backup and nobody was told. Opting out takes an
80
+ // explicit `manual` — see services/backup-schedule.ts.
81
+ test('no schedule declared → daily, so a year-old backup is stale', async () => {
79
82
  const result = await auditBackups({
80
83
  modules: [
81
84
  makeModule('lunacycle', {
@@ -85,7 +88,9 @@ describe('auditBackups', () => {
85
88
  ],
86
89
  now: () => NOW,
87
90
  });
88
- expect(result).toEqual([]);
91
+ expect(result).toHaveLength(1);
92
+ expect(result[0]).toMatchObject({ code: 'backup_stale', subject: 'lunacycle' });
93
+ expect(result[0].message).toContain('daily');
89
94
  });
90
95
 
91
96
  test('daily schedule: 26h-old is stale', async () => {
@@ -8,14 +8,17 @@
8
8
  * scheduled run doesn't flag drift on every audit.
9
9
  *
10
10
  * Modules without an `on_backup` hook are skipped — there's nothing
11
- * to back up. Modules whose schedule is `manual` (or unset) skip the
12
- * staleness check (the user decides cadence) but still get a
13
- * `backup_missing` finding if no backup has ever been recorded.
11
+ * to back up. Modules whose schedule is explicitly `manual` skip the
12
+ * staleness check (the operator decides cadence) but still get a
13
+ * `backup_missing` finding if no backup has ever been recorded. An
14
+ * unset schedule is `daily`, not `manual` — see
15
+ * [[services/backup-schedule.ts]] for why that default matters.
14
16
  *
15
17
  * Time is injected so tests can pin "now" deterministically.
16
18
  */
17
19
 
18
20
  import type { ModuleManifest } from '../../manifest/schema';
21
+ import { effectiveBackupSchedule } from '../backup-schedule';
19
22
  import type { DriftFinding } from './types';
20
23
 
21
24
  export interface InstalledModuleBackupInfo {
@@ -50,10 +53,7 @@ const DAY = 24 * HOUR;
50
53
  * - daily → 25h (24h + 1h grace)
51
54
  * - weekly → 8d (7d + 1d grace)
52
55
  * - monthly → 32d (~30d + 2d grace)
53
- * - manual → null (no staleness check; user-driven cadence)
54
- *
55
- * `undefined` (no `backup:` block in manifest) is treated as
56
- * `manual` — author opted out of declaring a cadence.
56
+ * - manual → null (no staleness check; operator-driven cadence)
57
57
  */
58
58
  const SCHEDULE_THRESHOLDS = {
59
59
  hourly: 2 * HOUR,
@@ -63,21 +63,13 @@ const SCHEDULE_THRESHOLDS = {
63
63
  manual: null,
64
64
  } as const;
65
65
 
66
- type ScheduleKey = keyof typeof SCHEDULE_THRESHOLDS;
67
-
68
66
  function moduleHasBackupHook(manifest: ModuleManifest): boolean {
69
67
  return Boolean(manifest.hooks?.on_backup);
70
68
  }
71
69
 
72
- function scheduleFor(manifest: ModuleManifest): ScheduleKey {
73
- const s = manifest.backup?.schedule;
74
- if (s === 'hourly' || s === 'daily' || s === 'weekly' || s === 'monthly') return s;
75
- return 'manual';
76
- }
77
-
78
70
  function thresholdFor(manifest: ModuleManifest, override: number | undefined): number | null {
79
71
  if (override !== undefined) return override;
80
- return SCHEDULE_THRESHOLDS[scheduleFor(manifest)];
72
+ return SCHEDULE_THRESHOLDS[effectiveBackupSchedule(manifest)];
81
73
  }
82
74
 
83
75
  function formatAge(ms: number): string {
@@ -109,7 +101,7 @@ export async function auditBackups(deps: BackupsAuditDeps): Promise<DriftFinding
109
101
  category: 'backups',
110
102
  severity: 'drift',
111
103
  code: 'backup_missing',
112
- message: `${m.id}: no successful backup recorded (schedule: ${scheduleFor(m.manifest)})`,
104
+ message: `${m.id}: no successful backup recorded (schedule: ${effectiveBackupSchedule(m.manifest)})`,
113
105
  remediation: `celilo backup create ${m.id} --force`,
114
106
  actionable: true,
115
107
  subject: m.id,
@@ -126,7 +118,7 @@ export async function auditBackups(deps: BackupsAuditDeps): Promise<DriftFinding
126
118
  category: 'backups',
127
119
  severity: 'drift',
128
120
  code: 'backup_stale',
129
- message: `${m.id}: last successful backup is ${formatAge(age)} old (schedule: ${scheduleFor(m.manifest)}, threshold: ${formatAge(threshold)})`,
121
+ message: `${m.id}: last successful backup is ${formatAge(age)} old (schedule: ${effectiveBackupSchedule(m.manifest)}, threshold: ${formatAge(threshold)})`,
130
122
  remediation: `celilo backup create ${m.id} --force`,
131
123
  actionable: true,
132
124
  subject: m.id,