@celilo/cli 0.14.2 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. package/CELILO_CORE_MODULES.md +1 -1
  2. package/CELILO_SUBSYSTEMS.md +18 -2
  3. package/drizzle/0018_drop_alert_policy_snapshot.sql +46 -0
  4. package/drizzle/meta/_journal.json +8 -1
  5. package/package.json +4 -4
  6. package/src/capabilities/public-web-publish.test.ts +15 -15
  7. package/src/cli/commands/alerts-list.ts +10 -0
  8. package/src/cli/commands/alerts-poll.ts +12 -6
  9. package/src/cli/commands/alerts-sweep.ts +22 -81
  10. package/src/cli/commands/backup-sweep.ts +65 -0
  11. package/src/cli/commands/module-operations.test.ts +93 -0
  12. package/src/cli/commands/module-operations.ts +134 -0
  13. package/src/cli/commands/module-upgrade.test.ts +32 -20
  14. package/src/cli/commands/module-upgrade.ts +37 -32
  15. package/src/cli/commands/monitor.ts +26 -6
  16. package/src/cli/commands/system-audit.ts +9 -68
  17. package/src/cli/completion.ts +18 -1
  18. package/src/cli/index.ts +11 -0
  19. package/src/db/schema.ts +5 -3
  20. package/src/hooks/capability-loader.ts +0 -12
  21. package/src/manifest/schema.ts +4 -1
  22. package/src/module/packaging/build.ts +4 -0
  23. package/src/services/alerting/builtin-source.ts +18 -51
  24. package/src/services/alerting/delivery-loop.test.ts +5 -1
  25. package/src/services/alerting/format.test.ts +0 -1
  26. package/src/services/alerting/inbound-poller.test.ts +44 -8
  27. package/src/services/alerting/inbound-poller.ts +65 -28
  28. package/src/services/alerting/notify-deps.ts +113 -0
  29. package/src/services/alerting/run-monitor.ts +0 -1
  30. package/src/services/alerting/store.test.ts +1 -1
  31. package/src/services/alerting/store.ts +0 -2
  32. package/src/services/alerting/sweep-runner.test.ts +11 -2
  33. package/src/services/alerting/sweep-runner.ts +14 -7
  34. package/src/services/audit/backup-source.ts +54 -0
  35. package/src/services/audit/backups.test.ts +7 -2
  36. package/src/services/audit/backups.ts +10 -18
  37. package/src/services/backup-cipher.test.ts +188 -0
  38. package/src/services/backup-cipher.ts +178 -0
  39. package/src/services/backup-create.ts +20 -30
  40. package/src/services/backup-envelope-roundtrip.test.ts +6 -26
  41. package/src/services/backup-restore.ts +10 -16
  42. package/src/services/backup-schedule.ts +35 -0
  43. package/src/services/backup-sweep.test.ts +148 -0
  44. package/src/services/backup-sweep.ts +124 -0
  45. package/src/services/deploy-posture.ts +15 -2
  46. package/src/services/machine-probe.test.ts +50 -0
  47. package/src/services/machine-probe.ts +73 -0
  48. package/src/services/module-operations.test.ts +67 -6
  49. package/src/services/module-operations.ts +69 -19
  50. package/src/services/module-subscriptions.test.ts +33 -2
  51. package/src/services/module-subscriptions.ts +10 -1
  52. package/src/services/module-validator/typescript-build.test.ts +20 -1
  53. package/src/services/module-validator/typescript-build.ts +9 -5
  54. package/src/services/restore-from-file.ts +6 -21
  55. package/src/templates/generator.test.ts +88 -0
  56. package/src/templates/generator.ts +119 -16
package/src/cli/index.ts CHANGED
@@ -66,6 +66,7 @@ import { handleModuleHealth } from './commands/module-health';
66
66
  import { handleModuleImport } from './commands/module-import';
67
67
  import { handleModuleList } from './commands/module-list';
68
68
  import { handleModuleLogs } from './commands/module-logs';
69
+ import { handleModuleOperations } from './commands/module-operations';
69
70
  import { handleModulePublish } from './commands/module-publish';
70
71
  import { handleModuleRemove } from './commands/module-remove';
71
72
  import { handleModuleSearch } from './commands/module-search';
@@ -747,6 +748,9 @@ Subcommands:
747
748
  --storage <id> Use specific storage destination
748
749
  --no-interactive Non-interactive mode (for cron)
749
750
 
751
+ sweep Back up every module whose declared schedule is due
752
+ (run by the event bus on timer.tick.1h, not by hand)
753
+
750
754
  list [module-id] List available backups
751
755
  Options:
752
756
  --limit <n> Number of backups to show (default: 20)
@@ -1466,6 +1470,8 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
1466
1470
  return handleModuleLogs(parsed.args, parsed.flags);
1467
1471
  case 'health':
1468
1472
  return handleModuleHealth(parsed.args, parsed.flags);
1473
+ case 'operations':
1474
+ return handleModuleOperations(parsed.args, parsed.flags);
1469
1475
  case 'remove':
1470
1476
  return handleModuleRemove(parsed.args, parsed.flags);
1471
1477
  case 'update':
@@ -1864,6 +1870,11 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
1864
1870
  return handleBackupCreate(parsed.args, parsed.flags);
1865
1871
  }
1866
1872
 
1873
+ if (parsed.subcommand === 'sweep') {
1874
+ const { handleBackupSweep } = await import('./commands/backup-sweep');
1875
+ return handleBackupSweep();
1876
+ }
1877
+
1867
1878
  if (parsed.subcommand === 'list') {
1868
1879
  const { handleBackupList } = await import('./commands/backup-list');
1869
1880
  return handleBackupList(parsed.args, parsed.flags);
package/src/db/schema.ts CHANGED
@@ -1048,9 +1048,11 @@ export const alerts = sqliteTable(
1048
1048
  deferredRouteId: text('deferred_route_id').references(() => routes.id, {
1049
1049
  onDelete: 'set null',
1050
1050
  }),
1051
- escalationPolicyId: text('escalation_policy_id').references(() => escalationPolicies.id, {
1052
- onDelete: 'set null',
1053
- }),
1051
+ // No escalation policy is stored here on purpose. It lives on the MONITOR
1052
+ // and is resolved fresh on every sweep, so assigning a policy takes effect
1053
+ // on alerts that are already firing. Snapshotting it at alert creation made
1054
+ // `escalation-policy assign` a no-op for exactly the alert an operator was
1055
+ // trying to route — the one already paging nobody (#481).
1054
1056
  message: text('message').notNull(),
1055
1057
  details: text('details'),
1056
1058
  resolvedAt: integer('resolved_at', { mode: 'timestamp' }),
@@ -361,17 +361,6 @@ export async function loadCapabilityFunctions(
361
361
  debugLog(`public_web: using firewall natIp ${firewallNatIp} for internal DNS`);
362
362
  }
363
363
 
364
- // Caddy's external (WAN) IP for the D1 public A record (M1 / #328). Caddy
365
- // learns it from its own firewall.exposeService at install and persists it
366
- // as the `public_ip` hook output (stored as a caddy secret). register_route
367
- // points a new hostname's PUBLIC record here, so it's reachable from one
368
- // deploy — no per-hostname DNAT (shared :443 ingress).
369
- const caddyExternalIp =
370
- typeof providerSecrets.public_ip === 'string' ? providerSecrets.public_ip : undefined;
371
- if (caddyExternalIp) {
372
- debugLog(`public_web: using caddy externalIp ${caddyExternalIp} for public DNS`);
373
- }
374
-
375
364
  // Caddy's configured hostnames — public_web rejects routes for any
376
365
  // hostname not in this list, throwing a structured error that runs
377
366
  // caddy's `managed_hostname` ensure interview. Empty list means
@@ -484,7 +473,6 @@ export async function loadCapabilityFunctions(
484
473
  dnsInternal: result.dns_internal as DnsInternalCapability | undefined,
485
474
  firewallNatIp,
486
475
  dnsRegistrar: result.dns_registrar as DnsRegistrarCapability | undefined,
487
- externalIp: caddyExternalIp,
488
476
  hostnames: caddyHostnames,
489
477
  caddyModuleId: provider.moduleId,
490
478
  dnsManagedDomains,
@@ -694,7 +694,10 @@ export const ModuleManifestSchema = z
694
694
 
695
695
  backup: z
696
696
  .object({
697
- schedule: z.enum(['hourly', 'daily', 'weekly', 'monthly', 'manual']).default('manual'),
697
+ // Absent means `daily`. Opting out of backups entirely is a real
698
+ // decision and takes an explicit `manual` — see
699
+ // [[services/backup-schedule.ts]].
700
+ schedule: z.enum(['hourly', 'daily', 'weekly', 'monthly', 'manual']).default('daily'),
698
701
  retention: z
699
702
  .object({
700
703
  count: z.number().int().positive().default(7),
@@ -59,6 +59,10 @@ const EXCLUDE_PATTERNS = [
59
59
  '.DS_Store',
60
60
  '*.netapp',
61
61
  '*.test.ts',
62
+ // Dev-only, same bucket as the tests: scripts/tsconfig.json exists so tsc can
63
+ // check hooks in CI. Nothing on a target ever runs tsc, and shipping it would
64
+ // make every module's packaged content change whenever the shared base moves.
65
+ 'tsconfig.json',
62
66
  'checksums.json',
63
67
  'signature.sig',
64
68
  ];
@@ -12,70 +12,37 @@
12
12
  * as "nothing is wrong".
13
13
  */
14
14
 
15
- import { execFile } from 'node:child_process';
16
- import { promisify } from 'node:util';
15
+ import type { DbClient } from '../../db/client';
16
+ import { loadBackupAuditInfo } from '../audit/backup-source';
17
+ import { auditBackups } from '../audit/backups';
17
18
  import { auditMachinesReachable } from '../audit/machines-reachable';
18
- import type { MachineReachableResult } from '../audit/machines-reachable';
19
19
  import type { DriftCategory, DriftFinding } from '../audit/types';
20
- import { listMachines } from '../machine-pool';
21
-
22
- const execFileAsync = promisify(execFile);
20
+ import { probeMachines } from '../machine-probe';
23
21
 
24
22
  /** Categories a monitor can currently schedule. */
25
- export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = ['machines_reachable'];
23
+ export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = [
24
+ 'machines_reachable',
25
+ 'backups',
26
+ ];
26
27
 
27
28
  export function isSchedulableBuiltin(category: string): category is DriftCategory {
28
29
  return (SCHEDULABLE_BUILTIN_CHECKS as readonly string[]).includes(category);
29
30
  }
30
31
 
31
- /**
32
- * SSH-probe every pool machine.
33
- *
34
- * `BatchMode=yes` prevents a password prompt from hanging the probe forever,
35
- * and `ConnectTimeout` bounds the wait on an unresponsive host — the exact
36
- * condition this check exists to detect must not be the one that wedges it.
37
- */
38
- async function probeMachines(): Promise<MachineReachableResult[]> {
39
- const machines = await listMachines();
40
- return Promise.all(
41
- machines.map(async (m): Promise<MachineReachableResult> => {
42
- try {
43
- await execFileAsync(
44
- 'ssh',
45
- [
46
- '-o',
47
- 'BatchMode=yes',
48
- '-o',
49
- 'ConnectTimeout=5',
50
- '-o',
51
- 'StrictHostKeyChecking=no',
52
- '-o',
53
- 'UserKnownHostsFile=/dev/null',
54
- `${m.sshUser}@${m.ipAddress}`,
55
- 'true',
56
- ],
57
- { timeout: 8000 },
58
- );
59
- return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
60
- } catch (err) {
61
- const e = err as { stderr?: string; message?: string };
62
- return {
63
- id: m.id,
64
- hostname: m.hostname,
65
- ipAddress: m.ipAddress,
66
- reachable: false,
67
- message: e.stderr?.trim() || e.message || 'SSH probe failed',
68
- };
69
- }
70
- }),
71
- );
72
- }
73
-
74
- export async function runBuiltinCheckForMonitor(category: DriftCategory): Promise<DriftFinding[]> {
32
+ export async function runBuiltinCheckForMonitor(
33
+ category: DriftCategory,
34
+ db: DbClient,
35
+ ): Promise<DriftFinding[]> {
75
36
  if (category === 'machines_reachable') {
76
37
  return auditMachinesReachable({ results: await probeMachines() });
77
38
  }
78
39
 
40
+ // Local DB reads only — cheap enough to run on every sweep, which is
41
+ // the whole reason this category is schedulable and most are not.
42
+ if (category === 'backups') {
43
+ return auditBackups({ modules: loadBackupAuditInfo(db) });
44
+ }
45
+
79
46
  throw new Error(
80
47
  `Built-in check "${category}" is not schedulable yet. Schedulable: ${SCHEDULABLE_BUILTIN_CHECKS.join(', ')}.`,
81
48
  );
@@ -68,6 +68,11 @@ beforeAll(async () => {
68
68
  ...process.env,
69
69
  SIGNAL_RPC_PORT: String(PORT),
70
70
  SIGNAL_KNOWN_RECIPIENTS: `${PETER},${WIFE}`,
71
+ // The flag the module's systemd unit must pass. The simulator defaults
72
+ // to signal-cli's real default (`on-start`), where the daemon drains
73
+ // replies into its own SSE stream and REFUSES `receive` — so a loop test
74
+ // may only read replies from a daemon started the way celilo deploys it.
75
+ SIGNAL_RECEIVE_MODE: 'manual',
71
76
  },
72
77
  stdout: 'pipe',
73
78
  stderr: 'pipe',
@@ -169,7 +174,6 @@ describe('delivery loop against the signal-cli simulator', () => {
169
174
  silencedUntil: null,
170
175
  escalationStep: 0,
171
176
  nextEscalationAt: null,
172
- escalationPolicyId: null,
173
177
  message: '/var 94% used',
174
178
  details: null,
175
179
  resolvedAt: null,
@@ -34,7 +34,6 @@ function alert(over: Partial<Alert> = {}): Alert {
34
34
  silencedUntil: null,
35
35
  escalationStep: 0,
36
36
  nextEscalationAt: null,
37
- escalationPolicyId: null,
38
37
  message: '/var 94% used',
39
38
  details: null,
40
39
  resolvedAt: null,
@@ -27,7 +27,7 @@ describe('pollInbound', () => {
27
27
 
28
28
  function deps(messages: InboundMessage[], over: Partial<InboundPollDeps> = {}): InboundPollDeps {
29
29
  return {
30
- receiveFrom: async () => ({ messages, cursor: 'cursor-1' }),
30
+ receiveFrom: async () => ({ status: 'received', messages, cursor: 'cursor-1' }),
31
31
  readCursor: (t) => cursors.get(t) ?? null,
32
32
  writeCursor: (t, c) => {
33
33
  cursors.set(t, c);
@@ -190,7 +190,7 @@ describe('pollInbound', () => {
190
190
  const delivery = page(peterRoute.id);
191
191
  const report = await pollInbound(db, deps([inbound(WIFE, delivery.token)]));
192
192
 
193
- expect(report.rejected).toBe(1);
193
+ expect(report.unheard).toEqual([{ senderAddress: WIFE, reason: 'wrong_sender' }]);
194
194
  expect(report.acked).toBe(0);
195
195
  expect(alertState()).toBe('firing');
196
196
  });
@@ -200,7 +200,7 @@ describe('pollInbound', () => {
200
200
  const delivery = page(peterRoute.id);
201
201
  const report = await pollInbound(db, deps([inbound(STRANGER, delivery.token)]));
202
202
 
203
- expect(report.ignored).toBe(1);
203
+ expect(report.unheard).toEqual([{ senderAddress: STRANGER, reason: 'unknown_sender' }]);
204
204
  expect(alertState()).toBe('firing');
205
205
  });
206
206
 
@@ -228,7 +228,7 @@ describe('pollInbound', () => {
228
228
  expect(first.acked).toBe(1);
229
229
  // The token is spent, so the replay is refused rather than acking again.
230
230
  expect(second.acked).toBe(0);
231
- expect(second.rejected).toBe(1);
231
+ expect(second.unheard).toEqual([{ senderAddress: PETER, reason: 'unknown_token' }]);
232
232
  expect(alertState()).toBe('acked');
233
233
  });
234
234
 
@@ -262,31 +262,67 @@ describe('pollInbound', () => {
262
262
  });
263
263
 
264
264
  const report = await pollInbound(db, deps([inbound(PETER, 'ack')]));
265
- expect(report.rejected).toBe(1);
265
+ expect(report.unheard).toEqual([{ senderAddress: PETER, reason: 'ambiguous' }]);
266
266
  expect(alertState()).toBe('firing');
267
267
  });
268
268
 
269
269
  // One dead transport must not stop the others being read.
270
270
  test('an unreachable transport is skipped without throwing', async () => {
271
- const report = await pollInbound(db, deps([], { receiveFrom: async () => null }));
271
+ const report = await pollInbound(
272
+ db,
273
+ deps([], { receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
274
+ );
272
275
  expect(report.transportsPolled).toBe(1);
273
276
  expect(report.messagesRead).toBe(0);
274
277
  });
275
278
 
279
+ /**
280
+ * The regression that cost a week. signal-cli's daemon refused celilo's read
281
+ * call outright ("Receive command cannot be used if messages are already
282
+ * being received") and the poller reported the same "0 message(s)" it
283
+ * reports when nobody has replied. The reason must reach the report, or the
284
+ * two are indistinguishable from outside.
285
+ */
286
+ test('a transport that REFUSES the read says so instead of looking quiet', async () => {
287
+ const refusal = 'Receive command cannot be used if messages are already being received.';
288
+ const report = await pollInbound(
289
+ db,
290
+ deps([], { receiveFrom: async () => ({ status: 'failed', error: refusal }) }),
291
+ );
292
+
293
+ expect(report.failures).toEqual([{ transportModuleId: 'signal', error: refusal }]);
294
+ // ...and a genuinely quiet transport must NOT look like a failure.
295
+ const quiet = await pollInbound(db, deps([]));
296
+ expect(quiet.failures).toEqual([]);
297
+ expect(quiet.messagesRead).toBe(0);
298
+ });
299
+
300
+ test('a unidirectional transport is not reported as a failure', async () => {
301
+ const report = await pollInbound(
302
+ db,
303
+ deps([], { receiveFrom: async () => ({ status: 'unidirectional' }) }),
304
+ );
305
+ expect(report.failures).toEqual([]);
306
+ expect(report.transportsPolled).toBe(1);
307
+ });
308
+
276
309
  test('the cursor is persisted after a successful poll', async () => {
277
310
  await pollInbound(db, deps([]));
278
311
  expect(cursors.get('signal')).toBe('cursor-1');
279
312
  });
280
313
 
281
314
  test('the cursor is NOT advanced when the transport could not be reached', async () => {
282
- await pollInbound(db, deps([], { receiveFrom: async () => null }));
315
+ await pollInbound(
316
+ db,
317
+ deps([], { receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
318
+ );
283
319
  expect(cursors.get('signal')).toBeUndefined();
284
320
  });
285
321
 
286
322
  test('gibberish is rejected without touching the alert', async () => {
287
323
  page(peterRoute.id);
288
324
  const report = await pollInbound(db, deps([inbound(PETER, 'what is going on')]));
289
- expect(report.rejected).toBe(1);
325
+ expect(report.unheard).toEqual([{ senderAddress: PETER, reason: 'unrecognised' }]);
290
326
  expect(alertState()).toBe('firing');
291
327
  });
292
328
 
@@ -25,12 +25,35 @@ import type { NotificationTransport } from './notifier';
25
25
  import { composeAckBroadcastBody } from './notifier';
26
26
  import { consumeDelivery } from './tokens';
27
27
 
28
+ /**
29
+ * What one attempt to read a transport produced.
30
+ *
31
+ * `unidirectional` and `failed` used to collapse into a bare null, and that
32
+ * cost a week: a transport whose read call was being REFUSED reported the same
33
+ * "0 message(s)" as a transport nobody had replied on. An operator could not
34
+ * tell "my reply never arrived" from "celilo cannot read this transport at
35
+ * all", and neither could anyone debugging it.
36
+ */
37
+ export type TransportReceive =
38
+ | { status: 'received'; messages: InboundMessage[]; cursor: string | null }
39
+ | { status: 'unidirectional' }
40
+ | { status: 'failed'; error: string };
41
+
42
+ /** A transport that could not be read, and why. */
43
+ export interface TransportFailure {
44
+ transportModuleId: string;
45
+ error: string;
46
+ }
47
+
48
+ /** A message that was read but not acted on, and why. */
49
+ export interface UnheardMessage {
50
+ senderAddress: string;
51
+ reason: 'unknown_sender' | 'unknown_token' | 'wrong_sender' | 'ambiguous' | 'unrecognised';
52
+ }
53
+
28
54
  export interface InboundPollDeps {
29
- /** Receive from one transport module, or null when it cannot be reached. */
30
- receiveFrom(
31
- transportModuleId: string,
32
- cursor: string | null,
33
- ): Promise<{ messages: InboundMessage[]; cursor: string | null } | null>;
55
+ /** Receive from one transport module. */
56
+ receiveFrom(transportModuleId: string, cursor: string | null): Promise<TransportReceive>;
34
57
  /** Persisted receive cursor per transport. */
35
58
  readCursor(transportModuleId: string): string | null;
36
59
  writeCursor(transportModuleId: string, cursor: string | null): void;
@@ -53,12 +76,14 @@ export interface InboundPollReport {
53
76
  transportsPolled: number;
54
77
  messagesRead: number;
55
78
  acked: number;
56
- ignored: number;
57
- rejected: number;
58
79
  /** Routes told that someone else took the alert. */
59
80
  broadcast: number;
60
81
  /** Deploy questions answered from a phone. */
61
82
  answered: number;
83
+ /** Transports that could not be read at all. Never silent — see above. */
84
+ failures: TransportFailure[];
85
+ /** Messages read and then discarded, with the reason each was discarded. */
86
+ unheard: UnheardMessage[];
62
87
  }
63
88
 
64
89
  /** Transports that have at least one route pointing at them. */
@@ -89,19 +114,24 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
89
114
  transportsPolled: 0,
90
115
  messagesRead: 0,
91
116
  acked: 0,
92
- ignored: 0,
93
- rejected: 0,
94
117
  broadcast: 0,
95
118
  answered: 0,
119
+ failures: [],
120
+ unheard: [],
96
121
  };
97
122
 
98
123
  for (const transportId of transportsWithRoutes(db)) {
99
124
  const received = await deps.receiveFrom(transportId, deps.readCursor(transportId));
100
125
  report.transportsPolled++;
101
- // A transport that cannot be reached is not an error here its own
102
- // health check is what reports that, and one dead transport must not stop
103
- // the others from being read.
104
- if (!received) continue;
126
+ // One dead transport must not stop the others being read but it is
127
+ // RECORDED rather than skipped in silence, because "cannot read" and
128
+ // "nothing to read" are the two things an operator most needs to tell
129
+ // apart at 3am.
130
+ if (received.status === 'failed') {
131
+ report.failures.push({ transportModuleId: transportId, error: received.error });
132
+ continue;
133
+ }
134
+ if (received.status === 'unidirectional') continue;
105
135
 
106
136
  for (const message of received.messages) {
107
137
  report.messagesRead++;
@@ -113,12 +143,8 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
113
143
  outstandingForRoute: (routeId) => outstandingAlertDeliveries(db, routeId, deps.now()),
114
144
  });
115
145
 
116
- if (outcome.action === 'ignored') {
117
- report.ignored++;
118
- continue;
119
- }
120
- if (outcome.action === 'rejected') {
121
- report.rejected++;
146
+ if (outcome.action === 'ignored' || outcome.action === 'rejected') {
147
+ report.unheard.push({ senderAddress: message.senderAddress, reason: outcome.reason });
122
148
  continue;
123
149
  }
124
150
 
@@ -127,7 +153,7 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
127
153
  if (outcome.delivery.kind === 'interview') {
128
154
  const value = parseInterviewAnswer(message.body, outcome.delivery.token);
129
155
  if (!value || !deps.answerInterview) {
130
- report.rejected++;
156
+ report.unheard.push({ senderAddress: message.senderAddress, reason: 'unrecognised' });
131
157
  continue;
132
158
  }
133
159
  consumeDelivery(db, outcome.delivery.id, deps.now());
@@ -213,12 +239,23 @@ function outstandingAlertDeliveries(db: DbClient, routeId: string, now: Date) {
213
239
  *
214
240
  * A transport with no `receive` is unidirectional — that is not an error, it
215
241
  * just means replies cannot arrive, which the route's `can_ack` already
216
- * records.
242
+ * records. Anything else going wrong IS an error and is returned as one.
243
+ *
244
+ * This used to be a bare `catch {}` returning null, on the reasoning that the
245
+ * transport's own health check would report an unreachable daemon. That was
246
+ * wrong twice over: a health check that only proves the daemon answers cannot
247
+ * see a read call being REFUSED by a daemon that is otherwise perfectly
248
+ * healthy, and the swallowed error was the only place the reason existed. The
249
+ * live signal-cli case was exactly that shape — `receive` refused with
250
+ * "Receive command cannot be used if messages are already being received."
251
+ * while every health check passed and every poll reported zero messages.
217
252
  */
218
253
  export function makeReceiver(db: DbClient) {
219
- return async (transportModuleId: string, cursor: string | null) => {
254
+ return async (transportModuleId: string, cursor: string | null): Promise<TransportReceive> => {
220
255
  const module = db.select().from(modules).where(eq(modules.id, transportModuleId)).get();
221
- if (!module) return null;
256
+ if (!module) {
257
+ return { status: 'failed', error: `no module '${transportModuleId}' is installed` };
258
+ }
222
259
 
223
260
  try {
224
261
  const { logger } = createCapturingLogger();
@@ -226,11 +263,11 @@ export function makeReceiver(db: DbClient) {
226
263
  const notification = (capabilities as Record<string, unknown>).notification as
227
264
  | NotificationCapability
228
265
  | undefined;
229
- if (!notification?.receive) return null;
230
- return await notification.receive(cursor);
231
- } catch {
232
- // Unreachable transport: the transport's own health check reports it.
233
- return null;
266
+ if (!notification?.receive) return { status: 'unidirectional' };
267
+ const result = await notification.receive(cursor);
268
+ return { status: 'received', messages: result.messages, cursor: result.cursor };
269
+ } catch (error) {
270
+ return { status: 'failed', error: error instanceof Error ? error.message : String(error) };
234
271
  }
235
272
  };
236
273
  }
@@ -0,0 +1,113 @@
1
+ /**
2
+ * Resolving an alert to the people it should page.
3
+ *
4
+ * The escalation policy is a property of the MONITOR, and it is read here on
5
+ * every sweep rather than copied onto the alert when the alert is created.
6
+ * That ordering is the whole point: an operator only reaches for
7
+ * `escalation-policy assign` once something is already firing and reaching
8
+ * nobody, so a policy that only applied to alerts created afterwards would be
9
+ * a no-op for the one alert they were trying to fix (#481).
10
+ *
11
+ * The cost of resolving late is that a policy edited mid-escalation changes
12
+ * where the remaining steps go. That is the behaviour an operator asks for
13
+ * when they edit it — the alert's own progress through the chain
14
+ * (`escalationStep`, `nextEscalationAt`) is what stays on the alert.
15
+ *
16
+ * Lives here rather than in the sweep command so the sweep's tests can drive
17
+ * the real resolution instead of a fake, which is how the snapshot bug
18
+ * survived: every existing test stubbed `notifyDepsFor`.
19
+ */
20
+
21
+ import { eq } from 'drizzle-orm';
22
+ import type { DbClient } from '../../db/client';
23
+ import { type Alert, type EscalationPolicy, escalationPolicies, monitors } from '../../db/schema';
24
+ import type { NotifyDeps } from './notifier';
25
+ import { listPeople, listPolicySteps, listRoutes } from './people';
26
+ import { mintDelivery } from './tokens';
27
+ import { loadNotificationTransport } from './transport-loader';
28
+
29
+ /**
30
+ * How long a reply token stays valid. A day is long enough that someone who
31
+ * sees a page overnight can still act on it in the morning, and short enough
32
+ * that a token found later is useless.
33
+ */
34
+ const TOKEN_TTL_MS = 24 * 60 * 60_000;
35
+
36
+ /**
37
+ * The escalation policy an alert resolves to right now, or null.
38
+ *
39
+ * The single answer to "who would this page", shared by the sweep and by every
40
+ * command that has to show it. Two readers computing it separately is how a
41
+ * CLI ends up confidently reporting a policy the sweep does not use.
42
+ */
43
+ export function policyForAlert(
44
+ db: DbClient,
45
+ alert: Pick<Alert, 'monitorId'>,
46
+ ): EscalationPolicy | null {
47
+ const monitor = db.select().from(monitors).where(eq(monitors.id, alert.monitorId)).get();
48
+ if (!monitor?.escalationPolicyId) return null;
49
+ return (
50
+ db
51
+ .select()
52
+ .from(escalationPolicies)
53
+ .where(eq(escalationPolicies.id, monitor.escalationPolicyId))
54
+ .get() ?? null
55
+ );
56
+ }
57
+
58
+ /**
59
+ * Assemble everything needed to page for one alert, or null when nobody can be.
60
+ *
61
+ * Returning null rather than an empty policy is deliberate: "no escalation
62
+ * policy assigned" and "policy exists but nobody is eligible" are different
63
+ * situations, and only the second is worth an escalation decision.
64
+ */
65
+ export function buildNotifyDeps(db: DbClient, alert: Alert, now: Date): NotifyDeps | null {
66
+ const policy = policyForAlert(db, alert);
67
+ if (!policy) return null;
68
+
69
+ const steps = listPolicySteps(db, policy.id).map((step) => ({
70
+ stepIndex: step.stepIndex,
71
+ routeId: step.routeId,
72
+ delayMinutes: step.delayMinutes,
73
+ }));
74
+ if (steps.length === 0) return null;
75
+
76
+ const routeRows = listRoutes(db);
77
+ const people = listPeople(db);
78
+
79
+ return {
80
+ steps,
81
+ routes: new Map(
82
+ routeRows.map((r) => [
83
+ r.id,
84
+ { id: r.id, severityFloor: r.severityFloor, enabled: r.enabled },
85
+ ]),
86
+ ),
87
+ routeDetails: new Map(routeRows.map((r) => [r.id, r])),
88
+ quietHoursByPerson: new Map(
89
+ people
90
+ .filter((p) => p.quietHoursStart && p.quietHoursEnd)
91
+ .map((p) => [
92
+ p.id,
93
+ {
94
+ personId: p.id,
95
+ start: p.quietHoursStart,
96
+ end: p.quietHoursEnd,
97
+ timezone: p.timezone,
98
+ },
99
+ ]),
100
+ ),
101
+ bypassQuietHours: policy.bypassQuietHours,
102
+ transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
103
+ mintToken: (alertId, routeId) =>
104
+ mintDelivery(db, {
105
+ kind: 'alert',
106
+ targetId: alertId,
107
+ routeId,
108
+ now,
109
+ ttlMs: TOKEN_TTL_MS,
110
+ }).token,
111
+ now,
112
+ };
113
+ }
@@ -151,7 +151,6 @@ export async function runOneMonitor(
151
151
 
152
152
  const { createdIds, resolvedIds } = applyReconcileActions(db, actions, {
153
153
  monitorId: monitor.id,
154
- escalationPolicyId: monitor.escalationPolicyId,
155
154
  now,
156
155
  });
157
156
 
@@ -27,7 +27,7 @@ describe('alert store', () => {
27
27
  let dir: string;
28
28
  let db: DbClient;
29
29
 
30
- const context = { monitorId: MONITOR, escalationPolicyId: null, now: NOW };
30
+ const context = { monitorId: MONITOR, now: NOW };
31
31
 
32
32
  const create = (key: string): ReconcileAction => ({
33
33
  type: 'create',
@@ -28,7 +28,6 @@ export function loadLiveAlerts(db: DbClient, monitorId: string): LiveAlert[] {
28
28
 
29
29
  export interface ApplyContext {
30
30
  monitorId: string;
31
- escalationPolicyId: string | null;
32
31
  now: Date;
33
32
  }
34
33
 
@@ -66,7 +65,6 @@ export function applyReconcileActions(
66
65
  lastSeenAt: context.now,
67
66
  graceUntil: action.graceUntil,
68
67
  escalationStep: 0,
69
- escalationPolicyId: context.escalationPolicyId,
70
68
  message: action.message,
71
69
  details: action.details,
72
70
  })
@@ -238,7 +238,16 @@ describe('runSweep', () => {
238
238
 
239
239
  expect(liveAlerts()).toHaveLength(1);
240
240
  expect(report.notified).toBe(0);
241
- expect(report.noPolicy).toBe(1);
241
+ expect(report.noPolicy).toEqual([{ alertKey: PORT_CHECK, monitor: MODULE }]);
242
+ });
243
+
244
+ // A count told the operator something was unroutable but not WHICH thing,
245
+ // so the remedy printed next to it could not be aimed anywhere (#481).
246
+ test('the unroutable alert is named, along with the monitor to assign to', async () => {
247
+ const report = await runSweep(db, currentMonitors(), deps());
248
+
249
+ expect(report.noPolicy[0].alertKey).toBe(PORT_CHECK);
250
+ expect(report.noPolicy[0].monitor).toBe(MODULE);
242
251
  });
243
252
 
244
253
  test('escalation declining to notify records WHICH reason', async () => {
@@ -270,7 +279,7 @@ describe('runSweep', () => {
270
279
  );
271
280
 
272
281
  expect(report.notified).toBe(0);
273
- expect(report.noPolicy).toBe(0);
282
+ expect(report.noPolicy).toEqual([]);
274
283
  expect(report.skipped.within_grace).toBe(1);
275
284
  });
276
285