@celilo/cli 0.14.2 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +1 -1
- package/CELILO_SUBSYSTEMS.md +18 -2
- package/drizzle/0018_drop_alert_policy_snapshot.sql +46 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +4 -4
- package/src/capabilities/public-web-publish.test.ts +15 -15
- package/src/cli/commands/alerts-list.ts +10 -0
- package/src/cli/commands/alerts-poll.ts +12 -6
- package/src/cli/commands/alerts-sweep.ts +22 -81
- package/src/cli/commands/backup-sweep.ts +65 -0
- package/src/cli/commands/module-operations.test.ts +93 -0
- package/src/cli/commands/module-operations.ts +134 -0
- package/src/cli/commands/module-upgrade.test.ts +32 -20
- package/src/cli/commands/module-upgrade.ts +37 -32
- package/src/cli/commands/monitor.ts +26 -6
- package/src/cli/commands/system-audit.ts +9 -68
- package/src/cli/completion.ts +18 -1
- package/src/cli/index.ts +11 -0
- package/src/db/schema.ts +5 -3
- package/src/hooks/capability-loader.ts +0 -12
- package/src/manifest/schema.ts +4 -1
- package/src/module/packaging/build.ts +4 -0
- package/src/services/alerting/builtin-source.ts +18 -51
- package/src/services/alerting/delivery-loop.test.ts +5 -1
- package/src/services/alerting/format.test.ts +0 -1
- package/src/services/alerting/inbound-poller.test.ts +44 -8
- package/src/services/alerting/inbound-poller.ts +65 -28
- package/src/services/alerting/notify-deps.ts +113 -0
- package/src/services/alerting/run-monitor.ts +0 -1
- package/src/services/alerting/store.test.ts +1 -1
- package/src/services/alerting/store.ts +0 -2
- package/src/services/alerting/sweep-runner.test.ts +11 -2
- package/src/services/alerting/sweep-runner.ts +14 -7
- package/src/services/audit/backup-source.ts +54 -0
- package/src/services/audit/backups.test.ts +7 -2
- package/src/services/audit/backups.ts +10 -18
- package/src/services/backup-cipher.test.ts +188 -0
- package/src/services/backup-cipher.ts +178 -0
- package/src/services/backup-create.ts +20 -30
- package/src/services/backup-envelope-roundtrip.test.ts +6 -26
- package/src/services/backup-restore.ts +10 -16
- package/src/services/backup-schedule.ts +35 -0
- package/src/services/backup-sweep.test.ts +148 -0
- package/src/services/backup-sweep.ts +124 -0
- package/src/services/deploy-posture.ts +15 -2
- package/src/services/machine-probe.test.ts +50 -0
- package/src/services/machine-probe.ts +73 -0
- package/src/services/module-operations.test.ts +67 -6
- package/src/services/module-operations.ts +69 -19
- package/src/services/module-subscriptions.test.ts +33 -2
- package/src/services/module-subscriptions.ts +10 -1
- package/src/services/module-validator/typescript-build.test.ts +20 -1
- package/src/services/module-validator/typescript-build.ts +9 -5
- package/src/services/restore-from-file.ts +6 -21
- package/src/templates/generator.test.ts +88 -0
- package/src/templates/generator.ts +119 -16
package/src/cli/index.ts
CHANGED
|
@@ -66,6 +66,7 @@ import { handleModuleHealth } from './commands/module-health';
|
|
|
66
66
|
import { handleModuleImport } from './commands/module-import';
|
|
67
67
|
import { handleModuleList } from './commands/module-list';
|
|
68
68
|
import { handleModuleLogs } from './commands/module-logs';
|
|
69
|
+
import { handleModuleOperations } from './commands/module-operations';
|
|
69
70
|
import { handleModulePublish } from './commands/module-publish';
|
|
70
71
|
import { handleModuleRemove } from './commands/module-remove';
|
|
71
72
|
import { handleModuleSearch } from './commands/module-search';
|
|
@@ -747,6 +748,9 @@ Subcommands:
|
|
|
747
748
|
--storage <id> Use specific storage destination
|
|
748
749
|
--no-interactive Non-interactive mode (for cron)
|
|
749
750
|
|
|
751
|
+
sweep Back up every module whose declared schedule is due
|
|
752
|
+
(run by the event bus on timer.tick.1h, not by hand)
|
|
753
|
+
|
|
750
754
|
list [module-id] List available backups
|
|
751
755
|
Options:
|
|
752
756
|
--limit <n> Number of backups to show (default: 20)
|
|
@@ -1466,6 +1470,8 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
|
|
|
1466
1470
|
return handleModuleLogs(parsed.args, parsed.flags);
|
|
1467
1471
|
case 'health':
|
|
1468
1472
|
return handleModuleHealth(parsed.args, parsed.flags);
|
|
1473
|
+
case 'operations':
|
|
1474
|
+
return handleModuleOperations(parsed.args, parsed.flags);
|
|
1469
1475
|
case 'remove':
|
|
1470
1476
|
return handleModuleRemove(parsed.args, parsed.flags);
|
|
1471
1477
|
case 'update':
|
|
@@ -1864,6 +1870,11 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
|
|
|
1864
1870
|
return handleBackupCreate(parsed.args, parsed.flags);
|
|
1865
1871
|
}
|
|
1866
1872
|
|
|
1873
|
+
if (parsed.subcommand === 'sweep') {
|
|
1874
|
+
const { handleBackupSweep } = await import('./commands/backup-sweep');
|
|
1875
|
+
return handleBackupSweep();
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1867
1878
|
if (parsed.subcommand === 'list') {
|
|
1868
1879
|
const { handleBackupList } = await import('./commands/backup-list');
|
|
1869
1880
|
return handleBackupList(parsed.args, parsed.flags);
|
package/src/db/schema.ts
CHANGED
|
@@ -1048,9 +1048,11 @@ export const alerts = sqliteTable(
|
|
|
1048
1048
|
deferredRouteId: text('deferred_route_id').references(() => routes.id, {
|
|
1049
1049
|
onDelete: 'set null',
|
|
1050
1050
|
}),
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1051
|
+
// No escalation policy is stored here on purpose. It lives on the MONITOR
|
|
1052
|
+
// and is resolved fresh on every sweep, so assigning a policy takes effect
|
|
1053
|
+
// on alerts that are already firing. Snapshotting it at alert creation made
|
|
1054
|
+
// `escalation-policy assign` a no-op for exactly the alert an operator was
|
|
1055
|
+
// trying to route — the one already paging nobody (#481).
|
|
1054
1056
|
message: text('message').notNull(),
|
|
1055
1057
|
details: text('details'),
|
|
1056
1058
|
resolvedAt: integer('resolved_at', { mode: 'timestamp' }),
|
|
@@ -361,17 +361,6 @@ export async function loadCapabilityFunctions(
|
|
|
361
361
|
debugLog(`public_web: using firewall natIp ${firewallNatIp} for internal DNS`);
|
|
362
362
|
}
|
|
363
363
|
|
|
364
|
-
// Caddy's external (WAN) IP for the D1 public A record (M1 / #328). Caddy
|
|
365
|
-
// learns it from its own firewall.exposeService at install and persists it
|
|
366
|
-
// as the `public_ip` hook output (stored as a caddy secret). register_route
|
|
367
|
-
// points a new hostname's PUBLIC record here, so it's reachable from one
|
|
368
|
-
// deploy — no per-hostname DNAT (shared :443 ingress).
|
|
369
|
-
const caddyExternalIp =
|
|
370
|
-
typeof providerSecrets.public_ip === 'string' ? providerSecrets.public_ip : undefined;
|
|
371
|
-
if (caddyExternalIp) {
|
|
372
|
-
debugLog(`public_web: using caddy externalIp ${caddyExternalIp} for public DNS`);
|
|
373
|
-
}
|
|
374
|
-
|
|
375
364
|
// Caddy's configured hostnames — public_web rejects routes for any
|
|
376
365
|
// hostname not in this list, throwing a structured error that runs
|
|
377
366
|
// caddy's `managed_hostname` ensure interview. Empty list means
|
|
@@ -484,7 +473,6 @@ export async function loadCapabilityFunctions(
|
|
|
484
473
|
dnsInternal: result.dns_internal as DnsInternalCapability | undefined,
|
|
485
474
|
firewallNatIp,
|
|
486
475
|
dnsRegistrar: result.dns_registrar as DnsRegistrarCapability | undefined,
|
|
487
|
-
externalIp: caddyExternalIp,
|
|
488
476
|
hostnames: caddyHostnames,
|
|
489
477
|
caddyModuleId: provider.moduleId,
|
|
490
478
|
dnsManagedDomains,
|
package/src/manifest/schema.ts
CHANGED
|
@@ -694,7 +694,10 @@ export const ModuleManifestSchema = z
|
|
|
694
694
|
|
|
695
695
|
backup: z
|
|
696
696
|
.object({
|
|
697
|
-
|
|
697
|
+
// Absent means `daily`. Opting out of backups entirely is a real
|
|
698
|
+
// decision and takes an explicit `manual` — see
|
|
699
|
+
// [[services/backup-schedule.ts]].
|
|
700
|
+
schedule: z.enum(['hourly', 'daily', 'weekly', 'monthly', 'manual']).default('daily'),
|
|
698
701
|
retention: z
|
|
699
702
|
.object({
|
|
700
703
|
count: z.number().int().positive().default(7),
|
|
@@ -59,6 +59,10 @@ const EXCLUDE_PATTERNS = [
|
|
|
59
59
|
'.DS_Store',
|
|
60
60
|
'*.netapp',
|
|
61
61
|
'*.test.ts',
|
|
62
|
+
// Dev-only, same bucket as the tests: scripts/tsconfig.json exists so tsc can
|
|
63
|
+
// check hooks in CI. Nothing on a target ever runs tsc, and shipping it would
|
|
64
|
+
// make every module's packaged content change whenever the shared base moves.
|
|
65
|
+
'tsconfig.json',
|
|
62
66
|
'checksums.json',
|
|
63
67
|
'signature.sig',
|
|
64
68
|
];
|
|
@@ -12,70 +12,37 @@
|
|
|
12
12
|
* as "nothing is wrong".
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
15
|
+
import type { DbClient } from '../../db/client';
|
|
16
|
+
import { loadBackupAuditInfo } from '../audit/backup-source';
|
|
17
|
+
import { auditBackups } from '../audit/backups';
|
|
17
18
|
import { auditMachinesReachable } from '../audit/machines-reachable';
|
|
18
|
-
import type { MachineReachableResult } from '../audit/machines-reachable';
|
|
19
19
|
import type { DriftCategory, DriftFinding } from '../audit/types';
|
|
20
|
-
import {
|
|
21
|
-
|
|
22
|
-
const execFileAsync = promisify(execFile);
|
|
20
|
+
import { probeMachines } from '../machine-probe';
|
|
23
21
|
|
|
24
22
|
/** Categories a monitor can currently schedule. */
|
|
25
|
-
export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = [
|
|
23
|
+
export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = [
|
|
24
|
+
'machines_reachable',
|
|
25
|
+
'backups',
|
|
26
|
+
];
|
|
26
27
|
|
|
27
28
|
export function isSchedulableBuiltin(category: string): category is DriftCategory {
|
|
28
29
|
return (SCHEDULABLE_BUILTIN_CHECKS as readonly string[]).includes(category);
|
|
29
30
|
}
|
|
30
31
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
* and `ConnectTimeout` bounds the wait on an unresponsive host — the exact
|
|
36
|
-
* condition this check exists to detect must not be the one that wedges it.
|
|
37
|
-
*/
|
|
38
|
-
async function probeMachines(): Promise<MachineReachableResult[]> {
|
|
39
|
-
const machines = await listMachines();
|
|
40
|
-
return Promise.all(
|
|
41
|
-
machines.map(async (m): Promise<MachineReachableResult> => {
|
|
42
|
-
try {
|
|
43
|
-
await execFileAsync(
|
|
44
|
-
'ssh',
|
|
45
|
-
[
|
|
46
|
-
'-o',
|
|
47
|
-
'BatchMode=yes',
|
|
48
|
-
'-o',
|
|
49
|
-
'ConnectTimeout=5',
|
|
50
|
-
'-o',
|
|
51
|
-
'StrictHostKeyChecking=no',
|
|
52
|
-
'-o',
|
|
53
|
-
'UserKnownHostsFile=/dev/null',
|
|
54
|
-
`${m.sshUser}@${m.ipAddress}`,
|
|
55
|
-
'true',
|
|
56
|
-
],
|
|
57
|
-
{ timeout: 8000 },
|
|
58
|
-
);
|
|
59
|
-
return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
|
|
60
|
-
} catch (err) {
|
|
61
|
-
const e = err as { stderr?: string; message?: string };
|
|
62
|
-
return {
|
|
63
|
-
id: m.id,
|
|
64
|
-
hostname: m.hostname,
|
|
65
|
-
ipAddress: m.ipAddress,
|
|
66
|
-
reachable: false,
|
|
67
|
-
message: e.stderr?.trim() || e.message || 'SSH probe failed',
|
|
68
|
-
};
|
|
69
|
-
}
|
|
70
|
-
}),
|
|
71
|
-
);
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
export async function runBuiltinCheckForMonitor(category: DriftCategory): Promise<DriftFinding[]> {
|
|
32
|
+
export async function runBuiltinCheckForMonitor(
|
|
33
|
+
category: DriftCategory,
|
|
34
|
+
db: DbClient,
|
|
35
|
+
): Promise<DriftFinding[]> {
|
|
75
36
|
if (category === 'machines_reachable') {
|
|
76
37
|
return auditMachinesReachable({ results: await probeMachines() });
|
|
77
38
|
}
|
|
78
39
|
|
|
40
|
+
// Local DB reads only — cheap enough to run on every sweep, which is
|
|
41
|
+
// the whole reason this category is schedulable and most are not.
|
|
42
|
+
if (category === 'backups') {
|
|
43
|
+
return auditBackups({ modules: loadBackupAuditInfo(db) });
|
|
44
|
+
}
|
|
45
|
+
|
|
79
46
|
throw new Error(
|
|
80
47
|
`Built-in check "${category}" is not schedulable yet. Schedulable: ${SCHEDULABLE_BUILTIN_CHECKS.join(', ')}.`,
|
|
81
48
|
);
|
|
@@ -68,6 +68,11 @@ beforeAll(async () => {
|
|
|
68
68
|
...process.env,
|
|
69
69
|
SIGNAL_RPC_PORT: String(PORT),
|
|
70
70
|
SIGNAL_KNOWN_RECIPIENTS: `${PETER},${WIFE}`,
|
|
71
|
+
// The flag the module's systemd unit must pass. The simulator defaults
|
|
72
|
+
// to signal-cli's real default (`on-start`), where the daemon drains
|
|
73
|
+
// replies into its own SSE stream and REFUSES `receive` — so a loop test
|
|
74
|
+
// may only read replies from a daemon started the way celilo deploys it.
|
|
75
|
+
SIGNAL_RECEIVE_MODE: 'manual',
|
|
71
76
|
},
|
|
72
77
|
stdout: 'pipe',
|
|
73
78
|
stderr: 'pipe',
|
|
@@ -169,7 +174,6 @@ describe('delivery loop against the signal-cli simulator', () => {
|
|
|
169
174
|
silencedUntil: null,
|
|
170
175
|
escalationStep: 0,
|
|
171
176
|
nextEscalationAt: null,
|
|
172
|
-
escalationPolicyId: null,
|
|
173
177
|
message: '/var 94% used',
|
|
174
178
|
details: null,
|
|
175
179
|
resolvedAt: null,
|
|
@@ -27,7 +27,7 @@ describe('pollInbound', () => {
|
|
|
27
27
|
|
|
28
28
|
function deps(messages: InboundMessage[], over: Partial<InboundPollDeps> = {}): InboundPollDeps {
|
|
29
29
|
return {
|
|
30
|
-
receiveFrom: async () => ({ messages, cursor: 'cursor-1' }),
|
|
30
|
+
receiveFrom: async () => ({ status: 'received', messages, cursor: 'cursor-1' }),
|
|
31
31
|
readCursor: (t) => cursors.get(t) ?? null,
|
|
32
32
|
writeCursor: (t, c) => {
|
|
33
33
|
cursors.set(t, c);
|
|
@@ -190,7 +190,7 @@ describe('pollInbound', () => {
|
|
|
190
190
|
const delivery = page(peterRoute.id);
|
|
191
191
|
const report = await pollInbound(db, deps([inbound(WIFE, delivery.token)]));
|
|
192
192
|
|
|
193
|
-
expect(report.
|
|
193
|
+
expect(report.unheard).toEqual([{ senderAddress: WIFE, reason: 'wrong_sender' }]);
|
|
194
194
|
expect(report.acked).toBe(0);
|
|
195
195
|
expect(alertState()).toBe('firing');
|
|
196
196
|
});
|
|
@@ -200,7 +200,7 @@ describe('pollInbound', () => {
|
|
|
200
200
|
const delivery = page(peterRoute.id);
|
|
201
201
|
const report = await pollInbound(db, deps([inbound(STRANGER, delivery.token)]));
|
|
202
202
|
|
|
203
|
-
expect(report.
|
|
203
|
+
expect(report.unheard).toEqual([{ senderAddress: STRANGER, reason: 'unknown_sender' }]);
|
|
204
204
|
expect(alertState()).toBe('firing');
|
|
205
205
|
});
|
|
206
206
|
|
|
@@ -228,7 +228,7 @@ describe('pollInbound', () => {
|
|
|
228
228
|
expect(first.acked).toBe(1);
|
|
229
229
|
// The token is spent, so the replay is refused rather than acking again.
|
|
230
230
|
expect(second.acked).toBe(0);
|
|
231
|
-
expect(second.
|
|
231
|
+
expect(second.unheard).toEqual([{ senderAddress: PETER, reason: 'unknown_token' }]);
|
|
232
232
|
expect(alertState()).toBe('acked');
|
|
233
233
|
});
|
|
234
234
|
|
|
@@ -262,31 +262,67 @@ describe('pollInbound', () => {
|
|
|
262
262
|
});
|
|
263
263
|
|
|
264
264
|
const report = await pollInbound(db, deps([inbound(PETER, 'ack')]));
|
|
265
|
-
expect(report.
|
|
265
|
+
expect(report.unheard).toEqual([{ senderAddress: PETER, reason: 'ambiguous' }]);
|
|
266
266
|
expect(alertState()).toBe('firing');
|
|
267
267
|
});
|
|
268
268
|
|
|
269
269
|
// One dead transport must not stop the others being read.
|
|
270
270
|
test('an unreachable transport is skipped without throwing', async () => {
|
|
271
|
-
const report = await pollInbound(
|
|
271
|
+
const report = await pollInbound(
|
|
272
|
+
db,
|
|
273
|
+
deps([], { receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
|
|
274
|
+
);
|
|
272
275
|
expect(report.transportsPolled).toBe(1);
|
|
273
276
|
expect(report.messagesRead).toBe(0);
|
|
274
277
|
});
|
|
275
278
|
|
|
279
|
+
/**
|
|
280
|
+
* The regression that cost a week. signal-cli's daemon refused celilo's read
|
|
281
|
+
* call outright ("Receive command cannot be used if messages are already
|
|
282
|
+
* being received") and the poller reported the same "0 message(s)" it
|
|
283
|
+
* reports when nobody has replied. The reason must reach the report, or the
|
|
284
|
+
* two are indistinguishable from outside.
|
|
285
|
+
*/
|
|
286
|
+
test('a transport that REFUSES the read says so instead of looking quiet', async () => {
|
|
287
|
+
const refusal = 'Receive command cannot be used if messages are already being received.';
|
|
288
|
+
const report = await pollInbound(
|
|
289
|
+
db,
|
|
290
|
+
deps([], { receiveFrom: async () => ({ status: 'failed', error: refusal }) }),
|
|
291
|
+
);
|
|
292
|
+
|
|
293
|
+
expect(report.failures).toEqual([{ transportModuleId: 'signal', error: refusal }]);
|
|
294
|
+
// ...and a genuinely quiet transport must NOT look like a failure.
|
|
295
|
+
const quiet = await pollInbound(db, deps([]));
|
|
296
|
+
expect(quiet.failures).toEqual([]);
|
|
297
|
+
expect(quiet.messagesRead).toBe(0);
|
|
298
|
+
});
|
|
299
|
+
|
|
300
|
+
test('a unidirectional transport is not reported as a failure', async () => {
|
|
301
|
+
const report = await pollInbound(
|
|
302
|
+
db,
|
|
303
|
+
deps([], { receiveFrom: async () => ({ status: 'unidirectional' }) }),
|
|
304
|
+
);
|
|
305
|
+
expect(report.failures).toEqual([]);
|
|
306
|
+
expect(report.transportsPolled).toBe(1);
|
|
307
|
+
});
|
|
308
|
+
|
|
276
309
|
test('the cursor is persisted after a successful poll', async () => {
|
|
277
310
|
await pollInbound(db, deps([]));
|
|
278
311
|
expect(cursors.get('signal')).toBe('cursor-1');
|
|
279
312
|
});
|
|
280
313
|
|
|
281
314
|
test('the cursor is NOT advanced when the transport could not be reached', async () => {
|
|
282
|
-
await pollInbound(
|
|
315
|
+
await pollInbound(
|
|
316
|
+
db,
|
|
317
|
+
deps([], { receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
|
|
318
|
+
);
|
|
283
319
|
expect(cursors.get('signal')).toBeUndefined();
|
|
284
320
|
});
|
|
285
321
|
|
|
286
322
|
test('gibberish is rejected without touching the alert', async () => {
|
|
287
323
|
page(peterRoute.id);
|
|
288
324
|
const report = await pollInbound(db, deps([inbound(PETER, 'what is going on')]));
|
|
289
|
-
expect(report.
|
|
325
|
+
expect(report.unheard).toEqual([{ senderAddress: PETER, reason: 'unrecognised' }]);
|
|
290
326
|
expect(alertState()).toBe('firing');
|
|
291
327
|
});
|
|
292
328
|
|
|
@@ -25,12 +25,35 @@ import type { NotificationTransport } from './notifier';
|
|
|
25
25
|
import { composeAckBroadcastBody } from './notifier';
|
|
26
26
|
import { consumeDelivery } from './tokens';
|
|
27
27
|
|
|
28
|
+
/**
|
|
29
|
+
* What one attempt to read a transport produced.
|
|
30
|
+
*
|
|
31
|
+
* `unidirectional` and `failed` used to collapse into a bare null, and that
|
|
32
|
+
* cost a week: a transport whose read call was being REFUSED reported the same
|
|
33
|
+
* "0 message(s)" as a transport nobody had replied on. An operator could not
|
|
34
|
+
* tell "my reply never arrived" from "celilo cannot read this transport at
|
|
35
|
+
* all", and neither could anyone debugging it.
|
|
36
|
+
*/
|
|
37
|
+
export type TransportReceive =
|
|
38
|
+
| { status: 'received'; messages: InboundMessage[]; cursor: string | null }
|
|
39
|
+
| { status: 'unidirectional' }
|
|
40
|
+
| { status: 'failed'; error: string };
|
|
41
|
+
|
|
42
|
+
/** A transport that could not be read, and why. */
|
|
43
|
+
export interface TransportFailure {
|
|
44
|
+
transportModuleId: string;
|
|
45
|
+
error: string;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** A message that was read but not acted on, and why. */
|
|
49
|
+
export interface UnheardMessage {
|
|
50
|
+
senderAddress: string;
|
|
51
|
+
reason: 'unknown_sender' | 'unknown_token' | 'wrong_sender' | 'ambiguous' | 'unrecognised';
|
|
52
|
+
}
|
|
53
|
+
|
|
28
54
|
export interface InboundPollDeps {
|
|
29
|
-
/** Receive from one transport module
|
|
30
|
-
receiveFrom(
|
|
31
|
-
transportModuleId: string,
|
|
32
|
-
cursor: string | null,
|
|
33
|
-
): Promise<{ messages: InboundMessage[]; cursor: string | null } | null>;
|
|
55
|
+
/** Receive from one transport module. */
|
|
56
|
+
receiveFrom(transportModuleId: string, cursor: string | null): Promise<TransportReceive>;
|
|
34
57
|
/** Persisted receive cursor per transport. */
|
|
35
58
|
readCursor(transportModuleId: string): string | null;
|
|
36
59
|
writeCursor(transportModuleId: string, cursor: string | null): void;
|
|
@@ -53,12 +76,14 @@ export interface InboundPollReport {
|
|
|
53
76
|
transportsPolled: number;
|
|
54
77
|
messagesRead: number;
|
|
55
78
|
acked: number;
|
|
56
|
-
ignored: number;
|
|
57
|
-
rejected: number;
|
|
58
79
|
/** Routes told that someone else took the alert. */
|
|
59
80
|
broadcast: number;
|
|
60
81
|
/** Deploy questions answered from a phone. */
|
|
61
82
|
answered: number;
|
|
83
|
+
/** Transports that could not be read at all. Never silent — see above. */
|
|
84
|
+
failures: TransportFailure[];
|
|
85
|
+
/** Messages read and then discarded, with the reason each was discarded. */
|
|
86
|
+
unheard: UnheardMessage[];
|
|
62
87
|
}
|
|
63
88
|
|
|
64
89
|
/** Transports that have at least one route pointing at them. */
|
|
@@ -89,19 +114,24 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
|
|
|
89
114
|
transportsPolled: 0,
|
|
90
115
|
messagesRead: 0,
|
|
91
116
|
acked: 0,
|
|
92
|
-
ignored: 0,
|
|
93
|
-
rejected: 0,
|
|
94
117
|
broadcast: 0,
|
|
95
118
|
answered: 0,
|
|
119
|
+
failures: [],
|
|
120
|
+
unheard: [],
|
|
96
121
|
};
|
|
97
122
|
|
|
98
123
|
for (const transportId of transportsWithRoutes(db)) {
|
|
99
124
|
const received = await deps.receiveFrom(transportId, deps.readCursor(transportId));
|
|
100
125
|
report.transportsPolled++;
|
|
101
|
-
//
|
|
102
|
-
//
|
|
103
|
-
// the
|
|
104
|
-
|
|
126
|
+
// One dead transport must not stop the others being read — but it is
|
|
127
|
+
// RECORDED rather than skipped in silence, because "cannot read" and
|
|
128
|
+
// "nothing to read" are the two things an operator most needs to tell
|
|
129
|
+
// apart at 3am.
|
|
130
|
+
if (received.status === 'failed') {
|
|
131
|
+
report.failures.push({ transportModuleId: transportId, error: received.error });
|
|
132
|
+
continue;
|
|
133
|
+
}
|
|
134
|
+
if (received.status === 'unidirectional') continue;
|
|
105
135
|
|
|
106
136
|
for (const message of received.messages) {
|
|
107
137
|
report.messagesRead++;
|
|
@@ -113,12 +143,8 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
|
|
|
113
143
|
outstandingForRoute: (routeId) => outstandingAlertDeliveries(db, routeId, deps.now()),
|
|
114
144
|
});
|
|
115
145
|
|
|
116
|
-
if (outcome.action === 'ignored') {
|
|
117
|
-
report.
|
|
118
|
-
continue;
|
|
119
|
-
}
|
|
120
|
-
if (outcome.action === 'rejected') {
|
|
121
|
-
report.rejected++;
|
|
146
|
+
if (outcome.action === 'ignored' || outcome.action === 'rejected') {
|
|
147
|
+
report.unheard.push({ senderAddress: message.senderAddress, reason: outcome.reason });
|
|
122
148
|
continue;
|
|
123
149
|
}
|
|
124
150
|
|
|
@@ -127,7 +153,7 @@ export async function pollInbound(db: DbClient, deps: InboundPollDeps): Promise<
|
|
|
127
153
|
if (outcome.delivery.kind === 'interview') {
|
|
128
154
|
const value = parseInterviewAnswer(message.body, outcome.delivery.token);
|
|
129
155
|
if (!value || !deps.answerInterview) {
|
|
130
|
-
report.
|
|
156
|
+
report.unheard.push({ senderAddress: message.senderAddress, reason: 'unrecognised' });
|
|
131
157
|
continue;
|
|
132
158
|
}
|
|
133
159
|
consumeDelivery(db, outcome.delivery.id, deps.now());
|
|
@@ -213,12 +239,23 @@ function outstandingAlertDeliveries(db: DbClient, routeId: string, now: Date) {
|
|
|
213
239
|
*
|
|
214
240
|
* A transport with no `receive` is unidirectional — that is not an error, it
|
|
215
241
|
* just means replies cannot arrive, which the route's `can_ack` already
|
|
216
|
-
* records.
|
|
242
|
+
* records. Anything else going wrong IS an error and is returned as one.
|
|
243
|
+
*
|
|
244
|
+
* This used to be a bare `catch {}` returning null, on the reasoning that the
|
|
245
|
+
* transport's own health check would report an unreachable daemon. That was
|
|
246
|
+
* wrong twice over: a health check that only proves the daemon answers cannot
|
|
247
|
+
* see a read call being REFUSED by a daemon that is otherwise perfectly
|
|
248
|
+
* healthy, and the swallowed error was the only place the reason existed. The
|
|
249
|
+
* live signal-cli case was exactly that shape — `receive` refused with
|
|
250
|
+
* "Receive command cannot be used if messages are already being received."
|
|
251
|
+
* while every health check passed and every poll reported zero messages.
|
|
217
252
|
*/
|
|
218
253
|
export function makeReceiver(db: DbClient) {
|
|
219
|
-
return async (transportModuleId: string, cursor: string | null) => {
|
|
254
|
+
return async (transportModuleId: string, cursor: string | null): Promise<TransportReceive> => {
|
|
220
255
|
const module = db.select().from(modules).where(eq(modules.id, transportModuleId)).get();
|
|
221
|
-
if (!module)
|
|
256
|
+
if (!module) {
|
|
257
|
+
return { status: 'failed', error: `no module '${transportModuleId}' is installed` };
|
|
258
|
+
}
|
|
222
259
|
|
|
223
260
|
try {
|
|
224
261
|
const { logger } = createCapturingLogger();
|
|
@@ -226,11 +263,11 @@ export function makeReceiver(db: DbClient) {
|
|
|
226
263
|
const notification = (capabilities as Record<string, unknown>).notification as
|
|
227
264
|
| NotificationCapability
|
|
228
265
|
| undefined;
|
|
229
|
-
if (!notification?.receive) return
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
return
|
|
266
|
+
if (!notification?.receive) return { status: 'unidirectional' };
|
|
267
|
+
const result = await notification.receive(cursor);
|
|
268
|
+
return { status: 'received', messages: result.messages, cursor: result.cursor };
|
|
269
|
+
} catch (error) {
|
|
270
|
+
return { status: 'failed', error: error instanceof Error ? error.message : String(error) };
|
|
234
271
|
}
|
|
235
272
|
};
|
|
236
273
|
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Resolving an alert to the people it should page.
|
|
3
|
+
*
|
|
4
|
+
* The escalation policy is a property of the MONITOR, and it is read here on
|
|
5
|
+
* every sweep rather than copied onto the alert when the alert is created.
|
|
6
|
+
* That ordering is the whole point: an operator only reaches for
|
|
7
|
+
* `escalation-policy assign` once something is already firing and reaching
|
|
8
|
+
* nobody, so a policy that only applied to alerts created afterwards would be
|
|
9
|
+
* a no-op for the one alert they were trying to fix (#481).
|
|
10
|
+
*
|
|
11
|
+
* The cost of resolving late is that a policy edited mid-escalation changes
|
|
12
|
+
* where the remaining steps go. That is the behaviour an operator asks for
|
|
13
|
+
* when they edit it — the alert's own progress through the chain
|
|
14
|
+
* (`escalationStep`, `nextEscalationAt`) is what stays on the alert.
|
|
15
|
+
*
|
|
16
|
+
* Lives here rather than in the sweep command so the sweep's tests can drive
|
|
17
|
+
* the real resolution instead of a fake, which is how the snapshot bug
|
|
18
|
+
* survived: every existing test stubbed `notifyDepsFor`.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import { eq } from 'drizzle-orm';
|
|
22
|
+
import type { DbClient } from '../../db/client';
|
|
23
|
+
import { type Alert, type EscalationPolicy, escalationPolicies, monitors } from '../../db/schema';
|
|
24
|
+
import type { NotifyDeps } from './notifier';
|
|
25
|
+
import { listPeople, listPolicySteps, listRoutes } from './people';
|
|
26
|
+
import { mintDelivery } from './tokens';
|
|
27
|
+
import { loadNotificationTransport } from './transport-loader';
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* How long a reply token stays valid. A day is long enough that someone who
|
|
31
|
+
* sees a page overnight can still act on it in the morning, and short enough
|
|
32
|
+
* that a token found later is useless.
|
|
33
|
+
*/
|
|
34
|
+
const TOKEN_TTL_MS = 24 * 60 * 60_000;
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* The escalation policy an alert resolves to right now, or null.
|
|
38
|
+
*
|
|
39
|
+
* The single answer to "who would this page", shared by the sweep and by every
|
|
40
|
+
* command that has to show it. Two readers computing it separately is how a
|
|
41
|
+
* CLI ends up confidently reporting a policy the sweep does not use.
|
|
42
|
+
*/
|
|
43
|
+
export function policyForAlert(
|
|
44
|
+
db: DbClient,
|
|
45
|
+
alert: Pick<Alert, 'monitorId'>,
|
|
46
|
+
): EscalationPolicy | null {
|
|
47
|
+
const monitor = db.select().from(monitors).where(eq(monitors.id, alert.monitorId)).get();
|
|
48
|
+
if (!monitor?.escalationPolicyId) return null;
|
|
49
|
+
return (
|
|
50
|
+
db
|
|
51
|
+
.select()
|
|
52
|
+
.from(escalationPolicies)
|
|
53
|
+
.where(eq(escalationPolicies.id, monitor.escalationPolicyId))
|
|
54
|
+
.get() ?? null
|
|
55
|
+
);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Assemble everything needed to page for one alert, or null when nobody can be.
|
|
60
|
+
*
|
|
61
|
+
* Returning null rather than an empty policy is deliberate: "no escalation
|
|
62
|
+
* policy assigned" and "policy exists but nobody is eligible" are different
|
|
63
|
+
* situations, and only the second is worth an escalation decision.
|
|
64
|
+
*/
|
|
65
|
+
export function buildNotifyDeps(db: DbClient, alert: Alert, now: Date): NotifyDeps | null {
|
|
66
|
+
const policy = policyForAlert(db, alert);
|
|
67
|
+
if (!policy) return null;
|
|
68
|
+
|
|
69
|
+
const steps = listPolicySteps(db, policy.id).map((step) => ({
|
|
70
|
+
stepIndex: step.stepIndex,
|
|
71
|
+
routeId: step.routeId,
|
|
72
|
+
delayMinutes: step.delayMinutes,
|
|
73
|
+
}));
|
|
74
|
+
if (steps.length === 0) return null;
|
|
75
|
+
|
|
76
|
+
const routeRows = listRoutes(db);
|
|
77
|
+
const people = listPeople(db);
|
|
78
|
+
|
|
79
|
+
return {
|
|
80
|
+
steps,
|
|
81
|
+
routes: new Map(
|
|
82
|
+
routeRows.map((r) => [
|
|
83
|
+
r.id,
|
|
84
|
+
{ id: r.id, severityFloor: r.severityFloor, enabled: r.enabled },
|
|
85
|
+
]),
|
|
86
|
+
),
|
|
87
|
+
routeDetails: new Map(routeRows.map((r) => [r.id, r])),
|
|
88
|
+
quietHoursByPerson: new Map(
|
|
89
|
+
people
|
|
90
|
+
.filter((p) => p.quietHoursStart && p.quietHoursEnd)
|
|
91
|
+
.map((p) => [
|
|
92
|
+
p.id,
|
|
93
|
+
{
|
|
94
|
+
personId: p.id,
|
|
95
|
+
start: p.quietHoursStart,
|
|
96
|
+
end: p.quietHoursEnd,
|
|
97
|
+
timezone: p.timezone,
|
|
98
|
+
},
|
|
99
|
+
]),
|
|
100
|
+
),
|
|
101
|
+
bypassQuietHours: policy.bypassQuietHours,
|
|
102
|
+
transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
|
|
103
|
+
mintToken: (alertId, routeId) =>
|
|
104
|
+
mintDelivery(db, {
|
|
105
|
+
kind: 'alert',
|
|
106
|
+
targetId: alertId,
|
|
107
|
+
routeId,
|
|
108
|
+
now,
|
|
109
|
+
ttlMs: TOKEN_TTL_MS,
|
|
110
|
+
}).token,
|
|
111
|
+
now,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
@@ -27,7 +27,7 @@ describe('alert store', () => {
|
|
|
27
27
|
let dir: string;
|
|
28
28
|
let db: DbClient;
|
|
29
29
|
|
|
30
|
-
const context = { monitorId: MONITOR,
|
|
30
|
+
const context = { monitorId: MONITOR, now: NOW };
|
|
31
31
|
|
|
32
32
|
const create = (key: string): ReconcileAction => ({
|
|
33
33
|
type: 'create',
|
|
@@ -28,7 +28,6 @@ export function loadLiveAlerts(db: DbClient, monitorId: string): LiveAlert[] {
|
|
|
28
28
|
|
|
29
29
|
export interface ApplyContext {
|
|
30
30
|
monitorId: string;
|
|
31
|
-
escalationPolicyId: string | null;
|
|
32
31
|
now: Date;
|
|
33
32
|
}
|
|
34
33
|
|
|
@@ -66,7 +65,6 @@ export function applyReconcileActions(
|
|
|
66
65
|
lastSeenAt: context.now,
|
|
67
66
|
graceUntil: action.graceUntil,
|
|
68
67
|
escalationStep: 0,
|
|
69
|
-
escalationPolicyId: context.escalationPolicyId,
|
|
70
68
|
message: action.message,
|
|
71
69
|
details: action.details,
|
|
72
70
|
})
|
|
@@ -238,7 +238,16 @@ describe('runSweep', () => {
|
|
|
238
238
|
|
|
239
239
|
expect(liveAlerts()).toHaveLength(1);
|
|
240
240
|
expect(report.notified).toBe(0);
|
|
241
|
-
expect(report.noPolicy).
|
|
241
|
+
expect(report.noPolicy).toEqual([{ alertKey: PORT_CHECK, monitor: MODULE }]);
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
// A count told the operator something was unroutable but not WHICH thing,
|
|
245
|
+
// so the remedy printed next to it could not be aimed anywhere (#481).
|
|
246
|
+
test('the unroutable alert is named, along with the monitor to assign to', async () => {
|
|
247
|
+
const report = await runSweep(db, currentMonitors(), deps());
|
|
248
|
+
|
|
249
|
+
expect(report.noPolicy[0].alertKey).toBe(PORT_CHECK);
|
|
250
|
+
expect(report.noPolicy[0].monitor).toBe(MODULE);
|
|
242
251
|
});
|
|
243
252
|
|
|
244
253
|
test('escalation declining to notify records WHICH reason', async () => {
|
|
@@ -270,7 +279,7 @@ describe('runSweep', () => {
|
|
|
270
279
|
);
|
|
271
280
|
|
|
272
281
|
expect(report.notified).toBe(0);
|
|
273
|
-
expect(report.noPolicy).
|
|
282
|
+
expect(report.noPolicy).toEqual([]);
|
|
274
283
|
expect(report.skipped.within_grace).toBe(1);
|
|
275
284
|
});
|
|
276
285
|
|