@celilo/cli 0.13.3 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +3 -0
- package/CELILO_SUBSYSTEMS.md +71 -2
- package/docs/ALERTING.md +298 -0
- package/docs/INDEX.md +103 -0
- package/drizzle/0016_trusted_sources.sql +10 -0
- package/drizzle/0017_alerting.sql +127 -0
- package/drizzle/meta/_journal.json +15 -1
- package/package.json +3 -2
- package/schemas/system_config.json +9 -0
- package/src/capabilities/well-known.ts +11 -1
- package/src/cli/commands/alerts-act.ts +107 -0
- package/src/cli/commands/alerts-list.ts +62 -0
- package/src/cli/commands/alerts-poll.ts +129 -0
- package/src/cli/commands/alerts-sweep.ts +156 -0
- package/src/cli/commands/module-list.ts +50 -3
- package/src/cli/commands/module-show.ts +11 -3
- package/src/cli/commands/monitor.ts +178 -0
- package/src/cli/commands/notify-config.ts +453 -0
- package/src/cli/commands/system-audit.ts +2 -0
- package/src/cli/commands/system-update.ts +1 -0
- package/src/cli/completion.ts +26 -0
- package/src/cli/generate-zsh-completion.ts +2 -0
- package/src/cli/index.ts +58 -0
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/db/schema.ts +371 -2
- package/src/hooks/capability-loader.ts +158 -46
- package/src/hooks/capability-map-coverage.test.ts +101 -0
- package/src/manifest/schema.ts +77 -4
- package/src/services/alerting/ack.test.ts +212 -0
- package/src/services/alerting/ack.ts +119 -0
- package/src/services/alerting/builtin-monitors.test.ts +132 -0
- package/src/services/alerting/builtin-monitors.ts +84 -0
- package/src/services/alerting/builtin-source.ts +82 -0
- package/src/services/alerting/coverage-source.ts +38 -0
- package/src/services/alerting/deferral.test.ts +161 -0
- package/src/services/alerting/delivery-loop.test.ts +396 -0
- package/src/services/alerting/deploy-hooks.test.ts +125 -0
- package/src/services/alerting/deploy-hooks.ts +111 -0
- package/src/services/alerting/escalation.test.ts +207 -0
- package/src/services/alerting/escalation.ts +151 -0
- package/src/services/alerting/format.test.ts +193 -0
- package/src/services/alerting/format.ts +150 -0
- package/src/services/alerting/health-coverage.ts +81 -0
- package/src/services/alerting/inbound-poller.test.ts +298 -0
- package/src/services/alerting/inbound-poller.ts +236 -0
- package/src/services/alerting/inbound.test.ts +201 -0
- package/src/services/alerting/inbound.ts +112 -0
- package/src/services/alerting/interview-responder.test.ts +169 -0
- package/src/services/alerting/interview-responder.ts +158 -0
- package/src/services/alerting/keys.test.ts +155 -0
- package/src/services/alerting/keys.ts +190 -0
- package/src/services/alerting/monitors.ts +185 -0
- package/src/services/alerting/notification-responder.test.ts +290 -0
- package/src/services/alerting/notification-responder.ts +260 -0
- package/src/services/alerting/notifier.ts +219 -0
- package/src/services/alerting/people.ts +178 -0
- package/src/services/alerting/quiet-hours.test.ts +140 -0
- package/src/services/alerting/quiet-hours.ts +99 -0
- package/src/services/alerting/reconcile.test.ts +190 -0
- package/src/services/alerting/reconcile.ts +166 -0
- package/src/services/alerting/run-monitor.test.ts +185 -0
- package/src/services/alerting/run-monitor.ts +177 -0
- package/src/services/alerting/store.test.ts +222 -0
- package/src/services/alerting/store.ts +289 -0
- package/src/services/alerting/suppression.test.ts +228 -0
- package/src/services/alerting/suppression.ts +142 -0
- package/src/services/alerting/sweep-runner.test.ts +229 -0
- package/src/services/alerting/sweep-runner.ts +204 -0
- package/src/services/alerting/sweep.test.ts +61 -0
- package/src/services/alerting/sweep.ts +41 -0
- package/src/services/alerting/tokens.test.ts +152 -0
- package/src/services/alerting/tokens.ts +119 -0
- package/src/services/alerting/transport-loader.ts +48 -0
- package/src/services/audit/index.test.ts +1 -0
- package/src/services/audit/index.ts +3 -0
- package/src/services/audit/trusted-sources.test.ts +137 -0
- package/src/services/audit/trusted-sources.ts +124 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/firewall-reach.ts +83 -0
- package/src/services/health-runner.test.ts +50 -0
- package/src/services/health-runner.ts +116 -82
- package/src/services/machine-pool.ts +2 -1
- package/src/services/module-deploy.ts +17 -0
- package/src/services/system-config-validator.test.ts +31 -1
- package/src/services/trusted-sources.test.ts +221 -0
- package/src/services/trusted-sources.ts +159 -0
- package/src/services/update/orchestrator.test.ts +1 -0
- package/src/templates/generator.ts +6 -29
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
2
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { eq } from 'drizzle-orm';
|
|
6
|
+
import type { DbClient } from '../../db/client';
|
|
7
|
+
import { type Alert, type Route, alerts, modules, monitors } from '../../db/schema';
|
|
8
|
+
import { setupTestDatabase } from '../../test-utils/setup-test-db';
|
|
9
|
+
import { moduleCheckAlertKey } from './keys';
|
|
10
|
+
import { deliverDeferred } from './notifier';
|
|
11
|
+
import { createPerson, createRoute } from './people';
|
|
12
|
+
import { clearDeferral, deferNotification, dueDeferrals, recordStepTaken } from './store';
|
|
13
|
+
|
|
14
|
+
const NIGHT = new Date('2026-07-28T03:00:00Z');
|
|
15
|
+
const MORNING = new Date('2026-07-28T07:00:00Z');
|
|
16
|
+
const KEY = moduleCheckAlertKey('caddy', 'disk-space');
|
|
17
|
+
|
|
18
|
+
describe('quiet-hours deferral', () => {
|
|
19
|
+
let dir: string;
|
|
20
|
+
let db: DbClient;
|
|
21
|
+
let route: Route;
|
|
22
|
+
|
|
23
|
+
const sent: { address: string; body: string }[] = [];
|
|
24
|
+
const transport = {
|
|
25
|
+
send: async (request: { address: string; body: string }) => {
|
|
26
|
+
sent.push(request);
|
|
27
|
+
return { messageId: `m-${sent.length}` };
|
|
28
|
+
},
|
|
29
|
+
};
|
|
30
|
+
const deps = { transportFor: () => transport, mintToken: () => 'TOKEN1', now: MORNING };
|
|
31
|
+
|
|
32
|
+
const alert = () => db.select().from(alerts).where(eq(alerts.id, 'alert-1')).get() as Alert;
|
|
33
|
+
|
|
34
|
+
beforeEach(async () => {
|
|
35
|
+
dir = mkdtempSync(join(tmpdir(), 'deferral-'));
|
|
36
|
+
const dbPath = join(dir, 'celilo.db');
|
|
37
|
+
process.env.CELILO_DB_PATH = dbPath;
|
|
38
|
+
db = await setupTestDatabase(dbPath);
|
|
39
|
+
sent.length = 0;
|
|
40
|
+
|
|
41
|
+
db.insert(modules)
|
|
42
|
+
.values({
|
|
43
|
+
id: 'signal',
|
|
44
|
+
name: 'Signal',
|
|
45
|
+
version: '0.1.0',
|
|
46
|
+
manifestData: {},
|
|
47
|
+
sourcePath: '/tmp/signal',
|
|
48
|
+
})
|
|
49
|
+
.run();
|
|
50
|
+
db.insert(monitors)
|
|
51
|
+
.values({
|
|
52
|
+
id: 'mon-1',
|
|
53
|
+
kind: 'module_hook',
|
|
54
|
+
target: 'caddy',
|
|
55
|
+
intervalMinutes: 15,
|
|
56
|
+
severity: 'critical',
|
|
57
|
+
})
|
|
58
|
+
.run();
|
|
59
|
+
db.insert(alerts)
|
|
60
|
+
.values({
|
|
61
|
+
id: 'alert-1',
|
|
62
|
+
key: KEY,
|
|
63
|
+
activeKey: KEY,
|
|
64
|
+
monitorId: 'mon-1',
|
|
65
|
+
state: 'firing',
|
|
66
|
+
severity: 'critical',
|
|
67
|
+
graceUntil: NIGHT,
|
|
68
|
+
message: '/var 94% used',
|
|
69
|
+
})
|
|
70
|
+
.run();
|
|
71
|
+
|
|
72
|
+
const peter = createPerson(db, { name: 'peter', timezone: 'UTC' });
|
|
73
|
+
route = createRoute(db, {
|
|
74
|
+
personId: peter.id,
|
|
75
|
+
transportModuleId: 'signal',
|
|
76
|
+
address: '+15550001',
|
|
77
|
+
canAck: true,
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
afterEach(() => {
|
|
82
|
+
db.$client.close();
|
|
83
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
84
|
+
try {
|
|
85
|
+
rmSync(dir, { recursive: true, force: true });
|
|
86
|
+
} catch {
|
|
87
|
+
/* ignore */
|
|
88
|
+
}
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
// D13: the clock is what advances, not the message.
|
|
92
|
+
test('a step taken during quiet hours still advances the chain', () => {
|
|
93
|
+
recordStepTaken(db, 'alert-1', { stepIndex: 0, nextStepDueAt: MORNING });
|
|
94
|
+
expect(alert().escalationStep).toBe(1);
|
|
95
|
+
expect(alert().nextEscalationAt?.getTime()).toBe(MORNING.getTime());
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
test('a deferral is held until the window ends', () => {
|
|
99
|
+
deferNotification(db, 'alert-1', { routeId: route.id, until: MORNING });
|
|
100
|
+
expect(dueDeferrals(db, NIGHT)).toHaveLength(0);
|
|
101
|
+
expect(dueDeferrals(db, MORNING)).toHaveLength(1);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
// Otherwise a second step behind a longer window would push the message
|
|
105
|
+
// later than the first person's window already allowed.
|
|
106
|
+
test('the earliest pending deferral wins', () => {
|
|
107
|
+
deferNotification(db, 'alert-1', { routeId: route.id, until: MORNING });
|
|
108
|
+
deferNotification(db, 'alert-1', {
|
|
109
|
+
routeId: route.id,
|
|
110
|
+
until: new Date('2026-07-28T09:00:00Z'),
|
|
111
|
+
});
|
|
112
|
+
expect(alert().deferredUntil?.getTime()).toBe(MORNING.getTime());
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
test('a message held overnight is delivered when the window ends', async () => {
|
|
116
|
+
deferNotification(db, 'alert-1', { routeId: route.id, until: MORNING });
|
|
117
|
+
const outcome = await deliverDeferred(alert(), route, deps);
|
|
118
|
+
|
|
119
|
+
expect(outcome.result).toBe('sent');
|
|
120
|
+
expect(sent).toHaveLength(1);
|
|
121
|
+
expect(sent[0].body).toContain(KEY);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
// The whole reason delivery is decided at window end rather than scheduled.
|
|
125
|
+
test('an alert that resolved overnight is not announced in the morning', async () => {
|
|
126
|
+
deferNotification(db, 'alert-1', { routeId: route.id, until: MORNING });
|
|
127
|
+
db.update(alerts)
|
|
128
|
+
.set({ state: 'resolved', activeKey: null, resolvedAt: NIGHT })
|
|
129
|
+
.where(eq(alerts.id, 'alert-1'))
|
|
130
|
+
.run();
|
|
131
|
+
|
|
132
|
+
const outcome = await deliverDeferred(alert(), route, deps);
|
|
133
|
+
expect(outcome).toEqual({ result: 'skipped', reason: 'resolved' });
|
|
134
|
+
expect(sent).toHaveLength(0);
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
test('an alert acked overnight is not re-announced', async () => {
|
|
138
|
+
deferNotification(db, 'alert-1', { routeId: route.id, until: MORNING });
|
|
139
|
+
db.update(alerts).set({ state: 'acked' }).where(eq(alerts.id, 'alert-1')).run();
|
|
140
|
+
|
|
141
|
+
expect((await deliverDeferred(alert(), route, deps)).result).toBe('skipped');
|
|
142
|
+
expect(sent).toHaveLength(0);
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
test('an alert silenced past the window end stays quiet', async () => {
|
|
146
|
+
deferNotification(db, 'alert-1', { routeId: route.id, until: MORNING });
|
|
147
|
+
db.update(alerts)
|
|
148
|
+
.set({ silencedUntil: new Date('2026-07-28T12:00:00Z') })
|
|
149
|
+
.where(eq(alerts.id, 'alert-1'))
|
|
150
|
+
.run();
|
|
151
|
+
|
|
152
|
+
expect((await deliverDeferred(alert(), route, deps)).result).toBe('skipped');
|
|
153
|
+
expect(sent).toHaveLength(0);
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
test('clearing a deferral takes it out of the due set', () => {
|
|
157
|
+
deferNotification(db, 'alert-1', { routeId: route.id, until: MORNING });
|
|
158
|
+
clearDeferral(db, 'alert-1');
|
|
159
|
+
expect(dueDeferrals(db, MORNING)).toHaveLength(0);
|
|
160
|
+
});
|
|
161
|
+
});
|
|
@@ -0,0 +1,396 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The delivery loop, end to end, against the signal-cli simulator.
|
|
3
|
+
*
|
|
4
|
+
* Everything from "an alert is firing" to "the operator's reply acknowledged
|
|
5
|
+
* it" runs here against a real database and a real HTTP server speaking
|
|
6
|
+
* signal-cli's JSON-RPC dialect — including its awkward parts.
|
|
7
|
+
*
|
|
8
|
+
* What this proves: escalation selection, quiet-hours deferral, token minting,
|
|
9
|
+
* the send itself, sender authentication on the way back, and the all-clear
|
|
10
|
+
* reaching only those who were paged.
|
|
11
|
+
*
|
|
12
|
+
* What it does NOT prove: that signal-cli's real wire contract matches the
|
|
13
|
+
* simulator's. Both encode the same assumption, so a wrong assumption passes.
|
|
14
|
+
* That needs one verification against a real daemon.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
|
|
18
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
19
|
+
import { tmpdir } from 'node:os';
|
|
20
|
+
import { join } from 'node:path';
|
|
21
|
+
import type { DbClient } from '../../db/client';
|
|
22
|
+
import { type Alert, type Route, modules } from '../../db/schema';
|
|
23
|
+
import { setupTestDatabase } from '../../test-utils/setup-test-db';
|
|
24
|
+
import type { EscalationStep, RouteForEscalation } from './escalation';
|
|
25
|
+
import { interpretInbound } from './inbound';
|
|
26
|
+
import {
|
|
27
|
+
type NotificationTransport,
|
|
28
|
+
type NotifyDeps,
|
|
29
|
+
notifyAlert,
|
|
30
|
+
notifyResolved,
|
|
31
|
+
} from './notifier';
|
|
32
|
+
import { createPerson, createRoute } from './people';
|
|
33
|
+
import { deliveriesForAlert, findLiveDelivery, mintDelivery } from './tokens';
|
|
34
|
+
|
|
35
|
+
const PETER = '+15550001111';
|
|
36
|
+
const WIFE = '+15550002222';
|
|
37
|
+
const PORT = 18101;
|
|
38
|
+
const BASE = `http://127.0.0.1:${PORT}`;
|
|
39
|
+
// Verified: the real daemon serves JSON-RPC only at this path (404 otherwise).
|
|
40
|
+
const RPC = `${BASE}/api/v1/rpc`;
|
|
41
|
+
|
|
42
|
+
const NOW = new Date('2026-07-28T12:00:00Z');
|
|
43
|
+
const at = (minutes: number) => new Date(NOW.getTime() + minutes * 60_000);
|
|
44
|
+
|
|
45
|
+
let simulator: ReturnType<typeof Bun.spawn>;
|
|
46
|
+
|
|
47
|
+
async function control(path: string, body?: unknown) {
|
|
48
|
+
const response = await fetch(`${BASE}/_control/${path}`, {
|
|
49
|
+
method: 'POST',
|
|
50
|
+
headers: { 'content-type': 'application/json' },
|
|
51
|
+
body: body ? JSON.stringify(body) : undefined,
|
|
52
|
+
});
|
|
53
|
+
return response.json();
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
async function sentMessages(): Promise<{ recipient: string; message: string }[]> {
|
|
57
|
+
const response = await fetch(`${BASE}/_control/sent`);
|
|
58
|
+
return ((await response.json()) as { sent: { recipient: string; message: string }[] }).sent;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
beforeAll(async () => {
|
|
62
|
+
const simulatorPath = join(
|
|
63
|
+
import.meta.dir,
|
|
64
|
+
'../../../../../packages/e2e/simulators/signal-cli/server.ts',
|
|
65
|
+
);
|
|
66
|
+
simulator = Bun.spawn(['bun', simulatorPath], {
|
|
67
|
+
env: {
|
|
68
|
+
...process.env,
|
|
69
|
+
SIGNAL_RPC_PORT: String(PORT),
|
|
70
|
+
SIGNAL_KNOWN_RECIPIENTS: `${PETER},${WIFE}`,
|
|
71
|
+
},
|
|
72
|
+
stdout: 'pipe',
|
|
73
|
+
stderr: 'pipe',
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
// Wait for it to bind rather than sleeping a guessed interval.
|
|
77
|
+
for (let i = 0; i < 100; i++) {
|
|
78
|
+
try {
|
|
79
|
+
await fetch(`${BASE}/_control/sent`);
|
|
80
|
+
return;
|
|
81
|
+
} catch {
|
|
82
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
throw new Error('signal simulator did not start');
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
afterAll(() => {
|
|
89
|
+
simulator?.kill();
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
describe('delivery loop against the signal-cli simulator', () => {
|
|
93
|
+
let dir: string;
|
|
94
|
+
let db: DbClient;
|
|
95
|
+
let peterRoute: Route;
|
|
96
|
+
let wifeRoute: Route;
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* A minimal JSON-RPC client for the simulator.
|
|
100
|
+
*
|
|
101
|
+
* Deliberately NOT the signal module's own client: importing a module script
|
|
102
|
+
* here would pull it into apps/celilo's typecheck, where @celilo/capabilities
|
|
103
|
+
* resolves to the vendored npm copy rather than the workspace source. The
|
|
104
|
+
* module's client is driven against this same simulator from the module's own
|
|
105
|
+
* test, where it resolves correctly.
|
|
106
|
+
*/
|
|
107
|
+
const notifier: NotificationTransport & { receive(): Promise<{ from: string; body: string }[]> } =
|
|
108
|
+
{
|
|
109
|
+
async send({ address, body, token }) {
|
|
110
|
+
const message = token ? `${body}\n\nreply ${token}` : body;
|
|
111
|
+
const response = await fetch(RPC, {
|
|
112
|
+
method: 'POST',
|
|
113
|
+
headers: { 'content-type': 'application/json' },
|
|
114
|
+
body: JSON.stringify({
|
|
115
|
+
jsonrpc: '2.0',
|
|
116
|
+
id: 1,
|
|
117
|
+
method: 'send',
|
|
118
|
+
params: { recipient: [address], message },
|
|
119
|
+
}),
|
|
120
|
+
});
|
|
121
|
+
const parsed = (await response.json()) as {
|
|
122
|
+
result?: { timestamp: number };
|
|
123
|
+
error?: { message: string };
|
|
124
|
+
};
|
|
125
|
+
// signal-cli reports failures inside an HTTP 200 — checking response.ok
|
|
126
|
+
// would record a rejected page as delivered.
|
|
127
|
+
if (parsed.error) throw new Error(parsed.error.message);
|
|
128
|
+
return { messageId: String(parsed.result?.timestamp) };
|
|
129
|
+
},
|
|
130
|
+
|
|
131
|
+
async receive() {
|
|
132
|
+
const response = await fetch(RPC, {
|
|
133
|
+
method: 'POST',
|
|
134
|
+
headers: { 'content-type': 'application/json' },
|
|
135
|
+
body: JSON.stringify({ jsonrpc: '2.0', id: 2, method: 'receive', params: {} }),
|
|
136
|
+
});
|
|
137
|
+
const parsed = (await response.json()) as {
|
|
138
|
+
result?: {
|
|
139
|
+
envelope?: { sourceNumber?: string; dataMessage?: { message?: string } };
|
|
140
|
+
}[];
|
|
141
|
+
};
|
|
142
|
+
// Receipts and typing indicators are not replies.
|
|
143
|
+
return (parsed.result ?? [])
|
|
144
|
+
.filter((e) => e.envelope?.dataMessage?.message)
|
|
145
|
+
.map((e) => ({
|
|
146
|
+
from: e.envelope?.sourceNumber as string,
|
|
147
|
+
body: e.envelope?.dataMessage?.message as string,
|
|
148
|
+
}));
|
|
149
|
+
},
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
function alert(over: Partial<Alert> = {}): Alert {
|
|
153
|
+
return {
|
|
154
|
+
id: 'alert-1',
|
|
155
|
+
key: 'module:caddy/check:disk-space',
|
|
156
|
+
activeKey: 'module:caddy/check:disk-space',
|
|
157
|
+
monitorId: 'mon-1',
|
|
158
|
+
state: 'firing',
|
|
159
|
+
severity: 'critical',
|
|
160
|
+
firstFiredAt: NOW,
|
|
161
|
+
lastSeenAt: NOW,
|
|
162
|
+
graceUntil: NOW,
|
|
163
|
+
suppressedByAlertId: null,
|
|
164
|
+
suppressedByWindowId: null,
|
|
165
|
+
unsuppressedAt: null,
|
|
166
|
+
awaitingConfirmation: false,
|
|
167
|
+
ackedBy: null,
|
|
168
|
+
ackedAt: null,
|
|
169
|
+
silencedUntil: null,
|
|
170
|
+
escalationStep: 0,
|
|
171
|
+
nextEscalationAt: null,
|
|
172
|
+
escalationPolicyId: null,
|
|
173
|
+
message: '/var 94% used',
|
|
174
|
+
details: null,
|
|
175
|
+
resolvedAt: null,
|
|
176
|
+
...over,
|
|
177
|
+
} as Alert;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function deps(over: Partial<NotifyDeps> = {}): NotifyDeps {
|
|
181
|
+
const steps: EscalationStep[] = [
|
|
182
|
+
{ stepIndex: 0, routeId: peterRoute.id, delayMinutes: 0 },
|
|
183
|
+
{ stepIndex: 1, routeId: wifeRoute.id, delayMinutes: 10 },
|
|
184
|
+
];
|
|
185
|
+
const routes = new Map<string, RouteForEscalation>([
|
|
186
|
+
[peterRoute.id, { id: peterRoute.id, severityFloor: 'warning', enabled: true }],
|
|
187
|
+
[wifeRoute.id, { id: wifeRoute.id, severityFloor: 'critical', enabled: true }],
|
|
188
|
+
]);
|
|
189
|
+
return {
|
|
190
|
+
steps,
|
|
191
|
+
routes,
|
|
192
|
+
routeDetails: new Map([
|
|
193
|
+
[peterRoute.id, peterRoute],
|
|
194
|
+
[wifeRoute.id, wifeRoute],
|
|
195
|
+
]),
|
|
196
|
+
quietHoursByPerson: new Map(),
|
|
197
|
+
bypassQuietHours: false,
|
|
198
|
+
transportFor: () => notifier,
|
|
199
|
+
mintToken: (alertId, routeId) =>
|
|
200
|
+
mintDelivery(db, {
|
|
201
|
+
kind: 'alert',
|
|
202
|
+
targetId: alertId,
|
|
203
|
+
routeId,
|
|
204
|
+
now: NOW,
|
|
205
|
+
ttlMs: 86_400_000,
|
|
206
|
+
}).token,
|
|
207
|
+
now: NOW,
|
|
208
|
+
...over,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
beforeEach(async () => {
|
|
213
|
+
dir = mkdtempSync(join(tmpdir(), 'delivery-'));
|
|
214
|
+
const dbPath = join(dir, 'celilo.db');
|
|
215
|
+
process.env.CELILO_DB_PATH = dbPath;
|
|
216
|
+
db = await setupTestDatabase(dbPath);
|
|
217
|
+
await control('reset');
|
|
218
|
+
|
|
219
|
+
db.insert(modules)
|
|
220
|
+
.values({
|
|
221
|
+
id: 'signal',
|
|
222
|
+
name: 'Signal',
|
|
223
|
+
version: '0.1.0',
|
|
224
|
+
manifestData: {},
|
|
225
|
+
sourcePath: '/tmp/signal',
|
|
226
|
+
})
|
|
227
|
+
.run();
|
|
228
|
+
|
|
229
|
+
const peter = createPerson(db, { name: 'peter', timezone: 'America/Los_Angeles' });
|
|
230
|
+
const wife = createPerson(db, { name: 'wife', timezone: 'America/Los_Angeles' });
|
|
231
|
+
peterRoute = createRoute(db, {
|
|
232
|
+
personId: peter.id,
|
|
233
|
+
transportModuleId: 'signal',
|
|
234
|
+
address: PETER,
|
|
235
|
+
canAck: true,
|
|
236
|
+
});
|
|
237
|
+
wifeRoute = createRoute(db, {
|
|
238
|
+
personId: wife.id,
|
|
239
|
+
transportModuleId: 'signal',
|
|
240
|
+
address: WIFE,
|
|
241
|
+
severityFloor: 'critical',
|
|
242
|
+
canAck: true,
|
|
243
|
+
});
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
afterEach(() => {
|
|
247
|
+
db.$client.close();
|
|
248
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
249
|
+
try {
|
|
250
|
+
rmSync(dir, { recursive: true, force: true });
|
|
251
|
+
} catch {
|
|
252
|
+
/* ignore */
|
|
253
|
+
}
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
test('a firing alert pages the primary with a reply token', async () => {
|
|
257
|
+
const outcome = await notifyAlert(alert(), deps());
|
|
258
|
+
expect(outcome).toMatchObject({ result: 'sent', routeId: peterRoute.id, stepIndex: 0 });
|
|
259
|
+
|
|
260
|
+
const messages = await sentMessages();
|
|
261
|
+
expect(messages).toHaveLength(1);
|
|
262
|
+
expect(messages[0].recipient).toBe(PETER);
|
|
263
|
+
// Key first: on a phone, the first line is the notification preview.
|
|
264
|
+
expect(messages[0].message).toContain('module:caddy/check:disk-space');
|
|
265
|
+
expect(messages[0].message).toMatch(/reply [0-9A-Z]{6}$/);
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
test('the token in the message is the one recorded for that delivery', async () => {
|
|
269
|
+
await notifyAlert(alert(), deps());
|
|
270
|
+
const [message] = await sentMessages();
|
|
271
|
+
const token = /reply ([0-9A-Z]{6})/.exec(message.message)?.[1] as string;
|
|
272
|
+
|
|
273
|
+
const delivery = findLiveDelivery(db, token, NOW);
|
|
274
|
+
expect(delivery?.routeId).toBe(peterRoute.id);
|
|
275
|
+
expect(delivery?.targetId).toBe('alert-1');
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
// The whole point of the unification: the reply comes back over the same
|
|
279
|
+
// transport and identifies both the alert and the person.
|
|
280
|
+
test('the operator replies with the token and it authenticates', async () => {
|
|
281
|
+
await notifyAlert(alert(), deps());
|
|
282
|
+
const [message] = await sentMessages();
|
|
283
|
+
const token = /reply ([0-9A-Z]{6})/.exec(message.message)?.[1] as string;
|
|
284
|
+
|
|
285
|
+
await control('inbound', { from: PETER, body: token });
|
|
286
|
+
const received = await notifier.receive();
|
|
287
|
+
|
|
288
|
+
// Receipts and typing indicators the simulator interleaves are not replies.
|
|
289
|
+
expect(received).toHaveLength(1);
|
|
290
|
+
expect(received[0].from).toBe(PETER);
|
|
291
|
+
|
|
292
|
+
const outcome = interpretInbound(db, {
|
|
293
|
+
senderAddress: received[0].from,
|
|
294
|
+
body: received[0].body,
|
|
295
|
+
now: NOW,
|
|
296
|
+
outstandingForRoute: () => [],
|
|
297
|
+
});
|
|
298
|
+
expect(outcome).toMatchObject({ action: 'ack' });
|
|
299
|
+
expect(outcome.action === 'ack' && outcome.route.id).toBe(peterRoute.id);
|
|
300
|
+
});
|
|
301
|
+
|
|
302
|
+
// A token replayed from another phone is refused even though the token is
|
|
303
|
+
// perfectly valid.
|
|
304
|
+
test("the wife replying with peter's token is refused", async () => {
|
|
305
|
+
await notifyAlert(alert(), deps());
|
|
306
|
+
const [message] = await sentMessages();
|
|
307
|
+
const token = /reply ([0-9A-Z]{6})/.exec(message.message)?.[1] as string;
|
|
308
|
+
|
|
309
|
+
await control('inbound', { from: WIFE, body: token });
|
|
310
|
+
const received = await notifier.receive();
|
|
311
|
+
|
|
312
|
+
const outcome = interpretInbound(db, {
|
|
313
|
+
senderAddress: received[0].from,
|
|
314
|
+
body: received[0].body,
|
|
315
|
+
now: NOW,
|
|
316
|
+
outstandingForRoute: () => [],
|
|
317
|
+
});
|
|
318
|
+
expect(outcome).toEqual({ action: 'rejected', reason: 'wrong_sender' });
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
test('escalation reaches the secondary once the delay elapses', async () => {
|
|
322
|
+
const escalated = alert({ escalationStep: 1 });
|
|
323
|
+
const outcome = await notifyAlert(escalated, deps({ now: at(10) }));
|
|
324
|
+
expect(outcome).toMatchObject({ result: 'sent', routeId: wifeRoute.id, stepIndex: 1 });
|
|
325
|
+
expect((await sentMessages())[0].recipient).toBe(WIFE);
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
test('quiet hours defer delivery — nothing is sent', async () => {
|
|
329
|
+
// 04:00 Pacific, inside a 22:00-07:00 window. The alert fired an hour
|
|
330
|
+
// earlier so its grace window has long elapsed — otherwise the run skips
|
|
331
|
+
// on grace and never reaches the quiet-hours branch at all.
|
|
332
|
+
const insideWindow = new Date('2026-07-28T11:00:00Z');
|
|
333
|
+
const fired = new Date(insideWindow.getTime() - 60 * 60_000);
|
|
334
|
+
|
|
335
|
+
const outcome = await notifyAlert(
|
|
336
|
+
alert({ firstFiredAt: fired, graceUntil: fired }),
|
|
337
|
+
deps({
|
|
338
|
+
quietHoursByPerson: new Map([
|
|
339
|
+
[
|
|
340
|
+
peterRoute.personId,
|
|
341
|
+
{
|
|
342
|
+
personId: peterRoute.personId,
|
|
343
|
+
start: '22:00',
|
|
344
|
+
end: '07:00',
|
|
345
|
+
timezone: 'America/Los_Angeles',
|
|
346
|
+
},
|
|
347
|
+
],
|
|
348
|
+
]),
|
|
349
|
+
now: insideWindow,
|
|
350
|
+
}),
|
|
351
|
+
);
|
|
352
|
+
|
|
353
|
+
expect(outcome).toMatchObject({ result: 'deferred', reason: 'quiet_hours' });
|
|
354
|
+
// Deferred means deferred: the transport was never called.
|
|
355
|
+
expect(await sentMessages()).toEqual([]);
|
|
356
|
+
expect(outcome.result === 'deferred' && outcome.until.toISOString()).toBe(
|
|
357
|
+
'2026-07-28T14:00:00.000Z',
|
|
358
|
+
);
|
|
359
|
+
});
|
|
360
|
+
|
|
361
|
+
// A transport failure must surface. The operator believes they are covered.
|
|
362
|
+
test('a send to an unregistered number is reported as failed, not sent', async () => {
|
|
363
|
+
const strangerRoute = { ...peterRoute, address: '+15559998888' } as Route;
|
|
364
|
+
const outcome = await notifyAlert(
|
|
365
|
+
alert(),
|
|
366
|
+
deps({ routeDetails: new Map([[peterRoute.id, strangerRoute]]) }),
|
|
367
|
+
);
|
|
368
|
+
expect(outcome).toMatchObject({ result: 'failed' });
|
|
369
|
+
expect(outcome.result === 'failed' && outcome.error).toContain('Unregistered user');
|
|
370
|
+
});
|
|
371
|
+
|
|
372
|
+
test('a revoked device link surfaces as a failure rather than silence', async () => {
|
|
373
|
+
await control('unlink');
|
|
374
|
+
const outcome = await notifyAlert(alert(), deps());
|
|
375
|
+
expect(outcome).toMatchObject({ result: 'failed' });
|
|
376
|
+
expect(outcome.result === 'failed' && outcome.error).toMatch(/not registered|link/i);
|
|
377
|
+
await control('relink');
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
test('the all-clear goes only to routes that were paged', async () => {
|
|
381
|
+
await notifyAlert(alert(), deps());
|
|
382
|
+
await control('reset');
|
|
383
|
+
|
|
384
|
+
const paged = deliveriesForAlert(db, 'alert-1');
|
|
385
|
+
expect(paged).toHaveLength(1);
|
|
386
|
+
|
|
387
|
+
await notifyResolved({ id: 'alert-1', key: 'module:caddy/check:disk-space' }, [peterRoute], {
|
|
388
|
+
transportFor: () => notifier,
|
|
389
|
+
});
|
|
390
|
+
|
|
391
|
+
const messages = await sentMessages();
|
|
392
|
+
expect(messages).toHaveLength(1);
|
|
393
|
+
expect(messages[0].recipient).toBe(PETER);
|
|
394
|
+
expect(messages[0].message).toContain('RESOLVED');
|
|
395
|
+
});
|
|
396
|
+
});
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
2
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import type { DbClient } from '../../db/client';
|
|
6
|
+
import { modules, monitors } from '../../db/schema';
|
|
7
|
+
import { setupTestDatabase } from '../../test-utils/setup-test-db';
|
|
8
|
+
import {
|
|
9
|
+
closeDeployWindows,
|
|
10
|
+
ensureMonitorOnDeploy,
|
|
11
|
+
modulesInDeployWindow,
|
|
12
|
+
openDeployWindow,
|
|
13
|
+
} from './deploy-hooks';
|
|
14
|
+
|
|
15
|
+
const NOW = new Date('2026-07-28T12:00:00Z');
|
|
16
|
+
const LATER = new Date('2026-07-28T12:05:00Z');
|
|
17
|
+
|
|
18
|
+
describe('deploy hooks', () => {
|
|
19
|
+
let dir: string;
|
|
20
|
+
let db: DbClient;
|
|
21
|
+
|
|
22
|
+
const addModule = (id: string, manifestData: Record<string, unknown> = {}) =>
|
|
23
|
+
db
|
|
24
|
+
.insert(modules)
|
|
25
|
+
.values({ id, name: id, version: '1.0.0', manifestData, sourcePath: `/tmp/${id}` })
|
|
26
|
+
.run();
|
|
27
|
+
|
|
28
|
+
beforeEach(async () => {
|
|
29
|
+
dir = mkdtempSync(join(tmpdir(), 'deploy-hooks-'));
|
|
30
|
+
const dbPath = join(dir, 'celilo.db');
|
|
31
|
+
process.env.CELILO_DB_PATH = dbPath;
|
|
32
|
+
db = await setupTestDatabase(dbPath);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
afterEach(() => {
|
|
36
|
+
db.$client.close();
|
|
37
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
38
|
+
try {
|
|
39
|
+
rmSync(dir, { recursive: true, force: true });
|
|
40
|
+
} catch {
|
|
41
|
+
/* ignore */
|
|
42
|
+
}
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
describe('suppression windows', () => {
|
|
46
|
+
test('opening a window puts the module in the suppressed set', () => {
|
|
47
|
+
addModule('caddy');
|
|
48
|
+
openDeployWindow(db, 'caddy', NOW);
|
|
49
|
+
expect(modulesInDeployWindow(db).has('caddy')).toBe(true);
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
test('closing it takes the module back out', () => {
|
|
53
|
+
addModule('caddy');
|
|
54
|
+
openDeployWindow(db, 'caddy', NOW);
|
|
55
|
+
expect(closeDeployWindows(db, 'caddy', LATER)).toBe(1);
|
|
56
|
+
expect(modulesInDeployWindow(db).has('caddy')).toBe(false);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
// A redeploy during a deploy must not need two closes to become audible.
|
|
60
|
+
test('opening twice reuses the same window', () => {
|
|
61
|
+
addModule('caddy');
|
|
62
|
+
const first = openDeployWindow(db, 'caddy', NOW);
|
|
63
|
+
const second = openDeployWindow(db, 'caddy', NOW);
|
|
64
|
+
expect(second).toBe(first);
|
|
65
|
+
expect(closeDeployWindows(db, 'caddy', LATER)).toBe(1);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
// Closing by module rather than by id is what makes a crashed deploy
|
|
69
|
+
// recoverable — its window id is gone, but the module is still known.
|
|
70
|
+
test('a window orphaned by a crash is closed on the next attempt', () => {
|
|
71
|
+
addModule('caddy');
|
|
72
|
+
openDeployWindow(db, 'caddy', NOW);
|
|
73
|
+
// Simulate a crash: nobody closed it. The next deploy closes it anyway.
|
|
74
|
+
expect(closeDeployWindows(db, 'caddy', LATER)).toBe(1);
|
|
75
|
+
expect(modulesInDeployWindow(db).size).toBe(0);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test('one module deploying does not silence another', () => {
|
|
79
|
+
addModule('caddy');
|
|
80
|
+
addModule('forgejo');
|
|
81
|
+
openDeployWindow(db, 'caddy', NOW);
|
|
82
|
+
const suppressed = modulesInDeployWindow(db);
|
|
83
|
+
expect(suppressed.has('caddy')).toBe(true);
|
|
84
|
+
expect(suppressed.has('forgejo')).toBe(false);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
// Alerting bookkeeping must never fail a deploy.
|
|
88
|
+
test('an unknown module does not throw', () => {
|
|
89
|
+
expect(() => openDeployWindow(db, 'nope', NOW)).not.toThrow();
|
|
90
|
+
expect(() => closeDeployWindows(db, 'nope', NOW)).not.toThrow();
|
|
91
|
+
});
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
describe('monitor auto-create', () => {
|
|
95
|
+
test('a module declaring an interval gets a monitor', () => {
|
|
96
|
+
addModule('caddy', { hooks: { health_check: { script: 'x', interval: '15m' } } });
|
|
97
|
+
expect(ensureMonitorOnDeploy(db, 'caddy')).toBe(true);
|
|
98
|
+
|
|
99
|
+
const monitor = db.select().from(monitors).all()[0];
|
|
100
|
+
expect(monitor.target).toBe('caddy');
|
|
101
|
+
expect(monitor.intervalMinutes).toBe(15);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test('a module without an interval gets nothing', () => {
|
|
105
|
+
addModule('caddy', { hooks: { health_check: { script: 'x' } } });
|
|
106
|
+
expect(ensureMonitorOnDeploy(db, 'caddy')).toBe(false);
|
|
107
|
+
expect(db.select().from(monitors).all()).toEqual([]);
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
// The manifest interval is a SUGGESTION. An operator's setting has to
|
|
111
|
+
// survive module upgrades.
|
|
112
|
+
test('an existing monitor is never overwritten', () => {
|
|
113
|
+
addModule('caddy', { hooks: { health_check: { script: 'x', interval: '15m' } } });
|
|
114
|
+
ensureMonitorOnDeploy(db, 'caddy');
|
|
115
|
+
db.update(monitors).set({ intervalMinutes: 60 }).run();
|
|
116
|
+
|
|
117
|
+
expect(ensureMonitorOnDeploy(db, 'caddy')).toBe(false);
|
|
118
|
+
expect(db.select().from(monitors).all()[0].intervalMinutes).toBe(60);
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
test('an unknown module does not throw', () => {
|
|
122
|
+
expect(ensureMonitorOnDeploy(db, 'nope')).toBe(false);
|
|
123
|
+
});
|
|
124
|
+
});
|
|
125
|
+
});
|