@celilo/cli 0.13.3 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/CELILO_CORE_MODULES.md +3 -0
  2. package/CELILO_SUBSYSTEMS.md +71 -2
  3. package/docs/ALERTING.md +298 -0
  4. package/docs/INDEX.md +103 -0
  5. package/drizzle/0016_trusted_sources.sql +10 -0
  6. package/drizzle/0017_alerting.sql +127 -0
  7. package/drizzle/meta/_journal.json +15 -1
  8. package/package.json +3 -2
  9. package/schemas/system_config.json +9 -0
  10. package/src/cli/commands/alerts-act.ts +107 -0
  11. package/src/cli/commands/alerts-list.ts +62 -0
  12. package/src/cli/commands/alerts-poll.ts +129 -0
  13. package/src/cli/commands/alerts-sweep.ts +156 -0
  14. package/src/cli/commands/module-list.ts +50 -3
  15. package/src/cli/commands/monitor.ts +178 -0
  16. package/src/cli/commands/notify-config.ts +453 -0
  17. package/src/cli/commands/system-audit.ts +2 -0
  18. package/src/cli/commands/system-update.ts +1 -0
  19. package/src/cli/completion.ts +26 -0
  20. package/src/cli/generate-zsh-completion.ts +2 -0
  21. package/src/cli/index.ts +58 -0
  22. package/src/cli/tui/audit-state.ts +2 -0
  23. package/src/db/schema.ts +358 -0
  24. package/src/hooks/capability-loader.ts +158 -46
  25. package/src/hooks/capability-map-coverage.test.ts +101 -0
  26. package/src/manifest/schema.ts +60 -1
  27. package/src/services/alerting/ack.test.ts +212 -0
  28. package/src/services/alerting/ack.ts +119 -0
  29. package/src/services/alerting/builtin-monitors.test.ts +132 -0
  30. package/src/services/alerting/builtin-monitors.ts +84 -0
  31. package/src/services/alerting/builtin-source.ts +82 -0
  32. package/src/services/alerting/coverage-source.ts +38 -0
  33. package/src/services/alerting/deferral.test.ts +161 -0
  34. package/src/services/alerting/delivery-loop.test.ts +396 -0
  35. package/src/services/alerting/deploy-hooks.test.ts +125 -0
  36. package/src/services/alerting/deploy-hooks.ts +111 -0
  37. package/src/services/alerting/escalation.test.ts +207 -0
  38. package/src/services/alerting/escalation.ts +151 -0
  39. package/src/services/alerting/format.test.ts +193 -0
  40. package/src/services/alerting/format.ts +150 -0
  41. package/src/services/alerting/health-coverage.ts +81 -0
  42. package/src/services/alerting/inbound-poller.test.ts +298 -0
  43. package/src/services/alerting/inbound-poller.ts +236 -0
  44. package/src/services/alerting/inbound.test.ts +201 -0
  45. package/src/services/alerting/inbound.ts +112 -0
  46. package/src/services/alerting/interview-responder.test.ts +169 -0
  47. package/src/services/alerting/interview-responder.ts +158 -0
  48. package/src/services/alerting/keys.test.ts +155 -0
  49. package/src/services/alerting/keys.ts +190 -0
  50. package/src/services/alerting/monitors.ts +185 -0
  51. package/src/services/alerting/notification-responder.test.ts +290 -0
  52. package/src/services/alerting/notification-responder.ts +260 -0
  53. package/src/services/alerting/notifier.ts +219 -0
  54. package/src/services/alerting/people.ts +178 -0
  55. package/src/services/alerting/quiet-hours.test.ts +140 -0
  56. package/src/services/alerting/quiet-hours.ts +99 -0
  57. package/src/services/alerting/reconcile.test.ts +190 -0
  58. package/src/services/alerting/reconcile.ts +166 -0
  59. package/src/services/alerting/run-monitor.test.ts +185 -0
  60. package/src/services/alerting/run-monitor.ts +177 -0
  61. package/src/services/alerting/store.test.ts +222 -0
  62. package/src/services/alerting/store.ts +289 -0
  63. package/src/services/alerting/suppression.test.ts +228 -0
  64. package/src/services/alerting/suppression.ts +142 -0
  65. package/src/services/alerting/sweep-runner.test.ts +229 -0
  66. package/src/services/alerting/sweep-runner.ts +204 -0
  67. package/src/services/alerting/sweep.test.ts +61 -0
  68. package/src/services/alerting/sweep.ts +41 -0
  69. package/src/services/alerting/tokens.test.ts +152 -0
  70. package/src/services/alerting/tokens.ts +119 -0
  71. package/src/services/alerting/transport-loader.ts +48 -0
  72. package/src/services/audit/index.test.ts +1 -0
  73. package/src/services/audit/index.ts +3 -0
  74. package/src/services/audit/trusted-sources.test.ts +137 -0
  75. package/src/services/audit/trusted-sources.ts +124 -0
  76. package/src/services/audit/types.ts +2 -1
  77. package/src/services/firewall-reach.ts +83 -0
  78. package/src/services/health-runner.test.ts +50 -0
  79. package/src/services/health-runner.ts +116 -82
  80. package/src/services/module-deploy.ts +17 -0
  81. package/src/services/system-config-validator.test.ts +31 -1
  82. package/src/services/trusted-sources.test.ts +221 -0
  83. package/src/services/trusted-sources.ts +159 -0
  84. package/src/services/update/orchestrator.test.ts +1 -0
  85. package/src/templates/generator.ts +6 -29
@@ -0,0 +1,127 @@
1
+ CREATE TABLE `people` (
2
+ `id` text PRIMARY KEY NOT NULL,
3
+ `name` text NOT NULL,
4
+ `timezone` text NOT NULL,
5
+ `quiet_hours_start` text,
6
+ `quiet_hours_end` text,
7
+ `created_at` integer DEFAULT (unixepoch()) NOT NULL,
8
+ `updated_at` integer DEFAULT (unixepoch()) NOT NULL
9
+ );
10
+ --> statement-breakpoint
11
+ CREATE UNIQUE INDEX `people_name_unique` ON `people` (`name`);--> statement-breakpoint
12
+ CREATE TABLE `routes` (
13
+ `id` text PRIMARY KEY NOT NULL,
14
+ `person_id` text NOT NULL,
15
+ `transport_module_id` text NOT NULL,
16
+ `address` text NOT NULL,
17
+ `severity_floor` text DEFAULT 'warning' NOT NULL,
18
+ `can_ack` integer DEFAULT false NOT NULL,
19
+ `enabled` integer DEFAULT true NOT NULL,
20
+ `verified_at` integer,
21
+ `created_at` integer DEFAULT (unixepoch()) NOT NULL,
22
+ `updated_at` integer DEFAULT (unixepoch()) NOT NULL,
23
+ FOREIGN KEY (`person_id`) REFERENCES `people`(`id`) ON UPDATE no action ON DELETE cascade,
24
+ FOREIGN KEY (`transport_module_id`) REFERENCES `modules`(`id`) ON UPDATE no action ON DELETE cascade
25
+ );
26
+ --> statement-breakpoint
27
+ CREATE UNIQUE INDEX `routes_person_id_transport_module_id_address_unique` ON `routes` (`person_id`,`transport_module_id`,`address`);--> statement-breakpoint
28
+ CREATE TABLE `escalation_policies` (
29
+ `id` text PRIMARY KEY NOT NULL,
30
+ `name` text NOT NULL,
31
+ `bypass_quiet_hours` integer DEFAULT false NOT NULL,
32
+ `created_at` integer DEFAULT (unixepoch()) NOT NULL,
33
+ `updated_at` integer DEFAULT (unixepoch()) NOT NULL
34
+ );
35
+ --> statement-breakpoint
36
+ CREATE UNIQUE INDEX `escalation_policies_name_unique` ON `escalation_policies` (`name`);--> statement-breakpoint
37
+ CREATE TABLE `escalation_steps` (
38
+ `policy_id` text NOT NULL,
39
+ `step_index` integer NOT NULL,
40
+ `route_id` text NOT NULL,
41
+ `delay_minutes` integer NOT NULL,
42
+ PRIMARY KEY(`policy_id`, `step_index`),
43
+ FOREIGN KEY (`policy_id`) REFERENCES `escalation_policies`(`id`) ON UPDATE no action ON DELETE cascade,
44
+ FOREIGN KEY (`route_id`) REFERENCES `routes`(`id`) ON UPDATE no action ON DELETE cascade
45
+ );
46
+ --> statement-breakpoint
47
+ CREATE TABLE `monitors` (
48
+ `id` text PRIMARY KEY NOT NULL,
49
+ `kind` text NOT NULL,
50
+ `target` text NOT NULL,
51
+ `interval_minutes` integer NOT NULL,
52
+ `severity` text DEFAULT 'critical' NOT NULL,
53
+ `suppressible` integer DEFAULT true NOT NULL,
54
+ `enabled` integer DEFAULT true NOT NULL,
55
+ `escalation_policy_id` text,
56
+ `last_run_at` integer,
57
+ `created_at` integer DEFAULT (unixepoch()) NOT NULL,
58
+ `updated_at` integer DEFAULT (unixepoch()) NOT NULL,
59
+ FOREIGN KEY (`escalation_policy_id`) REFERENCES `escalation_policies`(`id`) ON UPDATE no action ON DELETE set null
60
+ );
61
+ --> statement-breakpoint
62
+ CREATE UNIQUE INDEX `monitors_kind_target_unique` ON `monitors` (`kind`,`target`);--> statement-breakpoint
63
+ CREATE TABLE `monitor_runs` (
64
+ `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
65
+ `monitor_id` text NOT NULL,
66
+ `ran_at` integer DEFAULT (unixepoch()) NOT NULL,
67
+ `outcome` text NOT NULL,
68
+ `error_message` text,
69
+ FOREIGN KEY (`monitor_id`) REFERENCES `monitors`(`id`) ON UPDATE no action ON DELETE cascade
70
+ );
71
+ --> statement-breakpoint
72
+ CREATE TABLE `suppression_windows` (
73
+ `id` text PRIMARY KEY NOT NULL,
74
+ `source` text NOT NULL,
75
+ `scope_module_id` text NOT NULL,
76
+ `started_at` integer DEFAULT (unixepoch()) NOT NULL,
77
+ `ends_at` integer,
78
+ FOREIGN KEY (`scope_module_id`) REFERENCES `modules`(`id`) ON UPDATE no action ON DELETE cascade
79
+ );
80
+ --> statement-breakpoint
81
+ CREATE TABLE `alerts` (
82
+ `id` text PRIMARY KEY NOT NULL,
83
+ `key` text NOT NULL,
84
+ `active_key` text,
85
+ `monitor_id` text NOT NULL,
86
+ `state` text DEFAULT 'pending' NOT NULL,
87
+ `severity` text NOT NULL,
88
+ `first_fired_at` integer DEFAULT (unixepoch()) NOT NULL,
89
+ `last_seen_at` integer DEFAULT (unixepoch()) NOT NULL,
90
+ `grace_until` integer NOT NULL,
91
+ `suppressed_by_alert_id` text,
92
+ `suppressed_by_window_id` text,
93
+ `unsuppressed_at` integer,
94
+ `awaiting_confirmation` integer DEFAULT false NOT NULL,
95
+ `acked_by` text,
96
+ `acked_at` integer,
97
+ `silenced_until` integer,
98
+ `escalation_step` integer DEFAULT 0 NOT NULL,
99
+ `next_escalation_at` integer,
100
+ `deferred_until` integer,
101
+ `deferred_route_id` text,
102
+ `escalation_policy_id` text,
103
+ `message` text NOT NULL,
104
+ `details` text,
105
+ `resolved_at` integer,
106
+ FOREIGN KEY (`monitor_id`) REFERENCES `monitors`(`id`) ON UPDATE no action ON DELETE cascade,
107
+ FOREIGN KEY (`suppressed_by_window_id`) REFERENCES `suppression_windows`(`id`) ON UPDATE no action ON DELETE set null,
108
+ FOREIGN KEY (`acked_by`) REFERENCES `people`(`id`) ON UPDATE no action ON DELETE set null,
109
+ FOREIGN KEY (`deferred_route_id`) REFERENCES `routes`(`id`) ON UPDATE no action ON DELETE set null,
110
+ FOREIGN KEY (`escalation_policy_id`) REFERENCES `escalation_policies`(`id`) ON UPDATE no action ON DELETE set null
111
+ );
112
+ --> statement-breakpoint
113
+ CREATE UNIQUE INDEX `alerts_live_key_idx` ON `alerts` (`active_key`);--> statement-breakpoint
114
+ CREATE INDEX `alerts_key_idx` ON `alerts` (`key`);--> statement-breakpoint
115
+ CREATE TABLE `notification_deliveries` (
116
+ `id` text PRIMARY KEY NOT NULL,
117
+ `token` text NOT NULL,
118
+ `kind` text NOT NULL,
119
+ `target_id` text NOT NULL,
120
+ `route_id` text NOT NULL,
121
+ `sent_at` integer DEFAULT (unixepoch()) NOT NULL,
122
+ `expires_at` integer NOT NULL,
123
+ `consumed_at` integer,
124
+ FOREIGN KEY (`route_id`) REFERENCES `routes`(`id`) ON UPDATE no action ON DELETE cascade
125
+ );
126
+ --> statement-breakpoint
127
+ CREATE UNIQUE INDEX `notification_deliveries_token_unique` ON `notification_deliveries` (`token`);
@@ -113,6 +113,20 @@
113
113
  "when": 1783100000000,
114
114
  "tag": "0015_port_forwards",
115
115
  "breakpoints": true
116
+ },
117
+ {
118
+ "idx": 16,
119
+ "version": "6",
120
+ "when": 1783200000000,
121
+ "tag": "0016_trusted_sources",
122
+ "breakpoints": true
123
+ },
124
+ {
125
+ "idx": 17,
126
+ "version": "6",
127
+ "when": 1783300000000,
128
+ "tag": "0017_alerting",
129
+ "breakpoints": true
116
130
  }
117
131
  ]
118
- }
132
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@celilo/cli",
3
- "version": "0.13.3",
3
+ "version": "0.14.0",
4
4
  "description": "Celilo — home lab orchestration CLI",
5
5
  "type": "module",
6
6
  "bin": {
@@ -13,6 +13,7 @@
13
13
  "schemas/",
14
14
  "tsconfig.json",
15
15
  "AGENTS.md",
16
+ "docs/",
16
17
  "CELILO_SUBSYSTEMS.md",
17
18
  "CELILO_CORE_MODULES.md",
18
19
  "MODULE_PRIMITIVES.md"
@@ -56,7 +57,7 @@
56
57
  },
57
58
  "dependencies": {
58
59
  "@aws-sdk/client-s3": "^3.1024.0",
59
- "@celilo/capabilities": "^0.7.1",
60
+ "@celilo/capabilities": "^0.9.0",
60
61
  "@celilo/cli-display": "^0.1.9",
61
62
  "@celilo/core": "^0.1.0",
62
63
  "@celilo/event-bus": "^0.1.8",
@@ -74,6 +74,11 @@
74
74
  "maximum": 4094,
75
75
  "description": "VLAN tag for internal zone (not defaulted; internal is untagged)"
76
76
  },
77
+ "network.vpn.subnet": {
78
+ "type": "string",
79
+ "pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$",
80
+ "description": "WireGuard VPN client subnet CIDR (remote-access tunnel; consumed by technitium's split-horizon → in-zone caddy view, #275, and by the firewall's trusted sources once the wireguard module registers it)"
81
+ },
77
82
  "network.secure-mgmt.subnet": {
78
83
  "type": "string",
79
84
  "pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$",
@@ -90,6 +95,10 @@
90
95
  "maximum": 4094,
91
96
  "description": "VLAN tag for the control-plane zone (not defaulted)"
92
97
  },
98
+ "firewall.trusted_subnets": {
99
+ "type": "string",
100
+ "description": "Operator-declared trusted source subnets (comma-separated CIDRs) permitted to reach every managed zone. ADDITIVE to the control-plane subnet celilo derives and to the sources modules register — it cannot drop control-plane trust, which would block the SSH every hook and converge runs over."
101
+ },
93
102
  "dns.primary": {
94
103
  "type": "string",
95
104
  "pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$",
@@ -0,0 +1,107 @@
1
+ /**
2
+ * `celilo alerts ack|silence|resolve` — the operator's side of the ack loop.
3
+ *
4
+ * Alerts are addressed by KEY, not id. The key is what `celilo alerts list`
5
+ * prints and what arrives in a page, so it is the only identifier an operator
6
+ * ever has in hand.
7
+ */
8
+
9
+ import { getDb } from '../../db/client';
10
+ import { parseIntervalMinutes } from '../../manifest/schema';
11
+ import {
12
+ acknowledgeAlert,
13
+ findLiveAlertByKey,
14
+ resolveAlertManually,
15
+ silenceAlert,
16
+ } from '../../services/alerting/ack';
17
+ import { findPerson, listPeople } from '../../services/alerting/people';
18
+ import type { CommandResult } from '../types';
19
+
20
+ function resolveKey(key: string | undefined) {
21
+ if (!key) return { error: 'An alert key is required. Run "celilo alerts list" to see them.' };
22
+ const db = getDb();
23
+ const alert = findLiveAlertByKey(db, key);
24
+ if (!alert) {
25
+ return {
26
+ error: `No live alert with key "${key}".\n\nRun "celilo alerts list" to see current alerts.`,
27
+ };
28
+ }
29
+ return { db, alert };
30
+ }
31
+
32
+ export async function handleAlertsAck(
33
+ args: string[],
34
+ flags: Record<string, boolean | string> = {},
35
+ ): Promise<CommandResult> {
36
+ const found = resolveKey(args[0]);
37
+ if (found.error !== undefined) return { success: false, error: found.error };
38
+ const { db, alert } = found;
39
+
40
+ // Who is acknowledging matters: it stops escalation AND tells everyone else
41
+ // who was paged that somebody has it.
42
+ const who = typeof flags.as === 'string' ? flags.as : undefined;
43
+ const people = listPeople(db);
44
+ if (!who) {
45
+ if (people.length !== 1) {
46
+ return {
47
+ success: false,
48
+ error: `Who is acknowledging? Pass --as <person>.\n\nKnown people: ${people.map((p) => p.name).join(', ') || '(none configured)'}`,
49
+ };
50
+ }
51
+ }
52
+ const person = who ? findPerson(db, who) : people[0];
53
+ if (!person) return { success: false, error: `No person named "${who}".` };
54
+
55
+ const result = acknowledgeAlert(db, alert.id, person.id, new Date());
56
+ if (!result) return { success: false, error: 'Alert vanished while acknowledging.' };
57
+
58
+ const broadcast = result.broadcastTo.length
59
+ ? ` — ${result.broadcastTo.length} other route(s) will be told`
60
+ : '';
61
+ return { success: true, message: `${alert.key} acknowledged by ${person.name}${broadcast}` };
62
+ }
63
+
64
+ export async function handleAlertsSilence(
65
+ args: string[],
66
+ flags: Record<string, boolean | string> = {},
67
+ ): Promise<CommandResult> {
68
+ const found = resolveKey(args[0]);
69
+ if (found.error !== undefined) return { success: false, error: found.error };
70
+ const { db, alert } = found;
71
+
72
+ const forFlag = typeof flags.for === 'string' ? flags.for : '2h';
73
+ const minutes = parseIntervalMinutes(forFlag);
74
+ if (minutes === null) {
75
+ return {
76
+ success: false,
77
+ error: `Invalid --for "${forFlag}". Use a duration like 2h, 30m, 1d.`,
78
+ };
79
+ }
80
+
81
+ const until = new Date(Date.now() + minutes * 60_000);
82
+ silenceAlert(db, alert.id, until);
83
+ return {
84
+ success: true,
85
+ message: `${alert.key} silenced for ${forFlag} (until ${until.toISOString()})`,
86
+ };
87
+ }
88
+
89
+ export async function handleAlertsResolve(args: string[]): Promise<CommandResult> {
90
+ const found = resolveKey(args[0]);
91
+ if (found.error !== undefined) return { success: false, error: found.error };
92
+ const { db, alert } = found;
93
+
94
+ resolveAlertManually(db, alert.id, new Date());
95
+ return {
96
+ success: true,
97
+ // Resolve does not override the monitor, and saying so is the whole point
98
+ // of the message — an operator who thinks it does will trust a cleared
99
+ // alert that is about to come back.
100
+ //
101
+ // What it IS for: you have just fixed something and the check runs every
102
+ // fifteen minutes. This stops the stale `firing` row escalating at you
103
+ // while you wait for the next run to agree.
104
+ message: `${alert.key} resolved.
105
+ The monitor is still the authority: if the check still fails, this fires again on the next sweep.`,
106
+ };
107
+ }
@@ -0,0 +1,62 @@
1
+ /**
2
+ * `celilo alerts list` — what is currently wrong, and what is already accounted for.
3
+ *
4
+ * Thin adapter (Rule 10.5): read, delegate, format, return. The ordering,
5
+ * wording, and the "explained by something upstream" column all live in
6
+ * services/alerting/format.ts where they are unit-tested.
7
+ */
8
+
9
+ import { getDb } from '../../db/client';
10
+ import { renderAlertTable, toAlertRow } from '../../services/alerting/format';
11
+ import { loadAllLiveAlerts } from '../../services/alerting/store';
12
+ import type { CommandResult } from '../types';
13
+
14
+ export async function handleAlertsList(
15
+ _args: string[],
16
+ flags: Record<string, boolean | string> = {},
17
+ ): Promise<CommandResult> {
18
+ const db = getDb();
19
+ const now = new Date();
20
+ const live = loadAllLiveAlerts(db);
21
+
22
+ if (flags.json) {
23
+ return {
24
+ success: true,
25
+ message: JSON.stringify(
26
+ live.map((alert) => ({
27
+ key: alert.key,
28
+ state: alert.state,
29
+ severity: alert.severity,
30
+ firstFiredAt: alert.firstFiredAt,
31
+ lastSeenAt: alert.lastSeenAt,
32
+ suppressed: alert.state === 'suppressed',
33
+ suppressedByAlertId: alert.suppressedByAlertId,
34
+ suppressedByWindowId: alert.suppressedByWindowId,
35
+ awaitingConfirmation: alert.awaitingConfirmation,
36
+ ackedBy: alert.ackedBy,
37
+ silencedUntil: alert.silencedUntil,
38
+ message: alert.message,
39
+ })),
40
+ null,
41
+ 2,
42
+ ),
43
+ rawOutput: true,
44
+ };
45
+ }
46
+
47
+ const table = renderAlertTable(live.map((alert) => toAlertRow(alert, now)));
48
+ console.log('');
49
+ console.log(table);
50
+ console.log('');
51
+
52
+ const firing = live.filter((a) => a.state === 'firing').length;
53
+ const suppressed = live.filter((a) => a.state === 'suppressed').length;
54
+
55
+ return {
56
+ success: true,
57
+ message:
58
+ live.length === 0
59
+ ? 'No alerts'
60
+ : `${firing} firing, ${suppressed} suppressed, ${live.length} live`,
61
+ };
62
+ }
@@ -0,0 +1,129 @@
1
+ /**
2
+ * `celilo alerts poll` — read replies off every bidirectional transport once.
3
+ *
4
+ * Runs far more often than the five-minute sweep: "I texted ack and nothing
5
+ * happened" is the experience this exists to prevent. Registered on
6
+ * `timer.tick.5m` like the sweep, but also safe to run by hand or from a
7
+ * tighter loop.
8
+ *
9
+ * The receive cursor is persisted in system config rather than memory, so a
10
+ * restart resumes where it left off instead of re-reading or skipping.
11
+ */
12
+
13
+ import { eq } from 'drizzle-orm';
14
+ import { getEventBusPath } from '../../config/paths';
15
+ import { getDb } from '../../db/client';
16
+ import { systemConfig } from '../../db/schema';
17
+ import { makeReceiver, pollInbound } from '../../services/alerting/inbound-poller';
18
+ import { startNotificationResponder } from '../../services/alerting/notification-responder';
19
+ import { listRoutes } from '../../services/alerting/people';
20
+ import { loadNotificationTransport } from '../../services/alerting/transport-loader';
21
+ import type { CommandResult } from '../types';
22
+
23
+ /** How long a reply token stays usable. A deploy may wait overnight. */
24
+ const INTERVIEW_TTL_MS = 24 * 60 * 60_000;
25
+
26
+ const CURSOR_PREFIX = 'alerting.inbound_cursor.';
27
+
28
+ function readCursor(db: ReturnType<typeof getDb>, transportModuleId: string): string | null {
29
+ const key = `${CURSOR_PREFIX}${transportModuleId}`;
30
+ const row = db.select().from(systemConfig).where(eq(systemConfig.key, key)).get();
31
+ return row?.value ?? null;
32
+ }
33
+
34
+ function writeCursor(
35
+ db: ReturnType<typeof getDb>,
36
+ transportModuleId: string,
37
+ cursor: string | null,
38
+ ): void {
39
+ const key = `${CURSOR_PREFIX}${transportModuleId}`;
40
+ if (cursor === null) return;
41
+ const existing = db.select().from(systemConfig).where(eq(systemConfig.key, key)).get();
42
+ if (existing) {
43
+ db.update(systemConfig).set({ value: cursor }).where(eq(systemConfig.key, key)).run();
44
+ } else {
45
+ db.insert(systemConfig)
46
+ .values({
47
+ key,
48
+ value: cursor,
49
+ description: `Inbound receive cursor for ${transportModuleId}`,
50
+ })
51
+ .run();
52
+ }
53
+ }
54
+
55
+ export async function handleAlertsPoll(
56
+ flags: Record<string, string | boolean> = {},
57
+ ): Promise<CommandResult> {
58
+ const db = getDb();
59
+ const askErrors: string[] = [];
60
+
61
+ // `--ask-after 0` asks immediately. Useful for an operator who wants a
62
+ // question escalated now, and for tests that would otherwise wait out the
63
+ // grace.
64
+ const raw = flags['ask-after'];
65
+ const askAfterSeconds = typeof raw === 'string' ? Number.parseInt(raw, 10) : null;
66
+ if (raw !== undefined && (askAfterSeconds === null || Number.isNaN(askAfterSeconds))) {
67
+ return { success: false, error: '--ask-after expects a number of seconds' };
68
+ }
69
+
70
+ // The responder both DELIVERS unanswered questions and publishes the
71
+ // answers. Both halves belong here: a question delivered by a process that
72
+ // then exits is still answerable, because what was asked lives in the
73
+ // deliveries table rather than in that process's memory.
74
+ const responder = startNotificationResponder({
75
+ db,
76
+ busDbPath: getEventBusPath(),
77
+ routes: listRoutes(db).filter((route) => route.enabled && route.canAck),
78
+ transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
79
+ // A CLI invocation is never the terminal responder for someone else's
80
+ // deploy — that responder attaches to its own deploy's stdin.
81
+ hasTty: () => false,
82
+ now: () => new Date(),
83
+ ttlMs: INTERVIEW_TTL_MS,
84
+ askAfterMs: askAfterSeconds === null ? undefined : askAfterSeconds * 1000,
85
+ onSendFailed: (eventType, error) => askErrors.push(`${eventType}: ${error}`),
86
+ });
87
+
88
+ let ask = { asked: 0, failed: 0 };
89
+ try {
90
+ ask = await responder.poll();
91
+ } catch {
92
+ // A bus that cannot be opened must not stop alert replies being read —
93
+ // those are the 3am path.
94
+ }
95
+
96
+ const report = await pollInbound(db, {
97
+ receiveFrom: makeReceiver(db),
98
+ readCursor: (t) => readCursor(db, t),
99
+ writeCursor: (t, c) => writeCursor(db, t, c),
100
+ now: () => new Date(),
101
+ transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
102
+ answerInterview: (eventId, value) => responder.answer(eventId, value),
103
+ });
104
+
105
+ responder.stop();
106
+
107
+ // Rejections are worth surfacing rather than counting silently: a run of
108
+ // them means someone is replying and not being heard, which looks identical
109
+ // to the alerting system being broken.
110
+ const parts = [
111
+ `${report.transportsPolled} transport(s)`,
112
+ `${report.messagesRead} message(s)`,
113
+ `${report.acked} acked`,
114
+ ];
115
+ if (report.rejected > 0) parts.push(`${report.rejected} rejected`);
116
+ if (report.ignored > 0) parts.push(`${report.ignored} from unknown senders`);
117
+ if (report.broadcast > 0) parts.push(`${report.broadcast} told someone has it`);
118
+ if (report.answered > 0) parts.push(`${report.answered} deploy question(s) answered`);
119
+ if (ask.asked > 0) parts.push(`${ask.asked} question(s) asked`);
120
+ // Never silent: a deploy blocked on a question nobody received is
121
+ // indistinguishable from a deploy that is merely slow.
122
+ if (ask.failed > 0) parts.push(`${ask.failed} question(s) COULD NOT BE DELIVERED`);
123
+
124
+ const message = `inbound poll: ${parts.join(', ')}`;
125
+ return {
126
+ success: true,
127
+ message: askErrors.length > 0 ? `${message}\n ${askErrors.join('\n ')}` : message,
128
+ };
129
+ }
@@ -0,0 +1,156 @@
1
+ /**
2
+ * `celilo alerts sweep` — one pass of the alerting loop.
3
+ *
4
+ * Invoked by the event-bus dispatcher on `timer.tick.5m`, not by a human. It
5
+ * is a normal command so the dispatcher's existing subprocess isolation,
6
+ * retry, and timeout apply unchanged — the sweep gets those for free rather
7
+ * than reimplementing them.
8
+ *
9
+ * Thin adapter (Rule 10.5): compose the real dependencies, call runSweep,
10
+ * report counts.
11
+ */
12
+
13
+ import { eq } from 'drizzle-orm';
14
+ import { getDb } from '../../db/client';
15
+ import { type Alert, escalationPolicies, modules } from '../../db/schema';
16
+ import { runBuiltinCheckForMonitor } from '../../services/alerting/builtin-source';
17
+ import { loadModuleCoverage } from '../../services/alerting/coverage-source';
18
+ import { modulesInDeployWindow } from '../../services/alerting/deploy-hooks';
19
+ import { listMonitors } from '../../services/alerting/monitors';
20
+ import type { NotifyDeps } from '../../services/alerting/notifier';
21
+ import { listPeople, listPolicySteps, listRoutes } from '../../services/alerting/people';
22
+ import type { SuppressionTopology } from '../../services/alerting/suppression';
23
+ import { runSweep } from '../../services/alerting/sweep-runner';
24
+ import { mintDelivery } from '../../services/alerting/tokens';
25
+ import { loadNotificationTransport } from '../../services/alerting/transport-loader';
26
+ import { getModuleSystems } from '../../services/deployed-systems';
27
+ import { runModuleHealthCheck } from '../../services/health-runner';
28
+ import type { CommandResult } from '../types';
29
+
30
+ /** Grace window before a newly-fired alert may notify. */
31
+ const DEFAULT_GRACE_MS = 60_000;
32
+
33
+ /**
34
+ * How long a reply token stays valid. A day is long enough that someone who
35
+ * sees a page overnight can still act on it in the morning, and short enough
36
+ * that a token found later is useless.
37
+ */
38
+ const TOKEN_TTL_MS = 24 * 60 * 60_000;
39
+
40
+ /**
41
+ * Deployment topology for suppression, read once per sweep.
42
+ *
43
+ * Derived from what celilo already records — asking an operator to restate it
44
+ * would guarantee it drifts, and a stale suppression graph hides real
45
+ * failures rather than merely being untidy.
46
+ */
47
+ function loadTopology(db: ReturnType<typeof getDb>): SuppressionTopology {
48
+ const moduleRows = db.select({ id: modules.id }).from(modules).all();
49
+ const moduleSystems = moduleRows.flatMap((module) =>
50
+ getModuleSystems(module.id, db).map((system) => ({
51
+ moduleId: module.id,
52
+ hostname: system.hostname,
53
+ zone: system.zone,
54
+ infraType: system.infrastructure.type,
55
+ })),
56
+ );
57
+ // Zone-capability providers are wired in a follow-up; an empty list means
58
+ // only the machine edge is active, which is the MVP scope.
59
+ return { moduleSystems, zoneProviders: [] };
60
+ }
61
+
62
+ /**
63
+ * Assemble everything needed to page for one alert, or null when nobody can be.
64
+ *
65
+ * Returning null rather than an empty policy is deliberate: "no escalation
66
+ * policy assigned" and "policy exists but nobody is eligible" are different
67
+ * situations, and only the second is worth an escalation decision.
68
+ */
69
+ function buildNotifyDeps(db: ReturnType<typeof getDb>, alert: Alert): NotifyDeps | null {
70
+ if (!alert.escalationPolicyId) return null;
71
+
72
+ const policy = db
73
+ .select()
74
+ .from(escalationPolicies)
75
+ .where(eq(escalationPolicies.id, alert.escalationPolicyId))
76
+ .get();
77
+ if (!policy) return null;
78
+
79
+ const steps = listPolicySteps(db, policy.id).map((step) => ({
80
+ stepIndex: step.stepIndex,
81
+ routeId: step.routeId,
82
+ delayMinutes: step.delayMinutes,
83
+ }));
84
+ if (steps.length === 0) return null;
85
+
86
+ const routeRows = listRoutes(db);
87
+ const people = listPeople(db);
88
+
89
+ return {
90
+ steps,
91
+ routes: new Map(
92
+ routeRows.map((r) => [
93
+ r.id,
94
+ { id: r.id, severityFloor: r.severityFloor, enabled: r.enabled },
95
+ ]),
96
+ ),
97
+ routeDetails: new Map(routeRows.map((r) => [r.id, r])),
98
+ quietHoursByPerson: new Map(
99
+ people
100
+ .filter((p) => p.quietHoursStart && p.quietHoursEnd)
101
+ .map((p) => [
102
+ p.id,
103
+ {
104
+ personId: p.id,
105
+ start: p.quietHoursStart,
106
+ end: p.quietHoursEnd,
107
+ timezone: p.timezone,
108
+ },
109
+ ]),
110
+ ),
111
+ bypassQuietHours: policy.bypassQuietHours,
112
+ transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
113
+ mintToken: (alertId, routeId) =>
114
+ mintDelivery(db, {
115
+ kind: 'alert',
116
+ targetId: alertId,
117
+ routeId,
118
+ now: new Date(),
119
+ ttlMs: TOKEN_TTL_MS,
120
+ }).token,
121
+ now: new Date(),
122
+ };
123
+ }
124
+
125
+ export async function handleAlertsSweep(): Promise<CommandResult> {
126
+ const db = getDb();
127
+ const monitorRows = listMonitors(db);
128
+
129
+ const suppressibleByMonitor = new Map(monitorRows.map((m) => [m.id, m.suppressible]));
130
+
131
+ const report = await runSweep(db, monitorRows, {
132
+ monitorDeps: {
133
+ runModuleCheck: (moduleId: string) =>
134
+ runModuleHealthCheck(moduleId, db, { unattended: true, noInteractive: true }),
135
+ runBuiltinCheck: (category) => runBuiltinCheckForMonitor(category),
136
+ loadModuleCoverage: () => loadModuleCoverage(db),
137
+ now: () => new Date(),
138
+ graceMs: DEFAULT_GRACE_MS,
139
+ },
140
+ loadTopology: () => loadTopology(db),
141
+ loadDeployWindowModules: () => modulesInDeployWindow(db),
142
+ isSuppressible: (alert) => suppressibleByMonitor.get(alert.monitorId) ?? true,
143
+ notifyDepsFor: (alert) => buildNotifyDeps(db, alert),
144
+ now: () => new Date(),
145
+ });
146
+
147
+ const parts = [
148
+ `${report.monitorsRun} run`,
149
+ `${report.monitorsErrored} errored`,
150
+ `${report.promoted} promoted`,
151
+ `${report.suppressed} suppressed`,
152
+ `${report.unsuppressed} unsuppressed`,
153
+ `${report.notified} notified`,
154
+ ];
155
+ return { success: true, message: `alert sweep: ${parts.join(', ')}` };
156
+ }