@celilo/cli 0.13.2 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +3 -0
- package/CELILO_SUBSYSTEMS.md +71 -2
- package/docs/ALERTING.md +298 -0
- package/docs/INDEX.md +103 -0
- package/drizzle/0016_trusted_sources.sql +10 -0
- package/drizzle/0017_alerting.sql +127 -0
- package/drizzle/meta/_journal.json +15 -1
- package/package.json +3 -2
- package/schemas/system_config.json +9 -0
- package/src/ansible/inventory.ts +2 -2
- package/src/cli/commands/alerts-act.ts +107 -0
- package/src/cli/commands/alerts-list.ts +62 -0
- package/src/cli/commands/alerts-poll.ts +129 -0
- package/src/cli/commands/alerts-sweep.ts +156 -0
- package/src/cli/commands/module-list.ts +50 -3
- package/src/cli/commands/monitor.ts +178 -0
- package/src/cli/commands/notify-config.ts +453 -0
- package/src/cli/commands/system-audit.ts +2 -0
- package/src/cli/commands/system-update.ts +1 -0
- package/src/cli/completion.ts +26 -0
- package/src/cli/generate-zsh-completion.ts +2 -0
- package/src/cli/index.ts +58 -0
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/db/schema.ts +358 -0
- package/src/hooks/capability-loader.ts +158 -46
- package/src/hooks/capability-map-coverage.test.ts +101 -0
- package/src/manifest/schema.ts +60 -1
- package/src/services/alerting/ack.test.ts +212 -0
- package/src/services/alerting/ack.ts +119 -0
- package/src/services/alerting/builtin-monitors.test.ts +132 -0
- package/src/services/alerting/builtin-monitors.ts +84 -0
- package/src/services/alerting/builtin-source.ts +82 -0
- package/src/services/alerting/coverage-source.ts +38 -0
- package/src/services/alerting/deferral.test.ts +161 -0
- package/src/services/alerting/delivery-loop.test.ts +396 -0
- package/src/services/alerting/deploy-hooks.test.ts +125 -0
- package/src/services/alerting/deploy-hooks.ts +111 -0
- package/src/services/alerting/escalation.test.ts +207 -0
- package/src/services/alerting/escalation.ts +151 -0
- package/src/services/alerting/format.test.ts +193 -0
- package/src/services/alerting/format.ts +150 -0
- package/src/services/alerting/health-coverage.ts +81 -0
- package/src/services/alerting/inbound-poller.test.ts +298 -0
- package/src/services/alerting/inbound-poller.ts +236 -0
- package/src/services/alerting/inbound.test.ts +201 -0
- package/src/services/alerting/inbound.ts +112 -0
- package/src/services/alerting/interview-responder.test.ts +169 -0
- package/src/services/alerting/interview-responder.ts +158 -0
- package/src/services/alerting/keys.test.ts +155 -0
- package/src/services/alerting/keys.ts +190 -0
- package/src/services/alerting/monitors.ts +185 -0
- package/src/services/alerting/notification-responder.test.ts +290 -0
- package/src/services/alerting/notification-responder.ts +260 -0
- package/src/services/alerting/notifier.ts +219 -0
- package/src/services/alerting/people.ts +178 -0
- package/src/services/alerting/quiet-hours.test.ts +140 -0
- package/src/services/alerting/quiet-hours.ts +99 -0
- package/src/services/alerting/reconcile.test.ts +190 -0
- package/src/services/alerting/reconcile.ts +166 -0
- package/src/services/alerting/run-monitor.test.ts +185 -0
- package/src/services/alerting/run-monitor.ts +177 -0
- package/src/services/alerting/store.test.ts +222 -0
- package/src/services/alerting/store.ts +289 -0
- package/src/services/alerting/suppression.test.ts +228 -0
- package/src/services/alerting/suppression.ts +142 -0
- package/src/services/alerting/sweep-runner.test.ts +229 -0
- package/src/services/alerting/sweep-runner.ts +204 -0
- package/src/services/alerting/sweep.test.ts +61 -0
- package/src/services/alerting/sweep.ts +41 -0
- package/src/services/alerting/tokens.test.ts +152 -0
- package/src/services/alerting/tokens.ts +119 -0
- package/src/services/alerting/transport-loader.ts +48 -0
- package/src/services/aspect-runner.ts +2 -2
- package/src/services/audit/index.test.ts +1 -0
- package/src/services/audit/index.ts +3 -0
- package/src/services/audit/trusted-sources.test.ts +137 -0
- package/src/services/audit/trusted-sources.ts +124 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/firewall-reach.ts +83 -0
- package/src/services/health-runner.test.ts +50 -0
- package/src/services/health-runner.ts +116 -82
- package/src/services/module-deploy.ts +32 -3
- package/src/services/ssh-key-manager.test.ts +14 -0
- package/src/services/ssh-key-manager.ts +12 -0
- package/src/services/system-config-validator.test.ts +31 -1
- package/src/services/trusted-sources.test.ts +221 -0
- package/src/services/trusted-sources.ts +159 -0
- package/src/services/update/orchestrator.test.ts +1 -0
- package/src/templates/generator.ts +6 -29
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
CREATE TABLE `people` (
|
|
2
|
+
`id` text PRIMARY KEY NOT NULL,
|
|
3
|
+
`name` text NOT NULL,
|
|
4
|
+
`timezone` text NOT NULL,
|
|
5
|
+
`quiet_hours_start` text,
|
|
6
|
+
`quiet_hours_end` text,
|
|
7
|
+
`created_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
8
|
+
`updated_at` integer DEFAULT (unixepoch()) NOT NULL
|
|
9
|
+
);
|
|
10
|
+
--> statement-breakpoint
|
|
11
|
+
CREATE UNIQUE INDEX `people_name_unique` ON `people` (`name`);--> statement-breakpoint
|
|
12
|
+
CREATE TABLE `routes` (
|
|
13
|
+
`id` text PRIMARY KEY NOT NULL,
|
|
14
|
+
`person_id` text NOT NULL,
|
|
15
|
+
`transport_module_id` text NOT NULL,
|
|
16
|
+
`address` text NOT NULL,
|
|
17
|
+
`severity_floor` text DEFAULT 'warning' NOT NULL,
|
|
18
|
+
`can_ack` integer DEFAULT false NOT NULL,
|
|
19
|
+
`enabled` integer DEFAULT true NOT NULL,
|
|
20
|
+
`verified_at` integer,
|
|
21
|
+
`created_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
22
|
+
`updated_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
23
|
+
FOREIGN KEY (`person_id`) REFERENCES `people`(`id`) ON UPDATE no action ON DELETE cascade,
|
|
24
|
+
FOREIGN KEY (`transport_module_id`) REFERENCES `modules`(`id`) ON UPDATE no action ON DELETE cascade
|
|
25
|
+
);
|
|
26
|
+
--> statement-breakpoint
|
|
27
|
+
CREATE UNIQUE INDEX `routes_person_id_transport_module_id_address_unique` ON `routes` (`person_id`,`transport_module_id`,`address`);--> statement-breakpoint
|
|
28
|
+
CREATE TABLE `escalation_policies` (
|
|
29
|
+
`id` text PRIMARY KEY NOT NULL,
|
|
30
|
+
`name` text NOT NULL,
|
|
31
|
+
`bypass_quiet_hours` integer DEFAULT false NOT NULL,
|
|
32
|
+
`created_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
33
|
+
`updated_at` integer DEFAULT (unixepoch()) NOT NULL
|
|
34
|
+
);
|
|
35
|
+
--> statement-breakpoint
|
|
36
|
+
CREATE UNIQUE INDEX `escalation_policies_name_unique` ON `escalation_policies` (`name`);--> statement-breakpoint
|
|
37
|
+
CREATE TABLE `escalation_steps` (
|
|
38
|
+
`policy_id` text NOT NULL,
|
|
39
|
+
`step_index` integer NOT NULL,
|
|
40
|
+
`route_id` text NOT NULL,
|
|
41
|
+
`delay_minutes` integer NOT NULL,
|
|
42
|
+
PRIMARY KEY(`policy_id`, `step_index`),
|
|
43
|
+
FOREIGN KEY (`policy_id`) REFERENCES `escalation_policies`(`id`) ON UPDATE no action ON DELETE cascade,
|
|
44
|
+
FOREIGN KEY (`route_id`) REFERENCES `routes`(`id`) ON UPDATE no action ON DELETE cascade
|
|
45
|
+
);
|
|
46
|
+
--> statement-breakpoint
|
|
47
|
+
CREATE TABLE `monitors` (
|
|
48
|
+
`id` text PRIMARY KEY NOT NULL,
|
|
49
|
+
`kind` text NOT NULL,
|
|
50
|
+
`target` text NOT NULL,
|
|
51
|
+
`interval_minutes` integer NOT NULL,
|
|
52
|
+
`severity` text DEFAULT 'critical' NOT NULL,
|
|
53
|
+
`suppressible` integer DEFAULT true NOT NULL,
|
|
54
|
+
`enabled` integer DEFAULT true NOT NULL,
|
|
55
|
+
`escalation_policy_id` text,
|
|
56
|
+
`last_run_at` integer,
|
|
57
|
+
`created_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
58
|
+
`updated_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
59
|
+
FOREIGN KEY (`escalation_policy_id`) REFERENCES `escalation_policies`(`id`) ON UPDATE no action ON DELETE set null
|
|
60
|
+
);
|
|
61
|
+
--> statement-breakpoint
|
|
62
|
+
CREATE UNIQUE INDEX `monitors_kind_target_unique` ON `monitors` (`kind`,`target`);--> statement-breakpoint
|
|
63
|
+
CREATE TABLE `monitor_runs` (
|
|
64
|
+
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
|
|
65
|
+
`monitor_id` text NOT NULL,
|
|
66
|
+
`ran_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
67
|
+
`outcome` text NOT NULL,
|
|
68
|
+
`error_message` text,
|
|
69
|
+
FOREIGN KEY (`monitor_id`) REFERENCES `monitors`(`id`) ON UPDATE no action ON DELETE cascade
|
|
70
|
+
);
|
|
71
|
+
--> statement-breakpoint
|
|
72
|
+
CREATE TABLE `suppression_windows` (
|
|
73
|
+
`id` text PRIMARY KEY NOT NULL,
|
|
74
|
+
`source` text NOT NULL,
|
|
75
|
+
`scope_module_id` text NOT NULL,
|
|
76
|
+
`started_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
77
|
+
`ends_at` integer,
|
|
78
|
+
FOREIGN KEY (`scope_module_id`) REFERENCES `modules`(`id`) ON UPDATE no action ON DELETE cascade
|
|
79
|
+
);
|
|
80
|
+
--> statement-breakpoint
|
|
81
|
+
CREATE TABLE `alerts` (
|
|
82
|
+
`id` text PRIMARY KEY NOT NULL,
|
|
83
|
+
`key` text NOT NULL,
|
|
84
|
+
`active_key` text,
|
|
85
|
+
`monitor_id` text NOT NULL,
|
|
86
|
+
`state` text DEFAULT 'pending' NOT NULL,
|
|
87
|
+
`severity` text NOT NULL,
|
|
88
|
+
`first_fired_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
89
|
+
`last_seen_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
90
|
+
`grace_until` integer NOT NULL,
|
|
91
|
+
`suppressed_by_alert_id` text,
|
|
92
|
+
`suppressed_by_window_id` text,
|
|
93
|
+
`unsuppressed_at` integer,
|
|
94
|
+
`awaiting_confirmation` integer DEFAULT false NOT NULL,
|
|
95
|
+
`acked_by` text,
|
|
96
|
+
`acked_at` integer,
|
|
97
|
+
`silenced_until` integer,
|
|
98
|
+
`escalation_step` integer DEFAULT 0 NOT NULL,
|
|
99
|
+
`next_escalation_at` integer,
|
|
100
|
+
`deferred_until` integer,
|
|
101
|
+
`deferred_route_id` text,
|
|
102
|
+
`escalation_policy_id` text,
|
|
103
|
+
`message` text NOT NULL,
|
|
104
|
+
`details` text,
|
|
105
|
+
`resolved_at` integer,
|
|
106
|
+
FOREIGN KEY (`monitor_id`) REFERENCES `monitors`(`id`) ON UPDATE no action ON DELETE cascade,
|
|
107
|
+
FOREIGN KEY (`suppressed_by_window_id`) REFERENCES `suppression_windows`(`id`) ON UPDATE no action ON DELETE set null,
|
|
108
|
+
FOREIGN KEY (`acked_by`) REFERENCES `people`(`id`) ON UPDATE no action ON DELETE set null,
|
|
109
|
+
FOREIGN KEY (`deferred_route_id`) REFERENCES `routes`(`id`) ON UPDATE no action ON DELETE set null,
|
|
110
|
+
FOREIGN KEY (`escalation_policy_id`) REFERENCES `escalation_policies`(`id`) ON UPDATE no action ON DELETE set null
|
|
111
|
+
);
|
|
112
|
+
--> statement-breakpoint
|
|
113
|
+
CREATE UNIQUE INDEX `alerts_live_key_idx` ON `alerts` (`active_key`);--> statement-breakpoint
|
|
114
|
+
CREATE INDEX `alerts_key_idx` ON `alerts` (`key`);--> statement-breakpoint
|
|
115
|
+
CREATE TABLE `notification_deliveries` (
|
|
116
|
+
`id` text PRIMARY KEY NOT NULL,
|
|
117
|
+
`token` text NOT NULL,
|
|
118
|
+
`kind` text NOT NULL,
|
|
119
|
+
`target_id` text NOT NULL,
|
|
120
|
+
`route_id` text NOT NULL,
|
|
121
|
+
`sent_at` integer DEFAULT (unixepoch()) NOT NULL,
|
|
122
|
+
`expires_at` integer NOT NULL,
|
|
123
|
+
`consumed_at` integer,
|
|
124
|
+
FOREIGN KEY (`route_id`) REFERENCES `routes`(`id`) ON UPDATE no action ON DELETE cascade
|
|
125
|
+
);
|
|
126
|
+
--> statement-breakpoint
|
|
127
|
+
CREATE UNIQUE INDEX `notification_deliveries_token_unique` ON `notification_deliveries` (`token`);
|
|
@@ -113,6 +113,20 @@
|
|
|
113
113
|
"when": 1783100000000,
|
|
114
114
|
"tag": "0015_port_forwards",
|
|
115
115
|
"breakpoints": true
|
|
116
|
+
},
|
|
117
|
+
{
|
|
118
|
+
"idx": 16,
|
|
119
|
+
"version": "6",
|
|
120
|
+
"when": 1783200000000,
|
|
121
|
+
"tag": "0016_trusted_sources",
|
|
122
|
+
"breakpoints": true
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
"idx": 17,
|
|
126
|
+
"version": "6",
|
|
127
|
+
"when": 1783300000000,
|
|
128
|
+
"tag": "0017_alerting",
|
|
129
|
+
"breakpoints": true
|
|
116
130
|
}
|
|
117
131
|
]
|
|
118
|
-
}
|
|
132
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@celilo/cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.14.0",
|
|
4
4
|
"description": "Celilo — home lab orchestration CLI",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
"schemas/",
|
|
14
14
|
"tsconfig.json",
|
|
15
15
|
"AGENTS.md",
|
|
16
|
+
"docs/",
|
|
16
17
|
"CELILO_SUBSYSTEMS.md",
|
|
17
18
|
"CELILO_CORE_MODULES.md",
|
|
18
19
|
"MODULE_PRIMITIVES.md"
|
|
@@ -56,7 +57,7 @@
|
|
|
56
57
|
},
|
|
57
58
|
"dependencies": {
|
|
58
59
|
"@aws-sdk/client-s3": "^3.1024.0",
|
|
59
|
-
"@celilo/capabilities": "^0.
|
|
60
|
+
"@celilo/capabilities": "^0.9.0",
|
|
60
61
|
"@celilo/cli-display": "^0.1.9",
|
|
61
62
|
"@celilo/core": "^0.1.0",
|
|
62
63
|
"@celilo/event-bus": "^0.1.8",
|
|
@@ -74,6 +74,11 @@
|
|
|
74
74
|
"maximum": 4094,
|
|
75
75
|
"description": "VLAN tag for internal zone (not defaulted; internal is untagged)"
|
|
76
76
|
},
|
|
77
|
+
"network.vpn.subnet": {
|
|
78
|
+
"type": "string",
|
|
79
|
+
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$",
|
|
80
|
+
"description": "WireGuard VPN client subnet CIDR (remote-access tunnel; consumed by technitium's split-horizon → in-zone caddy view, #275, and by the firewall's trusted sources once the wireguard module registers it)"
|
|
81
|
+
},
|
|
77
82
|
"network.secure-mgmt.subnet": {
|
|
78
83
|
"type": "string",
|
|
79
84
|
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}/\\d{1,2}$",
|
|
@@ -90,6 +95,10 @@
|
|
|
90
95
|
"maximum": 4094,
|
|
91
96
|
"description": "VLAN tag for the control-plane zone (not defaulted)"
|
|
92
97
|
},
|
|
98
|
+
"firewall.trusted_subnets": {
|
|
99
|
+
"type": "string",
|
|
100
|
+
"description": "Operator-declared trusted source subnets (comma-separated CIDRs) permitted to reach every managed zone. ADDITIVE to the control-plane subnet celilo derives and to the sources modules register — it cannot drop control-plane trust, which would block the SSH every hook and converge runs over."
|
|
101
|
+
},
|
|
93
102
|
"dns.primary": {
|
|
94
103
|
"type": "string",
|
|
95
104
|
"pattern": "^\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}$",
|
package/src/ansible/inventory.ts
CHANGED
|
@@ -6,7 +6,7 @@ import type { DbClient } from '../db/client';
|
|
|
6
6
|
import { machines, moduleConfigs, systemConfig } from '../db/schema';
|
|
7
7
|
import { getModuleSystems } from '../services/deployed-systems';
|
|
8
8
|
import { parseStoredConfigValue } from '../services/module-config';
|
|
9
|
-
import { getTempKeyPath } from '../services/ssh-key-manager';
|
|
9
|
+
import { LOCAL_MACHINE_IP, getTempKeyPath } from '../services/ssh-key-manager';
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
12
|
* Inventory generation result
|
|
@@ -395,7 +395,7 @@ export async function generateInventory(
|
|
|
395
395
|
|
|
396
396
|
// Build host definition from machine. The local box (127.0.0.1)
|
|
397
397
|
// uses Ansible's local connection — no SSH key/host.
|
|
398
|
-
const isLocal = machine.ipAddress ===
|
|
398
|
+
const isLocal = machine.ipAddress === LOCAL_MACHINE_IP;
|
|
399
399
|
host = {
|
|
400
400
|
hostname: moduleHostname || machine.hostname,
|
|
401
401
|
ansibleHost: machine.ipAddress,
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `celilo alerts ack|silence|resolve` — the operator's side of the ack loop.
|
|
3
|
+
*
|
|
4
|
+
* Alerts are addressed by KEY, not id. The key is what `celilo alerts list`
|
|
5
|
+
* prints and what arrives in a page, so it is the only identifier an operator
|
|
6
|
+
* ever has in hand.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { getDb } from '../../db/client';
|
|
10
|
+
import { parseIntervalMinutes } from '../../manifest/schema';
|
|
11
|
+
import {
|
|
12
|
+
acknowledgeAlert,
|
|
13
|
+
findLiveAlertByKey,
|
|
14
|
+
resolveAlertManually,
|
|
15
|
+
silenceAlert,
|
|
16
|
+
} from '../../services/alerting/ack';
|
|
17
|
+
import { findPerson, listPeople } from '../../services/alerting/people';
|
|
18
|
+
import type { CommandResult } from '../types';
|
|
19
|
+
|
|
20
|
+
function resolveKey(key: string | undefined) {
|
|
21
|
+
if (!key) return { error: 'An alert key is required. Run "celilo alerts list" to see them.' };
|
|
22
|
+
const db = getDb();
|
|
23
|
+
const alert = findLiveAlertByKey(db, key);
|
|
24
|
+
if (!alert) {
|
|
25
|
+
return {
|
|
26
|
+
error: `No live alert with key "${key}".\n\nRun "celilo alerts list" to see current alerts.`,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
return { db, alert };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export async function handleAlertsAck(
|
|
33
|
+
args: string[],
|
|
34
|
+
flags: Record<string, boolean | string> = {},
|
|
35
|
+
): Promise<CommandResult> {
|
|
36
|
+
const found = resolveKey(args[0]);
|
|
37
|
+
if (found.error !== undefined) return { success: false, error: found.error };
|
|
38
|
+
const { db, alert } = found;
|
|
39
|
+
|
|
40
|
+
// Who is acknowledging matters: it stops escalation AND tells everyone else
|
|
41
|
+
// who was paged that somebody has it.
|
|
42
|
+
const who = typeof flags.as === 'string' ? flags.as : undefined;
|
|
43
|
+
const people = listPeople(db);
|
|
44
|
+
if (!who) {
|
|
45
|
+
if (people.length !== 1) {
|
|
46
|
+
return {
|
|
47
|
+
success: false,
|
|
48
|
+
error: `Who is acknowledging? Pass --as <person>.\n\nKnown people: ${people.map((p) => p.name).join(', ') || '(none configured)'}`,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
const person = who ? findPerson(db, who) : people[0];
|
|
53
|
+
if (!person) return { success: false, error: `No person named "${who}".` };
|
|
54
|
+
|
|
55
|
+
const result = acknowledgeAlert(db, alert.id, person.id, new Date());
|
|
56
|
+
if (!result) return { success: false, error: 'Alert vanished while acknowledging.' };
|
|
57
|
+
|
|
58
|
+
const broadcast = result.broadcastTo.length
|
|
59
|
+
? ` — ${result.broadcastTo.length} other route(s) will be told`
|
|
60
|
+
: '';
|
|
61
|
+
return { success: true, message: `${alert.key} acknowledged by ${person.name}${broadcast}` };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export async function handleAlertsSilence(
|
|
65
|
+
args: string[],
|
|
66
|
+
flags: Record<string, boolean | string> = {},
|
|
67
|
+
): Promise<CommandResult> {
|
|
68
|
+
const found = resolveKey(args[0]);
|
|
69
|
+
if (found.error !== undefined) return { success: false, error: found.error };
|
|
70
|
+
const { db, alert } = found;
|
|
71
|
+
|
|
72
|
+
const forFlag = typeof flags.for === 'string' ? flags.for : '2h';
|
|
73
|
+
const minutes = parseIntervalMinutes(forFlag);
|
|
74
|
+
if (minutes === null) {
|
|
75
|
+
return {
|
|
76
|
+
success: false,
|
|
77
|
+
error: `Invalid --for "${forFlag}". Use a duration like 2h, 30m, 1d.`,
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const until = new Date(Date.now() + minutes * 60_000);
|
|
82
|
+
silenceAlert(db, alert.id, until);
|
|
83
|
+
return {
|
|
84
|
+
success: true,
|
|
85
|
+
message: `${alert.key} silenced for ${forFlag} (until ${until.toISOString()})`,
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export async function handleAlertsResolve(args: string[]): Promise<CommandResult> {
|
|
90
|
+
const found = resolveKey(args[0]);
|
|
91
|
+
if (found.error !== undefined) return { success: false, error: found.error };
|
|
92
|
+
const { db, alert } = found;
|
|
93
|
+
|
|
94
|
+
resolveAlertManually(db, alert.id, new Date());
|
|
95
|
+
return {
|
|
96
|
+
success: true,
|
|
97
|
+
// Resolve does not override the monitor, and saying so is the whole point
|
|
98
|
+
// of the message — an operator who thinks it does will trust a cleared
|
|
99
|
+
// alert that is about to come back.
|
|
100
|
+
//
|
|
101
|
+
// What it IS for: you have just fixed something and the check runs every
|
|
102
|
+
// fifteen minutes. This stops the stale `firing` row escalating at you
|
|
103
|
+
// while you wait for the next run to agree.
|
|
104
|
+
message: `${alert.key} resolved.
|
|
105
|
+
The monitor is still the authority: if the check still fails, this fires again on the next sweep.`,
|
|
106
|
+
};
|
|
107
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `celilo alerts list` — what is currently wrong, and what is already accounted for.
|
|
3
|
+
*
|
|
4
|
+
* Thin adapter (Rule 10.5): read, delegate, format, return. The ordering,
|
|
5
|
+
* wording, and the "explained by something upstream" column all live in
|
|
6
|
+
* services/alerting/format.ts where they are unit-tested.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { getDb } from '../../db/client';
|
|
10
|
+
import { renderAlertTable, toAlertRow } from '../../services/alerting/format';
|
|
11
|
+
import { loadAllLiveAlerts } from '../../services/alerting/store';
|
|
12
|
+
import type { CommandResult } from '../types';
|
|
13
|
+
|
|
14
|
+
export async function handleAlertsList(
|
|
15
|
+
_args: string[],
|
|
16
|
+
flags: Record<string, boolean | string> = {},
|
|
17
|
+
): Promise<CommandResult> {
|
|
18
|
+
const db = getDb();
|
|
19
|
+
const now = new Date();
|
|
20
|
+
const live = loadAllLiveAlerts(db);
|
|
21
|
+
|
|
22
|
+
if (flags.json) {
|
|
23
|
+
return {
|
|
24
|
+
success: true,
|
|
25
|
+
message: JSON.stringify(
|
|
26
|
+
live.map((alert) => ({
|
|
27
|
+
key: alert.key,
|
|
28
|
+
state: alert.state,
|
|
29
|
+
severity: alert.severity,
|
|
30
|
+
firstFiredAt: alert.firstFiredAt,
|
|
31
|
+
lastSeenAt: alert.lastSeenAt,
|
|
32
|
+
suppressed: alert.state === 'suppressed',
|
|
33
|
+
suppressedByAlertId: alert.suppressedByAlertId,
|
|
34
|
+
suppressedByWindowId: alert.suppressedByWindowId,
|
|
35
|
+
awaitingConfirmation: alert.awaitingConfirmation,
|
|
36
|
+
ackedBy: alert.ackedBy,
|
|
37
|
+
silencedUntil: alert.silencedUntil,
|
|
38
|
+
message: alert.message,
|
|
39
|
+
})),
|
|
40
|
+
null,
|
|
41
|
+
2,
|
|
42
|
+
),
|
|
43
|
+
rawOutput: true,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const table = renderAlertTable(live.map((alert) => toAlertRow(alert, now)));
|
|
48
|
+
console.log('');
|
|
49
|
+
console.log(table);
|
|
50
|
+
console.log('');
|
|
51
|
+
|
|
52
|
+
const firing = live.filter((a) => a.state === 'firing').length;
|
|
53
|
+
const suppressed = live.filter((a) => a.state === 'suppressed').length;
|
|
54
|
+
|
|
55
|
+
return {
|
|
56
|
+
success: true,
|
|
57
|
+
message:
|
|
58
|
+
live.length === 0
|
|
59
|
+
? 'No alerts'
|
|
60
|
+
: `${firing} firing, ${suppressed} suppressed, ${live.length} live`,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `celilo alerts poll` — read replies off every bidirectional transport once.
|
|
3
|
+
*
|
|
4
|
+
* Runs far more often than the five-minute sweep: "I texted ack and nothing
|
|
5
|
+
* happened" is the experience this exists to prevent. Registered on
|
|
6
|
+
* `timer.tick.5m` like the sweep, but also safe to run by hand or from a
|
|
7
|
+
* tighter loop.
|
|
8
|
+
*
|
|
9
|
+
* The receive cursor is persisted in system config rather than memory, so a
|
|
10
|
+
* restart resumes where it left off instead of re-reading or skipping.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { eq } from 'drizzle-orm';
|
|
14
|
+
import { getEventBusPath } from '../../config/paths';
|
|
15
|
+
import { getDb } from '../../db/client';
|
|
16
|
+
import { systemConfig } from '../../db/schema';
|
|
17
|
+
import { makeReceiver, pollInbound } from '../../services/alerting/inbound-poller';
|
|
18
|
+
import { startNotificationResponder } from '../../services/alerting/notification-responder';
|
|
19
|
+
import { listRoutes } from '../../services/alerting/people';
|
|
20
|
+
import { loadNotificationTransport } from '../../services/alerting/transport-loader';
|
|
21
|
+
import type { CommandResult } from '../types';
|
|
22
|
+
|
|
23
|
+
/** How long a reply token stays usable. A deploy may wait overnight. */
|
|
24
|
+
const INTERVIEW_TTL_MS = 24 * 60 * 60_000;
|
|
25
|
+
|
|
26
|
+
const CURSOR_PREFIX = 'alerting.inbound_cursor.';
|
|
27
|
+
|
|
28
|
+
function readCursor(db: ReturnType<typeof getDb>, transportModuleId: string): string | null {
|
|
29
|
+
const key = `${CURSOR_PREFIX}${transportModuleId}`;
|
|
30
|
+
const row = db.select().from(systemConfig).where(eq(systemConfig.key, key)).get();
|
|
31
|
+
return row?.value ?? null;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function writeCursor(
|
|
35
|
+
db: ReturnType<typeof getDb>,
|
|
36
|
+
transportModuleId: string,
|
|
37
|
+
cursor: string | null,
|
|
38
|
+
): void {
|
|
39
|
+
const key = `${CURSOR_PREFIX}${transportModuleId}`;
|
|
40
|
+
if (cursor === null) return;
|
|
41
|
+
const existing = db.select().from(systemConfig).where(eq(systemConfig.key, key)).get();
|
|
42
|
+
if (existing) {
|
|
43
|
+
db.update(systemConfig).set({ value: cursor }).where(eq(systemConfig.key, key)).run();
|
|
44
|
+
} else {
|
|
45
|
+
db.insert(systemConfig)
|
|
46
|
+
.values({
|
|
47
|
+
key,
|
|
48
|
+
value: cursor,
|
|
49
|
+
description: `Inbound receive cursor for ${transportModuleId}`,
|
|
50
|
+
})
|
|
51
|
+
.run();
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export async function handleAlertsPoll(
|
|
56
|
+
flags: Record<string, string | boolean> = {},
|
|
57
|
+
): Promise<CommandResult> {
|
|
58
|
+
const db = getDb();
|
|
59
|
+
const askErrors: string[] = [];
|
|
60
|
+
|
|
61
|
+
// `--ask-after 0` asks immediately. Useful for an operator who wants a
|
|
62
|
+
// question escalated now, and for tests that would otherwise wait out the
|
|
63
|
+
// grace.
|
|
64
|
+
const raw = flags['ask-after'];
|
|
65
|
+
const askAfterSeconds = typeof raw === 'string' ? Number.parseInt(raw, 10) : null;
|
|
66
|
+
if (raw !== undefined && (askAfterSeconds === null || Number.isNaN(askAfterSeconds))) {
|
|
67
|
+
return { success: false, error: '--ask-after expects a number of seconds' };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// The responder both DELIVERS unanswered questions and publishes the
|
|
71
|
+
// answers. Both halves belong here: a question delivered by a process that
|
|
72
|
+
// then exits is still answerable, because what was asked lives in the
|
|
73
|
+
// deliveries table rather than in that process's memory.
|
|
74
|
+
const responder = startNotificationResponder({
|
|
75
|
+
db,
|
|
76
|
+
busDbPath: getEventBusPath(),
|
|
77
|
+
routes: listRoutes(db).filter((route) => route.enabled && route.canAck),
|
|
78
|
+
transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
|
|
79
|
+
// A CLI invocation is never the terminal responder for someone else's
|
|
80
|
+
// deploy — that responder attaches to its own deploy's stdin.
|
|
81
|
+
hasTty: () => false,
|
|
82
|
+
now: () => new Date(),
|
|
83
|
+
ttlMs: INTERVIEW_TTL_MS,
|
|
84
|
+
askAfterMs: askAfterSeconds === null ? undefined : askAfterSeconds * 1000,
|
|
85
|
+
onSendFailed: (eventType, error) => askErrors.push(`${eventType}: ${error}`),
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
let ask = { asked: 0, failed: 0 };
|
|
89
|
+
try {
|
|
90
|
+
ask = await responder.poll();
|
|
91
|
+
} catch {
|
|
92
|
+
// A bus that cannot be opened must not stop alert replies being read —
|
|
93
|
+
// those are the 3am path.
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const report = await pollInbound(db, {
|
|
97
|
+
receiveFrom: makeReceiver(db),
|
|
98
|
+
readCursor: (t) => readCursor(db, t),
|
|
99
|
+
writeCursor: (t, c) => writeCursor(db, t, c),
|
|
100
|
+
now: () => new Date(),
|
|
101
|
+
transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
|
|
102
|
+
answerInterview: (eventId, value) => responder.answer(eventId, value),
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
responder.stop();
|
|
106
|
+
|
|
107
|
+
// Rejections are worth surfacing rather than counting silently: a run of
|
|
108
|
+
// them means someone is replying and not being heard, which looks identical
|
|
109
|
+
// to the alerting system being broken.
|
|
110
|
+
const parts = [
|
|
111
|
+
`${report.transportsPolled} transport(s)`,
|
|
112
|
+
`${report.messagesRead} message(s)`,
|
|
113
|
+
`${report.acked} acked`,
|
|
114
|
+
];
|
|
115
|
+
if (report.rejected > 0) parts.push(`${report.rejected} rejected`);
|
|
116
|
+
if (report.ignored > 0) parts.push(`${report.ignored} from unknown senders`);
|
|
117
|
+
if (report.broadcast > 0) parts.push(`${report.broadcast} told someone has it`);
|
|
118
|
+
if (report.answered > 0) parts.push(`${report.answered} deploy question(s) answered`);
|
|
119
|
+
if (ask.asked > 0) parts.push(`${ask.asked} question(s) asked`);
|
|
120
|
+
// Never silent: a deploy blocked on a question nobody received is
|
|
121
|
+
// indistinguishable from a deploy that is merely slow.
|
|
122
|
+
if (ask.failed > 0) parts.push(`${ask.failed} question(s) COULD NOT BE DELIVERED`);
|
|
123
|
+
|
|
124
|
+
const message = `inbound poll: ${parts.join(', ')}`;
|
|
125
|
+
return {
|
|
126
|
+
success: true,
|
|
127
|
+
message: askErrors.length > 0 ? `${message}\n ${askErrors.join('\n ')}` : message,
|
|
128
|
+
};
|
|
129
|
+
}
|
|
@@ -0,0 +1,156 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `celilo alerts sweep` — one pass of the alerting loop.
|
|
3
|
+
*
|
|
4
|
+
* Invoked by the event-bus dispatcher on `timer.tick.5m`, not by a human. It
|
|
5
|
+
* is a normal command so the dispatcher's existing subprocess isolation,
|
|
6
|
+
* retry, and timeout apply unchanged — the sweep gets those for free rather
|
|
7
|
+
* than reimplementing them.
|
|
8
|
+
*
|
|
9
|
+
* Thin adapter (Rule 10.5): compose the real dependencies, call runSweep,
|
|
10
|
+
* report counts.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { eq } from 'drizzle-orm';
|
|
14
|
+
import { getDb } from '../../db/client';
|
|
15
|
+
import { type Alert, escalationPolicies, modules } from '../../db/schema';
|
|
16
|
+
import { runBuiltinCheckForMonitor } from '../../services/alerting/builtin-source';
|
|
17
|
+
import { loadModuleCoverage } from '../../services/alerting/coverage-source';
|
|
18
|
+
import { modulesInDeployWindow } from '../../services/alerting/deploy-hooks';
|
|
19
|
+
import { listMonitors } from '../../services/alerting/monitors';
|
|
20
|
+
import type { NotifyDeps } from '../../services/alerting/notifier';
|
|
21
|
+
import { listPeople, listPolicySteps, listRoutes } from '../../services/alerting/people';
|
|
22
|
+
import type { SuppressionTopology } from '../../services/alerting/suppression';
|
|
23
|
+
import { runSweep } from '../../services/alerting/sweep-runner';
|
|
24
|
+
import { mintDelivery } from '../../services/alerting/tokens';
|
|
25
|
+
import { loadNotificationTransport } from '../../services/alerting/transport-loader';
|
|
26
|
+
import { getModuleSystems } from '../../services/deployed-systems';
|
|
27
|
+
import { runModuleHealthCheck } from '../../services/health-runner';
|
|
28
|
+
import type { CommandResult } from '../types';
|
|
29
|
+
|
|
30
|
+
/** Grace window before a newly-fired alert may notify. */
|
|
31
|
+
const DEFAULT_GRACE_MS = 60_000;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* How long a reply token stays valid. A day is long enough that someone who
|
|
35
|
+
* sees a page overnight can still act on it in the morning, and short enough
|
|
36
|
+
* that a token found later is useless.
|
|
37
|
+
*/
|
|
38
|
+
const TOKEN_TTL_MS = 24 * 60 * 60_000;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Deployment topology for suppression, read once per sweep.
|
|
42
|
+
*
|
|
43
|
+
* Derived from what celilo already records — asking an operator to restate it
|
|
44
|
+
* would guarantee it drifts, and a stale suppression graph hides real
|
|
45
|
+
* failures rather than merely being untidy.
|
|
46
|
+
*/
|
|
47
|
+
function loadTopology(db: ReturnType<typeof getDb>): SuppressionTopology {
|
|
48
|
+
const moduleRows = db.select({ id: modules.id }).from(modules).all();
|
|
49
|
+
const moduleSystems = moduleRows.flatMap((module) =>
|
|
50
|
+
getModuleSystems(module.id, db).map((system) => ({
|
|
51
|
+
moduleId: module.id,
|
|
52
|
+
hostname: system.hostname,
|
|
53
|
+
zone: system.zone,
|
|
54
|
+
infraType: system.infrastructure.type,
|
|
55
|
+
})),
|
|
56
|
+
);
|
|
57
|
+
// Zone-capability providers are wired in a follow-up; an empty list means
|
|
58
|
+
// only the machine edge is active, which is the MVP scope.
|
|
59
|
+
return { moduleSystems, zoneProviders: [] };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Assemble everything needed to page for one alert, or null when nobody can be.
|
|
64
|
+
*
|
|
65
|
+
* Returning null rather than an empty policy is deliberate: "no escalation
|
|
66
|
+
* policy assigned" and "policy exists but nobody is eligible" are different
|
|
67
|
+
* situations, and only the second is worth an escalation decision.
|
|
68
|
+
*/
|
|
69
|
+
function buildNotifyDeps(db: ReturnType<typeof getDb>, alert: Alert): NotifyDeps | null {
|
|
70
|
+
if (!alert.escalationPolicyId) return null;
|
|
71
|
+
|
|
72
|
+
const policy = db
|
|
73
|
+
.select()
|
|
74
|
+
.from(escalationPolicies)
|
|
75
|
+
.where(eq(escalationPolicies.id, alert.escalationPolicyId))
|
|
76
|
+
.get();
|
|
77
|
+
if (!policy) return null;
|
|
78
|
+
|
|
79
|
+
const steps = listPolicySteps(db, policy.id).map((step) => ({
|
|
80
|
+
stepIndex: step.stepIndex,
|
|
81
|
+
routeId: step.routeId,
|
|
82
|
+
delayMinutes: step.delayMinutes,
|
|
83
|
+
}));
|
|
84
|
+
if (steps.length === 0) return null;
|
|
85
|
+
|
|
86
|
+
const routeRows = listRoutes(db);
|
|
87
|
+
const people = listPeople(db);
|
|
88
|
+
|
|
89
|
+
return {
|
|
90
|
+
steps,
|
|
91
|
+
routes: new Map(
|
|
92
|
+
routeRows.map((r) => [
|
|
93
|
+
r.id,
|
|
94
|
+
{ id: r.id, severityFloor: r.severityFloor, enabled: r.enabled },
|
|
95
|
+
]),
|
|
96
|
+
),
|
|
97
|
+
routeDetails: new Map(routeRows.map((r) => [r.id, r])),
|
|
98
|
+
quietHoursByPerson: new Map(
|
|
99
|
+
people
|
|
100
|
+
.filter((p) => p.quietHoursStart && p.quietHoursEnd)
|
|
101
|
+
.map((p) => [
|
|
102
|
+
p.id,
|
|
103
|
+
{
|
|
104
|
+
personId: p.id,
|
|
105
|
+
start: p.quietHoursStart,
|
|
106
|
+
end: p.quietHoursEnd,
|
|
107
|
+
timezone: p.timezone,
|
|
108
|
+
},
|
|
109
|
+
]),
|
|
110
|
+
),
|
|
111
|
+
bypassQuietHours: policy.bypassQuietHours,
|
|
112
|
+
transportFor: (route) => loadNotificationTransport(db, route.transportModuleId),
|
|
113
|
+
mintToken: (alertId, routeId) =>
|
|
114
|
+
mintDelivery(db, {
|
|
115
|
+
kind: 'alert',
|
|
116
|
+
targetId: alertId,
|
|
117
|
+
routeId,
|
|
118
|
+
now: new Date(),
|
|
119
|
+
ttlMs: TOKEN_TTL_MS,
|
|
120
|
+
}).token,
|
|
121
|
+
now: new Date(),
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export async function handleAlertsSweep(): Promise<CommandResult> {
|
|
126
|
+
const db = getDb();
|
|
127
|
+
const monitorRows = listMonitors(db);
|
|
128
|
+
|
|
129
|
+
const suppressibleByMonitor = new Map(monitorRows.map((m) => [m.id, m.suppressible]));
|
|
130
|
+
|
|
131
|
+
const report = await runSweep(db, monitorRows, {
|
|
132
|
+
monitorDeps: {
|
|
133
|
+
runModuleCheck: (moduleId: string) =>
|
|
134
|
+
runModuleHealthCheck(moduleId, db, { unattended: true, noInteractive: true }),
|
|
135
|
+
runBuiltinCheck: (category) => runBuiltinCheckForMonitor(category),
|
|
136
|
+
loadModuleCoverage: () => loadModuleCoverage(db),
|
|
137
|
+
now: () => new Date(),
|
|
138
|
+
graceMs: DEFAULT_GRACE_MS,
|
|
139
|
+
},
|
|
140
|
+
loadTopology: () => loadTopology(db),
|
|
141
|
+
loadDeployWindowModules: () => modulesInDeployWindow(db),
|
|
142
|
+
isSuppressible: (alert) => suppressibleByMonitor.get(alert.monitorId) ?? true,
|
|
143
|
+
notifyDepsFor: (alert) => buildNotifyDeps(db, alert),
|
|
144
|
+
now: () => new Date(),
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
const parts = [
|
|
148
|
+
`${report.monitorsRun} run`,
|
|
149
|
+
`${report.monitorsErrored} errored`,
|
|
150
|
+
`${report.promoted} promoted`,
|
|
151
|
+
`${report.suppressed} suppressed`,
|
|
152
|
+
`${report.unsuppressed} unsuppressed`,
|
|
153
|
+
`${report.notified} notified`,
|
|
154
|
+
];
|
|
155
|
+
return { success: true, message: `alert sweep: ${parts.join(', ')}` };
|
|
156
|
+
}
|