@astrofoundry/pi-astro 0.38.1 → 0.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -34,8 +34,11 @@ export interface DiscordConfig {
|
|
|
34
34
|
* Reports the bridge can post on a schedule, without a model. `security` is the
|
|
35
35
|
* daily report and also turns on the urgent-alert check in its channel;
|
|
36
36
|
* `security-weekly` posts on Mondays only, in place of that day's daily report.
|
|
37
|
+
* `daily` is the morning run: at its time it posts the other reports in their
|
|
38
|
+
* own channels (their `time` is then unused), applies the updates, reads every
|
|
39
|
+
* specialist's status, and posts the summary in its channel.
|
|
37
40
|
*/
|
|
38
|
-
export const REPORT_NAMES = ["arcane", "tools", "security", "security-weekly"] as const;
|
|
41
|
+
export const REPORT_NAMES = ["arcane", "tools", "security", "security-weekly", "daily"] as const;
|
|
39
42
|
export type ReportName = (typeof REPORT_NAMES)[number];
|
|
40
43
|
|
|
41
44
|
/** One entry of `reports` in config.json: which report, where, and when. */
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import type { ArcaneReport } from "./daily-report.ts";
|
|
3
|
+
import { headlineOf } from "./security-monitor.ts";
|
|
4
|
+
import type { ToolsReport } from "./tools-report.ts";
|
|
5
|
+
import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, problemText, proxmoxStatus, summaryText, toolsSummary } from "./daily-run.ts";
|
|
6
|
+
|
|
7
|
+
const EDGE = `nginx active
|
|
8
|
+
wg-quick@wg0 active
|
|
9
|
+
crowdsec-firewall-bouncer active
|
|
10
|
+
wazuh-agent active
|
|
11
|
+
rsyslog active
|
|
12
|
+
wireguard handshake 21s ago
|
|
13
|
+
reboot required no
|
|
14
|
+
kernel 6.12.113+deb13-cloud-amd64
|
|
15
|
+
os Debian GNU/Linux 13 (trixie)
|
|
16
|
+
uptime up 3 days, 16 hours, 55 minutes`;
|
|
17
|
+
|
|
18
|
+
const UPDATES = `Listing...
|
|
19
|
+
google-cloud-cli/cloud-sdk-trixie 590.0.0-0 amd64 [upgradable from: 580.0.0-0]
|
|
20
|
+
tzdata/stable-updates 2026e-0+deb13u1 all [upgradable from: 2026c-0+deb13u1]
|
|
21
|
+
wazuh-agent/stable 4.14.9-1 amd64 [upgradable from: 4.14.7-1]`;
|
|
22
|
+
|
|
23
|
+
const PULSAR = `pve-manager/9.2.10/43df2e01f27a1a19 (running kernel: 7.0.14-12-pve)
|
|
24
|
+
16:24:19 up 59 days, 22:05, 13 users, load average: 0.27, 0.42, 0.51
|
|
25
|
+
total used free shared buff/cache available
|
|
26
|
+
Mem: 60Gi 47Gi 11Gi 41Mi 2.0Gi 13Gi
|
|
27
|
+
Swap: 8.0Gi 3.0Gi 5.0Gi
|
|
28
|
+
Filesystem Size Used Avail Use% Mounted on
|
|
29
|
+
/dev/mapper/pve-root 94G 12G 78G 13% /`;
|
|
30
|
+
|
|
31
|
+
const PULSAR_UPDATES = `Listing...
|
|
32
|
+
base-files/stable 13.8+deb13u7 amd64 [upgradable from: 13.8+deb13u6]
|
|
33
|
+
curl/stable-security 8.14.1-2+deb13u7 amd64 [upgradable from: 8.14.1-2+deb13u4]`;
|
|
34
|
+
|
|
35
|
+
const RADIUS = `● freeradius.service - FreeRADIUS multi-protocol policy server
|
|
36
|
+
Loaded: loaded (/usr/lib/systemd/system/freeradius.service; enabled; preset: enabled)
|
|
37
|
+
Active: active (running) since Mon 2026-10-19 10:39:07 UTC; 4 days ago`;
|
|
38
|
+
|
|
39
|
+
const TAILSCALE = `100.121.19.2 tailscale-37 astronaute@ linux -
|
|
40
|
+
100.72.6.108 cortex astronaute@ macOS offline, last seen 67d ago`;
|
|
41
|
+
|
|
42
|
+
const DMZ = `● pomerium.service - Pomerium identity-aware proxy
|
|
43
|
+
Loaded: loaded (/etc/systemd/system/pomerium.service; enabled; preset: enabled)
|
|
44
|
+
Active: active (running) since Thu 2026-09-24 17:21:24 CEST; 4 weeks 0 days ago
|
|
45
|
+
● nginx.service - A high performance web server
|
|
46
|
+
Active: active (running) since Thu 2026-09-24 17:21:24 CEST; 4 weeks 0 days ago
|
|
47
|
+
● lego-renew.timer - Renew certificates
|
|
48
|
+
Active: active (waiting) since Thu 2026-09-24 17:21:24 CEST; 4 weeks 0 days ago`;
|
|
49
|
+
|
|
50
|
+
const NOW = Date.parse("2026-10-24T03:00:00Z");
|
|
51
|
+
|
|
52
|
+
describe("morning status lines", () => {
|
|
53
|
+
it("reads handshake ages", () => {
|
|
54
|
+
expect(agoSeconds("21s ago")).toBe(21);
|
|
55
|
+
expect(agoSeconds("3 minutes ago")).toBe(180);
|
|
56
|
+
expect(agoSeconds("never")).toBeNull();
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it("summarises the front door and flags a down unit or a stale tunnel", () => {
|
|
60
|
+
expect(edgeStatus(EDGE, UPDATES)).toEqual({ area: "Edge", line: "front door: 5 services active, WireGuard 21s ago, no reboot pending, 3 upgradable packages" });
|
|
61
|
+
expect(edgeStatus(EDGE.replace("nginx active", "nginx failed"), UPDATES).problem).toBe("Edge: nginx not active on the front door");
|
|
62
|
+
expect(edgeStatus(EDGE.replace("21s ago", "9m ago"), null)).toMatchObject({ problem: "Edge: the WireGuard handshake is 9m ago", route: "admin" });
|
|
63
|
+
expect(edgeStatus(null, null)).toMatchObject({ line: "the check failed (vps status)", problem: "Edge: the status check failed" });
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it("reports Pulsar and its waiting host packages without flagging them", () => {
|
|
67
|
+
expect(proxmoxStatus(PULSAR, PULSAR_UPDATES)).toEqual({ area: "Proxmox", line: "Pulsar up 59 days, RAM 47Gi of 60Gi used, root disk 13%, 2 host package updates waiting (1 security), applied by hand" });
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
it("checks RADIUS and the Tailscale router", () => {
|
|
71
|
+
expect(networkStatus(RADIUS, TAILSCALE)).toEqual({ area: "Network", line: "RADIUS running, Tailscale router tailscale-37 online" });
|
|
72
|
+
expect(networkStatus(RADIUS.replace("active (running)", "failed (Result: exit-code)"), "100.121.19.2 tailscale-37 astronaute@ linux offline, last seen 1h ago")).toMatchObject({ problem: "Network: FreeRADIUS is not running; the Tailscale subnet router is offline" });
|
|
73
|
+
expect(networkStatus(null, TAILSCALE).problem).toBe("Network: a network status check failed");
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it("lists the DMZ units and flags one that is not active", () => {
|
|
77
|
+
expect(identityStatus(DMZ)).toEqual({ area: "Identity", line: "pomerium, nginx, lego-renew.timer active" });
|
|
78
|
+
expect(identityStatus(DMZ.replace("Active: active (running) since Thu 2026-09-24 17:21:24 CEST; 4 weeks 0 days ago\n● nginx", "Active: failed (Result: signal)\n● nginx")).problem).toBe("Identity: pomerium.service failed on the DMZ VM");
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it("checks both resolvers, Nexus, and SABnzbd", () => {
|
|
82
|
+
expect(dnsStatus({ status: "ok" }, { status: "ok" })).toEqual({ area: "DNS", line: "both resolvers answer" });
|
|
83
|
+
expect(dnsStatus({ status: "ok" }, null)).toMatchObject({ problem: "DNS: the secondary resolver does not answer" });
|
|
84
|
+
expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=200\nrerank \tstate = running \tpid = 81670 health=200")).toEqual({ area: "Inference", line: "Nexus embed, rerank running and healthy" });
|
|
85
|
+
expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=503")).toMatchObject({ problem: "Inference: embed not healthy on Nexus" });
|
|
86
|
+
expect(mediaStatus({ sabnzbd_paused: true, warnings: [] })).toEqual({ area: "Media", line: "SABnzbd paused" });
|
|
87
|
+
expect(mediaStatus({ sabnzbd_paused: false, warnings: [{ text: "login failed" }] })).toMatchObject({ line: "SABnzbd downloading, 1 warning", route: "media" });
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it("checks last night's Proxmox archives and the restic check", () => {
|
|
91
|
+
const archives = [
|
|
92
|
+
{ vmid: 100, ctime: NOW / 1000 - 3 * 3600, notes: "arcane" },
|
|
93
|
+
{ vmid: 103, ctime: NOW / 1000 - 2 * 86_400, notes: "identity" },
|
|
94
|
+
];
|
|
95
|
+
const restic = "Result=success\nExecMainStartTimestamp=Sun 2026-10-18 04:00:24 CEST";
|
|
96
|
+
expect(backupStatus(archives.slice(0, 1), restic, NOW)).toEqual({ area: "Backup", line: "the only guest backed up last night, restic check success 18 Oct 04:00" });
|
|
97
|
+
expect(backupStatus(archives, restic.replace("success", "exit-code"), NOW)).toMatchObject({ problem: "Backup: no Proxmox backup last night for identity; the restic check ended with exit-code", route: "admin" });
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
describe("morning summary", () => {
|
|
102
|
+
it("lists each report and area, then what needs the owners, without a mention", () => {
|
|
103
|
+
const text = summaryText({
|
|
104
|
+
date: "Fri 24 Oct",
|
|
105
|
+
guildId: "1",
|
|
106
|
+
routes: { admin: "11", media: "12" },
|
|
107
|
+
reports: [
|
|
108
|
+
{ area: "Security", line: "quiet day", needs: [], channelId: "21", messageId: "31" },
|
|
109
|
+
{ area: "Arcane", line: "3 stacks updated", needs: ["stirling-pdf major version"], channelId: "22", messageId: "32" },
|
|
110
|
+
],
|
|
111
|
+
areas: [
|
|
112
|
+
{ area: "DNS", line: "both resolvers answer" },
|
|
113
|
+
{ area: "Backup", line: "no backup last night for identity", problem: "Backup: no Proxmox backup last night for identity", route: "admin" },
|
|
114
|
+
],
|
|
115
|
+
});
|
|
116
|
+
expect(text).toBe(
|
|
117
|
+
[
|
|
118
|
+
"**Daily** · Fri 24 Oct · 🟠 2 items need you",
|
|
119
|
+
"Security · quiet day · [report](https://discord.com/channels/1/21/31)",
|
|
120
|
+
"Arcane · 3 stacks updated · [report](https://discord.com/channels/1/22/32)",
|
|
121
|
+
"DNS · both resolvers answer",
|
|
122
|
+
"Backup · no backup last night for identity",
|
|
123
|
+
"Needs you · 1) stirling-pdf major version (<#22>) 2) Backup: no Proxmox backup last night for identity (<#11>)",
|
|
124
|
+
].join("\n"),
|
|
125
|
+
);
|
|
126
|
+
expect(text).not.toContain("<@");
|
|
127
|
+
expect(summaryText({ date: "Fri 24 Oct", guildId: "1", routes: { admin: "11", media: "12" }, reports: [], areas: [] })).toBe("**Daily** · Fri 24 Oct · 🟢 nothing needs you");
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it("tags the owners for an area's problem in its channel", () => {
|
|
131
|
+
expect(problemText({ area: "Backup", line: "x", problem: "Backup: the restic check ended with exit-code", route: "admin" }, ["178096425509453824"])).toBe(
|
|
132
|
+
"⚠️ **Backup** · Backup: the restic check ended with exit-code\nFound by the morning run; the backup specialist can look into it here.\n<@178096425509453824>",
|
|
133
|
+
);
|
|
134
|
+
});
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
describe("report summaries", () => {
|
|
138
|
+
const tool = (over: Partial<ToolsReport["lines"][number]>): ToolsReport["lines"][number] => ({ key: "brew:jq", label: "jq", installed: "1.8.1", latest: "1.8.2", major: false, action: { kind: "brew", name: "jq" }, state: "offered", ...over }) as ToolsReport["lines"][number];
|
|
139
|
+
|
|
140
|
+
it("says what the tools report updated, what failed, and what waits for a button", () => {
|
|
141
|
+
const report = { date: "2026-10-24", lines: [tool({ state: "updated", plan: "auto" }), tool({ key: "brew:herdr", label: "herdr", latest: "0.10.0", state: "failed", plan: "auto" }), tool({ key: "pi", label: "Pi", latest: "1.4.0", plan: "button" }), tool({ key: "spc:node", label: "Node", latest: "26.1.0", major: true, plan: "update" })], messages: [{ id: "9", keys: [], head: true, all: false }] } as unknown as ToolsReport;
|
|
142
|
+
expect(toolsSummary(report, "2026-10-24", "5")).toEqual({
|
|
143
|
+
area: "Cortex tools",
|
|
144
|
+
line: "1 updated (jq 1.8.2), 1 failed (herdr), Pi 1.4.0, Node 26.1.0 wait for a button",
|
|
145
|
+
needs: ["Node 26.1.0 (major version)", "herdr automatic update failed"],
|
|
146
|
+
channelId: "5",
|
|
147
|
+
messageId: "9",
|
|
148
|
+
});
|
|
149
|
+
expect(toolsSummary(report, "2026-10-25", "5").line).toBe("no report today; the bridge log has the reason");
|
|
150
|
+
});
|
|
151
|
+
|
|
152
|
+
it("says what the Arcane report updated, rolled back, and leaves to the owner", () => {
|
|
153
|
+
const stack = (name: string, over: Partial<ArcaneReport["stacks"][number]>) => ({ stack: name, lines: [], actionable: true, state: "offered", ...over }) as ArcaneReport["stacks"][number];
|
|
154
|
+
const report = { date: "2026-10-24", stacks: [stack("media", { plan: "auto", state: "updated" }), stack("mealie", { plan: "auto", state: "rolled back" }), stack("stirling-pdf", { plan: "careful" }), stack("supabase", { actionable: false, held: "review first" })], messages: [{ id: "8", stacks: [], head: true }] } as unknown as ArcaneReport;
|
|
155
|
+
expect(arcaneSummary(report, "2026-10-24", "6")).toEqual({
|
|
156
|
+
area: "Arcane",
|
|
157
|
+
line: "1 stack updated (media), mealie rolled back, stirling-pdf waits for you, supabase held",
|
|
158
|
+
needs: ["stirling-pdf (Upgrade carefully)", "mealie automatic update rolled back"],
|
|
159
|
+
channelId: "6",
|
|
160
|
+
messageId: "8",
|
|
161
|
+
});
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
it("takes the security report's headline without its name", () => {
|
|
165
|
+
expect(headlineOf("**Security Daily Report:** quiet day (24 h to 05:00 CET)\nCrowdSec · 19 bans")).toBe("quiet day (24 h to 05:00 CET)");
|
|
166
|
+
});
|
|
167
|
+
});
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import type { ArcaneReport } from "./daily-report.ts";
|
|
2
|
+
import { escapeDiscord } from "./media.ts";
|
|
3
|
+
import { type BackupFacts, backupGuests, resticStatus } from "./security.ts";
|
|
4
|
+
import type { ToolsReport } from "./tools-report.ts";
|
|
5
|
+
|
|
6
|
+
/** Where an area's problems tag the owners: specialists without a channel of their own use the admin channel. */
|
|
7
|
+
export type Route = "admin" | "media";
|
|
8
|
+
|
|
9
|
+
/** One area of the morning summary: a line, and a problem that needs an owner, if any. */
|
|
10
|
+
export interface AreaStatus {
|
|
11
|
+
area: string;
|
|
12
|
+
line: string;
|
|
13
|
+
problem?: string;
|
|
14
|
+
route?: Route;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const HOUR = 3600;
|
|
18
|
+
|
|
19
|
+
function failed(area: string, route: Route, why: string): AreaStatus {
|
|
20
|
+
return { area, line: `the check failed (${why.slice(0, 120)})`, problem: `${area}: the status check failed`, route };
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Seconds from `21s ago`, `3m ago`, `2h ago`, `1d ago`; null when the text is not one of those. */
|
|
24
|
+
export function agoSeconds(text: string): number | null {
|
|
25
|
+
const m = /(\d+)\s*([smhd])\w* ago/.exec(text);
|
|
26
|
+
if (!m) return null;
|
|
27
|
+
return Number(m[1]) * { s: 1, m: 60, h: HOUR, d: 24 * HOUR }[m[2] as "s" | "m" | "h" | "d"];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** A WireGuard handshake older than this means the tunnel to the front door is down. */
|
|
31
|
+
const STALE_HANDSHAKE_S = 300;
|
|
32
|
+
|
|
33
|
+
/** `edge vps status` and `edge vps updates`: the front door's units, tunnel, reboot flag, and upgradable packages. */
|
|
34
|
+
export function edgeStatus(status: string | null, updates: string | null): AreaStatus {
|
|
35
|
+
if (status === null) return failed("Edge", "admin", "vps status");
|
|
36
|
+
const rows = new Map(status.split("\n").map((l) => /^(\S.*?)\s{2,}(.+)$/.exec(l.trim())).filter((m): m is RegExpExecArray => m !== null).map((m) => [m[1], m[2].trim()]));
|
|
37
|
+
const units = [...rows.entries()].filter(([, v]) => v === "active" || v === "inactive" || v === "failed" || v === "activating");
|
|
38
|
+
const down = units.filter(([, v]) => v !== "active").map(([k]) => k);
|
|
39
|
+
const handshake = rows.get("wireguard handshake") ?? "";
|
|
40
|
+
const age = agoSeconds(handshake);
|
|
41
|
+
const upgradable = updates === null ? null : updates.split("\n").filter((l) => l.includes("[upgradable from")).length;
|
|
42
|
+
const parts = [
|
|
43
|
+
down.length === 0 ? `${units.length} services active` : `down: ${down.join(", ")}`,
|
|
44
|
+
`WireGuard ${handshake || "unknown"}`,
|
|
45
|
+
rows.get("reboot required") === "yes" ? "reboot pending (it reboots itself at 02:30 UTC)" : "no reboot pending",
|
|
46
|
+
upgradable === null ? "updates unknown" : `${upgradable} upgradable package${upgradable === 1 ? "" : "s"}`,
|
|
47
|
+
];
|
|
48
|
+
const tunnel = age === null || age > STALE_HANDSHAKE_S;
|
|
49
|
+
const problem = down.length > 0 ? `Edge: ${down.join(", ")} not active on the front door` : tunnel ? `Edge: the WireGuard handshake is ${handshake || "missing"}` : undefined;
|
|
50
|
+
return { area: "Edge", line: `front door: ${parts.join(", ")}`, ...(problem ? { problem, route: "admin" as const } : {}) };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** `proxmox host-status` and `proxmox updates`: Pulsar's uptime, memory, root disk, and pending packages (report only). */
|
|
54
|
+
export function proxmoxStatus(status: string | null, updates: string | null): AreaStatus {
|
|
55
|
+
if (status === null) return failed("Proxmox", "admin", "host-status");
|
|
56
|
+
const up = /up\s+([^,]+(?:,\s*\d+:\d+)?)/.exec(status)?.[1]?.split(",")[0]?.trim();
|
|
57
|
+
const mem = /^Mem:\s+(\S+)\s+(\S+)/m.exec(status);
|
|
58
|
+
const root = /\s(\d+)%\s+\/\s*$/m.exec(status)?.[1];
|
|
59
|
+
const lines = updates === null ? null : updates.split("\n").filter((l) => l.includes("[upgradable from"));
|
|
60
|
+
const security = lines?.filter((l) => /-security\s/.test(l)).length ?? 0;
|
|
61
|
+
const parts = [
|
|
62
|
+
up ? `Pulsar up ${up}` : "Pulsar up",
|
|
63
|
+
mem ? `RAM ${mem[2]} of ${mem[1]} used` : null,
|
|
64
|
+
root ? `root disk ${root}%` : null,
|
|
65
|
+
lines === null ? "updates unknown" : `${lines.length} host package update${lines.length === 1 ? "" : "s"} waiting${security > 0 ? ` (${security} security)` : ""}, applied by hand`,
|
|
66
|
+
].filter(Boolean);
|
|
67
|
+
return { area: "Proxmox", line: parts.join(", ") };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** `network pulsar radius-status` and `tailscale-status`: FreeRADIUS running, and the subnet router online. */
|
|
71
|
+
export function networkStatus(radius: string | null, tailscale: string | null): AreaStatus {
|
|
72
|
+
const radiusUp = radius === null ? null : /Active:\s+active \(running\)/.test(radius);
|
|
73
|
+
const first = tailscale?.split("\n").find((l) => l.trim().length > 0)?.trim().split(/\s{2,}/) ?? null;
|
|
74
|
+
const routerUp = first === null ? null : !/offline/.test(first[first.length - 1] ?? "") && first.length >= 4;
|
|
75
|
+
const parts = [radiusUp === null ? "RADIUS unknown" : radiusUp ? "RADIUS running" : "RADIUS down", routerUp === null ? "Tailscale unknown" : routerUp ? `Tailscale router ${first?.[1] ?? ""} online` : "Tailscale router offline"];
|
|
76
|
+
const problems = [radiusUp === false ? "FreeRADIUS is not running" : null, routerUp === false ? "the Tailscale subnet router is offline" : null, radiusUp === null || routerUp === null ? "a network status check failed" : null].filter(Boolean);
|
|
77
|
+
return { area: "Network", line: parts.join(", "), ...(problems.length > 0 ? { problem: `Network: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** `identity dmz status`: Pomerium, nginx, and the certificate renewal timer on the DMZ VM. */
|
|
81
|
+
export function identityStatus(status: string | null): AreaStatus {
|
|
82
|
+
if (status === null) return failed("Identity", "admin", "dmz status");
|
|
83
|
+
const units = [...status.matchAll(/^[●○×]\s+(\S+)\s+-[^\n]*\n(?:.*\n)*?\s+Active:\s+(\S+)(?:\s+\((\w+)\))?/gm)].map((m) => ({ unit: m[1], active: m[2], sub: m[3] ?? "" }));
|
|
84
|
+
const bad = units.filter((u) => u.active !== "active").map((u) => `${u.unit} ${u.active}`);
|
|
85
|
+
const line = units.length === 0 ? "no unit status read" : bad.length === 0 ? `${units.map((u) => u.unit.replace(/\.service$/, "")).join(", ")} active` : `not active: ${bad.join(", ")}`;
|
|
86
|
+
return { area: "Identity", line, ...(bad.length > 0 || units.length === 0 ? { problem: `Identity: ${bad.length > 0 ? bad.join(", ") : "no unit status read"} on the DMZ VM`, route: "admin" as const } : {}) };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** `dns technitium <node> resolve <name>`: both resolvers answer an internal name. */
|
|
90
|
+
export function dnsStatus(primary: unknown, secondary: unknown): AreaStatus {
|
|
91
|
+
const ok = (r: unknown) => (r as { status?: unknown } | null)?.status === "ok";
|
|
92
|
+
const down = [ok(primary) ? null : "primary", ok(secondary) ? null : "secondary"].filter(Boolean);
|
|
93
|
+
return { area: "DNS", line: down.length === 0 ? "both resolvers answer" : `no answer from the ${down.join(" and ")} resolver`, ...(down.length > 0 ? { problem: `DNS: the ${down.join(" and ")} resolver does not answer`, route: "admin" as const } : {}) };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** `inference nexus-status`: each Nexus service running with HTTP health 200. */
|
|
97
|
+
export function inferenceStatus(status: string | null): AreaStatus {
|
|
98
|
+
if (status === null) return failed("Inference", "admin", "nexus-status");
|
|
99
|
+
const services = status
|
|
100
|
+
.split("\n")
|
|
101
|
+
.map((l) => /^(\S+)\s+state = (\S+).*?(?:health=(\d+))?\s*$/.exec(l.trim()))
|
|
102
|
+
.filter((m): m is RegExpExecArray => m !== null)
|
|
103
|
+
.map((m) => ({ name: m[1], running: m[2] === "running", healthy: m[3] === "200" }));
|
|
104
|
+
const bad = services.filter((s) => !s.running || !s.healthy).map((s) => s.name);
|
|
105
|
+
const line = services.length === 0 ? "no Nexus service listed" : bad.length === 0 ? `Nexus ${services.map((s) => s.name).join(", ")} running and healthy` : `Nexus not healthy: ${bad.join(", ")}`;
|
|
106
|
+
return { area: "Inference", line, ...(bad.length > 0 || services.length === 0 ? { problem: `Inference: ${bad.length > 0 ? `${bad.join(", ")} not healthy on Nexus` : "Nexus lists no service"}`, route: "admin" as const } : {}) };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** `media sabnzbd status`: paused or downloading, and SABnzbd's own warnings. */
|
|
110
|
+
export function mediaStatus(sab: unknown): AreaStatus {
|
|
111
|
+
const r = sab as { sabnzbd_paused?: unknown; warnings?: unknown } | null;
|
|
112
|
+
if (!r || typeof r.sabnzbd_paused !== "boolean") return failed("Media", "media", "sabnzbd status");
|
|
113
|
+
const warnings = Array.isArray(r.warnings) ? r.warnings.length : 0;
|
|
114
|
+
const line = `SABnzbd ${r.sabnzbd_paused ? "paused" : "downloading"}${warnings > 0 ? `, ${warnings} warning${warnings === 1 ? "" : "s"}` : ""}`;
|
|
115
|
+
return { area: "Media", line, ...(warnings > 0 ? { problem: `Media: SABnzbd shows ${warnings} warning${warnings === 1 ? "" : "s"}`, route: "media" as const } : {}) };
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/** Every guest is backed up nightly at 02:30; an archive older than this means last night's run missed it. */
|
|
119
|
+
const MISSED_BACKUP_S = 30 * HOUR;
|
|
120
|
+
|
|
121
|
+
/** `backup vzdump-archives` and `restic-check-status`: last night's Proxmox archives and the weekly restic check. */
|
|
122
|
+
export function backupStatus(archives: unknown, resticText: string | null, now: number): AreaStatus {
|
|
123
|
+
const guests = backupGuests(archives);
|
|
124
|
+
const restic: BackupFacts["restic"] = resticText === null ? null : resticStatus(resticText);
|
|
125
|
+
const missed = guests?.filter((g) => now / 1000 - g.last > MISSED_BACKUP_S) ?? [];
|
|
126
|
+
const proxmox = guests === null ? "Proxmox archives unknown" : missed.length === 0 ? `${guests.length === 1 ? "the only guest" : `all ${guests.length} guests`} backed up last night` : `no backup last night for ${missed.map((g) => escapeDiscord(g.name)).join(", ")}`;
|
|
127
|
+
const check = restic ? `restic check ${escapeDiscord(restic.result)} ${restic.at}` : "restic check unknown";
|
|
128
|
+
const problems = [missed.length > 0 ? `no Proxmox backup last night for ${missed.map((g) => g.name).join(", ")}` : null, restic && restic.result !== "success" ? `the restic check ended with ${restic.result}` : null, guests === null || restic === null ? "a backup status check failed" : null].filter(Boolean);
|
|
129
|
+
return { area: "Backup", line: `${proxmox}, ${check}`, ...(problems.length > 0 ? { problem: `Backup: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/** One report the run posted in its own channel, as the summary shows it. */
|
|
133
|
+
export interface ReportLine {
|
|
134
|
+
area: string;
|
|
135
|
+
line: string;
|
|
136
|
+
/** Items the report tagged the owners for. */
|
|
137
|
+
needs: string[];
|
|
138
|
+
channelId?: string;
|
|
139
|
+
messageId?: string;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function names(items: readonly string[]): string {
|
|
143
|
+
return items.map(escapeDiscord).join(", ");
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** The Cortex Tools Report after its automatic updates ran: what changed, what failed, and what waits for a button. */
|
|
147
|
+
export function toolsSummary(report: ToolsReport | null, date: string, channelId: string): ReportLine {
|
|
148
|
+
if (!report || report.date !== date) return { area: "Cortex tools", line: "no report today; the bridge log has the reason", needs: [], channelId };
|
|
149
|
+
const updated = report.lines.filter((l) => l.state === "updated");
|
|
150
|
+
const failed = report.lines.filter((l) => l.state === "failed");
|
|
151
|
+
const waiting = report.lines.filter((l) => (l.plan === "update" || l.plan === "button") && l.state === "offered");
|
|
152
|
+
const manual = report.lines.filter((l) => l.manual && l.state === "offered");
|
|
153
|
+
const parts = [
|
|
154
|
+
updated.length > 0 ? `${updated.length} updated (${names(updated.map((l) => `${l.label} ${l.latest}`))})` : null,
|
|
155
|
+
failed.length > 0 ? `${failed.length} failed (${names(failed.map((l) => l.label))})` : null,
|
|
156
|
+
waiting.length > 0 ? `${names(waiting.map((l) => `${l.label} ${l.latest}`))} wait${waiting.length === 1 ? "s" : ""} for a button` : null,
|
|
157
|
+
manual.length > 0 ? `${manual.length} by hand` : null,
|
|
158
|
+
].filter(Boolean);
|
|
159
|
+
const needs = [
|
|
160
|
+
...report.lines.filter((l) => l.plan === "update" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} (${l.major ? "major version" : "release notes"})`),
|
|
161
|
+
...failed.filter((l) => l.plan === "auto").map((l) => `${escapeDiscord(l.label)} automatic update failed`),
|
|
162
|
+
];
|
|
163
|
+
return { area: "Cortex tools", line: parts.join(", ") || "everything up to date", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/** The Arcane Daily Report after its automatic updates and checks: updated, rolled back, waiting for the owner, held. */
|
|
167
|
+
export function arcaneSummary(report: ArcaneReport | null, date: string, channelId: string): ReportLine {
|
|
168
|
+
if (!report || report.date !== date) return { area: "Arcane", line: "no report today; the bridge log has the reason", needs: [], channelId };
|
|
169
|
+
const by = (pick: (s: ArcaneReport["stacks"][number]) => boolean) => report.stacks.filter(pick).map((s) => s.stack);
|
|
170
|
+
const updated = by((s) => s.state === "updated");
|
|
171
|
+
const rolled = by((s) => s.state === "rolled back");
|
|
172
|
+
const failed = by((s) => s.state === "failed");
|
|
173
|
+
const waiting = report.stacks.filter((s) => (s.plan === "update" || s.plan === "careful") && (s.state === "offered" || s.state === "planning"));
|
|
174
|
+
const held = by((s) => s.held !== undefined);
|
|
175
|
+
const parts = [
|
|
176
|
+
updated.length > 0 ? `${updated.length} stack${updated.length === 1 ? "" : "s"} updated (${names(updated)})` : null,
|
|
177
|
+
rolled.length > 0 ? `${names(rolled)} rolled back` : null,
|
|
178
|
+
failed.length > 0 ? `${names(failed)} failed` : null,
|
|
179
|
+
waiting.length > 0 ? `${names(waiting.map((s) => s.stack))} wait${waiting.length === 1 ? "s" : ""} for you` : null,
|
|
180
|
+
held.length > 0 ? `${names(held)} held` : null,
|
|
181
|
+
].filter(Boolean);
|
|
182
|
+
const needs = [
|
|
183
|
+
...waiting.map((s) => `${escapeDiscord(s.stack)} (${s.plan === "careful" ? "Upgrade carefully" : "major version"})`),
|
|
184
|
+
...report.stacks.filter((s) => s.plan === "auto" && (s.state === "rolled back" || s.state === "failed")).map((s) => `${escapeDiscord(s.stack)} automatic update ${s.state}`),
|
|
185
|
+
];
|
|
186
|
+
return { area: "Arcane", line: parts.join(", ") || "no updates", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
export interface Summary {
|
|
190
|
+
date: string;
|
|
191
|
+
guildId: string;
|
|
192
|
+
reports: ReportLine[];
|
|
193
|
+
areas: AreaStatus[];
|
|
194
|
+
/** Channel ids of the routes, for links to the tagged messages. */
|
|
195
|
+
routes: Record<Route, string>;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function link(guildId: string, channelId: string, messageId: string): string {
|
|
199
|
+
return `https://discord.com/channels/${guildId}/${channelId}/${messageId}`;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/** The #daily message: one line per report and area, then what needs the owners, with no mention. */
|
|
203
|
+
export function summaryText(s: Summary): string {
|
|
204
|
+
const needs = [
|
|
205
|
+
...s.reports.flatMap((r) => r.needs.map((n) => `${n}${r.channelId ? ` (<#${r.channelId}>)` : ""}`)),
|
|
206
|
+
...s.areas.filter((a) => a.problem).map((a) => `${escapeDiscord(a.problem ?? "")} (<#${s.routes[a.route ?? "admin"]}>)`),
|
|
207
|
+
];
|
|
208
|
+
const head = `**Daily** · ${s.date} · ${needs.length === 0 ? "🟢 nothing needs you" : `🟠 ${needs.length} item${needs.length === 1 ? " needs" : "s need"} you`}`;
|
|
209
|
+
const reports = s.reports.map((r) => `${r.area} · ${r.line}${r.channelId && r.messageId ? ` · [report](${link(s.guildId, r.channelId, r.messageId)})` : ""}`);
|
|
210
|
+
const areas = s.areas.map((a) => `${a.area} · ${a.line}`);
|
|
211
|
+
return [head, ...reports, ...areas, ...(needs.length > 0 ? [`Needs you · ${needs.map((n, i) => `${i + 1}) ${n}`).join(" ")}`] : [])].join("\n");
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** The tagged message for one area's problem in its routed channel. */
|
|
215
|
+
export function problemText(a: AreaStatus, owners: readonly string[]): string {
|
|
216
|
+
return `⚠️ **${escapeDiscord(a.area)}** · ${escapeDiscord(a.problem ?? a.line)}\nFound by the morning run; the ${escapeDiscord(a.area.toLowerCase())} specialist can look into it here.\n${owners.map((id) => `<@${id}>`).join(" ")}`;
|
|
217
|
+
}
|
|
@@ -36,7 +36,7 @@ import {
|
|
|
36
36
|
import { GatewayClient } from "./gateway.ts";
|
|
37
37
|
import { type CommandOption, IS_COMPONENTS_V2, KeyedQueue, MEDIA, MEDIA_COMMANDS, type MediaKind, type Outcome, addOutcome, applyOutcome, commandArgs, hasActiveButton, listMessage, mediaAccess, mediaPointer, parseAddButton, parseAddReport, parseMediaList, parseQueueState, queueLine } from "./media.ts";
|
|
38
38
|
import { cancelledApprovalText, interruptedTaskText, readOpenWork, type OpenWork, type StopReason, writeOpenWork } from "./open-work.ts";
|
|
39
|
-
import { isDue, MONDAY, readReportState, writeReportState } from "./reports.ts";
|
|
39
|
+
import { isDue, MONDAY, readReportState, writeReportState, zonedWeekday } from "./reports.ts";
|
|
40
40
|
import { ACTIVITY_COMMAND, type Activity, addRestarts, astrogateActivity, herdrActivity, needsPrompt, type PendingRestart, readRestarts, restartCommand, SERVICE_LABEL, SERVICE_ORDER, writeRestarts } from "./restarts.ts";
|
|
41
41
|
import {
|
|
42
42
|
arcaneServerVersion,
|
|
@@ -77,6 +77,7 @@ import { ArcaneWatch } from "./arcane-watch-monitor.ts";
|
|
|
77
77
|
import { parseSnapshot } from "./arcane-watch.ts";
|
|
78
78
|
import { type Applied, autoFailureText, foremanTask, stackProblems, SUPERVISE_DELAY_MS, SUPERVISE_LIMIT_MS } from "./supervise.ts";
|
|
79
79
|
import { stackPlan, toolPlan } from "./update-policy.ts";
|
|
80
|
+
import { type AreaStatus, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, problemText, proxmoxStatus, type ReportLine, type Route, summaryText, toolsSummary } from "./daily-run.ts";
|
|
80
81
|
import { type InvestigationRequest, SecurityMonitor } from "./security-monitor.ts";
|
|
81
82
|
import { nasOutage, readSecurityState } from "./security.ts";
|
|
82
83
|
|
|
@@ -162,6 +163,9 @@ const UPDATE_TIMEOUT_MS = 30 * 60_000;
|
|
|
162
163
|
const REVIEWER = "astro.release-reviewer";
|
|
163
164
|
/** Key of a running Arcane stack update or manager upgrade among the tool runs; one at a time, because all of them push to the same GitOps checkout. */
|
|
164
165
|
const ARCANE_RUN = "arcane-update";
|
|
166
|
+
/** An internal name both resolvers must answer for the morning run's DNS line. */
|
|
167
|
+
const DNS_PROBE = "arcane.37pla.net";
|
|
168
|
+
|
|
165
169
|
/** The audit caller of updates the morning run applies on its own. */
|
|
166
170
|
const AUTO_CALLER = "daily-run";
|
|
167
171
|
|
|
@@ -477,19 +481,24 @@ class Bridge {
|
|
|
477
481
|
private async maybeReports(): Promise<void> {
|
|
478
482
|
if (this.stopping) return;
|
|
479
483
|
this.refreshConfig();
|
|
484
|
+
// With a morning run configured, it posts the other reports itself, in order; their own times are unused.
|
|
485
|
+
const daily = reportSchedule(this.config, "daily");
|
|
480
486
|
for (const schedule of this.config.reports) {
|
|
481
487
|
if (this.reportsRunning.has(schedule.report)) continue;
|
|
488
|
+
if (daily && schedule.report !== "daily") continue;
|
|
482
489
|
const { due, date } = isDue(schedule, readReportState(schedule.report), new Date(), schedule.report === "security-weekly" ? MONDAY : undefined);
|
|
483
490
|
if (!due) continue;
|
|
484
491
|
this.reportsRunning.add(schedule.report);
|
|
485
|
-
const run =
|
|
486
|
-
schedule.report === "
|
|
487
|
-
? this.
|
|
488
|
-
: schedule.report === "
|
|
489
|
-
? this.
|
|
490
|
-
: schedule.report === "
|
|
491
|
-
? this.
|
|
492
|
-
:
|
|
492
|
+
const run: Promise<unknown> =
|
|
493
|
+
schedule.report === "daily"
|
|
494
|
+
? this.dailyRun(schedule, date)
|
|
495
|
+
: schedule.report === "arcane"
|
|
496
|
+
? this.arcaneReport(schedule, date)
|
|
497
|
+
: schedule.report === "tools"
|
|
498
|
+
? this.toolsReport(schedule, date)
|
|
499
|
+
: schedule.report === "security"
|
|
500
|
+
? this.security.dailyReport(schedule, date)
|
|
501
|
+
: this.security.weeklyReport(schedule, date);
|
|
493
502
|
void run
|
|
494
503
|
.catch((err) => log(`${schedule.report} report ${date}: ${err instanceof Error ? err.message : String(err)}`))
|
|
495
504
|
.finally(() => this.reportsRunning.delete(schedule.report));
|
|
@@ -501,6 +510,83 @@ class Bridge {
|
|
|
501
510
|
* one section per stack with updates, a release-notes review for every
|
|
502
511
|
* major-version update, and an Update button per stack that is not held.
|
|
503
512
|
*/
|
|
513
|
+
/**
|
|
514
|
+
* The morning run: the security report, then the Cortex Tools Report and
|
|
515
|
+
* the Arcane Daily Report (each applying its automatic updates), then every
|
|
516
|
+
* specialist's status, and last the summary in the daily channel. A report
|
|
517
|
+
* that already ran today, before a restart, is not posted again.
|
|
518
|
+
*/
|
|
519
|
+
private async dailyRun(schedule: ReportSchedule, date: string): Promise<void> {
|
|
520
|
+
log(`daily ${date}: starting`);
|
|
521
|
+
const reports: ReportLine[] = [];
|
|
522
|
+
const done = (name: ReportName) => readReportState(name)?.date === date;
|
|
523
|
+
const security = reportSchedule(this.config, "security");
|
|
524
|
+
const weekly = reportSchedule(this.config, "security-weekly");
|
|
525
|
+
const monday = weekly !== undefined && zonedWeekday(schedule.timeZone, new Date()) === MONDAY;
|
|
526
|
+
const securitySchedule = monday ? weekly : security;
|
|
527
|
+
if (securitySchedule && !this.stopping) {
|
|
528
|
+
const name: ReportName = monday ? "security-weekly" : "security";
|
|
529
|
+
const posted = done(name) ? undefined : await (monday ? this.security.weeklyReport(securitySchedule, date) : this.security.dailyReport(securitySchedule, date)).catch((err) => log(`daily ${date}: security report failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
530
|
+
const area = monday ? "Security (weekly)" : "Security";
|
|
531
|
+
reports.push(posted ? { area, line: posted.headline, needs: posted.tagged ? [`the ${monday ? "weekly" : "daily"} security report`] : [], channelId: securitySchedule.channelId, messageId: posted.messageId } : { area, line: done(name) ? "already posted today" : "the report failed; the bridge log has the reason", needs: [], channelId: securitySchedule.channelId });
|
|
532
|
+
}
|
|
533
|
+
const tools = reportSchedule(this.config, "tools");
|
|
534
|
+
if (tools && !this.stopping) {
|
|
535
|
+
if (!done("tools")) await this.toolsReport(tools, date).catch((err) => log(`daily ${date}: tools report failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
536
|
+
reports.push(toolsSummary(readPostedReport(), date, tools.channelId));
|
|
537
|
+
}
|
|
538
|
+
const arcane = reportSchedule(this.config, "arcane");
|
|
539
|
+
if (arcane && !this.stopping) {
|
|
540
|
+
if (!done("arcane")) await this.arcaneReport(arcane, date).catch((err) => log(`daily ${date}: arcane report failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
541
|
+
reports.push(arcaneSummary(readPostedArcane(), date, arcane.channelId));
|
|
542
|
+
}
|
|
543
|
+
if (this.stopping) return;
|
|
544
|
+
const areas = await this.collectAreas();
|
|
545
|
+
const routes = this.routes();
|
|
546
|
+
const owners = this.config.owners;
|
|
547
|
+
for (const a of areas.filter((x) => x.problem)) {
|
|
548
|
+
await this.rest.createMessage(routes[a.route ?? "admin"], { content: problemText(a, owners), mentionUsers: owners }).catch((err) => log(`daily ${date}: ${a.area} problem message failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
549
|
+
}
|
|
550
|
+
const label = new Intl.DateTimeFormat("en-GB", { timeZone: schedule.timeZone, weekday: "short", day: "numeric", month: "short" }).format(new Date());
|
|
551
|
+
const text = summaryText({ date: label, guildId: this.config.guildId, reports, areas, routes });
|
|
552
|
+
// Recorded only now: a run a crash cut short runs again and skips the reports it already posted.
|
|
553
|
+
writeReportState("daily", { date, status: "sending" });
|
|
554
|
+
let first = "";
|
|
555
|
+
for (const chunk of chunkMessage(text)) first ||= (await this.rest.createMessage(schedule.channelId, { content: chunk })).id;
|
|
556
|
+
writeReportState("daily", { date, status: "sent", messageId: first });
|
|
557
|
+
log(`daily ${date}: posted`);
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/** Where an area's problem tags the owners: the media specialist's channel, else the admin channel. */
|
|
561
|
+
private routes(): Record<Route, string> {
|
|
562
|
+
const media = Object.entries(this.config.channels).find(([, c]) => c.specialists !== "*" && c.specialists.includes(MEDIA))?.[0];
|
|
563
|
+
return { admin: this.config.adminChannelId, media: media ?? this.config.adminChannelId };
|
|
564
|
+
}
|
|
565
|
+
|
|
566
|
+
/** Every configured specialist's status, read through its wrapper without a model. */
|
|
567
|
+
private async collectAreas(): Promise<AreaStatus[]> {
|
|
568
|
+
const present = new Set(this.specialists().names);
|
|
569
|
+
const text = (service: string, args: string[]) => this.runEntry(service, AUTO_CALLER, args, 3 * 60_000).catch(() => null);
|
|
570
|
+
const json = async (service: string, args: string[]) => {
|
|
571
|
+
const out = await text(service, args);
|
|
572
|
+
try {
|
|
573
|
+
return out === null ? null : (JSON.parse(out) as unknown);
|
|
574
|
+
} catch {
|
|
575
|
+
return null;
|
|
576
|
+
}
|
|
577
|
+
};
|
|
578
|
+
const areas: Promise<AreaStatus>[] = [];
|
|
579
|
+
if (present.has("backup")) areas.push((async () => backupStatus(await json("backup", ["vzdump-archives"]), await text("backup", ["restic-check-status"]), Date.now()))());
|
|
580
|
+
if (present.has("edge")) areas.push((async () => edgeStatus(await text("edge", ["vps", "status"]), await text("edge", ["vps", "updates"])))());
|
|
581
|
+
if (present.has("proxmox")) areas.push((async () => proxmoxStatus(await text("proxmox", ["host-status"]), await text("proxmox", ["updates"])))());
|
|
582
|
+
if (present.has("network")) areas.push((async () => networkStatus(await text("network", ["pulsar", "radius-status"]), await text("network", ["pulsar", "tailscale-status"])))());
|
|
583
|
+
if (present.has("identity")) areas.push((async () => identityStatus(await text("identity", ["dmz", "status"])))());
|
|
584
|
+
if (present.has("dns")) areas.push((async () => dnsStatus(await json("dns", ["technitium", "primary", "resolve", DNS_PROBE]), await json("dns", ["technitium", "secondary", "resolve", DNS_PROBE])))());
|
|
585
|
+
if (present.has("inference")) areas.push((async () => inferenceStatus(await text("inference", ["nexus-status"])))());
|
|
586
|
+
if (present.has(MEDIA)) areas.push((async () => mediaStatus(await json(MEDIA, ["sabnzbd", "status"])))());
|
|
587
|
+
return Promise.all(areas);
|
|
588
|
+
}
|
|
589
|
+
|
|
504
590
|
private async arcaneReport(schedule: ReportSchedule, date: string): Promise<void> {
|
|
505
591
|
if (!loadSpecialists().config) {
|
|
506
592
|
log("arcane report: specialists.json missing; no report");
|
|
@@ -90,6 +90,18 @@ export interface SecurityClick {
|
|
|
90
90
|
customId: string;
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
+
/** What a posted security report tells the morning summary. */
|
|
94
|
+
export interface PostedSummary {
|
|
95
|
+
headline: string;
|
|
96
|
+
tagged: boolean;
|
|
97
|
+
messageId: string;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** The report's first line without its bold name: `quiet day (24 h to 05:00 CET)`. */
|
|
101
|
+
export function headlineOf(text: string): string {
|
|
102
|
+
return text.split("\n")[0].replace(/^\*\*[^*]+\*\*\s*/, "");
|
|
103
|
+
}
|
|
104
|
+
|
|
93
105
|
const MINUTE = 60_000;
|
|
94
106
|
const DAY = 86_400_000;
|
|
95
107
|
export const POLL_MS = 2 * MINUTE;
|
|
@@ -456,7 +468,8 @@ export class SecurityMonitor {
|
|
|
456
468
|
return first;
|
|
457
469
|
}
|
|
458
470
|
|
|
459
|
-
|
|
471
|
+
/** Posts the Security Daily Report; returns its headline and whether it tagged the owners, for the morning summary. */
|
|
472
|
+
async dailyReport(s: ReportSchedule, date: string): Promise<PostedSummary | undefined> {
|
|
460
473
|
const now = this.deps.now();
|
|
461
474
|
if (reportSchedule(this.deps.config(), "security-weekly") && zonedWeekday(s.timeZone, new Date(now)) === MONDAY) {
|
|
462
475
|
writeReportState("security", { date, status: "skipped" });
|
|
@@ -475,12 +488,14 @@ export class SecurityMonitor {
|
|
|
475
488
|
if (this.deps.stopping()) return;
|
|
476
489
|
const data = { until: clockText(new Date(now).toISOString(), s.timeZone).replace(/^\d+ \w+ /, ""), crowdsec, wazuh, found: vulns?.found ?? null, closed: vulns?.closed ?? null, attention, ...images };
|
|
477
490
|
writeReportState("security", { date, status: "sending" });
|
|
478
|
-
const
|
|
491
|
+
const text = dailyText(data, this.owners());
|
|
492
|
+
const id = await this.postReport(s.channelId, text, dailyNeedsOwner(data));
|
|
479
493
|
writeReportState("security", { date, status: "sent", messageId: id });
|
|
480
494
|
this.deps.log(`security report ${date}: posted`);
|
|
495
|
+
return { headline: headlineOf(text), tagged: dailyNeedsOwner(data), messageId: id };
|
|
481
496
|
}
|
|
482
497
|
|
|
483
|
-
async weeklyReport(s: ReportSchedule, date: string): Promise<
|
|
498
|
+
async weeklyReport(s: ReportSchedule, date: string): Promise<PostedSummary | undefined> {
|
|
484
499
|
const now = this.deps.now();
|
|
485
500
|
this.deps.log(`security weekly report ${date}: collecting`);
|
|
486
501
|
const crowdsec = await this.read("alerts 7", "security", ["alerts", "7"], parseAlertSummary);
|
|
@@ -535,6 +550,7 @@ export class SecurityMonitor {
|
|
|
535
550
|
const id = await this.postReport(s.channelId, report.text, report.mention);
|
|
536
551
|
writeReportState("security-weekly", { date, status: "sent", messageId: id });
|
|
537
552
|
this.deps.log(`security weekly report ${date}: posted`);
|
|
553
|
+
return { headline: headlineOf(report.text), tagged: report.mention, messageId: id };
|
|
538
554
|
}
|
|
539
555
|
|
|
540
556
|
// ------------------------------------------------------------ buttons
|