@astrofoundry/pi-astro 0.40.74 → 0.40.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/astro-discord/approvals.ts +14 -0
- package/extensions/astro-discord/commands.ts +1 -1
- package/extensions/astro-discord/daily-run.test.ts +57 -49
- package/extensions/astro-discord/daily-run.ts +105 -48
- package/extensions/astro-discord/discord.test.ts +16 -1
- package/extensions/astro-discord/index.ts +32 -12
- package/extensions/astro-discord/reports.ts +2 -0
- package/package.json +1 -1
- package/skills/dns/SKILL.md +1 -1
- package/skills/proxmox/SKILL.md +1 -1
- package/specialists/entry/remote/spc-dns-node-entry.sh +2 -1
- package/specialists/entry/remote/spc-proxmox-pulsar-entry.sh +2 -1
- package/specialists/entry/remote/spc-proxmox-spacex-entry.sh +2 -1
|
@@ -3,6 +3,20 @@ import { createServer, type Server } from "node:http";
|
|
|
3
3
|
|
|
4
4
|
export type Decision = "approve" | "deny" | "timeout";
|
|
5
5
|
|
|
6
|
+
/** Longest command an approval message shows inline; Discord refuses messages above 2000 characters. */
|
|
7
|
+
export const APPROVAL_INLINE_CHARS = 1200;
|
|
8
|
+
const APPROVAL_PREVIEW_CHARS = 300;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* How an approval shows the command: whole when it is short, otherwise its start in the message and the whole
|
|
12
|
+
* command as an attached text file, so the owner can still read everything he approves.
|
|
13
|
+
*/
|
|
14
|
+
export function approvalCommand(args: readonly string[]): { inline: string; file?: { name: string; content: string } } {
|
|
15
|
+
const whole = args.join(" ");
|
|
16
|
+
if (whole.length <= APPROVAL_INLINE_CHARS) return { inline: whole };
|
|
17
|
+
return { inline: `${whole.slice(0, APPROVAL_PREVIEW_CHARS)}… (${whole.length} characters, the whole command is attached)`, file: { name: "command.txt", content: `${whole}\n` } };
|
|
18
|
+
}
|
|
19
|
+
|
|
6
20
|
export interface Ticket {
|
|
7
21
|
id: string;
|
|
8
22
|
agent: string;
|
|
@@ -53,7 +53,7 @@ function runCommand(specialist: string, text: string): Command {
|
|
|
53
53
|
/** Prompt paragraph appended for tasks that arrive from a chat channel. */
|
|
54
54
|
export const CHANNEL_RULES = `## Channel rules
|
|
55
55
|
|
|
56
|
-
This task arrived from a Discord channel and your answer is posted there. Answer in plain text
|
|
56
|
+
This task arrived from a Discord channel and your answer is posted there. Answer in plain text, facts first, no headings, with the fewest words that keep the meaning; this is mandatory, and 1500 characters is the ceiling. Write only what the owner needs to act or to know the outcome: the result, what you need from them, and the next step. Leave out what you read, checks that found nothing, what did not change, and comments on your own reasoning. Use short, plain sentences. When the task is ambiguous (which guest, which record, which host), do not act: ask one precise question and stop. Risky calls may wait for an operator's approval; if the tool reports a denial or a missing answer, report it and stop. When you need something from the owner (a go, a decision, an action, or a blocker you cannot remove), your first line starts with **Needs you:** and says what you need in one sentence; the bridge then tags them.`;
|
|
57
57
|
|
|
58
58
|
/** A reply whose first line asks the owner for something; the bridge tags the requester on it. */
|
|
59
59
|
export function needsYou(output: string): boolean {
|
|
@@ -57,9 +57,9 @@ describe("morning status lines", () => {
|
|
|
57
57
|
|
|
58
58
|
it("summarises the front door and flags a down unit or a stale tunnel", () => {
|
|
59
59
|
expect(edgeStatus(EDGE, UPDATES)).toEqual({ area: "Edge", line: "front door: 5 services active, WireGuard 21s ago, no reboot pending, 3 upgradable packages" });
|
|
60
|
-
expect(edgeStatus(EDGE.replace("nginx active", "nginx failed"), UPDATES).problem).toBe("Edge: nginx
|
|
61
|
-
expect(edgeStatus(EDGE.replace("21s ago", "9m ago"), null)).toMatchObject({ problem: "Edge:
|
|
62
|
-
expect(edgeStatus(null, null)).toMatchObject({ line: "the check failed (vps status)", problem: "Edge:
|
|
60
|
+
expect(edgeStatus(EDGE.replace("nginx active", "nginx failed"), UPDATES).problem).toBe("Edge: nginx down on the front door");
|
|
61
|
+
expect(edgeStatus(EDGE.replace("21s ago", "9m ago"), null)).toMatchObject({ problem: "Edge: WireGuard handshake 9m ago", route: "admin" });
|
|
62
|
+
expect(edgeStatus(null, null)).toMatchObject({ line: "the check failed (vps status)", problem: "Edge: `vps status` failed" });
|
|
63
63
|
});
|
|
64
64
|
|
|
65
65
|
it("reports both Proxmox hosts, the morning upgrade, a held major, and a pending reboot with its button", () => {
|
|
@@ -70,12 +70,12 @@ describe("morning status lines", () => {
|
|
|
70
70
|
{ name: "SPACE-X", id: "space-x", status: null, updates: null },
|
|
71
71
|
]);
|
|
72
72
|
expect(upgraded.line).toBe("Pulsar up 59 days, RAM 47Gi of 60Gi used, root disk 13%, 3 host updates installed, reboot needed; SPACE-X status unknown");
|
|
73
|
-
expect(upgraded.problem).toBe("Proxmox: Pulsar
|
|
73
|
+
expect(upgraded.problem).toBe("Proxmox: Pulsar: reboot needed (kernel 7.0.14-25-pve -> 7.0.14-26-pve); its guests stop for a few minutes; the status check failed for SPACE-X");
|
|
74
74
|
expect(upgraded.buttons).toEqual([{ label: "Reboot Pulsar", customId: "proxmox:reboot:pulsar" }]);
|
|
75
75
|
const major = proxmoxStatus([{ name: "SPACE-X", id: "space-x", status: `${PULSAR}\nreboot-needed: no`, updates: PULSAR_UPDATES, upgrade: "held: Proxmox VE 9.2.24 -> 10.0.1 is a major upgrade" }]);
|
|
76
|
-
expect(major).toMatchObject({ problem: "Proxmox: SPACE-X
|
|
76
|
+
expect(major).toMatchObject({ problem: "Proxmox: SPACE-X: Proxmox VE 9.2.24 -> 10.0.1 is a major upgrade, held for you", route: "admin" });
|
|
77
77
|
expect(major.line).toContain("a Proxmox VE major upgrade waits for you");
|
|
78
|
-
expect(proxmoxStatus([{ name: "Pulsar", id: "pulsar", status: PULSAR, updates: PULSAR_UPDATES, upgrade: null }]).problem).toBe("Proxmox:
|
|
78
|
+
expect(proxmoxStatus([{ name: "Pulsar", id: "pulsar", status: PULSAR, updates: PULSAR_UPDATES, upgrade: null }]).problem).toBe("Proxmox: Pulsar: host upgrade failed");
|
|
79
79
|
});
|
|
80
80
|
|
|
81
81
|
it("flags updates the nightly run left behind, reading guest-agent JSON too, and announces a Wazuh release from the held agent", () => {
|
|
@@ -84,17 +84,21 @@ describe("morning status lines", () => {
|
|
|
84
84
|
const held = "Listing...\nwazuh-agent/stable 4.14.10-1 amd64 [upgradable from: 4.14.9-1]";
|
|
85
85
|
expect(wazuhRelease({ arcane: vm, synapse: held, radius: null })).toEqual({ from: "4.14.9", to: "4.14.10" });
|
|
86
86
|
expect(wazuhRelease({ synapse: "Listing..." })).toBeNull();
|
|
87
|
-
expect(updatesStatus({ synapse: "Listing..." })).toEqual({ area: "Updates", line: "1 host up to date after the nightly run" });
|
|
88
|
-
const wazuh = updatesStatus({ technitium2: held, synapse: "Listing..." });
|
|
87
|
+
expect(updatesStatus({ synapse: "Listing..." })).toEqual([{ area: "Updates", line: "1 host up to date after the nightly run" }]);
|
|
88
|
+
const [wazuh] = updatesStatus({ technitium2: held, synapse: "Listing..." });
|
|
89
89
|
expect(wazuh.line).toBe("2 hosts up to date after the nightly run; Wazuh 4.14.10 is out (agents held at 4.14.9)");
|
|
90
|
-
expect(wazuh
|
|
90
|
+
expect(wazuh).toMatchObject({ problem: "Updates: Wazuh 4.14.10 is out (agents on 4.14.9). Claude Code on the Mac upgrades the manager first, then the agents", specialist: null });
|
|
91
91
|
const left = updatesStatus({ arcane: vm, radius: null });
|
|
92
|
-
expect(left).
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
expect(left
|
|
92
|
+
expect(left.map((a) => [a.specialist, a.items?.map((i) => i.text)])).toEqual([
|
|
93
|
+
["arcane", ["arcane: docker-ce did not install"]],
|
|
94
|
+
["network", ["radius: no update list"]],
|
|
95
|
+
[null, ["Wazuh 4.14.9 is out (agents on 4.14.7). Claude Code on the Mac upgrades the manager first, then the agents"]],
|
|
96
|
+
]);
|
|
97
|
+
expect(left[0]).toMatchObject({ line: "still waiting after the nightly run: arcane 1; no update list from radius; Wazuh 4.14.9 is out (agents held at 4.14.7)", route: "admin" });
|
|
98
|
+
const synapse = "Listing...\nlibcamera-ipa/stable 0.8.0+rpt20261104-1 arm64 [upgradable from: 0.7.2+rpt20261009-1]\nrpicam-apps-core/stable 1.15.0-1 arm64 [upgradable from: 1.14.0-1]\nnew: libcamera0.8\nremove: libcamera0.7";
|
|
99
|
+
expect(updatesStatus({ synapse })).toMatchObject([
|
|
100
|
+
{ problem: "Updates: synapse: libcamera-ipa, rpicam-apps-core kept back. Fix by hand: `apt-get full-upgrade` (adds libcamera0.8, removes libcamera0.7)", specialist: "dns" },
|
|
101
|
+
]);
|
|
98
102
|
expect(["dmz", "observability", "backup", "tailscale-mira (Mira)", "technitium3 (Mira)", "synapse", "hermes", "Pulsar guests"].map(guestOwner)).toEqual(["Identity", "Security", "Backup", "Network", "DNS", "DNS", "Proxmox", "Proxmox"]);
|
|
99
103
|
});
|
|
100
104
|
|
|
@@ -102,14 +106,14 @@ describe("morning status lines", () => {
|
|
|
102
106
|
expect(networkStatus(RADIUS, TAILSCALE)).toEqual({ area: "Network", line: "RADIUS running, Tailscale router tailscale-37 online" });
|
|
103
107
|
const both = `${TAILSCALE}\n100.95.12.39 astro-mb-m5-2 astronaute@ macOS offline, last seen 15h ago\n100.106.244.101 tailscale-mira astronaute@ linux -`;
|
|
104
108
|
expect(networkStatus(RADIUS, both)).toEqual({ area: "Network", line: "RADIUS running, Tailscale routers tailscale-37 and tailscale-mira online" });
|
|
105
|
-
expect(networkStatus(RADIUS, both.replace("tailscale-mira astronaute@ linux -", "tailscale-mira astronaute@ linux offline, last seen 2h ago"))).toMatchObject({ line: "RADIUS running, Tailscale offline: tailscale-mira", problem: "Network:
|
|
106
|
-
expect(networkStatus(RADIUS.replace("active (running)", "failed (Result: exit-code)"), "100.121.19.2 tailscale-37 astronaute@ linux offline, last seen 1h ago")).toMatchObject({ problem: "Network: FreeRADIUS
|
|
107
|
-
expect(networkStatus(null, TAILSCALE).problem).toBe("Network:
|
|
109
|
+
expect(networkStatus(RADIUS, both.replace("tailscale-mira astronaute@ linux -", "tailscale-mira astronaute@ linux offline, last seen 2h ago"))).toMatchObject({ line: "RADIUS running, Tailscale offline: tailscale-mira", problem: "Network: Tailscale router tailscale-mira offline" });
|
|
110
|
+
expect(networkStatus(RADIUS.replace("active (running)", "failed (Result: exit-code)"), "100.121.19.2 tailscale-37 astronaute@ linux offline, last seen 1h ago")).toMatchObject({ problem: "Network: FreeRADIUS down; Tailscale router tailscale-37 offline" });
|
|
111
|
+
expect(networkStatus(null, TAILSCALE).problem).toBe("Network: `pulsar radius-status` failed");
|
|
108
112
|
});
|
|
109
113
|
|
|
110
114
|
it("lists the DMZ units and flags one that is not active", () => {
|
|
111
115
|
expect(identityStatus(DMZ)).toEqual({ area: "Identity", line: "pomerium, nginx, lego-renew.timer active" });
|
|
112
|
-
expect(identityStatus(DMZ.replace("Active: active (running) since Thu 2026-09-24 17:21:24 CEST; 4 weeks 0 days ago\n● nginx", "Active: failed (Result: signal)\n● nginx")).problem).toBe("Identity: pomerium.service failed
|
|
116
|
+
expect(identityStatus(DMZ.replace("Active: active (running) since Thu 2026-09-24 17:21:24 CEST; 4 weeks 0 days ago\n● nginx", "Active: failed (Result: signal)\n● nginx")).problem).toBe("Identity: DMZ: pomerium.service failed");
|
|
113
117
|
});
|
|
114
118
|
|
|
115
119
|
it("offers an owner button when a newer Pomerium release is out", () => {
|
|
@@ -117,7 +121,7 @@ describe("morning status lines", () => {
|
|
|
117
121
|
expect(identityStatus(DMZ, { installed: "v0.33.6", newest: "v0.33.6", update: false, notes })).toEqual({ area: "Identity", line: "pomerium, nginx, lego-renew.timer active; Pomerium v0.33.6" });
|
|
118
122
|
const newer = identityStatus(DMZ, { installed: "v0.33.1", newest: "v0.33.6", update: true, notes });
|
|
119
123
|
expect(newer.line).toBe("pomerium, nginx, lego-renew.timer active; Pomerium v0.33.1, v0.33.6 available");
|
|
120
|
-
expect(newer.problem).toContain(`Pomerium v0.33.
|
|
124
|
+
expect(newer.problem).toContain(`Pomerium v0.33.1 → v0.33.6 (notes: <${notes}>)`);
|
|
121
125
|
expect(newer.buttons).toEqual([{ label: "Update Pomerium to v0.33.6", customId: pomeriumButtonId("v0.33.6") }]);
|
|
122
126
|
expect(newer.once).toBe("pomerium:v0.33.6");
|
|
123
127
|
const failedUnit = DMZ.replace("Active: active (running) since Thu 2026-09-24 17:21:24 CEST; 4 weeks 0 days ago\n● nginx", "Active: failed (Result: signal)\n● nginx");
|
|
@@ -127,28 +131,29 @@ describe("morning status lines", () => {
|
|
|
127
131
|
|
|
128
132
|
it("checks both resolvers, Nexus, and SABnzbd", () => {
|
|
129
133
|
expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: { status: "ok" }, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "technitium1, technitium2, technitium3 answer" });
|
|
130
|
-
expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: null, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "no answer from technitium2", problem: "DNS: technitium2
|
|
131
|
-
expect(dnsStatus({ technitium1: null, technitium2: { status: "error" }, technitium3: { status: "ok" } })).toMatchObject({ problem: "DNS: technitium1 and technitium2
|
|
134
|
+
expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: null, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "no answer from technitium2", problem: "DNS: technitium2 not answering", route: "admin" });
|
|
135
|
+
expect(dnsStatus({ technitium1: null, technitium2: { status: "error" }, technitium3: { status: "ok" } })).toMatchObject({ problem: "DNS: technitium1 and technitium2 not answering" });
|
|
132
136
|
});
|
|
133
137
|
|
|
134
138
|
it("flags any newer macOS of the installed major for its owner and notes a major without flagging it", () => {
|
|
135
139
|
const nexus = "26.7.2\nTitle: Command Line Tools for Xcode 26.6, Version: 26.6, Size: 920431KiB, Recommended: YES,\nTitle: Safari, Version: 27.2, Size: 250130KiB, Recommended: YES,\nTitle: macOS 27.2.1, Version: 27.2.1, Size: 17362949KiB, Recommended: YES, Action: restart,";
|
|
136
|
-
const cortex = "26.7.2\nTitle: macOS Tahoe 26.7.3, Version: 26.7.3, Size: 2506663KiB, Recommended: YES, Action: restart,\nTitle: macOS 27.2.1, Version: 27.2.1, Size: 17362949KiB, Recommended: YES, Action: restart,";
|
|
140
|
+
const cortex = "26.7.2\n* Label: macOS Tahoe 26.7.3-25H45\nTitle: macOS Tahoe 26.7.3, Version: 26.7.3, Size: 2506663KiB, Recommended: YES, Action: restart,\n* Label: macOS 27.2.1-26B131\nTitle: macOS 27.2.1, Version: 27.2.1, Size: 17362949KiB, Recommended: YES, Action: restart,";
|
|
137
141
|
expect(macosStatus([{ name: "Nexus", output: nexus }])).toEqual({ area: "macOS", line: "Nexus 26.7.2, macOS 27.2.1 offered (major, by hand)" });
|
|
138
|
-
const waiting =
|
|
142
|
+
const waiting = 'Cortex macOS 26.7.2 → 26.7.3. On Cortex: `sudo softwareupdate -i "macOS Tahoe 26.7.3-25H45" -R --user cortex` (restarts it)';
|
|
143
|
+
const nexusWaiting = 'Nexus macOS 26.7.2 → 26.7.3. On Nexus: `sudo softwareupdate -i "macOS Tahoe 26.7.3" -R --user astronaute` (restarts it)';
|
|
139
144
|
expect(macosStatus([{ name: "Cortex", output: cortex }, { name: "Nexus", output: nexus.replace("Title: Safari", "Title: macOS Tahoe 26.7.3, Version: 26.7.3, Size: 1KiB,\nTitle: Safari") }])).toEqual({
|
|
140
145
|
area: "macOS",
|
|
141
146
|
line: "Cortex 26.7.2, 26.7.3 waiting, macOS 27.2.1 offered (major, by hand); Nexus 26.7.2, 26.7.3 waiting, macOS 27.2.1 offered (major, by hand)",
|
|
142
|
-
problem: `macOS: ${waiting}; ${
|
|
147
|
+
problem: `macOS: ${waiting}; ${nexusWaiting}`,
|
|
143
148
|
items: [
|
|
144
149
|
{ owner: "Cortex", text: waiting },
|
|
145
|
-
{ owner: "Inference", text:
|
|
150
|
+
{ owner: "Inference", text: nexusWaiting },
|
|
146
151
|
],
|
|
147
152
|
route: "admin",
|
|
148
153
|
once: "macos:Cortex:26.7.3,Nexus:26.7.3",
|
|
149
154
|
});
|
|
150
155
|
const unknown = macosStatus([{ name: "Cortex", output: cortex }, { name: "Nexus", output: null }]);
|
|
151
|
-
expect(unknown.items?.at(-1)).toEqual({ owner: "Inference", text: "no macOS update list
|
|
156
|
+
expect(unknown.items?.at(-1)).toEqual({ owner: "Inference", text: "Nexus: no macOS update list" });
|
|
152
157
|
expect(unknown.once).toBeUndefined();
|
|
153
158
|
expect(macosStatus([{ name: "Cortex", output: "26.7.2" }]).line).toBe("Cortex 26.7.2");
|
|
154
159
|
});
|
|
@@ -163,10 +168,10 @@ describe("morning status lines", () => {
|
|
|
163
168
|
expect(technitiumPlan(update("16.0.0"), state(["15.6.2", "15.6.2", "15.6.2"]))).toEqual({ major: true, from: "15.6.2", to: "16.0.0" });
|
|
164
169
|
const all = { technitium1: { status: "ok" }, technitium2: { status: "ok" }, technitium3: { status: "ok" } };
|
|
165
170
|
expect(dnsStatus(all, { host: "technitium3", from: "15.6.2", to: "15.7.0", ok: true })).toEqual({ area: "DNS", line: "technitium1, technitium2, technitium3 answer; technitium3 upgraded 15.6.2 -> 15.7.0 (one node per morning, primary last)" });
|
|
166
|
-
expect(dnsStatus(all, { host: "technitium3", from: "15.6.2", to: "15.7.0", ok: false, why: "installer exited 1" }).problem).toBe("DNS:
|
|
167
|
-
expect(dnsStatus(all, { major: true, from: "15.6.2", to: "16.0.0" }).problem).toMatch(/^DNS: Technitium 16
|
|
171
|
+
expect(dnsStatus(all, { host: "technitium3", from: "15.6.2", to: "15.7.0", ok: false, why: "installer exited 1" }).problem).toBe("DNS: technitium3: upgrade to 15.7.0 failed (installer exited 1)");
|
|
172
|
+
expect(dnsStatus(all, { major: true, from: "15.6.2", to: "16.0.0" }).problem).toMatch(/^DNS: Technitium 15\.6\.2 → 16\.0\.0 is a major release: read its change log/);
|
|
168
173
|
expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=200\nrerank \tstate = running \tpid = 81670 health=200")).toEqual({ area: "Inference", line: "Nexus embed, rerank running and healthy" });
|
|
169
|
-
expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=503")).toMatchObject({ problem: "Inference: embed
|
|
174
|
+
expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=503")).toMatchObject({ problem: "Inference: embed unhealthy on Nexus" });
|
|
170
175
|
expect(homeStatus({ problems: [], unavailable: 3, version: "2026.11.1" })).toEqual({ area: "Home", line: "Home Assistant 2026.11.1, nothing to do, 3 entities unavailable" });
|
|
171
176
|
expect(homeStatus({ problems: ["1 update waits: Home Assistant Core 2026.11.1 → 2026.11.2", "the Nest integration (nest) did not load: setup_retry"], unavailable: 1, version: "2026.11.1" })).toEqual({
|
|
172
177
|
area: "Home",
|
|
@@ -178,18 +183,18 @@ describe("morning status lines", () => {
|
|
|
178
183
|
],
|
|
179
184
|
once: "home:1 update waits: Home Assistant Core 2026.11.1 → 2026.11.2|the Nest integration (nest) did not load: setup_retry",
|
|
180
185
|
});
|
|
181
|
-
expect(homeStatus(null)).toMatchObject({ area: "Home", problem: "Home:
|
|
186
|
+
expect(homeStatus(null)).toMatchObject({ area: "Home", problem: "Home: `problems` failed" });
|
|
182
187
|
const empty = { records: [], warnings: [] };
|
|
183
188
|
expect(mediaStatus({ sabnzbd_paused: true, warnings: [] }, empty, NOW)).toEqual({ area: "Media", line: "downloads paused" });
|
|
184
|
-
expect(mediaStatus({ sabnzbd_paused: false, warnings: [{ text: "login failed" }] }, empty, NOW)).toMatchObject({ line: "downloads running, 1 warning", problem: "Media:
|
|
185
|
-
expect(mediaStatus({ sabnzbd_paused: false, warnings: [] }, null, NOW)).toMatchObject({ line: "downloads running, import queue unknown", problem: "Media:
|
|
186
|
-
expect(mediaStatus({ sabnzbd_paused: false, warnings: [] }, { records: [], warnings: ["sonarr: sonarr answered 401 to GET /api/v3/queue"] }, NOW)).toMatchObject({ problem: "Media:
|
|
189
|
+
expect(mediaStatus({ sabnzbd_paused: false, warnings: [{ text: "login failed" }] }, empty, NOW)).toMatchObject({ line: "downloads running, 1 warning", problem: "Media: SABnzbd: 1 warning", route: "media" });
|
|
190
|
+
expect(mediaStatus({ sabnzbd_paused: false, warnings: [] }, null, NOW)).toMatchObject({ line: "downloads running, import queue unknown", problem: "Media: `queue` failed" });
|
|
191
|
+
expect(mediaStatus({ sabnzbd_paused: false, warnings: [] }, { records: [], warnings: ["sonarr: sonarr answered 401 to GET /api/v3/queue"] }, NOW)).toMatchObject({ problem: "Media: sonarr queue unreadable", route: "media" });
|
|
187
192
|
});
|
|
188
193
|
|
|
189
194
|
it("names imports that wait in the queue for more than a day, and tags them once", () => {
|
|
190
195
|
const record = (id: number, added: string) => ({ app: "radarr", id, title: "Primetime.2026.REPACK.2160p", state: "importPending", status: "warning", messages: ["Not a Custom Format upgrade for existing movie file(s)."], added, size: 13406787332 });
|
|
191
196
|
const queue = { records: [record(7, "2026-10-22T16:03:52Z"), record(8, "2026-10-23T20:00:00Z")], warnings: [] };
|
|
192
|
-
const item = "Primetime.2026.REPACK.2160p (radarr)
|
|
197
|
+
const item = "Primetime.2026.REPACK.2160p (radarr) stuck since 2026-10-22: Not a Custom Format upgrade for existing movie file(s). Reply \"remove\" to delete it (`queue-remove radarr 7`)";
|
|
193
198
|
expect(mediaStatus({ sabnzbd_paused: false, warnings: [] }, queue, NOW)).toEqual({
|
|
194
199
|
area: "Media",
|
|
195
200
|
line: "downloads running, 1 import stuck",
|
|
@@ -208,7 +213,7 @@ describe("morning status lines", () => {
|
|
|
208
213
|
];
|
|
209
214
|
const restic = "Result=success\nExecMainStartTimestamp=Sun 2026-10-18 04:00:24 CEST";
|
|
210
215
|
expect(backupStatus(archives.slice(0, 1), restic, NOW)).toEqual({ area: "Backup", line: "the only guest backed up last night, restic check success 18 Oct 04:00" });
|
|
211
|
-
expect(backupStatus(archives, restic.replace("success", "exit-code"), NOW)).toMatchObject({ problem: "Backup: no
|
|
216
|
+
expect(backupStatus(archives, restic.replace("success", "exit-code"), NOW)).toMatchObject({ problem: "Backup: no backup last night: identity; restic check: exit-code", route: "admin" });
|
|
212
217
|
});
|
|
213
218
|
});
|
|
214
219
|
|
|
@@ -225,18 +230,18 @@ describe("morning summary", () => {
|
|
|
225
230
|
],
|
|
226
231
|
areas: [
|
|
227
232
|
{ area: "DNS", line: "both resolvers answer" },
|
|
228
|
-
{ area: "Backup", line: "no backup last night for identity", problem: "Backup: no
|
|
233
|
+
{ area: "Backup", line: "no backup last night for identity", problem: "Backup: no backup last night: identity", route: "admin", messageId: "41" },
|
|
229
234
|
{ area: "macOS", line: "Cortex 26.7.2" },
|
|
230
|
-
{ area: "Updates", line: "x", problem: "Updates:
|
|
235
|
+
{ area: "Updates", line: "x", problem: "Updates: synapse: libcamera-ipa kept back. Fix by hand: `apt-get full-upgrade` (adds libcamera0.8)", items: [{ owner: "DNS", text: "synapse: libcamera-ipa kept back. Fix by hand: `apt-get full-upgrade` (adds libcamera0.8)" }], route: "admin" },
|
|
231
236
|
],
|
|
232
237
|
});
|
|
233
238
|
expect(text).toBe(
|
|
234
239
|
[
|
|
235
240
|
"**Daily** · Fri 24 Oct · 🟠 3 items need you",
|
|
236
241
|
"Arcane · stirling-pdf waits for you (major version) ([report](https://discord.com/channels/1/22/32))",
|
|
237
|
-
"
|
|
238
|
-
"
|
|
239
|
-
"All clear: Security, Cortex, Proxmox
|
|
242
|
+
"Backup · no backup last night: identity ([message](https://discord.com/channels/1/11/41))",
|
|
243
|
+
"DNS · synapse: libcamera-ipa kept back. Fix by hand: `apt-get full-upgrade` (adds libcamera0.8) (<#11>)",
|
|
244
|
+
"All clear: Security, Cortex, Proxmox",
|
|
240
245
|
].join("\n"),
|
|
241
246
|
);
|
|
242
247
|
expect(text).not.toContain("<@");
|
|
@@ -253,20 +258,23 @@ describe("morning summary", () => {
|
|
|
253
258
|
});
|
|
254
259
|
|
|
255
260
|
it("tags the owners for an area's problem in its channel", () => {
|
|
256
|
-
expect(problemText({ area: "Backup", line: "x", problem: "Backup:
|
|
257
|
-
"⚠️ **Backup** · Backup: the restic check ended with exit-code\nFound by the morning run; the backup specialist can look into it here.\n<@178096425509453824>",
|
|
258
|
-
);
|
|
261
|
+
expect(problemText({ area: "Backup", line: "x", problem: "Backup: restic check: exit-code", route: "admin" }, ["178096425509453824"])).toBe("⚠️ **Backup** · restic check: exit-code\nReplies go to the backup specialist.\n<@178096425509453824>");
|
|
259
262
|
});
|
|
260
263
|
|
|
261
|
-
it("names the specialist that
|
|
262
|
-
expect(problemText({ area: "Updates", line: "x", problem: "Updates: no update list
|
|
263
|
-
|
|
264
|
+
it("names the specialist that answers replies, the owner's for updates, none for macOS, and keeps commands as code", () => {
|
|
265
|
+
expect(problemText({ area: "Updates", line: "x", problem: "Updates: radius: no update list" }, ["1"])).toBe("⚠️ **Updates** · radius: no update list\nReplies go to the proxmox specialist.\n<@1>");
|
|
266
|
+
const [synapse] = updatesStatus({ synapse: "libcamera-ipa/stable 0.8.0 arm64 [upgradable from: 0.7.2]\nnew: libcamera0.8" });
|
|
267
|
+
expect(problemText(synapse, ["1"])).toBe("⚠️ **Updates** · synapse: libcamera-ipa kept back. Fix by hand: `apt-get full-upgrade` (adds libcamera0.8)\nReplies go to the dns specialist.\n<@1>");
|
|
268
|
+
const two = { area: "Home", line: "x", problem: "Home: a; b", items: [{ owner: "Home", text: "no_current_backup" }, { owner: "Home", text: "b" }] };
|
|
269
|
+
expect(problemText(two, ["1"])).toBe("⚠️ **Home**\n- no\\_current\\_backup\n- b\nReplies go to the home specialist.\n<@1>");
|
|
270
|
+
expect(problemText({ area: "macOS", line: "x", problem: "macOS: Nexus: no macOS update list" }, ["1"])).toBe("⚠️ **macOS** · Nexus: no macOS update list\n<@1>");
|
|
264
271
|
});
|
|
265
272
|
|
|
266
|
-
it("reads back the specialist a problem names, also after a button note", () => {
|
|
267
|
-
const text = problemText({ area: "Updates", line: "x", problem: "Updates: no update list
|
|
273
|
+
it("reads back the specialist a problem names, in both message formats, also after a button note", () => {
|
|
274
|
+
const text = problemText({ area: "Updates", line: "x", problem: "Updates: radius: no update list" }, ["1"]);
|
|
268
275
|
expect(problemSpecialist(text)).toBe("proxmox");
|
|
269
276
|
expect(problemSpecialist(`${text}\n♻️ SPACE-X reboots in one minute, started by <@1>; its guests restart with it.`)).toBe("proxmox");
|
|
277
|
+
expect(problemSpecialist("⚠️ **Updates** · x\nFound by the morning run; the proxmox specialist can look into it here.\n<@1>")).toBe("proxmox");
|
|
270
278
|
expect(problemSpecialist(problemText({ area: "macOS", line: "x", problem: "macOS: x" }, ["1"]))).toBeNull();
|
|
271
279
|
expect(problemSpecialist("Several specialists live here")).toBeNull();
|
|
272
280
|
});
|
|
@@ -33,12 +33,24 @@ export interface AreaStatus {
|
|
|
33
33
|
* the summary keeps listing it every morning.
|
|
34
34
|
*/
|
|
35
35
|
once?: string;
|
|
36
|
+
/** The specialist that answers replies to the problem's message, when not the area's own; null for none. */
|
|
37
|
+
specialist?: string | null;
|
|
38
|
+
/** The problem's tagged message, which its summary line links to. */
|
|
39
|
+
messageId?: string;
|
|
36
40
|
}
|
|
37
41
|
|
|
38
42
|
const HOUR = 3600;
|
|
39
43
|
|
|
40
44
|
function failed(area: string, route: Route, why: string): AreaStatus {
|
|
41
|
-
return { area, line: `the check failed (${why.slice(0, 120)})`, problem: `${area}:
|
|
45
|
+
return { area, line: `the check failed (${why.slice(0, 120)})`, problem: `${area}: \`${why}\` failed`, route };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Escapes Discord markdown outside inline code, so commands in a problem stay copyable code. */
|
|
49
|
+
export function escapeOutsideCode(text: string): string {
|
|
50
|
+
return text
|
|
51
|
+
.split(/(`[^`\n]+`)/)
|
|
52
|
+
.map((part, i) => (i % 2 === 1 ? part : escapeDiscord(part)))
|
|
53
|
+
.join("");
|
|
42
54
|
}
|
|
43
55
|
|
|
44
56
|
/** Seconds from `21s ago`, `3m ago`, `2h ago`, `1d ago`; null when the text is not one of those. */
|
|
@@ -67,7 +79,7 @@ export function edgeStatus(status: string | null, updates: string | null): AreaS
|
|
|
67
79
|
upgradable === null ? "updates unknown" : `${upgradable} upgradable package${upgradable === 1 ? "" : "s"}`,
|
|
68
80
|
];
|
|
69
81
|
const tunnel = age === null || age > STALE_HANDSHAKE_S;
|
|
70
|
-
const problem = down.length > 0 ? `Edge: ${down.join(", ")}
|
|
82
|
+
const problem = down.length > 0 ? `Edge: ${down.join(", ")} down on the front door` : tunnel ? `Edge: WireGuard handshake ${handshake || "missing"}` : undefined;
|
|
71
83
|
return { area: "Edge", line: `front door: ${parts.join(", ")}`, ...(problem ? { problem, route: "admin" as const } : {}) };
|
|
72
84
|
}
|
|
73
85
|
|
|
@@ -112,9 +124,9 @@ function proxmoxHostFacts(h: ProxmoxHost): ProxmoxFacts {
|
|
|
112
124
|
const rebootNeeded = reboot !== undefined && reboot !== "no";
|
|
113
125
|
const line = [up ? `${h.name} up ${up}` : `${h.name} up`, mem ? `RAM ${mem[2]} of ${mem[1]} used` : null, root ? `root disk ${root}%` : null, updates, rebootNeeded ? "reboot needed" : null].filter(Boolean).join(", ");
|
|
114
126
|
const problems = [
|
|
115
|
-
h.upgrade === null ?
|
|
116
|
-
held ? `${h.name}
|
|
117
|
-
rebootNeeded ? `${h.name}
|
|
127
|
+
h.upgrade === null ? `${h.name}: host upgrade failed` : null,
|
|
128
|
+
held ? `${h.name}: ${held}, held for you` : null,
|
|
129
|
+
rebootNeeded ? `${h.name}: reboot needed (${reboot}); its guests stop for a few minutes` : null,
|
|
118
130
|
].filter((p): p is string => p !== null);
|
|
119
131
|
return { line, problems, ...(rebootNeeded ? { button: { label: `Reboot ${h.name}`, customId: rebootButtonId(h.id) } } : {}) };
|
|
120
132
|
}
|
|
@@ -135,8 +147,8 @@ export function proxmoxStatus(hosts: readonly ProxmoxHost[]): AreaStatus {
|
|
|
135
147
|
/** Packages the nightly run leaves alone on purpose: a Wazuh agent must never be newer than the manager. */
|
|
136
148
|
const HELD_PACKAGES = new Set(["wazuh-agent"]);
|
|
137
149
|
|
|
138
|
-
/** The
|
|
139
|
-
function
|
|
150
|
+
/** The lines of an `updates` read, also when a VM's guest agent wraps them in JSON (`out-data`). */
|
|
151
|
+
function aptLines(output: string): string[] {
|
|
140
152
|
let text = output;
|
|
141
153
|
try {
|
|
142
154
|
const wrapped = (JSON.parse(output) as { "out-data"?: unknown } | null)?.["out-data"];
|
|
@@ -144,7 +156,19 @@ function upgradableLines(output: string): string[] {
|
|
|
144
156
|
} catch {
|
|
145
157
|
/* plain apt output */
|
|
146
158
|
}
|
|
147
|
-
return text.split("\n")
|
|
159
|
+
return text.split("\n");
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** The `[upgradable from` lines of `apt list --upgradable`. */
|
|
163
|
+
function upgradableLines(output: string): string[] {
|
|
164
|
+
return aptLines(output).filter((l) => l.includes("[upgradable from"));
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/** What a full upgrade would add and remove: the `new:` and `remove:` lines of the `updates` dry run. Packages wait when this is not empty. */
|
|
168
|
+
export function keptBack(output: string): { adds: string[]; removes: string[] } {
|
|
169
|
+
const lines = aptLines(output).map((l) => l.trim());
|
|
170
|
+
const pick = (prefix: string) => lines.filter((l) => l.startsWith(prefix)).map((l) => l.slice(prefix.length).trim());
|
|
171
|
+
return { adds: pick("new: "), removes: pick("remove: ") };
|
|
148
172
|
}
|
|
149
173
|
|
|
150
174
|
/** The upgradable packages of one host, held packages left out. */
|
|
@@ -177,15 +201,18 @@ export function guestOwner(name: string): string {
|
|
|
177
201
|
return "Proxmox";
|
|
178
202
|
}
|
|
179
203
|
|
|
180
|
-
/**
|
|
181
|
-
|
|
204
|
+
/**
|
|
205
|
+
* Each guest's and Synapse's `updates` after the 02:25 run: anything still waiting did not install.
|
|
206
|
+
* One problem per owner, so a reply reaches the specialist that runs the host; a kept-back upgrade names its fix.
|
|
207
|
+
*/
|
|
208
|
+
export function updatesStatus(hosts: Readonly<Record<string, string | null>>): AreaStatus[] {
|
|
182
209
|
const names = Object.keys(hosts);
|
|
183
210
|
const wazuh = wazuhRelease(hosts);
|
|
184
211
|
const unknown = names.filter((n) => hosts[n] === null);
|
|
185
212
|
const left = names.flatMap((n) => {
|
|
186
213
|
const out = hosts[n];
|
|
187
214
|
const packages = out === null ? [] : pendingPackageLines(out).map((l) => l.split("/")[0]);
|
|
188
|
-
return packages.length > 0 ? [{ name: n, packages }] : [];
|
|
215
|
+
return out !== null && packages.length > 0 ? [{ name: n, packages, ...keptBack(out) }] : [];
|
|
189
216
|
});
|
|
190
217
|
const checked = names.length - unknown.length;
|
|
191
218
|
const parts = [
|
|
@@ -193,15 +220,24 @@ export function updatesStatus(hosts: Readonly<Record<string, string | null>>): A
|
|
|
193
220
|
unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null,
|
|
194
221
|
wazuh ? `Wazuh ${wazuh.to} is out (agents held at ${wazuh.from})` : null,
|
|
195
222
|
].filter(Boolean);
|
|
196
|
-
const
|
|
197
|
-
const
|
|
198
|
-
const
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
];
|
|
204
|
-
|
|
223
|
+
const line = parts.join("; ");
|
|
224
|
+
const list = (items: readonly string[]) => `${items.slice(0, 4).join(", ")}${items.length > 4 ? ", ..." : ""}`;
|
|
225
|
+
const hostText = (h: (typeof left)[number]) => {
|
|
226
|
+
if (h.adds.length === 0 && h.removes.length === 0) return `${h.name}: ${list(h.packages)} did not install`;
|
|
227
|
+
const changes = [h.adds.length > 0 ? `adds ${list(h.adds)}` : null, h.removes.length > 0 ? `removes ${list(h.removes)}` : null].filter(Boolean).join(", ");
|
|
228
|
+
return `${h.name}: ${list(h.packages)} kept back. Fix by hand: \`apt-get full-upgrade\` (${changes})`;
|
|
229
|
+
};
|
|
230
|
+
const items: SummaryItem[] = [...left.map((h) => ({ owner: guestOwner(h.name), text: hostText(h) })), ...unknown.map((n) => ({ owner: guestOwner(n), text: `${n}: no update list` }))];
|
|
231
|
+
const owners = [...new Set(items.map((i) => i.owner))];
|
|
232
|
+
const areas: AreaStatus[] = owners.map((owner) => {
|
|
233
|
+
const mine = items.filter((i) => i.owner === owner);
|
|
234
|
+
return { area: "Updates", line, problem: `Updates: ${mine.map((i) => i.text).join("; ")}`, items: mine, route: "admin", specialist: owner.toLowerCase() };
|
|
235
|
+
});
|
|
236
|
+
if (wazuh) {
|
|
237
|
+
const text = `Wazuh ${wazuh.to} is out (agents on ${wazuh.from}). Claude Code on the Mac upgrades the manager first, then the agents`;
|
|
238
|
+
areas.push({ area: "Updates", line, problem: `Updates: ${text}`, items: [{ owner: "Security", text }], route: "admin", specialist: null });
|
|
239
|
+
}
|
|
240
|
+
return areas.length > 0 ? areas : [{ area: "Updates", line }];
|
|
205
241
|
}
|
|
206
242
|
|
|
207
243
|
/** `network pulsar radius-status` and `tailscale-status`: FreeRADIUS running, and every `tailscale*` subnet router (37 and Mira) online. */
|
|
@@ -218,9 +254,10 @@ export function networkStatus(radius: string | null, tailscale: string | null):
|
|
|
218
254
|
routers === null ? "Tailscale unknown" : down.length === 0 ? `Tailscale router${routers.length === 1 ? "" : "s"} ${routers.map((r) => r.name).join(" and ")} online` : `Tailscale offline: ${down.join(", ")}`,
|
|
219
255
|
];
|
|
220
256
|
const problems = [
|
|
221
|
-
radiusUp === false ? "FreeRADIUS
|
|
222
|
-
down.length > 0 ? `
|
|
223
|
-
radiusUp === null
|
|
257
|
+
radiusUp === false ? "FreeRADIUS down" : null,
|
|
258
|
+
down.length > 0 ? `Tailscale router ${down.join(" and ")} offline` : null,
|
|
259
|
+
radiusUp === null ? "`pulsar radius-status` failed" : null,
|
|
260
|
+
routers === null ? "`pulsar tailscale-status` failed" : null,
|
|
224
261
|
].filter(Boolean);
|
|
225
262
|
return { area: "Network", line: parts.join(", "), ...(problems.length > 0 ? { problem: `Network: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
226
263
|
}
|
|
@@ -253,8 +290,8 @@ export function identityStatus(status: string | null, pomerium?: unknown): AreaS
|
|
|
253
290
|
const unitLine = units.length === 0 ? "no unit status read" : bad.length === 0 ? `${units.map((u) => u.unit.replace(/\.service$/, "")).join(", ")} active` : `not active: ${bad.join(", ")}`;
|
|
254
291
|
const releaseLine = pomerium === undefined ? null : release === null ? "Pomerium version unknown" : newer ? `Pomerium ${newer.installed}, ${newer.newest} available` : `Pomerium ${release.installed}`;
|
|
255
292
|
const problems = [
|
|
256
|
-
bad.length > 0 || units.length === 0 ?
|
|
257
|
-
newer ? `Pomerium ${newer.
|
|
293
|
+
bad.length > 0 || units.length === 0 ? `DMZ: ${bad.length > 0 ? bad.join(", ") : "no unit status read"}` : null,
|
|
294
|
+
newer ? `Pomerium ${newer.installed} → ${newer.newest} (notes: <${newer.notes}>). The button checks the sha256, restarts Pomerium for seconds, and reverts if login fails` : null,
|
|
258
295
|
].filter(Boolean);
|
|
259
296
|
return {
|
|
260
297
|
area: "Identity",
|
|
@@ -296,9 +333,9 @@ export function dnsStatus(answers: Readonly<Record<string, unknown>>, upgrade?:
|
|
|
296
333
|
const names = down.join(" and ");
|
|
297
334
|
const upgradeLine = upgrade === undefined ? null : "major" in upgrade ? `Technitium ${upgrade.to} is a major release and waits for you` : upgrade.ok ? `${upgrade.host} upgraded ${upgrade.from} -> ${upgrade.to} (one node per morning, primary last)` : `the upgrade of ${upgrade.host} to ${upgrade.to} failed`;
|
|
298
335
|
const problems = [
|
|
299
|
-
down.length > 0 ? `${names}
|
|
300
|
-
upgrade && "major" in upgrade ? `Technitium ${upgrade.to} is a major release
|
|
301
|
-
upgrade && "ok" in upgrade && !upgrade.ok ?
|
|
336
|
+
down.length > 0 ? `${names} not answering` : null,
|
|
337
|
+
upgrade && "major" in upgrade ? `Technitium ${upgrade.from} → ${upgrade.to} is a major release: read its change log, then reply "upgrade <node>", one node at a time, technitium1 last` : null,
|
|
338
|
+
upgrade && "ok" in upgrade && !upgrade.ok ? `${upgrade.host}: upgrade to ${upgrade.to} failed (${upgrade.why.slice(0, 200)})` : null,
|
|
302
339
|
].filter(Boolean);
|
|
303
340
|
const line = [down.length === 0 ? `${nodes.join(", ")} answer` : `no answer from ${names}`, upgradeLine].filter(Boolean).join("; ");
|
|
304
341
|
return { area: "DNS", line, ...(problems.length > 0 ? { problem: `DNS: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
@@ -314,7 +351,7 @@ export function inferenceStatus(status: string | null): AreaStatus {
|
|
|
314
351
|
.map((m) => ({ name: m[1], running: m[2] === "running", healthy: m[3] === "200" }));
|
|
315
352
|
const bad = services.filter((s) => !s.running || !s.healthy).map((s) => s.name);
|
|
316
353
|
const line = services.length === 0 ? "no Nexus service listed" : bad.length === 0 ? `Nexus ${services.map((s) => s.name).join(", ")} running and healthy` : `Nexus not healthy: ${bad.join(", ")}`;
|
|
317
|
-
return { area: "Inference", line, ...(bad.length > 0 || services.length === 0 ? { problem: `Inference: ${bad.length > 0 ? `${bad.join(", ")}
|
|
354
|
+
return { area: "Inference", line, ...(bad.length > 0 || services.length === 0 ? { problem: `Inference: ${bad.length > 0 ? `${bad.join(", ")} unhealthy on Nexus` : "Nexus lists no service"}`, route: "admin" as const } : {}) };
|
|
318
355
|
}
|
|
319
356
|
|
|
320
357
|
/**
|
|
@@ -368,12 +405,12 @@ export function mediaStatus(sab: unknown, queue: unknown, now: number): AreaStat
|
|
|
368
405
|
const stuck = imports?.filter((i) => now / 1000 - Date.parse(i.added) / 1000 > STUCK_IMPORT_S) ?? [];
|
|
369
406
|
const line = [`downloads ${r.sabnzbd_paused ? "paused" : "running"}${warnings > 0 ? `, ${warnings} warning${warnings === 1 ? "" : "s"}` : ""}`, imports === null ? "import queue unknown" : stuck.length > 0 ? `${stuck.length} import${stuck.length === 1 ? "" : "s"} stuck` : null].filter(Boolean).join(", ");
|
|
370
407
|
const items: SummaryItem[] = [
|
|
371
|
-
...(warnings > 0 ? [{ owner: "Media", text: `
|
|
372
|
-
...(imports === null ? [{ owner: "Media", text: "
|
|
373
|
-
...unread.map((w) => ({ owner: "Media", text:
|
|
408
|
+
...(warnings > 0 ? [{ owner: "Media", text: `SABnzbd: ${warnings} warning${warnings === 1 ? "" : "s"}` }] : []),
|
|
409
|
+
...(imports === null ? [{ owner: "Media", text: "`queue` failed" }] : []),
|
|
410
|
+
...unread.map((w) => ({ owner: "Media", text: `${w} queue unreadable` })),
|
|
374
411
|
...stuck.map((i) => ({
|
|
375
412
|
owner: "Media",
|
|
376
|
-
text: `${i.title} (${i.app})
|
|
413
|
+
text: `${i.title} (${i.app}) stuck since ${i.added.slice(0, 10)}: ${i.messages[0] ? i.messages[0].slice(0, 160).replace(/\.+$/, "") : i.state}. Reply "remove" to delete it (\`queue-remove ${i.app} ${i.id}\`)`,
|
|
377
414
|
})),
|
|
378
415
|
];
|
|
379
416
|
const onlyStuck = items.length > 0 && items.length === stuck.length;
|
|
@@ -395,7 +432,12 @@ export function backupStatus(archives: unknown, resticText: string | null, now:
|
|
|
395
432
|
const missed = guests?.filter((g) => now / 1000 - g.last > MISSED_BACKUP_S) ?? [];
|
|
396
433
|
const proxmox = guests === null ? "Proxmox archives unknown" : missed.length === 0 ? `${guests.length === 1 ? "the only guest" : `all ${guests.length} guests`} backed up last night` : `no backup last night for ${missed.map((g) => escapeDiscord(g.name)).join(", ")}`;
|
|
397
434
|
const check = restic ? `restic check ${escapeDiscord(restic.result)} ${restic.at}` : "restic check unknown";
|
|
398
|
-
const problems = [
|
|
435
|
+
const problems = [
|
|
436
|
+
missed.length > 0 ? `no backup last night: ${missed.map((g) => g.name).join(", ")}` : null,
|
|
437
|
+
restic && restic.result !== "success" ? `restic check: ${restic.result}` : null,
|
|
438
|
+
guests === null ? "`vzdump-archives` failed" : null,
|
|
439
|
+
restic === null ? "`restic-check-status` failed" : null,
|
|
440
|
+
].filter(Boolean);
|
|
399
441
|
return { area: "Backup", line: `${proxmox}, ${check}`, ...(problems.length > 0 ? { problem: `Backup: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
400
442
|
}
|
|
401
443
|
|
|
@@ -408,6 +450,9 @@ export interface MacUpdates {
|
|
|
408
450
|
/** The summary line each Mac's items go to: Cortex is this host, Nexus is the inference specialist's. */
|
|
409
451
|
const MAC_OWNER: Readonly<Record<string, string>> = { Cortex: "Cortex", Nexus: "Inference" };
|
|
410
452
|
|
|
453
|
+
/** The volume owner whose password authorizes a macOS install on each Mac: user `nexus` has no secure token. */
|
|
454
|
+
const MAC_VOLUME_OWNER: Readonly<Record<string, string>> = { Cortex: "cortex", Nexus: "astronaute" };
|
|
455
|
+
|
|
411
456
|
/**
|
|
412
457
|
* Each Mac's macOS version and offers. Apple silicon installs a macOS update
|
|
413
458
|
* only with a volume owner's password, so any newer update of the installed
|
|
@@ -421,14 +466,22 @@ export function macosStatus(macs: readonly MacUpdates[]): AreaStatus {
|
|
|
421
466
|
const owner = MAC_OWNER[m.name] ?? "Cortex";
|
|
422
467
|
if (m.output === null) {
|
|
423
468
|
parts.push(`${m.name} unknown`);
|
|
424
|
-
items.push({ owner, text:
|
|
469
|
+
items.push({ owner, text: `${m.name}: no macOS update list` });
|
|
425
470
|
continue;
|
|
426
471
|
}
|
|
427
472
|
const [installed = "", ...rest] = m.output.split("\n").map((l) => l.trim());
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
473
|
+
// Software Update prints each `* Label:` line right before its `Title:` line; installing needs the label.
|
|
474
|
+
const offers: { title: string; version: string; label: string }[] = [];
|
|
475
|
+
let label = "";
|
|
476
|
+
for (const l of rest) {
|
|
477
|
+
const named = /^\* Label: (.+)$/.exec(l);
|
|
478
|
+
if (named) label = named[1].trim();
|
|
479
|
+
const x = /Title: (macOS[^,]*), Version: ([\d.]+)/.exec(l);
|
|
480
|
+
if (x) {
|
|
481
|
+
offers.push({ title: x[1], version: x[2], label: label || x[1] });
|
|
482
|
+
label = "";
|
|
483
|
+
}
|
|
484
|
+
}
|
|
432
485
|
const major = Number(installed.split(".")[0]);
|
|
433
486
|
const compare = (a: string, b: string) => a.localeCompare(b, undefined, { numeric: true });
|
|
434
487
|
const newer = offers.filter((o) => Number(o.version.split(".")[0]) === major && compare(o.version, installed) > 0).sort((a, b) => compare(a.version, b.version));
|
|
@@ -436,7 +489,7 @@ export function macosStatus(macs: readonly MacUpdates[]): AreaStatus {
|
|
|
436
489
|
const latest = newer.at(-1);
|
|
437
490
|
parts.push(`${m.name} ${installed}${latest ? `, ${latest.version} waiting` : ""}${majors.length > 0 ? `, macOS ${majors[0].version} offered (major, by hand)` : ""}`);
|
|
438
491
|
if (latest) {
|
|
439
|
-
items.push({ owner, text: `${m.name}
|
|
492
|
+
items.push({ owner, text: `${m.name} macOS ${installed} → ${latest.version}. On ${m.name}: \`sudo softwareupdate -i "${latest.label}" -R --user ${MAC_VOLUME_OWNER[m.name] ?? "<volume owner>"}\` (restarts it)` });
|
|
440
493
|
waiting.push(`${m.name}:${latest.version}`);
|
|
441
494
|
}
|
|
442
495
|
}
|
|
@@ -555,9 +608,10 @@ export function summaryText(s: Summary): string {
|
|
|
555
608
|
add(ownerOf(r.area), r.needs.map((n) => `${n}${where}`));
|
|
556
609
|
}
|
|
557
610
|
for (const a of s.areas) {
|
|
558
|
-
const
|
|
611
|
+
const channel = s.routes[a.route ?? "admin"];
|
|
612
|
+
const where = a.messageId ? ` ([message](${link(s.guildId, channel, a.messageId)}))` : ` (<#${channel}>)`;
|
|
559
613
|
add(ownerOf(a.area), []);
|
|
560
|
-
for (const item of a.items ?? (a.problem ? [{ owner: ownerOf(a.area), text: a.problem.replace(`${a.area}: `, "") }] : [])) add(item.owner, [`${
|
|
614
|
+
for (const item of a.items ?? (a.problem ? [{ owner: ownerOf(a.area), text: a.problem.replace(`${a.area}: `, "") }] : [])) add(item.owner, [`${escapeOutsideCode(item.text)}${where}`]);
|
|
561
615
|
}
|
|
562
616
|
const count = [...lines.values()].reduce((n, items) => n + items.length, 0);
|
|
563
617
|
const head = `**Daily** · ${s.date} · ${count === 0 ? `🟢 ${ALL_CLEAR}` : `🟠 ${count} item${count === 1 ? " needs" : "s need"} you`}`;
|
|
@@ -583,14 +637,17 @@ export function problemsToPost(areas: readonly AreaStatus[], notifiedBefore: rea
|
|
|
583
637
|
/** Areas read by a specialist of another name, or by none: Cortex's macOS is this host's own. */
|
|
584
638
|
const AREA_SPECIALIST: Readonly<Record<string, string | null>> = { Updates: "proxmox", macOS: null };
|
|
585
639
|
|
|
586
|
-
/** The specialist a morning-run problem message names, so a reply to it goes there without a prefix. */
|
|
640
|
+
/** The specialist a morning-run problem message names, so a reply to it goes there without a prefix; messages before 0.40.76 use the longer line. */
|
|
587
641
|
export function problemSpecialist(content: string): string | null {
|
|
588
|
-
|
|
642
|
+
const m = /^(?:Replies go to the (\S+) specialist\.|Found by the morning run; the (\S+) specialist can look into it here\.)$/m.exec(content);
|
|
643
|
+
return m?.[1] ?? m?.[2] ?? null;
|
|
589
644
|
}
|
|
590
645
|
|
|
591
|
-
/** The tagged message for one area's problem in its routed channel. */
|
|
646
|
+
/** The tagged message for one area's problem in its routed channel: the problem, one item per line, and who answers replies. */
|
|
592
647
|
export function problemText(a: AreaStatus, owners: readonly string[]): string {
|
|
593
|
-
const specialist = Object.hasOwn(AREA_SPECIALIST, a.area) ? AREA_SPECIALIST[a.area] : a.area.toLowerCase();
|
|
594
|
-
const
|
|
595
|
-
|
|
648
|
+
const specialist = a.specialist !== undefined ? a.specialist : Object.hasOwn(AREA_SPECIALIST, a.area) ? AREA_SPECIALIST[a.area] : a.area.toLowerCase();
|
|
649
|
+
const texts = a.items && a.items.length > 0 ? a.items.map((i) => i.text) : [(a.problem ?? a.line).replace(`${a.area}: `, "")];
|
|
650
|
+
const head = `⚠️ **${escapeDiscord(a.area)}**`;
|
|
651
|
+
const body = texts.length === 1 ? `${head} · ${escapeOutsideCode(texts[0])}` : [head, ...texts.map((t) => `- ${escapeOutsideCode(t)}`)].join("\n");
|
|
652
|
+
return [body, specialist === null ? null : `Replies go to the ${escapeDiscord(specialist)} specialist.`, owners.map((id) => `<@${id}>`).join(" ")].filter((l) => l !== null).join("\n");
|
|
596
653
|
}
|
|
@@ -2,7 +2,7 @@ import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
|
2
2
|
import { tmpdir } from "node:os";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
5
|
-
import { ApprovalServer, type Ticket } from "./approvals.ts";
|
|
5
|
+
import { APPROVAL_INLINE_CHARS, ApprovalServer, approvalCommand, type Ticket } from "./approvals.ts";
|
|
6
6
|
import { chunkMessage, fenceState } from "./chunk.ts";
|
|
7
7
|
import { applyAdminCommand, describeConfig } from "./admin.ts";
|
|
8
8
|
import { helpText, needsPrefixText, needsYou, parseCommand } from "./commands.ts";
|
|
@@ -70,6 +70,21 @@ describe("config", () => {
|
|
|
70
70
|
});
|
|
71
71
|
});
|
|
72
72
|
|
|
73
|
+
describe("approval command", () => {
|
|
74
|
+
it("shows a short command whole and a long one as a preview with the whole command attached", () => {
|
|
75
|
+
expect(approvalCommand(["restart", "--confirm"])).toEqual({ inline: "restart --confirm" });
|
|
76
|
+
const body = JSON.stringify({ alias: "Hallway lights", actions: "x".repeat(3000) });
|
|
77
|
+
const long = approvalCommand(["rest", "post", "/api/config/automation/config/1734267132664", body]);
|
|
78
|
+
const whole = `rest post /api/config/automation/config/1734267132664 ${body}`;
|
|
79
|
+
expect(long.inline.length).toBeLessThan(APPROVAL_INLINE_CHARS);
|
|
80
|
+
expect(long.inline.startsWith("rest post /api/config/automation/config/1734267132664")).toBe(true);
|
|
81
|
+
expect(long.inline).toContain(`(${whole.length} characters, the whole command is attached)`);
|
|
82
|
+
expect(long.file).toEqual({ name: "command.txt", content: `${whole}\n` });
|
|
83
|
+
// The message around the command stays far below Discord's 2000-character limit.
|
|
84
|
+
expect(`🔒 **home** wants to run \`${long.inline}\` for <@123456789012345678>. <@123456789012345678>: approve or deny within 30 min.`.length).toBeLessThan(2000);
|
|
85
|
+
});
|
|
86
|
+
});
|
|
87
|
+
|
|
73
88
|
describe("chunk", () => {
|
|
74
89
|
it("splits on paragraphs and keeps fences balanced", () => {
|
|
75
90
|
const text = `${"a".repeat(1500)}\n\n${"b".repeat(1500)}`;
|
|
@@ -7,7 +7,7 @@ import { APPROVAL_TOKEN_ENV, APPROVAL_URL_ENV } from "../specialist-gate/index.t
|
|
|
7
7
|
import { type AgentConfig, defaultDirs, discoverAgents, resolveSkills } from "../astro-subagents/agents.ts";
|
|
8
8
|
import { AGENT_ENV, type ChildSession, type DispatchDefaults, type RunResult, childRemaining, currentDepth, finalOutput, isFailed, resultOutput, runAgent } from "../astro-subagents/child.ts";
|
|
9
9
|
import { pruneSessions, sessionDirFor } from "../astro-subagents/sessions.ts";
|
|
10
|
-
import { ApprovalServer, type Ticket } from "./approvals.ts";
|
|
10
|
+
import { ApprovalServer, approvalCommand, type Ticket } from "./approvals.ts";
|
|
11
11
|
import { chunkMessage } from "./chunk.ts";
|
|
12
12
|
import { applyAdminCommand } from "./admin.ts";
|
|
13
13
|
import { CHANNEL_RULES, helpText, needsPrefixText, needsYou, parseCommand } from "./commands.ts";
|
|
@@ -275,7 +275,7 @@ class Bridge {
|
|
|
275
275
|
this.config = config;
|
|
276
276
|
this.conversations = loadState();
|
|
277
277
|
this.rest = new DiscordRest(token);
|
|
278
|
-
this.approvals = new ApprovalServer({ onTicket: (t) => void this.askApproval(t), onExpired: (t) => void this.expireApproval(t) }, config.approvalTimeoutMinutes * 60_000);
|
|
278
|
+
this.approvals = new ApprovalServer({ onTicket: (t) => void this.askApproval(t).catch((err) => this.approvalFailed(t, err)), onExpired: (t) => void this.expireApproval(t).catch((err) => log(`approval ${t.id} expiry: ${err instanceof Error ? err.message : String(err)}`)) }, config.approvalTimeoutMinutes * 60_000);
|
|
279
279
|
this.security = new SecurityMonitor({
|
|
280
280
|
rest: this.rest,
|
|
281
281
|
runEntry: (service, caller, args, timeoutMs) => this.runEntry(service, caller, args, timeoutMs),
|
|
@@ -631,20 +631,30 @@ class Bridge {
|
|
|
631
631
|
const routes = this.routes();
|
|
632
632
|
const owners = this.config.owners;
|
|
633
633
|
// A problem that only waits for the owners tags them on its first morning; the summary lists it every day.
|
|
634
|
-
const
|
|
634
|
+
const priorState = readReportState("daily");
|
|
635
|
+
const prior = priorState?.notified;
|
|
636
|
+
const priorLinks = priorState?.links ?? {};
|
|
635
637
|
const { post, notified } = problemsToPost(areas, Array.isArray(prior) ? prior : []);
|
|
636
638
|
for (const a of areas.filter((x) => x.problem && !post.includes(x))) log(`daily ${date}: ${a.area} problem already posted (${a.once ?? ""})`);
|
|
639
|
+
// Each summary line links to its problem's message; a problem posted on an earlier morning keeps that message.
|
|
640
|
+
const posted = new Map<AreaStatus, string>();
|
|
637
641
|
for (const a of post) {
|
|
638
642
|
const components = a.buttons ? [{ type: 1, components: a.buttons.map((b) => ({ type: 2, style: 4, label: b.label, custom_id: b.customId })) }] : undefined;
|
|
639
|
-
await this.rest.createMessage(routes[a.route ?? "admin"], { content: problemText(a, owners), mentionUsers: owners, ...(components ? { components } : {}) }).catch((err) => log(`daily ${date}: ${a.area} problem message failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
643
|
+
const message = await this.rest.createMessage(routes[a.route ?? "admin"], { content: problemText(a, owners), mentionUsers: owners, ...(components ? { components } : {}) }).catch((err) => log(`daily ${date}: ${a.area} problem message failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
644
|
+
if (message) posted.set(a, message.id);
|
|
640
645
|
}
|
|
646
|
+
const linked = areas.map((a) => {
|
|
647
|
+
const messageId = posted.get(a) ?? (a.once !== undefined ? priorLinks[a.once] : undefined);
|
|
648
|
+
return messageId ? { ...a, messageId } : a;
|
|
649
|
+
});
|
|
650
|
+
const links = Object.fromEntries(linked.flatMap((a) => (a.once !== undefined && a.messageId ? [[a.once, a.messageId]] : [])));
|
|
641
651
|
const label = new Intl.DateTimeFormat("en-GB", { timeZone: schedule.timeZone, weekday: "short", day: "numeric", month: "short" }).format(new Date());
|
|
642
|
-
const text = summaryText({ date: label, guildId: this.config.guildId, reports, areas, routes });
|
|
652
|
+
const text = summaryText({ date: label, guildId: this.config.guildId, reports, areas: linked, routes });
|
|
643
653
|
// Recorded only now: a run a crash cut short runs again and skips the reports it already posted.
|
|
644
|
-
writeReportState("daily", { date, status: "sending", notified });
|
|
654
|
+
writeReportState("daily", { date, status: "sending", notified, links });
|
|
645
655
|
let first = "";
|
|
646
656
|
for (const chunk of chunkMessage(text)) first ||= (await this.rest.createMessage(schedule.channelId, { content: chunk })).id;
|
|
647
|
-
writeReportState("daily", { date, status: "sent", messageId: first, notified });
|
|
657
|
+
writeReportState("daily", { date, status: "sent", messageId: first, notified, links });
|
|
648
658
|
log(`daily ${date}: posted`);
|
|
649
659
|
}
|
|
650
660
|
|
|
@@ -667,7 +677,7 @@ class Bridge {
|
|
|
667
677
|
return null;
|
|
668
678
|
}
|
|
669
679
|
};
|
|
670
|
-
const areas: Promise<AreaStatus>[] = [];
|
|
680
|
+
const areas: Promise<AreaStatus | AreaStatus[]>[] = [];
|
|
671
681
|
if (present.has("backup")) areas.push((async () => backupStatus(await json("backup", ["vzdump-archives"]), await text("backup", ["restic-check-status"]), Date.now()))());
|
|
672
682
|
if (present.has("edge")) areas.push((async () => edgeStatus(await text("edge", ["vps", "status"]), await text("edge", ["vps", "updates"])))());
|
|
673
683
|
if (present.has("proxmox")) {
|
|
@@ -746,7 +756,7 @@ class Bridge {
|
|
|
746
756
|
]);
|
|
747
757
|
})(),
|
|
748
758
|
);
|
|
749
|
-
return Promise.all(areas);
|
|
759
|
+
return (await Promise.all(areas)).flat();
|
|
750
760
|
}
|
|
751
761
|
|
|
752
762
|
private async arcaneReport(schedule: ReportSchedule, date: string): Promise<void> {
|
|
@@ -1946,12 +1956,14 @@ class Bridge {
|
|
|
1946
1956
|
const owners = this.config.owners.map((id) => `<@${id}>`).join(" ");
|
|
1947
1957
|
const liveView = (await this.settingsFor(task.channelId))?.settings.liveView;
|
|
1948
1958
|
const site = liveView ? captchaSite(ticket.args) : null;
|
|
1959
|
+
const command = approvalCommand(ticket.args);
|
|
1949
1960
|
const content =
|
|
1950
1961
|
site && liveView
|
|
1951
1962
|
? `${captchaText(site, liveView, owners)} Within ${this.config.approvalTimeoutMinutes} min.`
|
|
1952
|
-
: `🔒 **${short}** wants to run ${code(
|
|
1963
|
+
: `🔒 **${short}** wants to run ${code(command.inline)} for <@${task.userId}>. ${owners}: approve or deny within ${this.config.approvalTimeoutMinutes} min.`;
|
|
1953
1964
|
const posted = await this.rest.createMessage(task.channelId, {
|
|
1954
1965
|
content,
|
|
1966
|
+
...(command.file && !site ? { file: command.file } : {}),
|
|
1955
1967
|
replyTo: task.messageId,
|
|
1956
1968
|
components: approvalButtons(ticket.id, false, site ? CAPTCHA_LABELS : undefined),
|
|
1957
1969
|
mentionUsers: this.config.owners,
|
|
@@ -1960,12 +1972,20 @@ class Bridge {
|
|
|
1960
1972
|
this.saveOpenWork();
|
|
1961
1973
|
}
|
|
1962
1974
|
|
|
1975
|
+
/** A ticket that could not be shown is denied, so the specialist stops waiting; the bridge keeps running. */
|
|
1976
|
+
private approvalFailed(ticket: Ticket, err: unknown): void {
|
|
1977
|
+
log(`approval ${ticket.id} for ${ticket.agent} could not be posted; denying: ${err instanceof Error ? err.message : String(err)}`);
|
|
1978
|
+
this.approvals.decide(ticket.id, "deny", "bridge");
|
|
1979
|
+
const task = this.active.get(ticket.agent.replace(/^astro\./, ""));
|
|
1980
|
+
if (task) void this.rest.createMessage(task.channelId, { content: "⚠️ The approval request could not be posted, so the change was not run. Ask again.", replyTo: task.messageId }).catch(() => undefined);
|
|
1981
|
+
}
|
|
1982
|
+
|
|
1963
1983
|
private async expireApproval(ticket: Ticket): Promise<void> {
|
|
1964
1984
|
const posted = this.approvalMessages.get(ticket.id);
|
|
1965
1985
|
if (!posted) return;
|
|
1966
1986
|
this.approvalMessages.delete(ticket.id);
|
|
1967
1987
|
this.saveOpenWork();
|
|
1968
|
-
await this.rest.editMessage(posted.channelId, posted.messageId, `⌛ No decision within ${this.config.approvalTimeoutMinutes} min; ${code(ticket.args.
|
|
1988
|
+
await this.rest.editMessage(posted.channelId, posted.messageId, `⌛ No decision within ${this.config.approvalTimeoutMinutes} min; ${code(approvalCommand(ticket.args).inline)} was not run.`).catch(() => undefined);
|
|
1969
1989
|
await this.rest.removeOwnReaction(posted.userMessage.channelId, posted.userMessage.messageId, REACTION.waiting).catch(() => undefined);
|
|
1970
1990
|
}
|
|
1971
1991
|
|
|
@@ -2033,7 +2053,7 @@ class Bridge {
|
|
|
2033
2053
|
this.saveOpenWork();
|
|
2034
2054
|
const site = captchaSite(ticket.args);
|
|
2035
2055
|
const verdict = decision === "approve" ? `✅ Approved by <@${userId}>` : `⛔ Denied by <@${userId}>`;
|
|
2036
|
-
await this.rest.updateInteractionMessage(interaction.id, interaction.token, site ? captchaVerdict(site, decision === "approve", userId) : `${verdict}: ${code(ticket.args.
|
|
2056
|
+
await this.rest.updateInteractionMessage(interaction.id, interaction.token, site ? captchaVerdict(site, decision === "approve", userId) : `${verdict}: ${code(approvalCommand(ticket.args).inline)}`);
|
|
2037
2057
|
if (posted) await this.rest.removeOwnReaction(posted.userMessage.channelId, posted.userMessage.messageId, REACTION.waiting).catch(() => undefined);
|
|
2038
2058
|
log(`approval ${ticket.id} ${decision} by ${userId}: ${ticket.args.join(" ")}`);
|
|
2039
2059
|
}
|
|
@@ -10,6 +10,8 @@ export interface ReportState {
|
|
|
10
10
|
messageId?: string;
|
|
11
11
|
/** The morning run: `once` keys of the problems it has tagged the owners for, so a waiting release tags them once. */
|
|
12
12
|
notified?: string[];
|
|
13
|
+
/** The morning run: the message of each `once` problem still open, so later summaries link to it. */
|
|
14
|
+
links?: Record<string, string>;
|
|
13
15
|
}
|
|
14
16
|
|
|
15
17
|
/** Local date and minutes past midnight in a time zone. */
|
package/package.json
CHANGED
package/skills/dns/SKILL.md
CHANGED
|
@@ -63,7 +63,7 @@ Fixed operations on each node's host through one forced-command SSH key (`spc-dn
|
|
|
63
63
|
|---|---|
|
|
64
64
|
| `status` | `dns` service state, start time, restart count, uptime, memory, root disk |
|
|
65
65
|
| `journal <since>` | the `dns` service journal since, e.g. `2 hours ago` |
|
|
66
|
-
| `updates` | upgradable OS packages |
|
|
66
|
+
| `updates` | upgradable OS packages, then `new:` and `remove:` lines: what `apt-get full-upgrade` would add or remove. Such lines mean packages are kept back; the nightly run never installs them, and no specialist can: say so and give the host and the command |
|
|
67
67
|
| `restart` | restart Technitium and print its state |
|
|
68
68
|
| `upgrade` | run the official installer (`download.technitium.com/dns/install.sh`), which upgrades Technitium and its ASP.NET runtime in place; up to 10 minutes |
|
|
69
69
|
|
package/skills/proxmox/SKILL.md
CHANGED
|
@@ -43,7 +43,7 @@ Power actions and snapshot changes return `{ upid, exitstatus }`; `OK` means don
|
|
|
43
43
|
| `updates` | upgradable host packages |
|
|
44
44
|
| `host-upgrade` | apt full-upgrade of the host (risky, needs approval); a new Proxmox VE major is held and printed as `held:`; ends with `reboot-needed:`. The morning run does this on its own each day |
|
|
45
45
|
| `host-reboot --confirm` | reboot the host in one minute (risky): every guest on it restarts. Only on the owner's explicit request; the morning run offers a Reboot button instead |
|
|
46
|
-
| `guest-exec <vmid> <status|journal|df|updates>` | fixed read inside a guest: failed units and uptime, warning journal of 24 h, disk usage, upgradable packages |
|
|
46
|
+
| `guest-exec <vmid> <status|journal|df|updates>` | fixed read inside a guest: failed units and uptime, warning journal of 24 h, disk usage, upgradable packages (`updates` adds `new:` and `remove:` lines when packages are kept back: what `apt-get full-upgrade` would add or remove; the nightly run never installs them, and no specialist can: say so and give the guest and the command) |
|
|
47
47
|
| `nightly [days]` | start, end, duration (UTC) and result of each nightly job of the last 1..14 days (default 7): vzdump tasks from the API, then from the journals of the host and every running guest the host timers, `apt-daily-upgrade`, the backup and dump timers, and Docker restarts. Answer questions about nightly run times with it |
|
|
48
48
|
|
|
49
49
|
## Guest change procedure
|
|
@@ -26,7 +26,8 @@ case "$name" in
|
|
|
26
26
|
journal)
|
|
27
27
|
printf '%s' "$rest" | grep -Eq '^[A-Za-z0-9 :+-]{1,40}$' || { echo "bad since" >&2; exit 2; }
|
|
28
28
|
run journalctl -u dns --since "$rest" --no-pager -n 200 ;;
|
|
29
|
-
updates
|
|
29
|
+
# updates: the dry run of a full upgrade names the new packages and removals that keep packages back; the nightly run never installs those.
|
|
30
|
+
updates) run sh -c 'apt list --upgradable 2>/dev/null; apt-get -s -o Debug::NoLocking=1 full-upgrade 2>/dev/null | sed -n -e "s/^Inst \([^ ]*\) (.*/new: \1/p" -e "s/^Remv \([^ ]*\) .*/remove: \1/p"' ;;
|
|
30
31
|
restart) run sh -c 'systemctl restart dns && sleep 3 && systemctl is-active dns' ;;
|
|
31
32
|
# The installer's exit code decides the result; its last 20 lines are shown either way.
|
|
32
33
|
upgrade) run sh -c 'f=$(mktemp) && curl -fsSL https://download.technitium.com/dns/install.sh -o "$f" || exit 1; rc=0; bash "$f" >"$f.log" 2>&1 || rc=$?; tail -20 "$f.log"; rm -f "$f" "$f.log"; [ "$rc" -eq 0 ] || { echo "installer exited $rc" >&2; exit "$rc"; }; sleep 5; systemctl is-active dns' ;;
|
|
@@ -25,12 +25,13 @@ reboot_needed() {
|
|
|
25
25
|
fi
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
# updates: the dry run of a full upgrade names the new packages and removals that keep packages back; the nightly run never installs those.
|
|
28
29
|
guest_cmd() {
|
|
29
30
|
case "$1" in
|
|
30
31
|
status) echo 'systemctl --failed --no-pager; uptime' ;;
|
|
31
32
|
journal) echo 'journalctl -p warning --since "24 hours ago" --no-pager -n 100' ;;
|
|
32
33
|
df) echo 'df -h -x tmpfs -x devtmpfs' ;;
|
|
33
|
-
updates) echo 'apt list --upgradable 2>/dev/null' ;;
|
|
34
|
+
updates) echo 'apt list --upgradable 2>/dev/null; apt-get -s -o Debug::NoLocking=1 full-upgrade 2>/dev/null | sed -n -e "s/^Inst \([^ ]*\) (.*/new: \1/p" -e "s/^Remv \([^ ]*\) .*/remove: \1/p"' ;;
|
|
34
35
|
*) return 1 ;;
|
|
35
36
|
esac
|
|
36
37
|
}
|
|
@@ -25,12 +25,13 @@ reboot_needed() {
|
|
|
25
25
|
fi
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
# updates: the dry run of a full upgrade names the new packages and removals that keep packages back; the nightly run never installs those.
|
|
28
29
|
guest_cmd() {
|
|
29
30
|
case "$1" in
|
|
30
31
|
status) echo 'systemctl --failed --no-pager; uptime' ;;
|
|
31
32
|
journal) echo 'journalctl -p warning --since "24 hours ago" --no-pager -n 100' ;;
|
|
32
33
|
df) echo 'df -h -x tmpfs -x devtmpfs' ;;
|
|
33
|
-
updates) echo 'apt list --upgradable 2>/dev/null' ;;
|
|
34
|
+
updates) echo 'apt list --upgradable 2>/dev/null; apt-get -s -o Debug::NoLocking=1 full-upgrade 2>/dev/null | sed -n -e "s/^Inst \([^ ]*\) (.*/new: \1/p" -e "s/^Remv \([^ ]*\) .*/remove: \1/p"' ;;
|
|
34
35
|
*) return 1 ;;
|
|
35
36
|
esac
|
|
36
37
|
}
|