@astrofoundry/pi-astro 0.40.33 → 0.40.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/astro-discord/daily-run.test.ts +32 -4
- package/extensions/astro-discord/daily-run.ts +119 -18
- package/extensions/astro-discord/index.ts +108 -3
- package/extensions/specialist-gate/approvals.test.ts +4 -0
- package/extensions/specialist-gate/index.ts +2 -2
- package/package.json +1 -1
- package/skills/network/SKILL.md +4 -0
- package/skills/proxmox/SKILL.md +3 -1
- package/specialists/entry/remote/spc-proxmox-pulsar-entry.sh +32 -1
- package/specialists/entry/remote/spc-proxmox-spacex-entry.sh +32 -1
- package/specialists/network/run.ts +29 -0
- package/specialists/proxmox/run.ts +6 -1
- package/specialists/wrappers.test.ts +21 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
2
|
import type { ArcaneReport } from "./daily-report.ts";
|
|
3
3
|
import type { ToolsReport } from "./tools-report.ts";
|
|
4
|
-
import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, problemText, proxmoxStatus, summaryText, toolsSummary } from "./daily-run.ts";
|
|
4
|
+
import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
5
5
|
|
|
6
6
|
const EDGE = `nginx active
|
|
7
7
|
wg-quick@wg0 active
|
|
@@ -62,13 +62,41 @@ describe("morning status lines", () => {
|
|
|
62
62
|
expect(edgeStatus(null, null)).toMatchObject({ line: "the check failed (vps status)", problem: "Edge: the status check failed" });
|
|
63
63
|
});
|
|
64
64
|
|
|
65
|
-
it("reports
|
|
66
|
-
|
|
65
|
+
it("reports both Proxmox hosts, the morning upgrade, a held major, and a pending reboot with its button", () => {
|
|
66
|
+
const held = "Listing...\nwazuh-agent/stable 4.14.9-1 amd64 [upgradable from: 4.14.7-1]";
|
|
67
|
+
expect(proxmoxStatus([{ name: "Pulsar", id: "pulsar", status: `${PULSAR}\nreboot-needed: no`, updates: held }])).toEqual({ area: "Proxmox", line: "Pulsar up 59 days, RAM 47Gi of 60Gi used, root disk 13%, up to date" });
|
|
68
|
+
const upgraded = proxmoxStatus([
|
|
69
|
+
{ name: "Pulsar", id: "pulsar", status: `${PULSAR}\nreboot-needed: kernel 7.0.14-25-pve -> 7.0.14-26-pve`, updates: held, upgrade: "3 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.\nreboot-needed: kernel 7.0.14-25-pve -> 7.0.14-26-pve" },
|
|
70
|
+
{ name: "SPACE-X", id: "space-x", status: null, updates: null },
|
|
71
|
+
]);
|
|
72
|
+
expect(upgraded.line).toBe("Pulsar up 59 days, RAM 47Gi of 60Gi used, root disk 13%, 3 host updates installed, reboot needed; SPACE-X status unknown");
|
|
73
|
+
expect(upgraded.problem).toBe("Proxmox: Pulsar needs a reboot (kernel 7.0.14-25-pve -> 7.0.14-26-pve); every guest on it restarts for a few minutes; the status check failed for SPACE-X");
|
|
74
|
+
expect(upgraded.buttons).toEqual([{ label: "Reboot Pulsar", customId: "proxmox:reboot:pulsar" }]);
|
|
75
|
+
const major = proxmoxStatus([{ name: "SPACE-X", id: "space-x", status: `${PULSAR}\nreboot-needed: no`, updates: PULSAR_UPDATES, upgrade: "held: Proxmox VE 9.2.24 -> 10.0.1 is a major upgrade" }]);
|
|
76
|
+
expect(major).toMatchObject({ problem: "Proxmox: SPACE-X waits for you: Proxmox VE 9.2.24 -> 10.0.1 is a major upgrade", route: "admin" });
|
|
77
|
+
expect(major.line).toContain("a Proxmox VE major upgrade waits for you");
|
|
78
|
+
expect(proxmoxStatus([{ name: "Pulsar", id: "pulsar", status: PULSAR, updates: PULSAR_UPDATES, upgrade: null }]).problem).toBe("Proxmox: the host upgrade on Pulsar failed");
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it("flags updates the nightly run left behind, reading guest-agent JSON too, and ignores the held Wazuh agent", () => {
|
|
82
|
+
const vm = JSON.stringify({ exitcode: 0, exited: 1, "out-data": "Listing...\ndocker-ce/trixie 5:29.9.2-1 amd64 [upgradable from: 5:29.7.2-1]\nwazuh-agent/stable 4.14.9-1 amd64 [upgradable from: 4.14.7-1]\n" });
|
|
83
|
+
expect(pendingPackageLines(vm)).toEqual(["docker-ce/trixie 5:29.9.2-1 amd64 [upgradable from: 5:29.7.2-1]"]);
|
|
84
|
+
const held = "Listing...\nwazuh-agent/stable 4.14.9-1 amd64 [upgradable from: 4.14.7-1]";
|
|
85
|
+
expect(updatesStatus({ technitium2: held, synapse: "Listing..." })).toEqual({ area: "Updates", line: "2 hosts up to date after the nightly run" });
|
|
86
|
+
expect(updatesStatus({ arcane: vm, "technitium3 (Mira)": held, radius: null })).toEqual({
|
|
87
|
+
area: "Updates",
|
|
88
|
+
line: "still waiting after the nightly run: arcane 1; no update list from radius",
|
|
89
|
+
problem: "Updates: arcane (docker-ce) did not install overnight; no update list from radius",
|
|
90
|
+
route: "admin",
|
|
91
|
+
});
|
|
67
92
|
});
|
|
68
93
|
|
|
69
94
|
it("checks RADIUS and the Tailscale router", () => {
|
|
70
95
|
expect(networkStatus(RADIUS, TAILSCALE)).toEqual({ area: "Network", line: "RADIUS running, Tailscale router tailscale-37 online" });
|
|
71
|
-
|
|
96
|
+
const both = `${TAILSCALE}\n100.95.12.39 astro-mb-m5-2 astronaute@ macOS offline, last seen 15h ago\n100.106.244.101 tailscale-mira astronaute@ linux -`;
|
|
97
|
+
expect(networkStatus(RADIUS, both)).toEqual({ area: "Network", line: "RADIUS running, Tailscale routers tailscale-37 and tailscale-mira online" });
|
|
98
|
+
expect(networkStatus(RADIUS, both.replace("tailscale-mira astronaute@ linux -", "tailscale-mira astronaute@ linux offline, last seen 2h ago"))).toMatchObject({ line: "RADIUS running, Tailscale offline: tailscale-mira", problem: "Network: the Tailscale subnet router tailscale-mira is offline" });
|
|
99
|
+
expect(networkStatus(RADIUS.replace("active (running)", "failed (Result: exit-code)"), "100.121.19.2 tailscale-37 astronaute@ linux offline, last seen 1h ago")).toMatchObject({ problem: "Network: FreeRADIUS is not running; the Tailscale subnet router tailscale-37 is offline" });
|
|
72
100
|
expect(networkStatus(null, TAILSCALE).problem).toBe("Network: a network status check failed");
|
|
73
101
|
});
|
|
74
102
|
|
|
@@ -6,12 +6,19 @@ import type { ToolsReport } from "./tools-report.ts";
|
|
|
6
6
|
/** Where an area's problems tag the owners: specialists without a channel of their own use the admin channel. */
|
|
7
7
|
export type Route = "admin" | "media";
|
|
8
8
|
|
|
9
|
+
/** An owner-only button posted with an area's problem. */
|
|
10
|
+
export interface AreaButton {
|
|
11
|
+
label: string;
|
|
12
|
+
customId: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
9
15
|
/** One area of the morning summary: a line, and a problem that needs an owner, if any. */
|
|
10
16
|
export interface AreaStatus {
|
|
11
17
|
area: string;
|
|
12
18
|
line: string;
|
|
13
19
|
problem?: string;
|
|
14
20
|
route?: Route;
|
|
21
|
+
buttons?: AreaButton[];
|
|
15
22
|
}
|
|
16
23
|
|
|
17
24
|
const HOUR = 3600;
|
|
@@ -50,30 +57,124 @@ export function edgeStatus(status: string | null, updates: string | null): AreaS
|
|
|
50
57
|
return { area: "Edge", line: `front door: ${parts.join(", ")}`, ...(problem ? { problem, route: "admin" as const } : {}) };
|
|
51
58
|
}
|
|
52
59
|
|
|
53
|
-
/**
|
|
54
|
-
export
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
60
|
+
/** One Proxmox host as the morning run saw it. */
|
|
61
|
+
export interface ProxmoxHost {
|
|
62
|
+
name: string;
|
|
63
|
+
/** `pulsar` or `space-x`: the target of the Reboot button. */
|
|
64
|
+
id: string;
|
|
65
|
+
status: string | null;
|
|
66
|
+
/** `updates` after any upgrade. */
|
|
67
|
+
updates: string | null;
|
|
68
|
+
/** What `host-upgrade` printed this morning: undefined when nothing was waiting, null when it failed. */
|
|
69
|
+
upgrade?: string | null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** The custom id of the Reboot button for one Proxmox host. */
|
|
73
|
+
export function rebootButtonId(id: string): string {
|
|
74
|
+
return `proxmox:reboot:${id}`;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
interface ProxmoxFacts {
|
|
78
|
+
line: string;
|
|
79
|
+
problems: string[];
|
|
80
|
+
button?: AreaButton;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function proxmoxHostFacts(h: ProxmoxHost): ProxmoxFacts {
|
|
84
|
+
if (h.status === null) return { line: `${h.name} status unknown`, problems: [`the status check failed for ${h.name}`] };
|
|
85
|
+
const up = /up\s+([^,]+(?:,\s*\d+:\d+)?)/.exec(h.status)?.[1]?.split(",")[0]?.trim();
|
|
86
|
+
const mem = /^Mem:\s+(\S+)\s+(\S+)/m.exec(h.status);
|
|
87
|
+
const root = /\s(\d+)%\s+\/\s*$/m.exec(h.status)?.[1];
|
|
88
|
+
const reboot = /^reboot-needed:\s*(.*)$/m.exec(h.status)?.[1]?.trim();
|
|
89
|
+
const waiting = h.updates === null ? null : pendingPackageLines(h.updates).length;
|
|
90
|
+
const held = h.upgrade ? /^held:\s*(.*)$/m.exec(h.upgrade)?.[1] : undefined;
|
|
91
|
+
const installed = h.upgrade ? /^(\d+) upgraded/m.exec(h.upgrade)?.[1] : undefined;
|
|
92
|
+
const updates =
|
|
93
|
+
h.upgrade === null
|
|
94
|
+
? "the host upgrade failed"
|
|
95
|
+
: held
|
|
96
|
+
? "a Proxmox VE major upgrade waits for you"
|
|
97
|
+
: [installed ? `${installed} host update${installed === "1" ? "" : "s"} installed` : null, waiting === null ? "updates unknown" : waiting > 0 ? `${waiting} still waiting` : installed ? null : "up to date"].filter(Boolean).join(", ");
|
|
98
|
+
const rebootNeeded = reboot !== undefined && reboot !== "no";
|
|
99
|
+
const line = [up ? `${h.name} up ${up}` : `${h.name} up`, mem ? `RAM ${mem[2]} of ${mem[1]} used` : null, root ? `root disk ${root}%` : null, updates, rebootNeeded ? "reboot needed" : null].filter(Boolean).join(", ");
|
|
100
|
+
const problems = [
|
|
101
|
+
h.upgrade === null ? `the host upgrade on ${h.name} failed` : null,
|
|
102
|
+
held ? `${h.name} waits for you: ${held}` : null,
|
|
103
|
+
rebootNeeded ? `${h.name} needs a reboot (${reboot}); every guest on it restarts for a few minutes` : null,
|
|
104
|
+
].filter((p): p is string => p !== null);
|
|
105
|
+
return { line, problems, ...(rebootNeeded ? { button: { label: `Reboot ${h.name}`, customId: rebootButtonId(h.id) } } : {}) };
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Both Proxmox hosts: uptime, memory, root disk, the morning's automatic upgrade, a held major, and a pending reboot with its button. */
|
|
109
|
+
export function proxmoxStatus(hosts: readonly ProxmoxHost[]): AreaStatus {
|
|
110
|
+
const facts = hosts.map(proxmoxHostFacts);
|
|
111
|
+
const problems = facts.flatMap((f) => f.problems);
|
|
112
|
+
const buttons = facts.flatMap((f) => (f.button ? [f.button] : []));
|
|
113
|
+
return {
|
|
114
|
+
area: "Proxmox",
|
|
115
|
+
line: facts.map((f) => f.line).join("; "),
|
|
116
|
+
...(problems.length > 0 ? { problem: `Proxmox: ${problems.join("; ")}`, route: "admin" as const } : {}),
|
|
117
|
+
...(buttons.length > 0 ? { buttons } : {}),
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Packages the nightly run leaves alone on purpose: a Wazuh agent must never be newer than the manager. */
|
|
122
|
+
const HELD_PACKAGES = new Set(["wazuh-agent"]);
|
|
123
|
+
|
|
124
|
+
/** The `[upgradable from` lines of `apt list --upgradable`, also when a VM's guest agent wraps them in JSON (`out-data`); held packages left out. */
|
|
125
|
+
export function pendingPackageLines(output: string): string[] {
|
|
126
|
+
let text = output;
|
|
127
|
+
try {
|
|
128
|
+
const wrapped = (JSON.parse(output) as { "out-data"?: unknown } | null)?.["out-data"];
|
|
129
|
+
if (typeof wrapped === "string") text = wrapped;
|
|
130
|
+
} catch {
|
|
131
|
+
/* plain apt output */
|
|
132
|
+
}
|
|
133
|
+
return text
|
|
134
|
+
.split("\n")
|
|
135
|
+
.filter((l) => l.includes("[upgradable from"))
|
|
136
|
+
.filter((l) => !HELD_PACKAGES.has(l.split("/")[0]));
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** Each guest's and Synapse's `updates` after the 04:30 run: anything still waiting did not install and needs a look. */
|
|
140
|
+
export function updatesStatus(hosts: Readonly<Record<string, string | null>>): AreaStatus {
|
|
141
|
+
const names = Object.keys(hosts);
|
|
142
|
+
const unknown = names.filter((n) => hosts[n] === null);
|
|
143
|
+
const left = names.flatMap((n) => {
|
|
144
|
+
const out = hosts[n];
|
|
145
|
+
const packages = out === null ? [] : pendingPackageLines(out).map((l) => l.split("/")[0]);
|
|
146
|
+
return packages.length > 0 ? [{ name: n, packages }] : [];
|
|
147
|
+
});
|
|
148
|
+
const checked = names.length - unknown.length;
|
|
61
149
|
const parts = [
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
150
|
+
left.length === 0 ? `${checked} host${checked === 1 ? "" : "s"} up to date after the nightly run` : `still waiting after the nightly run: ${left.map((h) => `${escapeDiscord(h.name)} ${h.packages.length}`).join(", ")}`,
|
|
151
|
+
unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null,
|
|
152
|
+
].filter(Boolean);
|
|
153
|
+
const problems = [
|
|
154
|
+
left.length > 0 ? `${left.map((h) => `${h.name} (${h.packages.slice(0, 4).join(", ")}${h.packages.length > 4 ? ", ..." : ""})`).join("; ")} did not install overnight` : null,
|
|
155
|
+
unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null,
|
|
66
156
|
].filter(Boolean);
|
|
67
|
-
return { area: "
|
|
157
|
+
return { area: "Updates", line: parts.join("; "), ...(problems.length > 0 ? { problem: `Updates: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
68
158
|
}
|
|
69
159
|
|
|
70
|
-
/** `network pulsar radius-status` and `tailscale-status`: FreeRADIUS running, and
|
|
160
|
+
/** `network pulsar radius-status` and `tailscale-status`: FreeRADIUS running, and every `tailscale*` subnet router (37 and Mira) online. */
|
|
71
161
|
export function networkStatus(radius: string | null, tailscale: string | null): AreaStatus {
|
|
72
162
|
const radiusUp = radius === null ? null : /Active:\s+active \(running\)/.test(radius);
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
163
|
+
const rows = tailscale
|
|
164
|
+
?.split("\n")
|
|
165
|
+
.map((l) => l.trim().split(/\s{2,}/))
|
|
166
|
+
.filter((c) => c.length >= 4 && /^tailscale/.test(c[1] ?? ""));
|
|
167
|
+
const routers = rows === undefined || rows.length === 0 ? null : rows.map((c) => ({ name: c[1], up: !/offline/.test(c[c.length - 1] ?? "") }));
|
|
168
|
+
const down = routers?.filter((r) => !r.up).map((r) => r.name) ?? [];
|
|
169
|
+
const parts = [
|
|
170
|
+
radiusUp === null ? "RADIUS unknown" : radiusUp ? "RADIUS running" : "RADIUS down",
|
|
171
|
+
routers === null ? "Tailscale unknown" : down.length === 0 ? `Tailscale router${routers.length === 1 ? "" : "s"} ${routers.map((r) => r.name).join(" and ")} online` : `Tailscale offline: ${down.join(", ")}`,
|
|
172
|
+
];
|
|
173
|
+
const problems = [
|
|
174
|
+
radiusUp === false ? "FreeRADIUS is not running" : null,
|
|
175
|
+
down.length > 0 ? `the Tailscale subnet router ${down.join(" and ")} ${down.length === 1 ? "is" : "are"} offline` : null,
|
|
176
|
+
radiusUp === null || routers === null ? "a network status check failed" : null,
|
|
177
|
+
].filter(Boolean);
|
|
77
178
|
return { area: "Network", line: parts.join(", "), ...(problems.length > 0 ? { problem: `Network: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
78
179
|
}
|
|
79
180
|
|
|
@@ -78,7 +78,7 @@ import { ArcaneWatch } from "./arcane-watch-monitor.ts";
|
|
|
78
78
|
import { parseSnapshot } from "./arcane-watch.ts";
|
|
79
79
|
import { type Applied, autoFailureText, foremanTask, stackProblems, stackUpdateAnswer, SUPERVISE_DELAY_MS, SUPERVISE_LIMIT_MS } from "./supervise.ts";
|
|
80
80
|
import { stackPlan, toolPlan } from "./update-policy.ts";
|
|
81
|
-
import { type AreaStatus, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, problemText, proxmoxStatus, type ReportLine, type Route, summaryText, toolsSummary } from "./daily-run.ts";
|
|
81
|
+
import { type AreaStatus, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, type ProxmoxHost, proxmoxStatus, type ReportLine, type Route, summaryText, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
82
82
|
import { type InvestigationRequest, SecurityMonitor } from "./security-monitor.ts";
|
|
83
83
|
import { nasOutage, readSecurityState } from "./security.ts";
|
|
84
84
|
|
|
@@ -174,6 +174,13 @@ const DNS_NODES = { technitium2: "technitium2.dns.home.arpa", technitium3: "tech
|
|
|
174
174
|
|
|
175
175
|
/** The audit caller of updates the morning run applies on its own. */
|
|
176
176
|
const AUTO_CALLER = "daily-run";
|
|
177
|
+
/** The Proxmox hosts the morning run upgrades, with the wrapper prefix that selects each. */
|
|
178
|
+
const PROXMOX_HOSTS: readonly { name: string; id: string; prefix: string[] }[] = [
|
|
179
|
+
{ name: "Pulsar", id: "pulsar", prefix: [] },
|
|
180
|
+
{ name: "SPACE-X", id: "space-x", prefix: ["space-x"] },
|
|
181
|
+
];
|
|
182
|
+
/** The wrapper allows a host upgrade 20 minutes; the bridge waits a little longer for its answer. */
|
|
183
|
+
const HOST_UPGRADE_MS = 21 * 60_000;
|
|
177
184
|
|
|
178
185
|
function pause(ms: number): Promise<void> {
|
|
179
186
|
return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)).unref());
|
|
@@ -230,6 +237,8 @@ class Bridge {
|
|
|
230
237
|
private readonly mediaMessages = new Map<string, MessageComponent[]>();
|
|
231
238
|
/** Tool updates started from the Cortex Tools Report, by line key; they count as open work. */
|
|
232
239
|
private readonly toolRuns = new Map<string, { channelId: string; label: string }>();
|
|
240
|
+
/** Proxmox hosts that an owner's Reboot click is restarting. */
|
|
241
|
+
private readonly rebooting = new Set<string>();
|
|
233
242
|
private readonly toolClicks = new KeyedQueue();
|
|
234
243
|
private gateway: GatewayClient | undefined;
|
|
235
244
|
private stopping = false;
|
|
@@ -588,7 +597,8 @@ class Bridge {
|
|
|
588
597
|
const routes = this.routes();
|
|
589
598
|
const owners = this.config.owners;
|
|
590
599
|
for (const a of areas.filter((x) => x.problem)) {
|
|
591
|
-
|
|
600
|
+
const components = a.buttons ? [{ type: 1, components: a.buttons.map((b) => ({ type: 2, style: 4, label: b.label, custom_id: b.customId })) }] : undefined;
|
|
601
|
+
await this.rest.createMessage(routes[a.route ?? "admin"], { content: problemText(a, owners), mentionUsers: owners, ...(components ? { components } : {}) }).catch((err) => log(`daily ${date}: ${a.area} problem message failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
592
602
|
}
|
|
593
603
|
const label = new Intl.DateTimeFormat("en-GB", { timeZone: schedule.timeZone, weekday: "short", day: "numeric", month: "short" }).format(new Date());
|
|
594
604
|
const text = summaryText({ date: label, guildId: this.config.guildId, reports, areas, routes });
|
|
@@ -621,7 +631,41 @@ class Bridge {
|
|
|
621
631
|
const areas: Promise<AreaStatus>[] = [];
|
|
622
632
|
if (present.has("backup")) areas.push((async () => backupStatus(await json("backup", ["vzdump-archives"]), await text("backup", ["restic-check-status"]), Date.now()))());
|
|
623
633
|
if (present.has("edge")) areas.push((async () => edgeStatus(await text("edge", ["vps", "status"]), await text("edge", ["vps", "updates"])))());
|
|
624
|
-
if (present.has("proxmox"))
|
|
634
|
+
if (present.has("proxmox")) {
|
|
635
|
+
areas.push(
|
|
636
|
+
(async () => {
|
|
637
|
+
const hosts: ProxmoxHost[] = [];
|
|
638
|
+
for (const h of PROXMOX_HOSTS) {
|
|
639
|
+
const before = await text("proxmox", [...h.prefix, "updates"]);
|
|
640
|
+
// Host packages install on their own; a new Proxmox VE major is held by the entry script for the owner.
|
|
641
|
+
const upgrade = before !== null && pendingPackageLines(before).length > 0 ? await this.runEntry("proxmox", AUTO_CALLER, [...h.prefix, "host-upgrade"], HOST_UPGRADE_MS).catch((err) => (log(`daily: ${h.name} host-upgrade failed: ${err instanceof Error ? err.message : String(err)}`), null)) : undefined;
|
|
642
|
+
const updates = upgrade === undefined ? before : await text("proxmox", [...h.prefix, "updates"]);
|
|
643
|
+
hosts.push({ name: h.name, id: h.id, status: await text("proxmox", [...h.prefix, "host-status"]), updates, upgrade });
|
|
644
|
+
}
|
|
645
|
+
return proxmoxStatus(hosts);
|
|
646
|
+
})(),
|
|
647
|
+
);
|
|
648
|
+
areas.push(
|
|
649
|
+
(async () => {
|
|
650
|
+
const hosts: Record<string, string | null> = {};
|
|
651
|
+
// Guests one at a time: each call is an SSH session on the host.
|
|
652
|
+
for (const prefix of [[], ["space-x"]]) {
|
|
653
|
+
const guests = await json("proxmox", [...prefix, "guests"]);
|
|
654
|
+
if (!Array.isArray(guests)) {
|
|
655
|
+
hosts[prefix.length === 0 ? "Pulsar guests" : "SPACE-X guests (Mira)"] = null;
|
|
656
|
+
continue;
|
|
657
|
+
}
|
|
658
|
+
for (const g of guests as { vmid?: unknown; name?: unknown; status?: unknown }[]) {
|
|
659
|
+
if (g.status !== "running" || typeof g.vmid !== "number") continue;
|
|
660
|
+
const name = typeof g.name === "string" ? g.name : String(g.vmid);
|
|
661
|
+
hosts[prefix.length === 0 ? name : `${name} (Mira)`] = await text("proxmox", [...prefix, "guest-exec", String(g.vmid), "updates"]);
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
if (present.has("dns")) hosts.synapse = await text("dns", ["technitium", "host", "technitium1", "updates"]);
|
|
665
|
+
return updatesStatus(hosts);
|
|
666
|
+
})(),
|
|
667
|
+
);
|
|
668
|
+
}
|
|
625
669
|
if (present.has("network")) areas.push((async () => networkStatus(await text("network", ["pulsar", "radius-status"]), await text("network", ["pulsar", "tailscale-status"])))());
|
|
626
670
|
if (present.has("identity")) areas.push((async () => identityStatus(await text("identity", ["dmz", "status"])))());
|
|
627
671
|
if (present.has("dns")) {
|
|
@@ -1322,6 +1366,63 @@ class Bridge {
|
|
|
1322
1366
|
await this.restartService(p, userId);
|
|
1323
1367
|
}
|
|
1324
1368
|
|
|
1369
|
+
/**
|
|
1370
|
+
* Reboots one Proxmox host for an owner, when its status still says a reboot is needed,
|
|
1371
|
+
* then reports in the problem's channel whether it came back within ten minutes.
|
|
1372
|
+
*/
|
|
1373
|
+
private async onRebootButton(interaction: Interaction, id: string): Promise<void> {
|
|
1374
|
+
const host = PROXMOX_HOSTS.find((h) => h.id === id);
|
|
1375
|
+
const userId = interaction.member?.user?.id ?? interaction.user?.id ?? "";
|
|
1376
|
+
this.refreshConfig();
|
|
1377
|
+
if (!host) {
|
|
1378
|
+
await this.rest.ephemeralReply(interaction.id, interaction.token, "This button is no longer active.");
|
|
1379
|
+
return;
|
|
1380
|
+
}
|
|
1381
|
+
if (!isOwner(this.config, userId)) {
|
|
1382
|
+
await this.rest.ephemeralReply(interaction.id, interaction.token, "Only owners reboot Proxmox hosts.");
|
|
1383
|
+
return;
|
|
1384
|
+
}
|
|
1385
|
+
if (this.rebooting.has(host.id)) {
|
|
1386
|
+
await this.rest.ephemeralReply(interaction.id, interaction.token, `${host.name} is already rebooting.`);
|
|
1387
|
+
return;
|
|
1388
|
+
}
|
|
1389
|
+
const status = await this.runEntry("proxmox", userId, [...host.prefix, "host-status"], 3 * 60_000).catch(() => null);
|
|
1390
|
+
if (status !== null && /^reboot-needed:\s*no\s*$/m.test(status)) {
|
|
1391
|
+
await this.rest.ephemeralReply(interaction.id, interaction.token, `${host.name} no longer needs a reboot.`);
|
|
1392
|
+
return;
|
|
1393
|
+
}
|
|
1394
|
+
await this.rest.deferUpdate(interaction.id, interaction.token);
|
|
1395
|
+
const channelId = interaction.channel_id ?? this.config.adminChannelId;
|
|
1396
|
+
const messageId = interaction.message?.id;
|
|
1397
|
+
this.rebooting.add(host.id);
|
|
1398
|
+
try {
|
|
1399
|
+
log(`reboot ${host.name} by ${userId}`);
|
|
1400
|
+
await this.runEntry("proxmox", userId, [...host.prefix, "host-reboot", "--confirm"], 2 * 60_000);
|
|
1401
|
+
const note = `♻️ ${host.name} reboots in one minute, started by <@${userId}>; its guests restart with it.`;
|
|
1402
|
+
if (messageId) await this.rest.editMessage(channelId, messageId, `${interaction.message?.content ?? ""}\n${note}`.slice(0, 1900), []).catch(() => undefined);
|
|
1403
|
+
// The host goes down after one minute; ask again from minute four until minute fourteen.
|
|
1404
|
+
await new Promise((r) => setTimeout(r, 4 * 60_000));
|
|
1405
|
+
let back: string | null = null;
|
|
1406
|
+
for (let i = 0; i < 20 && back === null && !this.stopping; i++) {
|
|
1407
|
+
back = await this.runEntry("proxmox", AUTO_CALLER, [...host.prefix, "host-status"], 60_000).catch(() => null);
|
|
1408
|
+
if (back !== null && !/up\s+(\d+ min|0:0?\d)/.test(back)) back = null;
|
|
1409
|
+
if (back === null) await new Promise((r) => setTimeout(r, 30_000));
|
|
1410
|
+
}
|
|
1411
|
+
const kernel = back ? /running kernel: ([^)]+)\)/.exec(back)?.[1] : undefined;
|
|
1412
|
+
const reboot = back ? /^reboot-needed:\s*(.*)$/m.exec(back)?.[1] : undefined;
|
|
1413
|
+
await this.rest.createMessage(channelId, {
|
|
1414
|
+
content: back
|
|
1415
|
+
? `✅ ${host.name} is back${kernel ? ` on kernel ${kernel}` : ""}${reboot && reboot !== "no" ? `; it still reports reboot-needed: ${reboot}` : ""}.`
|
|
1416
|
+
: `❌ ${host.name} did not answer within ten minutes of the reboot. <@${userId}>`,
|
|
1417
|
+
...(back ? {} : { mentionUsers: [userId] }),
|
|
1418
|
+
});
|
|
1419
|
+
} catch (err) {
|
|
1420
|
+
await this.rest.createMessage(channelId, { content: `❌ The reboot of ${host.name} did not start: ${(err instanceof Error ? err.message : String(err)).slice(0, 300)}` }).catch(() => undefined);
|
|
1421
|
+
} finally {
|
|
1422
|
+
this.rebooting.delete(host.id);
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1325
1426
|
/** Runs one specialist wrapper through the sudo entry point, without a model; rejects with the wrapper's stderr. */
|
|
1326
1427
|
private runEntry(service: string, caller: string, args: string[], timeoutMs: number): Promise<string> {
|
|
1327
1428
|
const gate = loadSpecialists().config;
|
|
@@ -1726,6 +1827,10 @@ class Bridge {
|
|
|
1726
1827
|
await this.onArcaneButton(interaction, customId);
|
|
1727
1828
|
return;
|
|
1728
1829
|
}
|
|
1830
|
+
if (customId.startsWith("proxmox:reboot:")) {
|
|
1831
|
+
await this.onRebootButton(interaction, customId.slice("proxmox:reboot:".length));
|
|
1832
|
+
return;
|
|
1833
|
+
}
|
|
1729
1834
|
if (customId.startsWith("security:")) {
|
|
1730
1835
|
this.refreshConfig();
|
|
1731
1836
|
const userId = interaction.member?.user?.id ?? interaction.user?.id ?? "";
|
|
@@ -23,6 +23,7 @@ describe("approvals", () => {
|
|
|
23
23
|
expect(isRisky("network", ["unifi", "37", "nat"])).toBe(false);
|
|
24
24
|
expect(isRisky("network", ["unifi", "mira", "app-policy-delete", "0123456789abcdef01234567"])).toBe(true);
|
|
25
25
|
expect(isRisky("network", ["unifi", "mira", "client-fixed-ip", "0123456789abcdef01234567", "{}"])).toBe(true);
|
|
26
|
+
expect(isRisky("network", ["unifi", "mira", "syslog", '{"ip":"10.0.90.10"}'])).toBe(true);
|
|
26
27
|
expect(isRisky("network", ["unifi", "mira", "device-network", "0123456789abcdef01234567", "{}"])).toBe(true);
|
|
27
28
|
expect(isRisky("network", ["unifi", "mira", "app-get", "/api/s/default/stat/device"])).toBe(false);
|
|
28
29
|
expect(isRisky("arcane", ["projects", "list"])).toBe(false);
|
|
@@ -47,6 +48,9 @@ describe("approvals", () => {
|
|
|
47
48
|
expect(isRisky("proxmox", ["snapshot-create", "107", "before", "-h"])).toBe(true);
|
|
48
49
|
expect(isRisky("dns", ["--help"])).toBe(false);
|
|
49
50
|
expect(isRisky("proxmox", ["guests"])).toBe(false);
|
|
51
|
+
expect(isRisky("proxmox", ["space-x", "host-upgrade"])).toBe(true);
|
|
52
|
+
expect(isRisky("proxmox", ["host-reboot", "--confirm"])).toBe(true);
|
|
53
|
+
expect(isRisky("proxmox", ["space-x", "host-status"])).toBe(false);
|
|
50
54
|
expect(isRisky("dns", ["technitium", "host", "technitium1", "upgrade"])).toBe(true);
|
|
51
55
|
expect(isRisky("dns", ["technitium", "host", "technitium3", "restart"])).toBe(true);
|
|
52
56
|
expect(isRisky("dns", ["technitium", "host", "technitium2", "status"])).toBe(false);
|
|
@@ -33,7 +33,7 @@ export const RISKY: Readonly<Record<string, readonly RegExp[]>> = {
|
|
|
33
33
|
],
|
|
34
34
|
identity: [/^zitadel\s+(DELETE|POST|PUT|PATCH)\b/, /^dmz\s+(deploy-config|pomerium-restart|render|lego-renew)\b/],
|
|
35
35
|
// Every UniFi write, on either console.
|
|
36
|
-
network: [/^unifi\s+\S+\s+(write|nat-write|device-network|app-policy-delete|client-fixed-ip)\b/],
|
|
36
|
+
network: [/^unifi\s+\S+\s+(write|nat-write|device-network|app-policy-delete|client-fixed-ip|syslog)\b/],
|
|
37
37
|
dns: [
|
|
38
38
|
/^technitium\s+primary\s+(zone-create|zone-catalog|zone-delete|record-add|record-update|record-delete|cluster-init|cluster-delete|cluster-remove|server-domain)\b/,
|
|
39
39
|
/^technitium\s+node\s+\S+\s+(server-domain|stale-zone-delete)\b/,
|
|
@@ -43,7 +43,7 @@ export const RISKY: Readonly<Record<string, readonly RegExp[]>> = {
|
|
|
43
43
|
],
|
|
44
44
|
edge: [/^vps\s+deploy-nginx\b/],
|
|
45
45
|
backup: [/^(vzdump-run|host-backup-run|arcane-backup-run|gcs-run|restic-check-run|restic-restore|restore-clean)\b/],
|
|
46
|
-
proxmox: [/^(space-x\s+)?(set|snapshot-create|start)\b/],
|
|
46
|
+
proxmox: [/^(space-x\s+)?(set|snapshot-create|start|host-upgrade|host-reboot)\b/],
|
|
47
47
|
inference: [/^(nexus-restart|nexus-start)\b/],
|
|
48
48
|
// media has no entry: trendarr cannot delete anything, and adds and queue toggles run on the caller's request.
|
|
49
49
|
};
|
package/package.json
CHANGED
package/skills/network/SKILL.md
CHANGED
|
@@ -107,6 +107,10 @@ Behavior learned on the live consoles:
|
|
|
107
107
|
|
|
108
108
|
Change one device at a time, downstream devices and the AP serving the operator last, and wait until `devices` shows it `ONLINE` at the new address before the next. An AP takes the Management network and the static address in one step. A switch does not: on 2026-10-31 the Flex XG lost the controller after getting both at once, and reverted to its previous settings by itself about 10 minutes later. For a switch, first send `mgmt_network_id` with `config_network: {"type":"dhcp"}` and wait for `ONLINE`; then send the static address. If it stays in provisioning (`state` 5, `last_config_applied_successfully: false`), restart it. The official `RESTART` device action works only on an `ONLINE` device. A switch powered by PoE can be restarted from its upstream port: `write POST /v1/sites/<siteId>/devices/<upstreamId>/interfaces/ports/<n>/actions '{"action":"POWER_CYCLE"}'`. A restart interrupts everything behind the switch; at Mira that is SPACE-X and technitium3 on Flex XG port 2.
|
|
109
109
|
|
|
110
|
+
## `unifi <37|mira> syslog <json>`
|
|
111
|
+
|
|
112
|
+
Sets the console's activity log export, the classic setting `rsyslogd` (`app-get /api/s/default/rest/setting`, key `rsyslogd`). Network 10.6 shows a separate Integrations > System Logging / SIEM panel that does not change this setting; this setting decides whether the Network application's CEF events reach the SIEM server. Allowed keys: `this_controller`, `ip`, `port`, `contents`, `log_all_contents`, `enabled`; the wrapper keeps every other field of the live setting. 37 sends to `10.0.90.10:514` with `this_controller: false`.
|
|
113
|
+
|
|
110
114
|
## `unifi <37|mira> nat` and `unifi <37|mira> nat-write ...`
|
|
111
115
|
|
|
112
116
|
The gateway's NAT rules (Policy Table, type NAT), which the official API lacks. The wrapper calls the Network application's own path `/proxy/network/v2/api/site/default/nat` through the cloud connector with the account key `UNIFI_API_KEY`; `--local` uses the console's local key instead.
|
package/skills/proxmox/SKILL.md
CHANGED
|
@@ -41,6 +41,8 @@ Power actions and snapshot changes return `{ upid, exitstatus }`; `OK` means don
|
|
|
41
41
|
| `host-status` | `pveversion`, uptime, memory, root and `/var/lib/vz` usage, CPU and NVMe temperatures |
|
|
42
42
|
| `host-journal <since>` | warnings and errors from the host journal |
|
|
43
43
|
| `updates` | upgradable host packages |
|
|
44
|
+
| `host-upgrade` | apt full-upgrade of the host (risky, needs approval); a new Proxmox VE major is held and printed as `held:`; ends with `reboot-needed:`. The morning run does this on its own each day |
|
|
45
|
+
| `host-reboot --confirm` | reboot the host in one minute (risky): every guest on it restarts. Only on the owner's explicit request; the morning run offers a Reboot button instead |
|
|
44
46
|
| `guest-exec <vmid> <status|journal|df|updates>` | fixed read inside a guest: failed units and uptime, warning journal of 24 h, disk usage, upgradable packages |
|
|
45
47
|
|
|
46
48
|
## Guest change procedure
|
|
@@ -55,7 +57,7 @@ Memory and CPU changes on a running guest apply at the next start unless hotplug
|
|
|
55
57
|
|
|
56
58
|
## Not available
|
|
57
59
|
|
|
58
|
-
Creating, cloning, or destroying guests, disk resize or removal, storage changes,
|
|
60
|
+
Creating, cloning, or destroying guests, disk resize or removal, storage changes, backups (backup specialist), and anything inside a guest beyond the fixed reads. Describe the exact operator command (`qm`, `pct`, `pvesh`) and stop.
|
|
59
61
|
|
|
60
62
|
## When to stop and report
|
|
61
63
|
|
|
@@ -12,6 +12,19 @@ rest=""
|
|
|
12
12
|
[ "$cmd" != "$name" ] && rest="${cmd#* }"
|
|
13
13
|
since_ok() { printf '%s' "$1" | grep -Eq '^[A-Za-z0-9 :+-]{1,40}$'; }
|
|
14
14
|
|
|
15
|
+
# A newer Proxmox kernel than the running one, or a package's reboot flag, means the host needs a reboot.
|
|
16
|
+
reboot_needed() {
|
|
17
|
+
running="$(uname -r)"
|
|
18
|
+
newest="$(ls /boot/vmlinuz-*-pve 2>/dev/null | sed 's#^/boot/vmlinuz-##' | sort -V | tail -n 1)"
|
|
19
|
+
if [ -n "$newest" ] && [ "$newest" != "$running" ]; then
|
|
20
|
+
echo "reboot-needed: kernel $running -> $newest"
|
|
21
|
+
elif [ -f /run/reboot-required ]; then
|
|
22
|
+
echo "reboot-needed: $(tr '\n' ' ' < /run/reboot-required.pkgs 2>/dev/null)"
|
|
23
|
+
else
|
|
24
|
+
echo "reboot-needed: no"
|
|
25
|
+
fi
|
|
26
|
+
}
|
|
27
|
+
|
|
15
28
|
guest_cmd() {
|
|
16
29
|
case "$1" in
|
|
17
30
|
status) echo 'systemctl --failed --no-pager; uptime' ;;
|
|
@@ -28,9 +41,27 @@ case "$name" in
|
|
|
28
41
|
uptime
|
|
29
42
|
free -h
|
|
30
43
|
df -h / /var/lib/vz
|
|
31
|
-
sensors 2>/dev/null | grep -E 'Tctl|Composite|edge' || true
|
|
44
|
+
sensors 2>/dev/null | grep -E 'Tctl|Composite|edge' || true
|
|
45
|
+
reboot_needed ;;
|
|
32
46
|
host-journal) since_ok "$rest" || { echo "bad since" >&2; exit 2; }; journalctl -p warning --since "$rest" --no-pager -n 200 ;;
|
|
33
47
|
updates) apt list --upgradable 2>/dev/null ;;
|
|
48
|
+
host-upgrade)
|
|
49
|
+
[ -z "$rest" ] || { echo "host-upgrade takes no arguments" >&2; exit 2; }
|
|
50
|
+
export DEBIAN_FRONTEND=noninteractive
|
|
51
|
+
apt-get update -q >/dev/null
|
|
52
|
+
# A new Proxmox VE major waits for the owner.
|
|
53
|
+
now="$(dpkg-query -W -f='${Version}' pve-manager)"
|
|
54
|
+
next="$(apt-get -s full-upgrade | sed -n 's/^Inst pve-manager \[[^]]*\] (\([^ ]*\) .*/\1/p')"
|
|
55
|
+
if [ -n "$next" ] && [ "${next%%.*}" != "${now%%.*}" ]; then
|
|
56
|
+
echo "held: Proxmox VE $now -> $next is a major upgrade"
|
|
57
|
+
exit 0
|
|
58
|
+
fi
|
|
59
|
+
apt-get -y -q -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold full-upgrade 2>&1 | grep -E '^[0-9]+ upgraded|^E:' || true
|
|
60
|
+
reboot_needed ;;
|
|
61
|
+
host-reboot)
|
|
62
|
+
[ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
|
|
63
|
+
shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
|
|
64
|
+
echo "reboot in 1 minute" ;;
|
|
34
65
|
guest-exec)
|
|
35
66
|
vmid="${rest%% *}"; action="${rest#* }"
|
|
36
67
|
case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
|
|
@@ -12,6 +12,19 @@ rest=""
|
|
|
12
12
|
[ "$cmd" != "$name" ] && rest="${cmd#* }"
|
|
13
13
|
since_ok() { printf '%s' "$1" | grep -Eq '^[A-Za-z0-9 :+-]{1,40}$'; }
|
|
14
14
|
|
|
15
|
+
# A newer Proxmox kernel than the running one, or a package's reboot flag, means the host needs a reboot.
|
|
16
|
+
reboot_needed() {
|
|
17
|
+
running="$(uname -r)"
|
|
18
|
+
newest="$(ls /boot/vmlinuz-*-pve 2>/dev/null | sed 's#^/boot/vmlinuz-##' | sort -V | tail -n 1)"
|
|
19
|
+
if [ -n "$newest" ] && [ "$newest" != "$running" ]; then
|
|
20
|
+
echo "reboot-needed: kernel $running -> $newest"
|
|
21
|
+
elif [ -f /run/reboot-required ]; then
|
|
22
|
+
echo "reboot-needed: $(tr '\n' ' ' < /run/reboot-required.pkgs 2>/dev/null)"
|
|
23
|
+
else
|
|
24
|
+
echo "reboot-needed: no"
|
|
25
|
+
fi
|
|
26
|
+
}
|
|
27
|
+
|
|
15
28
|
guest_cmd() {
|
|
16
29
|
case "$1" in
|
|
17
30
|
status) echo 'systemctl --failed --no-pager; uptime' ;;
|
|
@@ -28,9 +41,27 @@ case "$name" in
|
|
|
28
41
|
uptime
|
|
29
42
|
free -h
|
|
30
43
|
df -h / /var/lib/vz
|
|
31
|
-
sensors 2>/dev/null | grep -E 'Tctl|Composite|edge' || true
|
|
44
|
+
sensors 2>/dev/null | grep -E 'Tctl|Composite|edge' || true
|
|
45
|
+
reboot_needed ;;
|
|
32
46
|
host-journal) since_ok "$rest" || { echo "bad since" >&2; exit 2; }; journalctl -p warning --since "$rest" --no-pager -n 200 ;;
|
|
33
47
|
updates) apt list --upgradable 2>/dev/null ;;
|
|
48
|
+
host-upgrade)
|
|
49
|
+
[ -z "$rest" ] || { echo "host-upgrade takes no arguments" >&2; exit 2; }
|
|
50
|
+
export DEBIAN_FRONTEND=noninteractive
|
|
51
|
+
apt-get update -q >/dev/null
|
|
52
|
+
# A new Proxmox VE major waits for the owner.
|
|
53
|
+
now="$(dpkg-query -W -f='${Version}' pve-manager)"
|
|
54
|
+
next="$(apt-get -s full-upgrade | sed -n 's/^Inst pve-manager \[[^]]*\] (\([^ ]*\) .*/\1/p')"
|
|
55
|
+
if [ -n "$next" ] && [ "${next%%.*}" != "${now%%.*}" ]; then
|
|
56
|
+
echo "held: Proxmox VE $now -> $next is a major upgrade"
|
|
57
|
+
exit 0
|
|
58
|
+
fi
|
|
59
|
+
apt-get -y -q -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold full-upgrade 2>&1 | grep -E '^[0-9]+ upgraded|^E:' || true
|
|
60
|
+
reboot_needed ;;
|
|
61
|
+
host-reboot)
|
|
62
|
+
[ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
|
|
63
|
+
shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
|
|
64
|
+
echo "reboot in 1 minute" ;;
|
|
34
65
|
guest-exec)
|
|
35
66
|
vmid="${rest%% *}"; action="${rest#* }"
|
|
36
67
|
case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
|
|
@@ -85,6 +85,7 @@ ${Object.values(UNIFI_READS)
|
|
|
85
85
|
set a device's management network and IP (keys mgmt_network_id, config_network)
|
|
86
86
|
unifi <37|mira> client-fixed-ip <clientId> <json> [--local]
|
|
87
87
|
set or clear a client's DHCP reservation (keys use_fixedip, fixed_ip, network_id)
|
|
88
|
+
unifi <37|mira> syslog <json> [--local] set the console's activity log export (keys this_controller, ip, port, contents, log_all_contents, enabled)
|
|
88
89
|
unifi <37|mira> app-policy-delete <policyId> <name> [--local]
|
|
89
90
|
delete a firewall policy the official API lists without an id
|
|
90
91
|
unifi <37|mira> nat [--local] the console's NAT rules (Policy Table; not in the official API)
|
|
@@ -253,6 +254,23 @@ export function clientFixedIpWrite(args: string[]): UnifiWrite {
|
|
|
253
254
|
return { method: "PUT", path: `/api/s/default/rest/user/${id}`, body };
|
|
254
255
|
}
|
|
255
256
|
|
|
257
|
+
export const SETTINGS_PATH = "/api/s/default/rest/setting";
|
|
258
|
+
const SYSLOG_KEYS = new Set(["this_controller", "ip", "port", "contents", "log_all_contents", "enabled"]);
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* `syslog <json>`: the console's activity log export (setting `rsyslogd`). Network 10.6 shows a separate
|
|
262
|
+
* System Logging integration that leaves this setting alone, yet this setting decides whether the
|
|
263
|
+
* Network application's CEF events reach a SIEM server.
|
|
264
|
+
*/
|
|
265
|
+
export function syslogChanges(args: string[]): Record<string, unknown> {
|
|
266
|
+
const [json, ...extra] = args;
|
|
267
|
+
if (json === undefined || extra.length > 0) throw new UsageError("syslog <json>");
|
|
268
|
+
const changes = JSON.parse(jsonBody(json)) as Record<string, unknown>;
|
|
269
|
+
const keys = Object.keys(changes);
|
|
270
|
+
if (Array.isArray(changes) || keys.length === 0 || keys.some((k) => !SYSLOG_KEYS.has(k))) throw new UsageError(`syslog body keys: ${[...SYSLOG_KEYS].join(", ")}`);
|
|
271
|
+
return changes;
|
|
272
|
+
}
|
|
273
|
+
|
|
256
274
|
export const POLICIES_PATH = "/v2/api/site/default/firewall-policies";
|
|
257
275
|
|
|
258
276
|
/** `app-policy-delete <policyId> <name>`: a firewall policy that the official API returns with `id: null` (rules converted from the old firewall). */
|
|
@@ -419,6 +437,17 @@ async function unifi(args: string[]): Promise<number> {
|
|
|
419
437
|
const call = clientFixedIpWrite(rest.slice(1));
|
|
420
438
|
return printAnswer(await siteRequest(site, readSite(site), local, call.path, call.method, call.body, "app"), `UniFi ${site} client`);
|
|
421
439
|
}
|
|
440
|
+
if (rest[0] === "syslog") {
|
|
441
|
+
const changes = syslogChanges(rest.slice(1));
|
|
442
|
+
const config = readSite(site);
|
|
443
|
+
const read = await siteRequest(site, config, local, SETTINGS_PATH, "GET", undefined, "app");
|
|
444
|
+
if (read.status < 200 || read.status >= 300) throw new ServiceError(`reading ${SETTINGS_PATH} answered ${read.status}; nothing changed`);
|
|
445
|
+
const current = ((JSON.parse(read.body) as { data?: Record<string, unknown>[] }).data ?? []).find((d) => d.key === "rsyslogd");
|
|
446
|
+
if (current === undefined || typeof current._id !== "string" || !NAT_ID.test(current._id)) throw new ServiceError(`no rsyslogd setting on ${site}; nothing changed`);
|
|
447
|
+
const kept = Object.fromEntries(Object.entries(current).filter(([k]) => k !== "_id" && k !== "site_id"));
|
|
448
|
+
const body = JSON.stringify({ ...kept, ...changes });
|
|
449
|
+
return printAnswer(await siteRequest(site, config, local, `${SETTINGS_PATH}/rsyslogd/${current._id}`, "PUT", body, "app"), `UniFi ${site} syslog`);
|
|
450
|
+
}
|
|
422
451
|
if (rest[0] === "app-policy-delete") {
|
|
423
452
|
const call = appPolicyDelete(rest.slice(1));
|
|
424
453
|
const config = readSite(site);
|
|
@@ -85,9 +85,13 @@ export const HOST_COMMANDS: Readonly<Record<string, { args: number; help: string
|
|
|
85
85
|
"host-status": { args: 0, help: "pveversion, uptime, load, memory, root disk, temperatures" },
|
|
86
86
|
"host-journal": { args: 1, help: "host-journal <since> warnings and errors of the host journal" },
|
|
87
87
|
updates: { args: 0, help: "upgradable packages on the host" },
|
|
88
|
+
"host-upgrade": { args: 0, help: "apt full-upgrade of the host; a new Proxmox VE major is held (prints held:); ends with reboot-needed" },
|
|
89
|
+
"host-reboot": { args: 1, help: "host-reboot --confirm reboot the host in one minute; every guest on it restarts" },
|
|
88
90
|
"guest-exec": { args: 2, help: "guest-exec <vmid> <status|journal|df|updates> fixed read inside a guest (pct exec or qm guest exec)" },
|
|
89
91
|
};
|
|
90
92
|
export const GUEST_EXEC = ["status", "journal", "df", "updates"];
|
|
93
|
+
/** A full upgrade with a new kernel and pve-manager can take several minutes on the host. */
|
|
94
|
+
export const HOST_UPGRADE_TIMEOUT_MS = 20 * 60_000;
|
|
91
95
|
const SINCE = /^[A-Za-z0-9 :\-+]{1,40}$/;
|
|
92
96
|
|
|
93
97
|
const HELP = `proxmox specialist (Proxmox VE on Pulsar at 37 and SPACE-X at Mira)
|
|
@@ -328,6 +332,7 @@ export function buildHostRemote(args: string[]): string {
|
|
|
328
332
|
if (!spec) throw new UsageError(`unknown host command: ${name ?? "(none)"}`);
|
|
329
333
|
if (rest.length !== spec.args) throw new UsageError(`${name} takes ${spec.args} argument(s)`);
|
|
330
334
|
if (name === "host-journal" && !SINCE.test(rest[0])) throw new UsageError("since: letters, digits, spaces, colon, plus, minus only");
|
|
335
|
+
if (name === "host-reboot" && rest[0] !== "--confirm") throw new UsageError("host-reboot restarts every guest on the host; add --confirm when the task asks for it explicitly");
|
|
331
336
|
if (name === "guest-exec") {
|
|
332
337
|
parseVmid(rest[0]);
|
|
333
338
|
if (!GUEST_EXEC.includes(rest[1])) throw new UsageError(`guest-exec action must be one of ${GUEST_EXEC.join(", ")}`);
|
|
@@ -345,7 +350,7 @@ async function host(config: Target, args: string[]): Promise<number> {
|
|
|
345
350
|
knownHostsFile: join(specialistsHome(), "config", "known_hosts"),
|
|
346
351
|
},
|
|
347
352
|
remote,
|
|
348
|
-
120_000,
|
|
353
|
+
args[0] === "host-upgrade" ? HOST_UPGRADE_TIMEOUT_MS : 120_000,
|
|
349
354
|
);
|
|
350
355
|
if (result.stdout.length > 0) printRaw(result.stdout);
|
|
351
356
|
if (result.code !== 0) throw new ServiceError(result.stderr.trim() || `${args[0]} exited ${result.code}`);
|
|
@@ -11,7 +11,7 @@ import { buildInferenceRemote, command as inferenceCommand } from "./inference/r
|
|
|
11
11
|
import { MAX_ADD_IDS, MAX_EXCLUDE_IDS, buildTrendarrCall, interpretRun, parseImdbIds, trendarrConfig, trendarrError, trendarrWarnings, command as mediaCommand } from "./media/run.ts";
|
|
12
12
|
import { ServiceError, UsageError } from "./lib/errors.ts";
|
|
13
13
|
import { safeRepoPath, safeWritePath } from "./lib/repo.ts";
|
|
14
|
-
import { appGetPath, appPolicyDelete, clientFixedIpWrite, buildPulsarRemote, cloudPath, deviceNetworkWrite, NAT_PATH, natWrite, parseQuery, parseSiteArgs, redactUnifi, unifiPath, unifiWrite, command as networkCommand } from "./network/run.ts";
|
|
14
|
+
import { appGetPath, appPolicyDelete, clientFixedIpWrite, syslogChanges, buildPulsarRemote, cloudPath, deviceNetworkWrite, NAT_PATH, natWrite, parseQuery, parseSiteArgs, redactUnifi, unifiPath, unifiWrite, command as networkCommand } from "./network/run.ts";
|
|
15
15
|
import { buildHostRemote, splitHost, needsConfirm, parseSetPairs, parseVmid, validateApiPath as validatePvePath, command as proxmoxCommand } from "./proxmox/run.ts";
|
|
16
16
|
import { banLength, buildObsRemote, compactDecisions, parseGoDuration, summariseAlerts, summariseVulns, summariseWazuh, vulnEvents, wazuhEvents, command as securityCommand } from "./security/run.ts";
|
|
17
17
|
|
|
@@ -653,6 +653,10 @@ describe("proxmox wrapper", () => {
|
|
|
653
653
|
expect(buildHostRemote(["guest-exec", "101", "df"])).toBe("guest-exec 101 df");
|
|
654
654
|
expect(() => buildHostRemote(["guest-exec", "101", "rm"])).toThrow(/action/);
|
|
655
655
|
expect(() => buildHostRemote(["host-journal", "1 hour ago; id"])).toThrow(/since/);
|
|
656
|
+
expect(buildHostRemote(["host-upgrade"])).toBe("host-upgrade");
|
|
657
|
+
expect(() => buildHostRemote(["host-upgrade", "now"])).toThrow(UsageError);
|
|
658
|
+
expect(buildHostRemote(["host-reboot", "--confirm"])).toBe("host-reboot --confirm");
|
|
659
|
+
expect(() => buildHostRemote(["host-reboot", "now"])).toThrow(/--confirm/);
|
|
656
660
|
expect(() => buildHostRemote(["shell"])).toThrow(/unknown host command/);
|
|
657
661
|
});
|
|
658
662
|
|
|
@@ -907,6 +911,22 @@ describe("network deletes check the live name", () => {
|
|
|
907
911
|
expect(calls.filter((c) => c.startsWith("DELETE"))).toHaveLength(1);
|
|
908
912
|
});
|
|
909
913
|
|
|
914
|
+
it("changes only the syslog keys and keeps the rest of the live setting", async () => {
|
|
915
|
+
expect(syslogChanges(['{"ip":"10.0.90.10","port":"514"}'])).toEqual({ ip: "10.0.90.10", port: "514" });
|
|
916
|
+
expect(() => syslogChanges(['{"x_secret":"a"}'])).toThrow(UsageError);
|
|
917
|
+
expect(() => syslogChanges(["{}"])).toThrow(UsageError);
|
|
918
|
+
const sent: { method: string; path: string; body?: string }[] = [];
|
|
919
|
+
vi.stubGlobal("fetch", async (url: string, init?: RequestInit) => {
|
|
920
|
+
sent.push({ method: init?.method ?? "GET", path: new URL(url).pathname, body: typeof init?.body === "string" ? init.body : undefined });
|
|
921
|
+
const data = init?.method === "PUT" ? [] : [{ _id: id, site_id: "s1", key: "rsyslogd", this_controller: true, enabled: true }];
|
|
922
|
+
return new Response(JSON.stringify({ meta: { rc: "ok" }, data }));
|
|
923
|
+
});
|
|
924
|
+
await networkCommand(["unifi", "mira", "syslog", '{"this_controller":false,"ip":"10.0.90.10","port":"514"}']);
|
|
925
|
+
const put = sent.find((r) => r.method === "PUT");
|
|
926
|
+
expect(put?.path).toMatch(new RegExp(`/rest/setting/rsyslogd/${id}$`));
|
|
927
|
+
expect(JSON.parse(put?.body ?? "{}")).toEqual({ key: "rsyslogd", this_controller: false, enabled: true, ip: "10.0.90.10", port: "514" });
|
|
928
|
+
});
|
|
929
|
+
|
|
910
930
|
it("refuses a NAT delete of a rule that is not there", async () => {
|
|
911
931
|
const calls = stubConsole([{ _id: "fedcba9876543210fedcba98", description: "IoT DNS redirect" }]);
|
|
912
932
|
await expect(networkCommand(["unifi", "mira", "nat-write", "DELETE", id, "IoT DNS redirect"])).rejects.toThrow(/is not in .*nothing deleted/);
|