@astrofoundry/pi-astro 0.40.34 → 0.40.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/astro-discord/daily-run.test.ts +41 -9
- package/extensions/astro-discord/daily-run.ts +119 -21
- package/extensions/astro-discord/index.ts +110 -8
- package/extensions/specialist-gate/approvals.test.ts +3 -0
- package/extensions/specialist-gate/index.ts +1 -1
- package/package.json +1 -1
- package/skills/dns/SKILL.md +1 -1
- package/skills/inference/SKILL.md +1 -0
- package/skills/proxmox/SKILL.md +3 -1
- package/specialists/dns/run.ts +3 -0
- package/specialists/entry/remote/spc-inference-nexus-entry.sh +1 -0
- package/specialists/entry/remote/spc-proxmox-pulsar-entry.sh +32 -1
- package/specialists/entry/remote/spc-proxmox-spacex-entry.sh +32 -1
- package/specialists/inference/run.ts +1 -0
- package/specialists/proxmox/run.ts +6 -1
- package/specialists/wrappers.test.ts +5 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
2
|
import type { ArcaneReport } from "./daily-report.ts";
|
|
3
3
|
import type { ToolsReport } from "./tools-report.ts";
|
|
4
|
-
import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
4
|
+
import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
5
5
|
|
|
6
6
|
const EDGE = `nginx active
|
|
7
7
|
wg-quick@wg0 active
|
|
@@ -62,15 +62,20 @@ describe("morning status lines", () => {
|
|
|
62
62
|
expect(edgeStatus(null, null)).toMatchObject({ line: "the check failed (vps status)", problem: "Edge: the status check failed" });
|
|
63
63
|
});
|
|
64
64
|
|
|
65
|
-
it("reports both Proxmox hosts and
|
|
66
|
-
const held =
|
|
67
|
-
expect(proxmoxStatus([{ name: "Pulsar", status: PULSAR
|
|
68
|
-
const
|
|
69
|
-
{ name: "Pulsar", status: PULSAR
|
|
70
|
-
{ name: "SPACE-X", status: null, updates: null },
|
|
65
|
+
it("reports both Proxmox hosts, the morning upgrade, a held major, and a pending reboot with its button", () => {
|
|
66
|
+
const held = "Listing...\nwazuh-agent/stable 4.14.9-1 amd64 [upgradable from: 4.14.7-1]";
|
|
67
|
+
expect(proxmoxStatus([{ name: "Pulsar", id: "pulsar", status: `${PULSAR}\nreboot-needed: no`, updates: held }])).toEqual({ area: "Proxmox", line: "Pulsar up 59 days, RAM 47Gi of 60Gi used, root disk 13%, up to date" });
|
|
68
|
+
const upgraded = proxmoxStatus([
|
|
69
|
+
{ name: "Pulsar", id: "pulsar", status: `${PULSAR}\nreboot-needed: kernel 7.0.14-25-pve -> 7.0.14-26-pve`, updates: held, upgrade: "3 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.\nreboot-needed: kernel 7.0.14-25-pve -> 7.0.14-26-pve" },
|
|
70
|
+
{ name: "SPACE-X", id: "space-x", status: null, updates: null },
|
|
71
71
|
]);
|
|
72
|
-
expect(
|
|
73
|
-
expect(
|
|
72
|
+
expect(upgraded.line).toBe("Pulsar up 59 days, RAM 47Gi of 60Gi used, root disk 13%, 3 host updates installed, reboot needed; SPACE-X status unknown");
|
|
73
|
+
expect(upgraded.problem).toBe("Proxmox: Pulsar needs a reboot (kernel 7.0.14-25-pve -> 7.0.14-26-pve); every guest on it restarts for a few minutes; the status check failed for SPACE-X");
|
|
74
|
+
expect(upgraded.buttons).toEqual([{ label: "Reboot Pulsar", customId: "proxmox:reboot:pulsar" }]);
|
|
75
|
+
const major = proxmoxStatus([{ name: "SPACE-X", id: "space-x", status: `${PULSAR}\nreboot-needed: no`, updates: PULSAR_UPDATES, upgrade: "held: Proxmox VE 9.2.24 -> 10.0.1 is a major upgrade" }]);
|
|
76
|
+
expect(major).toMatchObject({ problem: "Proxmox: SPACE-X waits for you: Proxmox VE 9.2.24 -> 10.0.1 is a major upgrade", route: "admin" });
|
|
77
|
+
expect(major.line).toContain("a Proxmox VE major upgrade waits for you");
|
|
78
|
+
expect(proxmoxStatus([{ name: "Pulsar", id: "pulsar", status: PULSAR, updates: PULSAR_UPDATES, upgrade: null }]).problem).toBe("Proxmox: the host upgrade on Pulsar failed");
|
|
74
79
|
});
|
|
75
80
|
|
|
76
81
|
it("flags updates the nightly run left behind, reading guest-agent JSON too, and ignores the held Wazuh agent", () => {
|
|
@@ -104,6 +109,33 @@ describe("morning status lines", () => {
|
|
|
104
109
|
expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: { status: "ok" }, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "technitium1, technitium2, technitium3 answer" });
|
|
105
110
|
expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: null, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "no answer from technitium2", problem: "DNS: technitium2 does not answer", route: "admin" });
|
|
106
111
|
expect(dnsStatus({ technitium1: null, technitium2: { status: "error" }, technitium3: { status: "ok" } })).toMatchObject({ problem: "DNS: technitium1 and technitium2 do not answer" });
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it("flags a Mac two macOS minors behind and notes a major without flagging it", () => {
|
|
115
|
+
const nexus = "26.5.2\nTitle: Command Line Tools for Xcode 26.6, Version: 26.6, Size: 920431KiB, Recommended: YES,\nTitle: macOS Tahoe 26.7.2, Version: 26.7.2, Size: 3868257KiB, Recommended: YES, Action: restart,\nTitle: macOS 27.2, Version: 27.2, Size: 11319983KiB, Recommended: YES, Action: restart,";
|
|
116
|
+
const cortex = "26.6.1\nTitle: macOS Tahoe 26.7.2, Version: 26.7.2, Size: 3168375KiB, Recommended: YES, Action: restart,";
|
|
117
|
+
expect(macosStatus([{ name: "Cortex", output: cortex }])).toEqual({ area: "macOS", line: "Cortex 26.6.1, 26.7.2 waiting" });
|
|
118
|
+
expect(macosStatus([{ name: "Nexus", output: nexus }, { name: "Cortex", output: null }])).toEqual({
|
|
119
|
+
area: "macOS",
|
|
120
|
+
line: "Nexus 26.5.2, 26.7.2 waiting, macOS 27.2 offered (major, by hand); Cortex unknown",
|
|
121
|
+
problem: "macOS: Nexus runs macOS 26.5.2 while 26.7.2 is out; automatic installs are not keeping up; no macOS update list from Cortex",
|
|
122
|
+
route: "admin",
|
|
123
|
+
});
|
|
124
|
+
expect(macosStatus([{ name: "Cortex", output: "26.7.2" }]).line).toBe("Cortex 26.7.2");
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
it("upgrades one Technitium node per morning, Mira first and the primary last, and holds a major", () => {
|
|
128
|
+
const update = (to: string) => ({ status: "ok", response: { updateAvailable: true, updateVersion: to, currentVersion: "15.6.2" } });
|
|
129
|
+
const state = (versions: [string, string, string]) => ({ status: "ok", response: { clusterNodes: versions.map((v, i) => ({ name: `technitium${i + 1}.dns.home.arpa`, version: v })) } });
|
|
130
|
+
expect(technitiumPlan({ status: "ok", response: { updateAvailable: false } }, state(["15.6.2", "15.6.2", "15.6.2"]))).toBeNull();
|
|
131
|
+
expect(technitiumPlan(update("15.7.0"), state(["15.6.2", "15.6.2", "15.6.2"]))).toEqual({ host: "technitium3", from: "15.6.2", to: "15.7.0" });
|
|
132
|
+
expect(technitiumPlan(update("15.7.0"), state(["15.6.2", "15.6.2", "15.7.0"]))).toEqual({ host: "technitium2", from: "15.6.2", to: "15.7.0" });
|
|
133
|
+
expect(technitiumPlan(update("15.7.0"), state(["15.6.2", "15.7.0", "15.7.0"]))).toEqual({ host: "technitium1", from: "15.6.2", to: "15.7.0" });
|
|
134
|
+
expect(technitiumPlan(update("16.0.0"), state(["15.6.2", "15.6.2", "15.6.2"]))).toEqual({ major: true, from: "15.6.2", to: "16.0.0" });
|
|
135
|
+
const all = { technitium1: { status: "ok" }, technitium2: { status: "ok" }, technitium3: { status: "ok" } };
|
|
136
|
+
expect(dnsStatus(all, { host: "technitium3", from: "15.6.2", to: "15.7.0", ok: true })).toEqual({ area: "DNS", line: "technitium1, technitium2, technitium3 answer; technitium3 upgraded 15.6.2 -> 15.7.0 (one node per morning, primary last)" });
|
|
137
|
+
expect(dnsStatus(all, { host: "technitium3", from: "15.6.2", to: "15.7.0", ok: false, why: "installer exited 1" }).problem).toBe("DNS: the upgrade of technitium3 to 15.7.0 failed (installer exited 1)");
|
|
138
|
+
expect(dnsStatus(all, { major: true, from: "15.6.2", to: "16.0.0" }).problem).toMatch(/^DNS: Technitium 16.0.0 is a major release \(running 15.6.2\)/);
|
|
107
139
|
expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=200\nrerank \tstate = running \tpid = 81670 health=200")).toEqual({ area: "Inference", line: "Nexus embed, rerank running and healthy" });
|
|
108
140
|
expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=503")).toMatchObject({ problem: "Inference: embed not healthy on Nexus" });
|
|
109
141
|
expect(mediaStatus({ sabnzbd_paused: true, warnings: [] })).toEqual({ area: "Media", line: "downloads paused" });
|
|
@@ -6,12 +6,19 @@ import type { ToolsReport } from "./tools-report.ts";
|
|
|
6
6
|
/** Where an area's problems tag the owners: specialists without a channel of their own use the admin channel. */
|
|
7
7
|
export type Route = "admin" | "media";
|
|
8
8
|
|
|
9
|
+
/** An owner-only button posted with an area's problem. */
|
|
10
|
+
export interface AreaButton {
|
|
11
|
+
label: string;
|
|
12
|
+
customId: string;
|
|
13
|
+
}
|
|
14
|
+
|
|
9
15
|
/** One area of the morning summary: a line, and a problem that needs an owner, if any. */
|
|
10
16
|
export interface AreaStatus {
|
|
11
17
|
area: string;
|
|
12
18
|
line: string;
|
|
13
19
|
problem?: string;
|
|
14
20
|
route?: Route;
|
|
21
|
+
buttons?: AreaButton[];
|
|
15
22
|
}
|
|
16
23
|
|
|
17
24
|
const HOUR = 3600;
|
|
@@ -50,35 +57,65 @@ export function edgeStatus(status: string | null, updates: string | null): AreaS
|
|
|
50
57
|
return { area: "Edge", line: `front door: ${parts.join(", ")}`, ...(problem ? { problem, route: "admin" as const } : {}) };
|
|
51
58
|
}
|
|
52
59
|
|
|
53
|
-
/** One Proxmox host
|
|
60
|
+
/** One Proxmox host as the morning run saw it. */
|
|
54
61
|
export interface ProxmoxHost {
|
|
55
62
|
name: string;
|
|
63
|
+
/** `pulsar` or `space-x`: the target of the Reboot button. */
|
|
64
|
+
id: string;
|
|
56
65
|
status: string | null;
|
|
66
|
+
/** `updates` after any upgrade. */
|
|
57
67
|
updates: string | null;
|
|
68
|
+
/** What `host-upgrade` printed this morning: undefined when nothing was waiting, null when it failed. */
|
|
69
|
+
upgrade?: string | null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** The custom id of the Reboot button for one Proxmox host. */
|
|
73
|
+
export function rebootButtonId(id: string): string {
|
|
74
|
+
return `proxmox:reboot:${id}`;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
interface ProxmoxFacts {
|
|
78
|
+
line: string;
|
|
79
|
+
problems: string[];
|
|
80
|
+
button?: AreaButton;
|
|
58
81
|
}
|
|
59
82
|
|
|
60
|
-
function
|
|
61
|
-
if (h.status === null) return `${h.name} status unknown
|
|
83
|
+
function proxmoxHostFacts(h: ProxmoxHost): ProxmoxFacts {
|
|
84
|
+
if (h.status === null) return { line: `${h.name} status unknown`, problems: [`the status check failed for ${h.name}`] };
|
|
62
85
|
const up = /up\s+([^,]+(?:,\s*\d+:\d+)?)/.exec(h.status)?.[1]?.split(",")[0]?.trim();
|
|
63
86
|
const mem = /^Mem:\s+(\S+)\s+(\S+)/m.exec(h.status);
|
|
64
87
|
const root = /\s(\d+)%\s+\/\s*$/m.exec(h.status)?.[1];
|
|
65
|
-
const
|
|
66
|
-
const
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
88
|
+
const reboot = /^reboot-needed:\s*(.*)$/m.exec(h.status)?.[1]?.trim();
|
|
89
|
+
const waiting = h.updates === null ? null : pendingPackageLines(h.updates).length;
|
|
90
|
+
const held = h.upgrade ? /^held:\s*(.*)$/m.exec(h.upgrade)?.[1] : undefined;
|
|
91
|
+
const installed = h.upgrade ? /^(\d+) upgraded/m.exec(h.upgrade)?.[1] : undefined;
|
|
92
|
+
const updates =
|
|
93
|
+
h.upgrade === null
|
|
94
|
+
? "the host upgrade failed"
|
|
95
|
+
: held
|
|
96
|
+
? "a Proxmox VE major upgrade waits for you"
|
|
97
|
+
: [installed ? `${installed} host update${installed === "1" ? "" : "s"} installed` : null, waiting === null ? "updates unknown" : waiting > 0 ? `${waiting} still waiting` : installed ? null : "up to date"].filter(Boolean).join(", ");
|
|
98
|
+
const rebootNeeded = reboot !== undefined && reboot !== "no";
|
|
99
|
+
const line = [up ? `${h.name} up ${up}` : `${h.name} up`, mem ? `RAM ${mem[2]} of ${mem[1]} used` : null, root ? `root disk ${root}%` : null, updates, rebootNeeded ? "reboot needed" : null].filter(Boolean).join(", ");
|
|
100
|
+
const problems = [
|
|
101
|
+
h.upgrade === null ? `the host upgrade on ${h.name} failed` : null,
|
|
102
|
+
held ? `${h.name} waits for you: ${held}` : null,
|
|
103
|
+
rebootNeeded ? `${h.name} needs a reboot (${reboot}); every guest on it restarts for a few minutes` : null,
|
|
104
|
+
].filter((p): p is string => p !== null);
|
|
105
|
+
return { line, problems, ...(rebootNeeded ? { button: { label: `Reboot ${h.name}`, customId: rebootButtonId(h.id) } } : {}) };
|
|
75
106
|
}
|
|
76
107
|
|
|
77
|
-
/**
|
|
108
|
+
/** Both Proxmox hosts: uptime, memory, root disk, the morning's automatic upgrade, a held major, and a pending reboot with its button. */
|
|
78
109
|
export function proxmoxStatus(hosts: readonly ProxmoxHost[]): AreaStatus {
|
|
79
|
-
const
|
|
80
|
-
const
|
|
81
|
-
|
|
110
|
+
const facts = hosts.map(proxmoxHostFacts);
|
|
111
|
+
const problems = facts.flatMap((f) => f.problems);
|
|
112
|
+
const buttons = facts.flatMap((f) => (f.button ? [f.button] : []));
|
|
113
|
+
return {
|
|
114
|
+
area: "Proxmox",
|
|
115
|
+
line: facts.map((f) => f.line).join("; "),
|
|
116
|
+
...(problems.length > 0 ? { problem: `Proxmox: ${problems.join("; ")}`, route: "admin" as const } : {}),
|
|
117
|
+
...(buttons.length > 0 ? { buttons } : {}),
|
|
118
|
+
};
|
|
82
119
|
}
|
|
83
120
|
|
|
84
121
|
/** Packages the nightly run leaves alone on purpose: a Wazuh agent must never be newer than the manager. */
|
|
@@ -150,14 +187,43 @@ export function identityStatus(status: string | null): AreaStatus {
|
|
|
150
187
|
return { area: "Identity", line, ...(bad.length > 0 || units.length === 0 ? { problem: `Identity: ${bad.length > 0 ? bad.join(", ") : "no unit status read"} on the DMZ VM`, route: "admin" as const } : {}) };
|
|
151
188
|
}
|
|
152
189
|
|
|
153
|
-
/**
|
|
154
|
-
|
|
190
|
+
/** Upgrade order: Mira's node first, the primary last, one node per morning. */
|
|
191
|
+
const TECHNITIUM_ORDER = ["technitium3", "technitium2", "technitium1"];
|
|
192
|
+
|
|
193
|
+
/** What the morning run does about a Technitium release: upgrade one node, or hold a major for the owner. */
|
|
194
|
+
export type TechnitiumPlan = { host: string; from: string; to: string } | { major: true; from: string; to: string };
|
|
195
|
+
|
|
196
|
+
/** `check-update` and `cluster-state` from the primary: the next node to upgrade, a held major, or nothing. */
|
|
197
|
+
export function technitiumPlan(update: unknown, state: unknown): TechnitiumPlan | null {
|
|
198
|
+
const u = (update as { response?: { updateAvailable?: unknown; updateVersion?: unknown; currentVersion?: unknown } } | null)?.response;
|
|
199
|
+
if (u?.updateAvailable !== true || typeof u.updateVersion !== "string" || typeof u.currentVersion !== "string") return null;
|
|
200
|
+
const to = u.updateVersion;
|
|
201
|
+
if (to.split(".")[0] !== u.currentVersion.split(".")[0]) return { major: true, from: u.currentVersion, to };
|
|
202
|
+
const nodes = (state as { response?: { clusterNodes?: { name?: unknown; version?: unknown }[] } } | null)?.response?.clusterNodes ?? [];
|
|
203
|
+
for (const host of TECHNITIUM_ORDER) {
|
|
204
|
+
const node = nodes.find((n) => typeof n.name === "string" && n.name.split(".")[0] === host);
|
|
205
|
+
if (node && typeof node.version === "string" && node.version !== to) return { host, from: node.version, to };
|
|
206
|
+
}
|
|
207
|
+
return null;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** The morning's Technitium upgrade, if one ran or a major waits. */
|
|
211
|
+
export type DnsUpgrade = { host: string; from: string; to: string; ok: true } | { host: string; from: string; to: string; ok: false; why: string } | { major: true; from: string; to: string };
|
|
212
|
+
|
|
213
|
+
/** `dns technitium primary|node <fqdn> resolve <name>`: every cluster node answers an internal name, keyed by node name; plus the morning's upgrade. */
|
|
214
|
+
export function dnsStatus(answers: Readonly<Record<string, unknown>>, upgrade?: DnsUpgrade): AreaStatus {
|
|
155
215
|
const ok = (r: unknown) => (r as { status?: unknown } | null)?.status === "ok";
|
|
156
216
|
const nodes = Object.keys(answers);
|
|
157
217
|
const down = nodes.filter((node) => !ok(answers[node]));
|
|
158
218
|
const names = down.join(" and ");
|
|
159
|
-
|
|
160
|
-
|
|
219
|
+
const upgradeLine = upgrade === undefined ? null : "major" in upgrade ? `Technitium ${upgrade.to} is a major release and waits for you` : upgrade.ok ? `${upgrade.host} upgraded ${upgrade.from} -> ${upgrade.to} (one node per morning, primary last)` : `the upgrade of ${upgrade.host} to ${upgrade.to} failed`;
|
|
220
|
+
const problems = [
|
|
221
|
+
down.length > 0 ? `${names} ${down.length === 1 ? "does" : "do"} not answer` : null,
|
|
222
|
+
upgrade && "major" in upgrade ? `Technitium ${upgrade.to} is a major release (running ${upgrade.from}); read its change log, then upgrade one node at a time with \`dns technitium host <node> upgrade\`` : null,
|
|
223
|
+
upgrade && "ok" in upgrade && !upgrade.ok ? `the upgrade of ${upgrade.host} to ${upgrade.to} failed (${upgrade.why.slice(0, 200)})` : null,
|
|
224
|
+
].filter(Boolean);
|
|
225
|
+
const line = [down.length === 0 ? `${nodes.join(", ")} answer` : `no answer from ${names}`, upgradeLine].filter(Boolean).join("; ");
|
|
226
|
+
return { area: "DNS", line, ...(problems.length > 0 ? { problem: `DNS: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
161
227
|
}
|
|
162
228
|
|
|
163
229
|
/** `inference nexus-status`: each Nexus service running with HTTP health 200. */
|
|
@@ -196,6 +262,38 @@ export function backupStatus(archives: unknown, resticText: string | null, now:
|
|
|
196
262
|
return { area: "Backup", line: `${proxmox}, ${check}`, ...(problems.length > 0 ? { problem: `Backup: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
197
263
|
}
|
|
198
264
|
|
|
265
|
+
/** `sw_vers -productVersion` on its first line, then Software Update's `Title:` lines, for one Mac. */
|
|
266
|
+
export interface MacUpdates {
|
|
267
|
+
name: string;
|
|
268
|
+
output: string | null;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/** A Mac this many minor releases behind the newest offered one is not keeping up with its automatic installs. */
|
|
272
|
+
const MACOS_MINORS_BEHIND = 2;
|
|
273
|
+
|
|
274
|
+
/** Each Mac's macOS version and offers: automatic installs cover minors; a major is noted for a manual upgrade. */
|
|
275
|
+
export function macosStatus(macs: readonly MacUpdates[]): AreaStatus {
|
|
276
|
+
const parts: string[] = [];
|
|
277
|
+
const problems: string[] = [];
|
|
278
|
+
for (const m of macs) {
|
|
279
|
+
if (m.output === null) {
|
|
280
|
+
parts.push(`${m.name} unknown`);
|
|
281
|
+
problems.push(`no macOS update list from ${m.name}`);
|
|
282
|
+
continue;
|
|
283
|
+
}
|
|
284
|
+
const [installed = "", ...rest] = m.output.split("\n").map((l) => l.trim());
|
|
285
|
+
const offers = rest.map((l) => /Title: macOS[^,]*, Version: ([\d.]+)/.exec(l)?.[1]).filter((v): v is string => v !== undefined);
|
|
286
|
+
const [major, minor] = installed.split(".").map(Number);
|
|
287
|
+
const sameMajor = offers.filter((v) => Number(v.split(".")[0]) === major);
|
|
288
|
+
const newestMinor = Math.max(minor ?? 0, ...sameMajor.map((v) => Number(v.split(".")[1] ?? 0)));
|
|
289
|
+
const majors = offers.filter((v) => Number(v.split(".")[0]) > major);
|
|
290
|
+
const latest = sameMajor.sort((a, b) => a.localeCompare(b, undefined, { numeric: true })).at(-1);
|
|
291
|
+
parts.push(`${m.name} ${installed}${latest ? `, ${latest} waiting` : ""}${majors.length > 0 ? `, macOS ${majors[0]} offered (major, by hand)` : ""}`);
|
|
292
|
+
if (newestMinor - (minor ?? 0) >= MACOS_MINORS_BEHIND) problems.push(`${m.name} runs macOS ${installed} while ${latest} is out; automatic installs are not keeping up`);
|
|
293
|
+
}
|
|
294
|
+
return { area: "macOS", line: parts.join("; "), ...(problems.length > 0 ? { problem: `macOS: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
295
|
+
}
|
|
296
|
+
|
|
199
297
|
/** One report the run posted in its own channel, as the summary shows it. */
|
|
200
298
|
export interface ReportLine {
|
|
201
299
|
area: string;
|
|
@@ -78,7 +78,7 @@ import { ArcaneWatch } from "./arcane-watch-monitor.ts";
|
|
|
78
78
|
import { parseSnapshot } from "./arcane-watch.ts";
|
|
79
79
|
import { type Applied, autoFailureText, foremanTask, stackProblems, stackUpdateAnswer, SUPERVISE_DELAY_MS, SUPERVISE_LIMIT_MS } from "./supervise.ts";
|
|
80
80
|
import { stackPlan, toolPlan } from "./update-policy.ts";
|
|
81
|
-
import { type AreaStatus, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, problemText, proxmoxStatus, type ReportLine, type Route, summaryText, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
81
|
+
import { type AreaStatus, arcaneSummary, backupStatus, type DnsUpgrade, dnsStatus, macosStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, type ProxmoxHost, proxmoxStatus, type ReportLine, type Route, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
82
82
|
import { type InvestigationRequest, SecurityMonitor } from "./security-monitor.ts";
|
|
83
83
|
import { nasOutage, readSecurityState } from "./security.ts";
|
|
84
84
|
|
|
@@ -174,6 +174,15 @@ const DNS_NODES = { technitium2: "technitium2.dns.home.arpa", technitium3: "tech
|
|
|
174
174
|
|
|
175
175
|
/** The audit caller of updates the morning run applies on its own. */
|
|
176
176
|
const AUTO_CALLER = "daily-run";
|
|
177
|
+
/** The Proxmox hosts the morning run upgrades, with the wrapper prefix that selects each. */
|
|
178
|
+
const PROXMOX_HOSTS: readonly { name: string; id: string; prefix: string[] }[] = [
|
|
179
|
+
{ name: "Pulsar", id: "pulsar", prefix: [] },
|
|
180
|
+
{ name: "SPACE-X", id: "space-x", prefix: ["space-x"] },
|
|
181
|
+
];
|
|
182
|
+
/** The dns wrapper allows a node upgrade 10 minutes; the bridge waits a little longer. */
|
|
183
|
+
const DNS_UPGRADE_MS = 11 * 60_000;
|
|
184
|
+
/** The wrapper allows a host upgrade 20 minutes; the bridge waits a little longer for its answer. */
|
|
185
|
+
const HOST_UPGRADE_MS = 21 * 60_000;
|
|
177
186
|
|
|
178
187
|
function pause(ms: number): Promise<void> {
|
|
179
188
|
return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)).unref());
|
|
@@ -230,6 +239,8 @@ class Bridge {
|
|
|
230
239
|
private readonly mediaMessages = new Map<string, MessageComponent[]>();
|
|
231
240
|
/** Tool updates started from the Cortex Tools Report, by line key; they count as open work. */
|
|
232
241
|
private readonly toolRuns = new Map<string, { channelId: string; label: string }>();
|
|
242
|
+
/** Proxmox hosts that an owner's Reboot click is restarting. */
|
|
243
|
+
private readonly rebooting = new Set<string>();
|
|
233
244
|
private readonly toolClicks = new KeyedQueue();
|
|
234
245
|
private gateway: GatewayClient | undefined;
|
|
235
246
|
private stopping = false;
|
|
@@ -588,7 +599,8 @@ class Bridge {
|
|
|
588
599
|
const routes = this.routes();
|
|
589
600
|
const owners = this.config.owners;
|
|
590
601
|
for (const a of areas.filter((x) => x.problem)) {
|
|
591
|
-
|
|
602
|
+
const components = a.buttons ? [{ type: 1, components: a.buttons.map((b) => ({ type: 2, style: 4, label: b.label, custom_id: b.customId })) }] : undefined;
|
|
603
|
+
await this.rest.createMessage(routes[a.route ?? "admin"], { content: problemText(a, owners), mentionUsers: owners, ...(components ? { components } : {}) }).catch((err) => log(`daily ${date}: ${a.area} problem message failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
592
604
|
}
|
|
593
605
|
const label = new Intl.DateTimeFormat("en-GB", { timeZone: schedule.timeZone, weekday: "short", day: "numeric", month: "short" }).format(new Date());
|
|
594
606
|
const text = summaryText({ date: label, guildId: this.config.guildId, reports, areas, routes });
|
|
@@ -623,11 +635,17 @@ class Bridge {
|
|
|
623
635
|
if (present.has("edge")) areas.push((async () => edgeStatus(await text("edge", ["vps", "status"]), await text("edge", ["vps", "updates"])))());
|
|
624
636
|
if (present.has("proxmox")) {
|
|
625
637
|
areas.push(
|
|
626
|
-
(async () =>
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
638
|
+
(async () => {
|
|
639
|
+
const hosts: ProxmoxHost[] = [];
|
|
640
|
+
for (const h of PROXMOX_HOSTS) {
|
|
641
|
+
const before = await text("proxmox", [...h.prefix, "updates"]);
|
|
642
|
+
// Host packages install on their own; a new Proxmox VE major is held by the entry script for the owner.
|
|
643
|
+
const upgrade = before !== null && pendingPackageLines(before).length > 0 ? await this.runEntry("proxmox", AUTO_CALLER, [...h.prefix, "host-upgrade"], HOST_UPGRADE_MS).catch((err) => (log(`daily: ${h.name} host-upgrade failed: ${err instanceof Error ? err.message : String(err)}`), null)) : undefined;
|
|
644
|
+
const updates = upgrade === undefined ? before : await text("proxmox", [...h.prefix, "updates"]);
|
|
645
|
+
hosts.push({ name: h.name, id: h.id, status: await text("proxmox", [...h.prefix, "host-status"]), updates, upgrade });
|
|
646
|
+
}
|
|
647
|
+
return proxmoxStatus(hosts);
|
|
648
|
+
})(),
|
|
631
649
|
);
|
|
632
650
|
areas.push(
|
|
633
651
|
(async () => {
|
|
@@ -655,17 +673,40 @@ class Bridge {
|
|
|
655
673
|
if (present.has("dns")) {
|
|
656
674
|
areas.push(
|
|
657
675
|
(async () => {
|
|
676
|
+
// A minor Technitium release upgrades one node per morning, before the answer checks below.
|
|
677
|
+
const plan = technitiumPlan(await json("dns", ["technitium", "primary", "check-update"]), await json("dns", ["technitium", "primary", "cluster-state"]));
|
|
678
|
+
let upgrade: DnsUpgrade | undefined;
|
|
679
|
+
if (plan && "major" in plan) upgrade = plan;
|
|
680
|
+
else if (plan) {
|
|
681
|
+
upgrade = await this.runEntry("dns", AUTO_CALLER, ["technitium", "host", plan.host, "upgrade"], DNS_UPGRADE_MS).then(
|
|
682
|
+
() => ({ ...plan, ok: true as const }),
|
|
683
|
+
(err) => ({ ...plan, ok: false as const, why: err instanceof Error ? err.message : String(err) }),
|
|
684
|
+
);
|
|
685
|
+
}
|
|
658
686
|
const [technitium1, technitium2, technitium3] = await Promise.all([
|
|
659
687
|
json("dns", ["technitium", "primary", "resolve", DNS_PROBE]),
|
|
660
688
|
json("dns", ["technitium", "node", DNS_NODES.technitium2, "resolve", DNS_PROBE]),
|
|
661
689
|
json("dns", ["technitium", "node", DNS_NODES.technitium3, "resolve", DNS_PROBE]),
|
|
662
690
|
]);
|
|
663
|
-
return dnsStatus({ technitium1, technitium2, technitium3 });
|
|
691
|
+
return dnsStatus({ technitium1, technitium2, technitium3 }, upgrade);
|
|
664
692
|
})(),
|
|
665
693
|
);
|
|
666
694
|
}
|
|
667
695
|
if (present.has("inference")) areas.push((async () => inferenceStatus(await text("inference", ["nexus-status"])))());
|
|
668
696
|
if (present.has(MEDIA)) areas.push((async () => mediaStatus(await json(MEDIA, ["sabnzbd", "status"])))());
|
|
697
|
+
areas.push(
|
|
698
|
+
(async () => {
|
|
699
|
+
// Cortex is this host; Nexus answers through the inference specialist.
|
|
700
|
+
const version = await this.execTool("/usr/bin/sw_vers", ["-productVersion"], { timeoutMs: 30_000, env: { PATH: TOOL_PATH } });
|
|
701
|
+
const offers = await this.execTool("/usr/sbin/softwareupdate", ["-l"], { timeoutMs: 3 * 60_000, env: { PATH: TOOL_PATH } });
|
|
702
|
+
const cortex = version.code === 0 ? `${version.stdout.trim()}\n${offers.stdout}${offers.stderr}` : null;
|
|
703
|
+
const nexus = present.has("inference") ? await text("inference", ["macos-updates"]) : null;
|
|
704
|
+
return macosStatus([
|
|
705
|
+
{ name: "Cortex", output: cortex },
|
|
706
|
+
...(present.has("inference") ? [{ name: "Nexus", output: nexus }] : []),
|
|
707
|
+
]);
|
|
708
|
+
})(),
|
|
709
|
+
);
|
|
669
710
|
return Promise.all(areas);
|
|
670
711
|
}
|
|
671
712
|
|
|
@@ -1350,6 +1391,63 @@ class Bridge {
|
|
|
1350
1391
|
await this.restartService(p, userId);
|
|
1351
1392
|
}
|
|
1352
1393
|
|
|
1394
|
+
/**
|
|
1395
|
+
* Reboots one Proxmox host for an owner, when its status still says a reboot is needed,
|
|
1396
|
+
* then reports in the problem's channel whether it came back within ten minutes.
|
|
1397
|
+
*/
|
|
1398
|
+
private async onRebootButton(interaction: Interaction, id: string): Promise<void> {
|
|
1399
|
+
const host = PROXMOX_HOSTS.find((h) => h.id === id);
|
|
1400
|
+
const userId = interaction.member?.user?.id ?? interaction.user?.id ?? "";
|
|
1401
|
+
this.refreshConfig();
|
|
1402
|
+
if (!host) {
|
|
1403
|
+
await this.rest.ephemeralReply(interaction.id, interaction.token, "This button is no longer active.");
|
|
1404
|
+
return;
|
|
1405
|
+
}
|
|
1406
|
+
if (!isOwner(this.config, userId)) {
|
|
1407
|
+
await this.rest.ephemeralReply(interaction.id, interaction.token, "Only owners reboot Proxmox hosts.");
|
|
1408
|
+
return;
|
|
1409
|
+
}
|
|
1410
|
+
if (this.rebooting.has(host.id)) {
|
|
1411
|
+
await this.rest.ephemeralReply(interaction.id, interaction.token, `${host.name} is already rebooting.`);
|
|
1412
|
+
return;
|
|
1413
|
+
}
|
|
1414
|
+
const status = await this.runEntry("proxmox", userId, [...host.prefix, "host-status"], 3 * 60_000).catch(() => null);
|
|
1415
|
+
if (status !== null && /^reboot-needed:\s*no\s*$/m.test(status)) {
|
|
1416
|
+
await this.rest.ephemeralReply(interaction.id, interaction.token, `${host.name} no longer needs a reboot.`);
|
|
1417
|
+
return;
|
|
1418
|
+
}
|
|
1419
|
+
await this.rest.deferUpdate(interaction.id, interaction.token);
|
|
1420
|
+
const channelId = interaction.channel_id ?? this.config.adminChannelId;
|
|
1421
|
+
const messageId = interaction.message?.id;
|
|
1422
|
+
this.rebooting.add(host.id);
|
|
1423
|
+
try {
|
|
1424
|
+
log(`reboot ${host.name} by ${userId}`);
|
|
1425
|
+
await this.runEntry("proxmox", userId, [...host.prefix, "host-reboot", "--confirm"], 2 * 60_000);
|
|
1426
|
+
const note = `♻️ ${host.name} reboots in one minute, started by <@${userId}>; its guests restart with it.`;
|
|
1427
|
+
if (messageId) await this.rest.editMessage(channelId, messageId, `${interaction.message?.content ?? ""}\n${note}`.slice(0, 1900), []).catch(() => undefined);
|
|
1428
|
+
// The host goes down after one minute; ask again from minute four until minute fourteen.
|
|
1429
|
+
await new Promise((r) => setTimeout(r, 4 * 60_000));
|
|
1430
|
+
let back: string | null = null;
|
|
1431
|
+
for (let i = 0; i < 20 && back === null && !this.stopping; i++) {
|
|
1432
|
+
back = await this.runEntry("proxmox", AUTO_CALLER, [...host.prefix, "host-status"], 60_000).catch(() => null);
|
|
1433
|
+
if (back !== null && !/up\s+(\d+ min|0:0?\d)/.test(back)) back = null;
|
|
1434
|
+
if (back === null) await new Promise((r) => setTimeout(r, 30_000));
|
|
1435
|
+
}
|
|
1436
|
+
const kernel = back ? /running kernel: ([^)]+)\)/.exec(back)?.[1] : undefined;
|
|
1437
|
+
const reboot = back ? /^reboot-needed:\s*(.*)$/m.exec(back)?.[1] : undefined;
|
|
1438
|
+
await this.rest.createMessage(channelId, {
|
|
1439
|
+
content: back
|
|
1440
|
+
? `✅ ${host.name} is back${kernel ? ` on kernel ${kernel}` : ""}${reboot && reboot !== "no" ? `; it still reports reboot-needed: ${reboot}` : ""}.`
|
|
1441
|
+
: `❌ ${host.name} did not answer within ten minutes of the reboot. <@${userId}>`,
|
|
1442
|
+
...(back ? {} : { mentionUsers: [userId] }),
|
|
1443
|
+
});
|
|
1444
|
+
} catch (err) {
|
|
1445
|
+
await this.rest.createMessage(channelId, { content: `❌ The reboot of ${host.name} did not start: ${(err instanceof Error ? err.message : String(err)).slice(0, 300)}` }).catch(() => undefined);
|
|
1446
|
+
} finally {
|
|
1447
|
+
this.rebooting.delete(host.id);
|
|
1448
|
+
}
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1353
1451
|
/** Runs one specialist wrapper through the sudo entry point, without a model; rejects with the wrapper's stderr. */
|
|
1354
1452
|
private runEntry(service: string, caller: string, args: string[], timeoutMs: number): Promise<string> {
|
|
1355
1453
|
const gate = loadSpecialists().config;
|
|
@@ -1754,6 +1852,10 @@ class Bridge {
|
|
|
1754
1852
|
await this.onArcaneButton(interaction, customId);
|
|
1755
1853
|
return;
|
|
1756
1854
|
}
|
|
1855
|
+
if (customId.startsWith("proxmox:reboot:")) {
|
|
1856
|
+
await this.onRebootButton(interaction, customId.slice("proxmox:reboot:".length));
|
|
1857
|
+
return;
|
|
1858
|
+
}
|
|
1757
1859
|
if (customId.startsWith("security:")) {
|
|
1758
1860
|
this.refreshConfig();
|
|
1759
1861
|
const userId = interaction.member?.user?.id ?? interaction.user?.id ?? "";
|
|
@@ -48,6 +48,9 @@ describe("approvals", () => {
|
|
|
48
48
|
expect(isRisky("proxmox", ["snapshot-create", "107", "before", "-h"])).toBe(true);
|
|
49
49
|
expect(isRisky("dns", ["--help"])).toBe(false);
|
|
50
50
|
expect(isRisky("proxmox", ["guests"])).toBe(false);
|
|
51
|
+
expect(isRisky("proxmox", ["space-x", "host-upgrade"])).toBe(true);
|
|
52
|
+
expect(isRisky("proxmox", ["host-reboot", "--confirm"])).toBe(true);
|
|
53
|
+
expect(isRisky("proxmox", ["space-x", "host-status"])).toBe(false);
|
|
51
54
|
expect(isRisky("dns", ["technitium", "host", "technitium1", "upgrade"])).toBe(true);
|
|
52
55
|
expect(isRisky("dns", ["technitium", "host", "technitium3", "restart"])).toBe(true);
|
|
53
56
|
expect(isRisky("dns", ["technitium", "host", "technitium2", "status"])).toBe(false);
|
|
@@ -43,7 +43,7 @@ export const RISKY: Readonly<Record<string, readonly RegExp[]>> = {
|
|
|
43
43
|
],
|
|
44
44
|
edge: [/^vps\s+deploy-nginx\b/],
|
|
45
45
|
backup: [/^(vzdump-run|host-backup-run|arcane-backup-run|gcs-run|restic-check-run|restic-restore|restore-clean)\b/],
|
|
46
|
-
proxmox: [/^(space-x\s+)?(set|snapshot-create|start)\b/],
|
|
46
|
+
proxmox: [/^(space-x\s+)?(set|snapshot-create|start|host-upgrade|host-reboot)\b/],
|
|
47
47
|
inference: [/^(nexus-restart|nexus-start)\b/],
|
|
48
48
|
// media has no entry: trendarr cannot delete anything, and adds and queue toggles run on the caller's request.
|
|
49
49
|
};
|
package/package.json
CHANGED
package/skills/dns/SKILL.md
CHANGED
|
@@ -66,7 +66,7 @@ Fixed operations on each node's host through one forced-command SSH key (`spc-dn
|
|
|
66
66
|
| `restart` | restart Technitium and print its state |
|
|
67
67
|
| `upgrade` | run the official installer (`download.technitium.com/dns/install.sh`), which upgrades Technitium and its ASP.NET runtime in place; up to 10 minutes |
|
|
68
68
|
|
|
69
|
-
Upgrade or restart one node at a time and check `cluster-state` and `resolve` on it before the next; the other two nodes keep answering. The wrapper holds one lock for `restart` and `upgrade` across all nodes and refuses a second one while it runs. Upgrade the secondaries first and technitium1 last, and keep all three on the same version.
|
|
69
|
+
Upgrade or restart one node at a time and check `cluster-state` and `resolve` on it before the next; the other two nodes keep answering. The morning run does this on its own for a minor release (`technitium primary check-update`): technitium3, then technitium2, then the primary, one node per morning; a new major waits for the owner. The wrapper holds one lock for `restart` and `upgrade` across all nodes and refuses a second one while it runs. Upgrade the secondaries first and technitium1 last, and keep all three on the same version.
|
|
70
70
|
|
|
71
71
|
```json
|
|
72
72
|
{ "args": ["technitium", "host", "technitium3", "status"] }
|
|
@@ -17,6 +17,7 @@ Two hosts serve AI workloads. Nexus (Mac Studio, `10.0.40.38`, user `nexus`) run
|
|
|
17
17
|
| `nexus-stop <embed|rerank> --confirm` | unload the service |
|
|
18
18
|
| `nexus-log <embed|rerank> <lines>` | tail of `~/llama-<service>.log` |
|
|
19
19
|
| `nexus-disk` | free space |
|
|
20
|
+
| `macos-updates` | installed macOS version and Software Update's offers (Nexus installs minor updates on its own; the morning run flags it two minors behind) |
|
|
20
21
|
|
|
21
22
|
A model load after restart takes up to a minute; `health=503` during that time is loading, not failure.
|
|
22
23
|
|
package/skills/proxmox/SKILL.md
CHANGED
|
@@ -41,6 +41,8 @@ Power actions and snapshot changes return `{ upid, exitstatus }`; `OK` means don
|
|
|
41
41
|
| `host-status` | `pveversion`, uptime, memory, root and `/var/lib/vz` usage, CPU and NVMe temperatures |
|
|
42
42
|
| `host-journal <since>` | warnings and errors from the host journal |
|
|
43
43
|
| `updates` | upgradable host packages |
|
|
44
|
+
| `host-upgrade` | apt full-upgrade of the host (risky, needs approval); a new Proxmox VE major is held and printed as `held:`; ends with `reboot-needed:`. The morning run does this on its own each day |
|
|
45
|
+
| `host-reboot --confirm` | reboot the host in one minute (risky): every guest on it restarts. Only on the owner's explicit request; the morning run offers a Reboot button instead |
|
|
44
46
|
| `guest-exec <vmid> <status|journal|df|updates>` | fixed read inside a guest: failed units and uptime, warning journal of 24 h, disk usage, upgradable packages |
|
|
45
47
|
|
|
46
48
|
## Guest change procedure
|
|
@@ -55,7 +57,7 @@ Memory and CPU changes on a running guest apply at the next start unless hotplug
|
|
|
55
57
|
|
|
56
58
|
## Not available
|
|
57
59
|
|
|
58
|
-
Creating, cloning, or destroying guests, disk resize or removal, storage changes,
|
|
60
|
+
Creating, cloning, or destroying guests, disk resize or removal, storage changes, backups (backup specialist), and anything inside a guest beyond the fixed reads. Describe the exact operator command (`qm`, `pct`, `pvesh`) and stop.
|
|
59
61
|
|
|
60
62
|
## When to stop and report
|
|
61
63
|
|
package/specialists/dns/run.ts
CHANGED
|
@@ -106,6 +106,7 @@ export const TECHNITIUM_READS: Readonly<Record<string, { args: [number, number];
|
|
|
106
106
|
stats: { args: [0, 1], help: `stats [${STATS_TYPES.join("|")}] dashboard counters` },
|
|
107
107
|
"cluster-state": { args: [0, 0], help: "cluster-state cluster nodes, their state, and this server's addresses" },
|
|
108
108
|
settings: { args: [0, 0], help: "settings all server settings (secrets redacted)" },
|
|
109
|
+
"check-update": { args: [0, 0], help: "check-update (primary) whether a newer Technitium version exists" },
|
|
109
110
|
};
|
|
110
111
|
|
|
111
112
|
/** Technitium writes; zone and cluster writes on the primary only, the other nodes follow through the catalog and the cluster. */
|
|
@@ -224,6 +225,8 @@ export function technitiumRequest(catalog: string, args: string[]): TechnitiumRe
|
|
|
224
225
|
return { path: "/api/admin/cluster/state", params: { includeServerIpAddresses: "true" }, primaryOnly: false };
|
|
225
226
|
case "settings":
|
|
226
227
|
return { path: "/api/settings/get", params: {}, primaryOnly: false };
|
|
228
|
+
case "check-update":
|
|
229
|
+
return { path: "/api/user/checkForUpdate", params: {}, primaryOnly: true };
|
|
227
230
|
case "cluster-init": {
|
|
228
231
|
if (!IP_LIST.test(rest[1])) throw new UsageError("cluster-init needs a comma separated list of IP addresses");
|
|
229
232
|
return {
|
|
@@ -35,5 +35,6 @@ case "$name" in
|
|
|
35
35
|
printf '%s' "$lines" | grep -Eq '^[1-9][0-9]{0,3}$' || { echo "bad lines" >&2; exit 2; }
|
|
36
36
|
tail -n "$lines" "$HOME/llama-$svc.log" ;;
|
|
37
37
|
nexus-disk) df -h / ;;
|
|
38
|
+
macos-updates) [ -z "$rest" ] || { echo "macos-updates takes no arguments" >&2; exit 2; }; sw_vers -productVersion; softwareupdate -l 2>&1 | grep -E 'Title:' || true ;;
|
|
38
39
|
*) echo "refused: $cmd" >&2; exit 2 ;;
|
|
39
40
|
esac
|
|
@@ -12,6 +12,19 @@ rest=""
|
|
|
12
12
|
[ "$cmd" != "$name" ] && rest="${cmd#* }"
|
|
13
13
|
since_ok() { printf '%s' "$1" | grep -Eq '^[A-Za-z0-9 :+-]{1,40}$'; }
|
|
14
14
|
|
|
15
|
+
# A newer Proxmox kernel than the running one, or a package's reboot flag, means the host needs a reboot.
|
|
16
|
+
reboot_needed() {
|
|
17
|
+
running="$(uname -r)"
|
|
18
|
+
newest="$(ls /boot/vmlinuz-*-pve 2>/dev/null | sed 's#^/boot/vmlinuz-##' | sort -V | tail -n 1)"
|
|
19
|
+
if [ -n "$newest" ] && [ "$newest" != "$running" ]; then
|
|
20
|
+
echo "reboot-needed: kernel $running -> $newest"
|
|
21
|
+
elif [ -f /run/reboot-required ]; then
|
|
22
|
+
echo "reboot-needed: $(tr '\n' ' ' < /run/reboot-required.pkgs 2>/dev/null)"
|
|
23
|
+
else
|
|
24
|
+
echo "reboot-needed: no"
|
|
25
|
+
fi
|
|
26
|
+
}
|
|
27
|
+
|
|
15
28
|
guest_cmd() {
|
|
16
29
|
case "$1" in
|
|
17
30
|
status) echo 'systemctl --failed --no-pager; uptime' ;;
|
|
@@ -28,9 +41,27 @@ case "$name" in
|
|
|
28
41
|
uptime
|
|
29
42
|
free -h
|
|
30
43
|
df -h / /var/lib/vz
|
|
31
|
-
sensors 2>/dev/null | grep -E 'Tctl|Composite|edge' || true
|
|
44
|
+
sensors 2>/dev/null | grep -E 'Tctl|Composite|edge' || true
|
|
45
|
+
reboot_needed ;;
|
|
32
46
|
host-journal) since_ok "$rest" || { echo "bad since" >&2; exit 2; }; journalctl -p warning --since "$rest" --no-pager -n 200 ;;
|
|
33
47
|
updates) apt list --upgradable 2>/dev/null ;;
|
|
48
|
+
host-upgrade)
|
|
49
|
+
[ -z "$rest" ] || { echo "host-upgrade takes no arguments" >&2; exit 2; }
|
|
50
|
+
export DEBIAN_FRONTEND=noninteractive
|
|
51
|
+
apt-get update -q >/dev/null
|
|
52
|
+
# A new Proxmox VE major waits for the owner.
|
|
53
|
+
now="$(dpkg-query -W -f='${Version}' pve-manager)"
|
|
54
|
+
next="$(apt-get -s full-upgrade | sed -n 's/^Inst pve-manager \[[^]]*\] (\([^ ]*\) .*/\1/p')"
|
|
55
|
+
if [ -n "$next" ] && [ "${next%%.*}" != "${now%%.*}" ]; then
|
|
56
|
+
echo "held: Proxmox VE $now -> $next is a major upgrade"
|
|
57
|
+
exit 0
|
|
58
|
+
fi
|
|
59
|
+
apt-get -y -q -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold full-upgrade 2>&1 | grep -E '^[0-9]+ upgraded|^E:' || true
|
|
60
|
+
reboot_needed ;;
|
|
61
|
+
host-reboot)
|
|
62
|
+
[ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
|
|
63
|
+
shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
|
|
64
|
+
echo "reboot in 1 minute" ;;
|
|
34
65
|
guest-exec)
|
|
35
66
|
vmid="${rest%% *}"; action="${rest#* }"
|
|
36
67
|
case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
|
|
@@ -12,6 +12,19 @@ rest=""
|
|
|
12
12
|
[ "$cmd" != "$name" ] && rest="${cmd#* }"
|
|
13
13
|
since_ok() { printf '%s' "$1" | grep -Eq '^[A-Za-z0-9 :+-]{1,40}$'; }
|
|
14
14
|
|
|
15
|
+
# A newer Proxmox kernel than the running one, or a package's reboot flag, means the host needs a reboot.
|
|
16
|
+
reboot_needed() {
|
|
17
|
+
running="$(uname -r)"
|
|
18
|
+
newest="$(ls /boot/vmlinuz-*-pve 2>/dev/null | sed 's#^/boot/vmlinuz-##' | sort -V | tail -n 1)"
|
|
19
|
+
if [ -n "$newest" ] && [ "$newest" != "$running" ]; then
|
|
20
|
+
echo "reboot-needed: kernel $running -> $newest"
|
|
21
|
+
elif [ -f /run/reboot-required ]; then
|
|
22
|
+
echo "reboot-needed: $(tr '\n' ' ' < /run/reboot-required.pkgs 2>/dev/null)"
|
|
23
|
+
else
|
|
24
|
+
echo "reboot-needed: no"
|
|
25
|
+
fi
|
|
26
|
+
}
|
|
27
|
+
|
|
15
28
|
guest_cmd() {
|
|
16
29
|
case "$1" in
|
|
17
30
|
status) echo 'systemctl --failed --no-pager; uptime' ;;
|
|
@@ -28,9 +41,27 @@ case "$name" in
|
|
|
28
41
|
uptime
|
|
29
42
|
free -h
|
|
30
43
|
df -h / /var/lib/vz
|
|
31
|
-
sensors 2>/dev/null | grep -E 'Tctl|Composite|edge' || true
|
|
44
|
+
sensors 2>/dev/null | grep -E 'Tctl|Composite|edge' || true
|
|
45
|
+
reboot_needed ;;
|
|
32
46
|
host-journal) since_ok "$rest" || { echo "bad since" >&2; exit 2; }; journalctl -p warning --since "$rest" --no-pager -n 200 ;;
|
|
33
47
|
updates) apt list --upgradable 2>/dev/null ;;
|
|
48
|
+
host-upgrade)
|
|
49
|
+
[ -z "$rest" ] || { echo "host-upgrade takes no arguments" >&2; exit 2; }
|
|
50
|
+
export DEBIAN_FRONTEND=noninteractive
|
|
51
|
+
apt-get update -q >/dev/null
|
|
52
|
+
# A new Proxmox VE major waits for the owner.
|
|
53
|
+
now="$(dpkg-query -W -f='${Version}' pve-manager)"
|
|
54
|
+
next="$(apt-get -s full-upgrade | sed -n 's/^Inst pve-manager \[[^]]*\] (\([^ ]*\) .*/\1/p')"
|
|
55
|
+
if [ -n "$next" ] && [ "${next%%.*}" != "${now%%.*}" ]; then
|
|
56
|
+
echo "held: Proxmox VE $now -> $next is a major upgrade"
|
|
57
|
+
exit 0
|
|
58
|
+
fi
|
|
59
|
+
apt-get -y -q -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold full-upgrade 2>&1 | grep -E '^[0-9]+ upgraded|^E:' || true
|
|
60
|
+
reboot_needed ;;
|
|
61
|
+
host-reboot)
|
|
62
|
+
[ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
|
|
63
|
+
shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
|
|
64
|
+
echo "reboot in 1 minute" ;;
|
|
34
65
|
guest-exec)
|
|
35
66
|
vmid="${rest%% *}"; action="${rest#* }"
|
|
36
67
|
case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
|
|
@@ -39,6 +39,7 @@ export const INFERENCE_COMMANDS: Readonly<Record<string, RemoteSpec>> = {
|
|
|
39
39
|
"nexus-stop": { args: 1, help: "nexus-stop <embed|rerank> --confirm launchctl bootout", timeoutMs: 2 * MINUTE, confirm: true },
|
|
40
40
|
"nexus-log": { args: 2, help: "nexus-log <embed|rerank> <lines>", timeoutMs: MINUTE },
|
|
41
41
|
"nexus-disk": { args: 0, help: "free space on the Nexus disk", timeoutMs: MINUTE },
|
|
42
|
+
"macos-updates": { args: 0, help: "installed macOS version and the updates Software Update offers", timeoutMs: 3 * MINUTE },
|
|
42
43
|
};
|
|
43
44
|
|
|
44
45
|
const NEXUS_SERVICES = ["embed", "rerank"];
|
|
@@ -85,9 +85,13 @@ export const HOST_COMMANDS: Readonly<Record<string, { args: number; help: string
|
|
|
85
85
|
"host-status": { args: 0, help: "pveversion, uptime, load, memory, root disk, temperatures" },
|
|
86
86
|
"host-journal": { args: 1, help: "host-journal <since> warnings and errors of the host journal" },
|
|
87
87
|
updates: { args: 0, help: "upgradable packages on the host" },
|
|
88
|
+
"host-upgrade": { args: 0, help: "apt full-upgrade of the host; a new Proxmox VE major is held (prints held:); ends with reboot-needed" },
|
|
89
|
+
"host-reboot": { args: 1, help: "host-reboot --confirm reboot the host in one minute; every guest on it restarts" },
|
|
88
90
|
"guest-exec": { args: 2, help: "guest-exec <vmid> <status|journal|df|updates> fixed read inside a guest (pct exec or qm guest exec)" },
|
|
89
91
|
};
|
|
90
92
|
export const GUEST_EXEC = ["status", "journal", "df", "updates"];
|
|
93
|
+
/** A full upgrade with a new kernel and pve-manager can take several minutes on the host. */
|
|
94
|
+
export const HOST_UPGRADE_TIMEOUT_MS = 20 * 60_000;
|
|
91
95
|
const SINCE = /^[A-Za-z0-9 :\-+]{1,40}$/;
|
|
92
96
|
|
|
93
97
|
const HELP = `proxmox specialist (Proxmox VE on Pulsar at 37 and SPACE-X at Mira)
|
|
@@ -328,6 +332,7 @@ export function buildHostRemote(args: string[]): string {
|
|
|
328
332
|
if (!spec) throw new UsageError(`unknown host command: ${name ?? "(none)"}`);
|
|
329
333
|
if (rest.length !== spec.args) throw new UsageError(`${name} takes ${spec.args} argument(s)`);
|
|
330
334
|
if (name === "host-journal" && !SINCE.test(rest[0])) throw new UsageError("since: letters, digits, spaces, colon, plus, minus only");
|
|
335
|
+
if (name === "host-reboot" && rest[0] !== "--confirm") throw new UsageError("host-reboot restarts every guest on the host; add --confirm when the task asks for it explicitly");
|
|
331
336
|
if (name === "guest-exec") {
|
|
332
337
|
parseVmid(rest[0]);
|
|
333
338
|
if (!GUEST_EXEC.includes(rest[1])) throw new UsageError(`guest-exec action must be one of ${GUEST_EXEC.join(", ")}`);
|
|
@@ -345,7 +350,7 @@ async function host(config: Target, args: string[]): Promise<number> {
|
|
|
345
350
|
knownHostsFile: join(specialistsHome(), "config", "known_hosts"),
|
|
346
351
|
},
|
|
347
352
|
remote,
|
|
348
|
-
120_000,
|
|
353
|
+
args[0] === "host-upgrade" ? HOST_UPGRADE_TIMEOUT_MS : 120_000,
|
|
349
354
|
);
|
|
350
355
|
if (result.stdout.length > 0) printRaw(result.stdout);
|
|
351
356
|
if (result.code !== 0) throw new ServiceError(result.stderr.trim() || `${args[0]} exited ${result.code}`);
|
|
@@ -216,6 +216,7 @@ describe("network wrapper", () => {
|
|
|
216
216
|
describe("dns wrapper", () => {
|
|
217
217
|
it("maps technitium commands to documented API paths", () => {
|
|
218
218
|
expect(technitiumRequest("catalog.invalid", ["zones"])).toEqual({ path: "/api/zones/list", params: {}, primaryOnly: false });
|
|
219
|
+
expect(technitiumRequest("catalog.invalid", ["check-update"])).toEqual({ path: "/api/user/checkForUpdate", params: {}, primaryOnly: true });
|
|
219
220
|
expect(technitiumRequest("catalog.invalid", ["zones", "*.37pla.net"])).toEqual({ path: "/api/zones/list", params: { filterName: "*.37pla.net" }, primaryOnly: false });
|
|
220
221
|
expect(technitiumRequest("catalog.invalid", ["records", "37pla.net"])).toEqual({
|
|
221
222
|
path: "/api/zones/records/get",
|
|
@@ -653,6 +654,10 @@ describe("proxmox wrapper", () => {
|
|
|
653
654
|
expect(buildHostRemote(["guest-exec", "101", "df"])).toBe("guest-exec 101 df");
|
|
654
655
|
expect(() => buildHostRemote(["guest-exec", "101", "rm"])).toThrow(/action/);
|
|
655
656
|
expect(() => buildHostRemote(["host-journal", "1 hour ago; id"])).toThrow(/since/);
|
|
657
|
+
expect(buildHostRemote(["host-upgrade"])).toBe("host-upgrade");
|
|
658
|
+
expect(() => buildHostRemote(["host-upgrade", "now"])).toThrow(UsageError);
|
|
659
|
+
expect(buildHostRemote(["host-reboot", "--confirm"])).toBe("host-reboot --confirm");
|
|
660
|
+
expect(() => buildHostRemote(["host-reboot", "now"])).toThrow(/--confirm/);
|
|
656
661
|
expect(() => buildHostRemote(["shell"])).toThrow(/unknown host command/);
|
|
657
662
|
});
|
|
658
663
|
|