@astrofoundry/pi-astro 0.40.34 → 0.40.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  import { describe, expect, it } from "vitest";
2
2
  import type { ArcaneReport } from "./daily-report.ts";
3
3
  import type { ToolsReport } from "./tools-report.ts";
4
- import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, toolsSummary, updatesStatus } from "./daily-run.ts";
4
+ import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
5
5
 
6
6
  const EDGE = `nginx active
7
7
  wg-quick@wg0 active
@@ -62,15 +62,20 @@ describe("morning status lines", () => {
62
62
  expect(edgeStatus(null, null)).toMatchObject({ line: "the check failed (vps status)", problem: "Edge: the status check failed" });
63
63
  });
64
64
 
65
- it("reports both Proxmox hosts and their waiting host packages without flagging them", () => {
66
- const held = `${PULSAR_UPDATES}\nwazuh-agent/stable 4.14.9-1 amd64 [upgradable from: 4.14.7-1]`;
67
- expect(proxmoxStatus([{ name: "Pulsar", status: PULSAR, updates: held }])).toEqual({ area: "Proxmox", line: "Pulsar up 59 days, RAM 47Gi of 60Gi used, root disk 13%, 2 host package updates waiting (1 security), applied by hand" });
68
- const both = proxmoxStatus([
69
- { name: "Pulsar", status: PULSAR, updates: PULSAR_UPDATES },
70
- { name: "SPACE-X", status: null, updates: null },
65
+ it("reports both Proxmox hosts, the morning upgrade, a held major, and a pending reboot with its button", () => {
66
+ const held = "Listing...\nwazuh-agent/stable 4.14.9-1 amd64 [upgradable from: 4.14.7-1]";
67
+ expect(proxmoxStatus([{ name: "Pulsar", id: "pulsar", status: `${PULSAR}\nreboot-needed: no`, updates: held }])).toEqual({ area: "Proxmox", line: "Pulsar up 59 days, RAM 47Gi of 60Gi used, root disk 13%, up to date" });
68
+ const upgraded = proxmoxStatus([
69
+ { name: "Pulsar", id: "pulsar", status: `${PULSAR}\nreboot-needed: kernel 7.0.14-25-pve -> 7.0.14-26-pve`, updates: held, upgrade: "3 upgraded, 1 newly installed, 0 to remove and 0 not upgraded.\nreboot-needed: kernel 7.0.14-25-pve -> 7.0.14-26-pve" },
70
+ { name: "SPACE-X", id: "space-x", status: null, updates: null },
71
71
  ]);
72
- expect(both.line).toBe("Pulsar up 59 days, RAM 47Gi of 60Gi used, root disk 13%, 2 host package updates waiting (1 security), applied by hand; SPACE-X status unknown");
73
- expect(both).toMatchObject({ problem: "Proxmox: the status check failed for SPACE-X", route: "admin" });
72
+ expect(upgraded.line).toBe("Pulsar up 59 days, RAM 47Gi of 60Gi used, root disk 13%, 3 host updates installed, reboot needed; SPACE-X status unknown");
73
+ expect(upgraded.problem).toBe("Proxmox: Pulsar needs a reboot (kernel 7.0.14-25-pve -> 7.0.14-26-pve); every guest on it restarts for a few minutes; the status check failed for SPACE-X");
74
+ expect(upgraded.buttons).toEqual([{ label: "Reboot Pulsar", customId: "proxmox:reboot:pulsar" }]);
75
+ const major = proxmoxStatus([{ name: "SPACE-X", id: "space-x", status: `${PULSAR}\nreboot-needed: no`, updates: PULSAR_UPDATES, upgrade: "held: Proxmox VE 9.2.24 -> 10.0.1 is a major upgrade" }]);
76
+ expect(major).toMatchObject({ problem: "Proxmox: SPACE-X waits for you: Proxmox VE 9.2.24 -> 10.0.1 is a major upgrade", route: "admin" });
77
+ expect(major.line).toContain("a Proxmox VE major upgrade waits for you");
78
+ expect(proxmoxStatus([{ name: "Pulsar", id: "pulsar", status: PULSAR, updates: PULSAR_UPDATES, upgrade: null }]).problem).toBe("Proxmox: the host upgrade on Pulsar failed");
74
79
  });
75
80
 
76
81
  it("flags updates the nightly run left behind, reading guest-agent JSON too, and ignores the held Wazuh agent", () => {
@@ -104,6 +109,33 @@ describe("morning status lines", () => {
104
109
  expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: { status: "ok" }, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "technitium1, technitium2, technitium3 answer" });
105
110
  expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: null, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "no answer from technitium2", problem: "DNS: technitium2 does not answer", route: "admin" });
106
111
  expect(dnsStatus({ technitium1: null, technitium2: { status: "error" }, technitium3: { status: "ok" } })).toMatchObject({ problem: "DNS: technitium1 and technitium2 do not answer" });
112
+ });
113
+
114
+ it("flags a Mac two macOS minors behind and notes a major without flagging it", () => {
115
+ const nexus = "26.5.2\nTitle: Command Line Tools for Xcode 26.6, Version: 26.6, Size: 920431KiB, Recommended: YES,\nTitle: macOS Tahoe 26.7.2, Version: 26.7.2, Size: 3868257KiB, Recommended: YES, Action: restart,\nTitle: macOS 27.2, Version: 27.2, Size: 11319983KiB, Recommended: YES, Action: restart,";
116
+ const cortex = "26.6.1\nTitle: macOS Tahoe 26.7.2, Version: 26.7.2, Size: 3168375KiB, Recommended: YES, Action: restart,";
117
+ expect(macosStatus([{ name: "Cortex", output: cortex }])).toEqual({ area: "macOS", line: "Cortex 26.6.1, 26.7.2 waiting" });
118
+ expect(macosStatus([{ name: "Nexus", output: nexus }, { name: "Cortex", output: null }])).toEqual({
119
+ area: "macOS",
120
+ line: "Nexus 26.5.2, 26.7.2 waiting, macOS 27.2 offered (major, by hand); Cortex unknown",
121
+ problem: "macOS: Nexus runs macOS 26.5.2 while 26.7.2 is out; automatic installs are not keeping up; no macOS update list from Cortex",
122
+ route: "admin",
123
+ });
124
+ expect(macosStatus([{ name: "Cortex", output: "26.7.2" }]).line).toBe("Cortex 26.7.2");
125
+ });
126
+
127
+ it("upgrades one Technitium node per morning, Mira first and the primary last, and holds a major", () => {
128
+ const update = (to: string) => ({ status: "ok", response: { updateAvailable: true, updateVersion: to, currentVersion: "15.6.2" } });
129
+ const state = (versions: [string, string, string]) => ({ status: "ok", response: { clusterNodes: versions.map((v, i) => ({ name: `technitium${i + 1}.dns.home.arpa`, version: v })) } });
130
+ expect(technitiumPlan({ status: "ok", response: { updateAvailable: false } }, state(["15.6.2", "15.6.2", "15.6.2"]))).toBeNull();
131
+ expect(technitiumPlan(update("15.7.0"), state(["15.6.2", "15.6.2", "15.6.2"]))).toEqual({ host: "technitium3", from: "15.6.2", to: "15.7.0" });
132
+ expect(technitiumPlan(update("15.7.0"), state(["15.6.2", "15.6.2", "15.7.0"]))).toEqual({ host: "technitium2", from: "15.6.2", to: "15.7.0" });
133
+ expect(technitiumPlan(update("15.7.0"), state(["15.6.2", "15.7.0", "15.7.0"]))).toEqual({ host: "technitium1", from: "15.6.2", to: "15.7.0" });
134
+ expect(technitiumPlan(update("16.0.0"), state(["15.6.2", "15.6.2", "15.6.2"]))).toEqual({ major: true, from: "15.6.2", to: "16.0.0" });
135
+ const all = { technitium1: { status: "ok" }, technitium2: { status: "ok" }, technitium3: { status: "ok" } };
136
+ expect(dnsStatus(all, { host: "technitium3", from: "15.6.2", to: "15.7.0", ok: true })).toEqual({ area: "DNS", line: "technitium1, technitium2, technitium3 answer; technitium3 upgraded 15.6.2 -> 15.7.0 (one node per morning, primary last)" });
137
+ expect(dnsStatus(all, { host: "technitium3", from: "15.6.2", to: "15.7.0", ok: false, why: "installer exited 1" }).problem).toBe("DNS: the upgrade of technitium3 to 15.7.0 failed (installer exited 1)");
138
+ expect(dnsStatus(all, { major: true, from: "15.6.2", to: "16.0.0" }).problem).toMatch(/^DNS: Technitium 16.0.0 is a major release \(running 15.6.2\)/);
107
139
  expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=200\nrerank \tstate = running \tpid = 81670 health=200")).toEqual({ area: "Inference", line: "Nexus embed, rerank running and healthy" });
108
140
  expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=503")).toMatchObject({ problem: "Inference: embed not healthy on Nexus" });
109
141
  expect(mediaStatus({ sabnzbd_paused: true, warnings: [] })).toEqual({ area: "Media", line: "downloads paused" });
@@ -6,12 +6,19 @@ import type { ToolsReport } from "./tools-report.ts";
6
6
  /** Where an area's problems tag the owners: specialists without a channel of their own use the admin channel. */
7
7
  export type Route = "admin" | "media";
8
8
 
9
+ /** An owner-only button posted with an area's problem. */
10
+ export interface AreaButton {
11
+ label: string;
12
+ customId: string;
13
+ }
14
+
9
15
  /** One area of the morning summary: a line, and a problem that needs an owner, if any. */
10
16
  export interface AreaStatus {
11
17
  area: string;
12
18
  line: string;
13
19
  problem?: string;
14
20
  route?: Route;
21
+ buttons?: AreaButton[];
15
22
  }
16
23
 
17
24
  const HOUR = 3600;
@@ -50,35 +57,65 @@ export function edgeStatus(status: string | null, updates: string | null): AreaS
50
57
  return { area: "Edge", line: `front door: ${parts.join(", ")}`, ...(problem ? { problem, route: "admin" as const } : {}) };
51
58
  }
52
59
 
53
- /** One Proxmox host's `host-status` and `updates` output. */
60
+ /** One Proxmox host as the morning run saw it. */
54
61
  export interface ProxmoxHost {
55
62
  name: string;
63
+ /** `pulsar` or `space-x`: the target of the Reboot button. */
64
+ id: string;
56
65
  status: string | null;
66
+ /** `updates` after any upgrade. */
57
67
  updates: string | null;
68
+ /** What `host-upgrade` printed this morning: undefined when nothing was waiting, null when it failed. */
69
+ upgrade?: string | null;
70
+ }
71
+
72
+ /** The custom id of the Reboot button for one Proxmox host. */
73
+ export function rebootButtonId(id: string): string {
74
+ return `proxmox:reboot:${id}`;
75
+ }
76
+
77
+ interface ProxmoxFacts {
78
+ line: string;
79
+ problems: string[];
80
+ button?: AreaButton;
58
81
  }
59
82
 
60
- function proxmoxHostLine(h: ProxmoxHost): string {
61
- if (h.status === null) return `${h.name} status unknown`;
83
+ function proxmoxHostFacts(h: ProxmoxHost): ProxmoxFacts {
84
+ if (h.status === null) return { line: `${h.name} status unknown`, problems: [`the status check failed for ${h.name}`] };
62
85
  const up = /up\s+([^,]+(?:,\s*\d+:\d+)?)/.exec(h.status)?.[1]?.split(",")[0]?.trim();
63
86
  const mem = /^Mem:\s+(\S+)\s+(\S+)/m.exec(h.status);
64
87
  const root = /\s(\d+)%\s+\/\s*$/m.exec(h.status)?.[1];
65
- const lines = h.updates === null ? null : pendingPackageLines(h.updates);
66
- const security = lines?.filter((l) => /-security\s/.test(l)).length ?? 0;
67
- return [
68
- up ? `${h.name} up ${up}` : `${h.name} up`,
69
- mem ? `RAM ${mem[2]} of ${mem[1]} used` : null,
70
- root ? `root disk ${root}%` : null,
71
- lines === null ? "updates unknown" : `${lines.length} host package update${lines.length === 1 ? "" : "s"} waiting${security > 0 ? ` (${security} security)` : ""}, applied by hand`,
72
- ]
73
- .filter(Boolean)
74
- .join(", ");
88
+ const reboot = /^reboot-needed:\s*(.*)$/m.exec(h.status)?.[1]?.trim();
89
+ const waiting = h.updates === null ? null : pendingPackageLines(h.updates).length;
90
+ const held = h.upgrade ? /^held:\s*(.*)$/m.exec(h.upgrade)?.[1] : undefined;
91
+ const installed = h.upgrade ? /^(\d+) upgraded/m.exec(h.upgrade)?.[1] : undefined;
92
+ const updates =
93
+ h.upgrade === null
94
+ ? "the host upgrade failed"
95
+ : held
96
+ ? "a Proxmox VE major upgrade waits for you"
97
+ : [installed ? `${installed} host update${installed === "1" ? "" : "s"} installed` : null, waiting === null ? "updates unknown" : waiting > 0 ? `${waiting} still waiting` : installed ? null : "up to date"].filter(Boolean).join(", ");
98
+ const rebootNeeded = reboot !== undefined && reboot !== "no";
99
+ const line = [up ? `${h.name} up ${up}` : `${h.name} up`, mem ? `RAM ${mem[2]} of ${mem[1]} used` : null, root ? `root disk ${root}%` : null, updates, rebootNeeded ? "reboot needed" : null].filter(Boolean).join(", ");
100
+ const problems = [
101
+ h.upgrade === null ? `the host upgrade on ${h.name} failed` : null,
102
+ held ? `${h.name} waits for you: ${held}` : null,
103
+ rebootNeeded ? `${h.name} needs a reboot (${reboot}); every guest on it restarts for a few minutes` : null,
104
+ ].filter((p): p is string => p !== null);
105
+ return { line, problems, ...(rebootNeeded ? { button: { label: `Reboot ${h.name}`, customId: rebootButtonId(h.id) } } : {}) };
75
106
  }
76
107
 
77
- /** `proxmox [space-x] host-status` and `updates`: each host's uptime, memory, root disk, and pending packages (report only). */
108
+ /** Both Proxmox hosts: uptime, memory, root disk, the morning's automatic upgrade, a held major, and a pending reboot with its button. */
78
109
  export function proxmoxStatus(hosts: readonly ProxmoxHost[]): AreaStatus {
79
- const unknown = hosts.filter((h) => h.status === null).map((h) => h.name);
80
- const line = hosts.map(proxmoxHostLine).join("; ");
81
- return { area: "Proxmox", line, ...(unknown.length > 0 ? { problem: `Proxmox: the status check failed for ${unknown.join(" and ")}`, route: "admin" as const } : {}) };
110
+ const facts = hosts.map(proxmoxHostFacts);
111
+ const problems = facts.flatMap((f) => f.problems);
112
+ const buttons = facts.flatMap((f) => (f.button ? [f.button] : []));
113
+ return {
114
+ area: "Proxmox",
115
+ line: facts.map((f) => f.line).join("; "),
116
+ ...(problems.length > 0 ? { problem: `Proxmox: ${problems.join("; ")}`, route: "admin" as const } : {}),
117
+ ...(buttons.length > 0 ? { buttons } : {}),
118
+ };
82
119
  }
83
120
 
84
121
  /** Packages the nightly run leaves alone on purpose: a Wazuh agent must never be newer than the manager. */
@@ -150,14 +187,43 @@ export function identityStatus(status: string | null): AreaStatus {
150
187
  return { area: "Identity", line, ...(bad.length > 0 || units.length === 0 ? { problem: `Identity: ${bad.length > 0 ? bad.join(", ") : "no unit status read"} on the DMZ VM`, route: "admin" as const } : {}) };
151
188
  }
152
189
 
153
- /** `dns technitium primary|node <fqdn> resolve <name>`: every cluster node answers an internal name, keyed by node name. */
154
- export function dnsStatus(answers: Readonly<Record<string, unknown>>): AreaStatus {
190
+ /** Upgrade order: Mira's node first, the primary last, one node per morning. */
191
+ const TECHNITIUM_ORDER = ["technitium3", "technitium2", "technitium1"];
192
+
193
+ /** What the morning run does about a Technitium release: upgrade one node, or hold a major for the owner. */
194
+ export type TechnitiumPlan = { host: string; from: string; to: string } | { major: true; from: string; to: string };
195
+
196
+ /** `check-update` and `cluster-state` from the primary: the next node to upgrade, a held major, or nothing. */
197
+ export function technitiumPlan(update: unknown, state: unknown): TechnitiumPlan | null {
198
+ const u = (update as { response?: { updateAvailable?: unknown; updateVersion?: unknown; currentVersion?: unknown } } | null)?.response;
199
+ if (u?.updateAvailable !== true || typeof u.updateVersion !== "string" || typeof u.currentVersion !== "string") return null;
200
+ const to = u.updateVersion;
201
+ if (to.split(".")[0] !== u.currentVersion.split(".")[0]) return { major: true, from: u.currentVersion, to };
202
+ const nodes = (state as { response?: { clusterNodes?: { name?: unknown; version?: unknown }[] } } | null)?.response?.clusterNodes ?? [];
203
+ for (const host of TECHNITIUM_ORDER) {
204
+ const node = nodes.find((n) => typeof n.name === "string" && n.name.split(".")[0] === host);
205
+ if (node && typeof node.version === "string" && node.version !== to) return { host, from: node.version, to };
206
+ }
207
+ return null;
208
+ }
209
+
210
+ /** The morning's Technitium upgrade, if one ran or a major waits. */
211
+ export type DnsUpgrade = { host: string; from: string; to: string; ok: true } | { host: string; from: string; to: string; ok: false; why: string } | { major: true; from: string; to: string };
212
+
213
+ /** `dns technitium primary|node <fqdn> resolve <name>`: every cluster node answers an internal name, keyed by node name; plus the morning's upgrade. */
214
+ export function dnsStatus(answers: Readonly<Record<string, unknown>>, upgrade?: DnsUpgrade): AreaStatus {
155
215
  const ok = (r: unknown) => (r as { status?: unknown } | null)?.status === "ok";
156
216
  const nodes = Object.keys(answers);
157
217
  const down = nodes.filter((node) => !ok(answers[node]));
158
218
  const names = down.join(" and ");
159
- if (down.length === 0) return { area: "DNS", line: `${nodes.join(", ")} answer` };
160
- return { area: "DNS", line: `no answer from ${names}`, problem: `DNS: ${names} ${down.length === 1 ? "does" : "do"} not answer`, route: "admin" };
219
+ const upgradeLine = upgrade === undefined ? null : "major" in upgrade ? `Technitium ${upgrade.to} is a major release and waits for you` : upgrade.ok ? `${upgrade.host} upgraded ${upgrade.from} -> ${upgrade.to} (one node per morning, primary last)` : `the upgrade of ${upgrade.host} to ${upgrade.to} failed`;
220
+ const problems = [
221
+ down.length > 0 ? `${names} ${down.length === 1 ? "does" : "do"} not answer` : null,
222
+ upgrade && "major" in upgrade ? `Technitium ${upgrade.to} is a major release (running ${upgrade.from}); read its change log, then upgrade one node at a time with \`dns technitium host <node> upgrade\`` : null,
223
+ upgrade && "ok" in upgrade && !upgrade.ok ? `the upgrade of ${upgrade.host} to ${upgrade.to} failed (${upgrade.why.slice(0, 200)})` : null,
224
+ ].filter(Boolean);
225
+ const line = [down.length === 0 ? `${nodes.join(", ")} answer` : `no answer from ${names}`, upgradeLine].filter(Boolean).join("; ");
226
+ return { area: "DNS", line, ...(problems.length > 0 ? { problem: `DNS: ${problems.join("; ")}`, route: "admin" as const } : {}) };
161
227
  }
162
228
 
163
229
  /** `inference nexus-status`: each Nexus service running with HTTP health 200. */
@@ -196,6 +262,38 @@ export function backupStatus(archives: unknown, resticText: string | null, now:
196
262
  return { area: "Backup", line: `${proxmox}, ${check}`, ...(problems.length > 0 ? { problem: `Backup: ${problems.join("; ")}`, route: "admin" as const } : {}) };
197
263
  }
198
264
 
265
+ /** `sw_vers -productVersion` on its first line, then Software Update's `Title:` lines, for one Mac. */
266
+ export interface MacUpdates {
267
+ name: string;
268
+ output: string | null;
269
+ }
270
+
271
+ /** A Mac this many minor releases behind the newest offered one is not keeping up with its automatic installs. */
272
+ const MACOS_MINORS_BEHIND = 2;
273
+
274
+ /** Each Mac's macOS version and offers: automatic installs cover minors; a major is noted for a manual upgrade. */
275
+ export function macosStatus(macs: readonly MacUpdates[]): AreaStatus {
276
+ const parts: string[] = [];
277
+ const problems: string[] = [];
278
+ for (const m of macs) {
279
+ if (m.output === null) {
280
+ parts.push(`${m.name} unknown`);
281
+ problems.push(`no macOS update list from ${m.name}`);
282
+ continue;
283
+ }
284
+ const [installed = "", ...rest] = m.output.split("\n").map((l) => l.trim());
285
+ const offers = rest.map((l) => /Title: macOS[^,]*, Version: ([\d.]+)/.exec(l)?.[1]).filter((v): v is string => v !== undefined);
286
+ const [major, minor] = installed.split(".").map(Number);
287
+ const sameMajor = offers.filter((v) => Number(v.split(".")[0]) === major);
288
+ const newestMinor = Math.max(minor ?? 0, ...sameMajor.map((v) => Number(v.split(".")[1] ?? 0)));
289
+ const majors = offers.filter((v) => Number(v.split(".")[0]) > major);
290
+ const latest = sameMajor.sort((a, b) => a.localeCompare(b, undefined, { numeric: true })).at(-1);
291
+ parts.push(`${m.name} ${installed}${latest ? `, ${latest} waiting` : ""}${majors.length > 0 ? `, macOS ${majors[0]} offered (major, by hand)` : ""}`);
292
+ if (newestMinor - (minor ?? 0) >= MACOS_MINORS_BEHIND) problems.push(`${m.name} runs macOS ${installed} while ${latest} is out; automatic installs are not keeping up`);
293
+ }
294
+ return { area: "macOS", line: parts.join("; "), ...(problems.length > 0 ? { problem: `macOS: ${problems.join("; ")}`, route: "admin" as const } : {}) };
295
+ }
296
+
199
297
  /** One report the run posted in its own channel, as the summary shows it. */
200
298
  export interface ReportLine {
201
299
  area: string;
@@ -78,7 +78,7 @@ import { ArcaneWatch } from "./arcane-watch-monitor.ts";
78
78
  import { parseSnapshot } from "./arcane-watch.ts";
79
79
  import { type Applied, autoFailureText, foremanTask, stackProblems, stackUpdateAnswer, SUPERVISE_DELAY_MS, SUPERVISE_LIMIT_MS } from "./supervise.ts";
80
80
  import { stackPlan, toolPlan } from "./update-policy.ts";
81
- import { type AreaStatus, arcaneSummary, backupStatus, dnsStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, problemText, proxmoxStatus, type ReportLine, type Route, summaryText, toolsSummary, updatesStatus } from "./daily-run.ts";
81
+ import { type AreaStatus, arcaneSummary, backupStatus, type DnsUpgrade, dnsStatus, macosStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, type ProxmoxHost, proxmoxStatus, type ReportLine, type Route, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
82
82
  import { type InvestigationRequest, SecurityMonitor } from "./security-monitor.ts";
83
83
  import { nasOutage, readSecurityState } from "./security.ts";
84
84
 
@@ -174,6 +174,15 @@ const DNS_NODES = { technitium2: "technitium2.dns.home.arpa", technitium3: "tech
174
174
 
175
175
  /** The audit caller of updates the morning run applies on its own. */
176
176
  const AUTO_CALLER = "daily-run";
177
+ /** The Proxmox hosts the morning run upgrades, with the wrapper prefix that selects each. */
178
+ const PROXMOX_HOSTS: readonly { name: string; id: string; prefix: string[] }[] = [
179
+ { name: "Pulsar", id: "pulsar", prefix: [] },
180
+ { name: "SPACE-X", id: "space-x", prefix: ["space-x"] },
181
+ ];
182
+ /** The dns wrapper allows a node upgrade 10 minutes; the bridge waits a little longer. */
183
+ const DNS_UPGRADE_MS = 11 * 60_000;
184
+ /** The wrapper allows a host upgrade 20 minutes; the bridge waits a little longer for its answer. */
185
+ const HOST_UPGRADE_MS = 21 * 60_000;
177
186
 
178
187
  function pause(ms: number): Promise<void> {
179
188
  return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)).unref());
@@ -230,6 +239,8 @@ class Bridge {
230
239
  private readonly mediaMessages = new Map<string, MessageComponent[]>();
231
240
  /** Tool updates started from the Cortex Tools Report, by line key; they count as open work. */
232
241
  private readonly toolRuns = new Map<string, { channelId: string; label: string }>();
242
+ /** Proxmox hosts that an owner's Reboot click is restarting. */
243
+ private readonly rebooting = new Set<string>();
233
244
  private readonly toolClicks = new KeyedQueue();
234
245
  private gateway: GatewayClient | undefined;
235
246
  private stopping = false;
@@ -588,7 +599,8 @@ class Bridge {
588
599
  const routes = this.routes();
589
600
  const owners = this.config.owners;
590
601
  for (const a of areas.filter((x) => x.problem)) {
591
- await this.rest.createMessage(routes[a.route ?? "admin"], { content: problemText(a, owners), mentionUsers: owners }).catch((err) => log(`daily ${date}: ${a.area} problem message failed: ${err instanceof Error ? err.message : String(err)}`));
602
+ const components = a.buttons ? [{ type: 1, components: a.buttons.map((b) => ({ type: 2, style: 4, label: b.label, custom_id: b.customId })) }] : undefined;
603
+ await this.rest.createMessage(routes[a.route ?? "admin"], { content: problemText(a, owners), mentionUsers: owners, ...(components ? { components } : {}) }).catch((err) => log(`daily ${date}: ${a.area} problem message failed: ${err instanceof Error ? err.message : String(err)}`));
592
604
  }
593
605
  const label = new Intl.DateTimeFormat("en-GB", { timeZone: schedule.timeZone, weekday: "short", day: "numeric", month: "short" }).format(new Date());
594
606
  const text = summaryText({ date: label, guildId: this.config.guildId, reports, areas, routes });
@@ -623,11 +635,17 @@ class Bridge {
623
635
  if (present.has("edge")) areas.push((async () => edgeStatus(await text("edge", ["vps", "status"]), await text("edge", ["vps", "updates"])))());
624
636
  if (present.has("proxmox")) {
625
637
  areas.push(
626
- (async () =>
627
- proxmoxStatus([
628
- { name: "Pulsar", status: await text("proxmox", ["host-status"]), updates: await text("proxmox", ["updates"]) },
629
- { name: "SPACE-X", status: await text("proxmox", ["space-x", "host-status"]), updates: await text("proxmox", ["space-x", "updates"]) },
630
- ]))(),
638
+ (async () => {
639
+ const hosts: ProxmoxHost[] = [];
640
+ for (const h of PROXMOX_HOSTS) {
641
+ const before = await text("proxmox", [...h.prefix, "updates"]);
642
+ // Host packages install on their own; a new Proxmox VE major is held by the entry script for the owner.
643
+ const upgrade = before !== null && pendingPackageLines(before).length > 0 ? await this.runEntry("proxmox", AUTO_CALLER, [...h.prefix, "host-upgrade"], HOST_UPGRADE_MS).catch((err) => (log(`daily: ${h.name} host-upgrade failed: ${err instanceof Error ? err.message : String(err)}`), null)) : undefined;
644
+ const updates = upgrade === undefined ? before : await text("proxmox", [...h.prefix, "updates"]);
645
+ hosts.push({ name: h.name, id: h.id, status: await text("proxmox", [...h.prefix, "host-status"]), updates, upgrade });
646
+ }
647
+ return proxmoxStatus(hosts);
648
+ })(),
631
649
  );
632
650
  areas.push(
633
651
  (async () => {
@@ -655,17 +673,40 @@ class Bridge {
655
673
  if (present.has("dns")) {
656
674
  areas.push(
657
675
  (async () => {
676
+ // A minor Technitium release upgrades one node per morning, before the answer checks below.
677
+ const plan = technitiumPlan(await json("dns", ["technitium", "primary", "check-update"]), await json("dns", ["technitium", "primary", "cluster-state"]));
678
+ let upgrade: DnsUpgrade | undefined;
679
+ if (plan && "major" in plan) upgrade = plan;
680
+ else if (plan) {
681
+ upgrade = await this.runEntry("dns", AUTO_CALLER, ["technitium", "host", plan.host, "upgrade"], DNS_UPGRADE_MS).then(
682
+ () => ({ ...plan, ok: true as const }),
683
+ (err) => ({ ...plan, ok: false as const, why: err instanceof Error ? err.message : String(err) }),
684
+ );
685
+ }
658
686
  const [technitium1, technitium2, technitium3] = await Promise.all([
659
687
  json("dns", ["technitium", "primary", "resolve", DNS_PROBE]),
660
688
  json("dns", ["technitium", "node", DNS_NODES.technitium2, "resolve", DNS_PROBE]),
661
689
  json("dns", ["technitium", "node", DNS_NODES.technitium3, "resolve", DNS_PROBE]),
662
690
  ]);
663
- return dnsStatus({ technitium1, technitium2, technitium3 });
691
+ return dnsStatus({ technitium1, technitium2, technitium3 }, upgrade);
664
692
  })(),
665
693
  );
666
694
  }
667
695
  if (present.has("inference")) areas.push((async () => inferenceStatus(await text("inference", ["nexus-status"])))());
668
696
  if (present.has(MEDIA)) areas.push((async () => mediaStatus(await json(MEDIA, ["sabnzbd", "status"])))());
697
+ areas.push(
698
+ (async () => {
699
+ // Cortex is this host; Nexus answers through the inference specialist.
700
+ const version = await this.execTool("/usr/bin/sw_vers", ["-productVersion"], { timeoutMs: 30_000, env: { PATH: TOOL_PATH } });
701
+ const offers = await this.execTool("/usr/sbin/softwareupdate", ["-l"], { timeoutMs: 3 * 60_000, env: { PATH: TOOL_PATH } });
702
+ const cortex = version.code === 0 ? `${version.stdout.trim()}\n${offers.stdout}${offers.stderr}` : null;
703
+ const nexus = present.has("inference") ? await text("inference", ["macos-updates"]) : null;
704
+ return macosStatus([
705
+ { name: "Cortex", output: cortex },
706
+ ...(present.has("inference") ? [{ name: "Nexus", output: nexus }] : []),
707
+ ]);
708
+ })(),
709
+ );
669
710
  return Promise.all(areas);
670
711
  }
671
712
 
@@ -1350,6 +1391,63 @@ class Bridge {
1350
1391
  await this.restartService(p, userId);
1351
1392
  }
1352
1393
 
1394
+ /**
1395
+ * Reboots one Proxmox host for an owner, when its status still says a reboot is needed,
1396
+ * then reports in the problem's channel whether it came back within ten minutes.
1397
+ */
1398
+ private async onRebootButton(interaction: Interaction, id: string): Promise<void> {
1399
+ const host = PROXMOX_HOSTS.find((h) => h.id === id);
1400
+ const userId = interaction.member?.user?.id ?? interaction.user?.id ?? "";
1401
+ this.refreshConfig();
1402
+ if (!host) {
1403
+ await this.rest.ephemeralReply(interaction.id, interaction.token, "This button is no longer active.");
1404
+ return;
1405
+ }
1406
+ if (!isOwner(this.config, userId)) {
1407
+ await this.rest.ephemeralReply(interaction.id, interaction.token, "Only owners reboot Proxmox hosts.");
1408
+ return;
1409
+ }
1410
+ if (this.rebooting.has(host.id)) {
1411
+ await this.rest.ephemeralReply(interaction.id, interaction.token, `${host.name} is already rebooting.`);
1412
+ return;
1413
+ }
1414
+ const status = await this.runEntry("proxmox", userId, [...host.prefix, "host-status"], 3 * 60_000).catch(() => null);
1415
+ if (status !== null && /^reboot-needed:\s*no\s*$/m.test(status)) {
1416
+ await this.rest.ephemeralReply(interaction.id, interaction.token, `${host.name} no longer needs a reboot.`);
1417
+ return;
1418
+ }
1419
+ await this.rest.deferUpdate(interaction.id, interaction.token);
1420
+ const channelId = interaction.channel_id ?? this.config.adminChannelId;
1421
+ const messageId = interaction.message?.id;
1422
+ this.rebooting.add(host.id);
1423
+ try {
1424
+ log(`reboot ${host.name} by ${userId}`);
1425
+ await this.runEntry("proxmox", userId, [...host.prefix, "host-reboot", "--confirm"], 2 * 60_000);
1426
+ const note = `♻️ ${host.name} reboots in one minute, started by <@${userId}>; its guests restart with it.`;
1427
+ if (messageId) await this.rest.editMessage(channelId, messageId, `${interaction.message?.content ?? ""}\n${note}`.slice(0, 1900), []).catch(() => undefined);
1428
+ // The host goes down after one minute; ask again from minute four until minute fourteen.
1429
+ await new Promise((r) => setTimeout(r, 4 * 60_000));
1430
+ let back: string | null = null;
1431
+ for (let i = 0; i < 20 && back === null && !this.stopping; i++) {
1432
+ back = await this.runEntry("proxmox", AUTO_CALLER, [...host.prefix, "host-status"], 60_000).catch(() => null);
1433
+ if (back !== null && !/up\s+(\d+ min|0:0?\d)/.test(back)) back = null;
1434
+ if (back === null) await new Promise((r) => setTimeout(r, 30_000));
1435
+ }
1436
+ const kernel = back ? /running kernel: ([^)]+)\)/.exec(back)?.[1] : undefined;
1437
+ const reboot = back ? /^reboot-needed:\s*(.*)$/m.exec(back)?.[1] : undefined;
1438
+ await this.rest.createMessage(channelId, {
1439
+ content: back
1440
+ ? `✅ ${host.name} is back${kernel ? ` on kernel ${kernel}` : ""}${reboot && reboot !== "no" ? `; it still reports reboot-needed: ${reboot}` : ""}.`
1441
+ : `❌ ${host.name} did not answer within ten minutes of the reboot. <@${userId}>`,
1442
+ ...(back ? {} : { mentionUsers: [userId] }),
1443
+ });
1444
+ } catch (err) {
1445
+ await this.rest.createMessage(channelId, { content: `❌ The reboot of ${host.name} did not start: ${(err instanceof Error ? err.message : String(err)).slice(0, 300)}` }).catch(() => undefined);
1446
+ } finally {
1447
+ this.rebooting.delete(host.id);
1448
+ }
1449
+ }
1450
+
1353
1451
  /** Runs one specialist wrapper through the sudo entry point, without a model; rejects with the wrapper's stderr. */
1354
1452
  private runEntry(service: string, caller: string, args: string[], timeoutMs: number): Promise<string> {
1355
1453
  const gate = loadSpecialists().config;
@@ -1754,6 +1852,10 @@ class Bridge {
1754
1852
  await this.onArcaneButton(interaction, customId);
1755
1853
  return;
1756
1854
  }
1855
+ if (customId.startsWith("proxmox:reboot:")) {
1856
+ await this.onRebootButton(interaction, customId.slice("proxmox:reboot:".length));
1857
+ return;
1858
+ }
1757
1859
  if (customId.startsWith("security:")) {
1758
1860
  this.refreshConfig();
1759
1861
  const userId = interaction.member?.user?.id ?? interaction.user?.id ?? "";
@@ -48,6 +48,9 @@ describe("approvals", () => {
48
48
  expect(isRisky("proxmox", ["snapshot-create", "107", "before", "-h"])).toBe(true);
49
49
  expect(isRisky("dns", ["--help"])).toBe(false);
50
50
  expect(isRisky("proxmox", ["guests"])).toBe(false);
51
+ expect(isRisky("proxmox", ["space-x", "host-upgrade"])).toBe(true);
52
+ expect(isRisky("proxmox", ["host-reboot", "--confirm"])).toBe(true);
53
+ expect(isRisky("proxmox", ["space-x", "host-status"])).toBe(false);
51
54
  expect(isRisky("dns", ["technitium", "host", "technitium1", "upgrade"])).toBe(true);
52
55
  expect(isRisky("dns", ["technitium", "host", "technitium3", "restart"])).toBe(true);
53
56
  expect(isRisky("dns", ["technitium", "host", "technitium2", "status"])).toBe(false);
@@ -43,7 +43,7 @@ export const RISKY: Readonly<Record<string, readonly RegExp[]>> = {
43
43
  ],
44
44
  edge: [/^vps\s+deploy-nginx\b/],
45
45
  backup: [/^(vzdump-run|host-backup-run|arcane-backup-run|gcs-run|restic-check-run|restic-restore|restore-clean)\b/],
46
- proxmox: [/^(space-x\s+)?(set|snapshot-create|start)\b/],
46
+ proxmox: [/^(space-x\s+)?(set|snapshot-create|start|host-upgrade|host-reboot)\b/],
47
47
  inference: [/^(nexus-restart|nexus-start)\b/],
48
48
  // media has no entry: trendarr cannot delete anything, and adds and queue toggles run on the caller's request.
49
49
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@astrofoundry/pi-astro",
3
- "version": "0.40.34",
3
+ "version": "0.40.36",
4
4
  "description": "Personal pi customizations (extensions, subagents, skills, prompts, themes) for the pi coding agent.",
5
5
  "keywords": [
6
6
  "pi-package"
@@ -66,7 +66,7 @@ Fixed operations on each node's host through one forced-command SSH key (`spc-dn
66
66
  | `restart` | restart Technitium and print its state |
67
67
  | `upgrade` | run the official installer (`download.technitium.com/dns/install.sh`), which upgrades Technitium and its ASP.NET runtime in place; up to 10 minutes |
68
68
 
69
- Upgrade or restart one node at a time and check `cluster-state` and `resolve` on it before the next; the other two nodes keep answering. The wrapper holds one lock for `restart` and `upgrade` across all nodes and refuses a second one while it runs. Upgrade the secondaries first and technitium1 last, and keep all three on the same version.
69
+ Upgrade or restart one node at a time and check `cluster-state` and `resolve` on it before the next; the other two nodes keep answering. The morning run does this on its own for a minor release (`technitium primary check-update`): technitium3, then technitium2, then the primary, one node per morning; a new major waits for the owner. The wrapper holds one lock for `restart` and `upgrade` across all nodes and refuses a second one while it runs. Upgrade the secondaries first and technitium1 last, and keep all three on the same version.
70
70
 
71
71
  ```json
72
72
  { "args": ["technitium", "host", "technitium3", "status"] }
@@ -17,6 +17,7 @@ Two hosts serve AI workloads. Nexus (Mac Studio, `10.0.40.38`, user `nexus`) run
17
17
  | `nexus-stop <embed|rerank> --confirm` | unload the service |
18
18
  | `nexus-log <embed|rerank> <lines>` | tail of `~/llama-<service>.log` |
19
19
  | `nexus-disk` | free space |
20
+ | `macos-updates` | installed macOS version and Software Update's offers (Nexus installs minor updates on its own; the morning run flags it two minors behind) |
20
21
 
21
22
  A model load after restart takes up to a minute; `health=503` during that time is loading, not failure.
22
23
 
@@ -41,6 +41,8 @@ Power actions and snapshot changes return `{ upid, exitstatus }`; `OK` means don
41
41
  | `host-status` | `pveversion`, uptime, memory, root and `/var/lib/vz` usage, CPU and NVMe temperatures |
42
42
  | `host-journal <since>` | warnings and errors from the host journal |
43
43
  | `updates` | upgradable host packages |
44
+ | `host-upgrade` | apt full-upgrade of the host (risky, needs approval); a new Proxmox VE major is held and printed as `held:`; ends with `reboot-needed:`. The morning run does this on its own each day |
45
+ | `host-reboot --confirm` | reboot the host in one minute (risky): every guest on it restarts. Only on the owner's explicit request; the morning run offers a Reboot button instead |
44
46
  | `guest-exec <vmid> <status|journal|df|updates>` | fixed read inside a guest: failed units and uptime, warning journal of 24 h, disk usage, upgradable packages |
45
47
 
46
48
  ## Guest change procedure
@@ -55,7 +57,7 @@ Memory and CPU changes on a running guest apply at the next start unless hotplug
55
57
 
56
58
  ## Not available
57
59
 
58
- Creating, cloning, or destroying guests, disk resize or removal, storage changes, host reboot or updates, backups (backup specialist), and anything inside a guest beyond the fixed reads. Describe the exact operator command (`qm`, `pct`, `pvesh`) and stop.
60
+ Creating, cloning, or destroying guests, disk resize or removal, storage changes, backups (backup specialist), and anything inside a guest beyond the fixed reads. Describe the exact operator command (`qm`, `pct`, `pvesh`) and stop.
59
61
 
60
62
  ## When to stop and report
61
63
 
@@ -106,6 +106,7 @@ export const TECHNITIUM_READS: Readonly<Record<string, { args: [number, number];
106
106
  stats: { args: [0, 1], help: `stats [${STATS_TYPES.join("|")}] dashboard counters` },
107
107
  "cluster-state": { args: [0, 0], help: "cluster-state cluster nodes, their state, and this server's addresses" },
108
108
  settings: { args: [0, 0], help: "settings all server settings (secrets redacted)" },
109
+ "check-update": { args: [0, 0], help: "check-update (primary) whether a newer Technitium version exists" },
109
110
  };
110
111
 
111
112
  /** Technitium writes; zone and cluster writes on the primary only, the other nodes follow through the catalog and the cluster. */
@@ -224,6 +225,8 @@ export function technitiumRequest(catalog: string, args: string[]): TechnitiumRe
224
225
  return { path: "/api/admin/cluster/state", params: { includeServerIpAddresses: "true" }, primaryOnly: false };
225
226
  case "settings":
226
227
  return { path: "/api/settings/get", params: {}, primaryOnly: false };
228
+ case "check-update":
229
+ return { path: "/api/user/checkForUpdate", params: {}, primaryOnly: true };
227
230
  case "cluster-init": {
228
231
  if (!IP_LIST.test(rest[1])) throw new UsageError("cluster-init needs a comma separated list of IP addresses");
229
232
  return {
@@ -35,5 +35,6 @@ case "$name" in
35
35
  printf '%s' "$lines" | grep -Eq '^[1-9][0-9]{0,3}$' || { echo "bad lines" >&2; exit 2; }
36
36
  tail -n "$lines" "$HOME/llama-$svc.log" ;;
37
37
  nexus-disk) df -h / ;;
38
+ macos-updates) [ -z "$rest" ] || { echo "macos-updates takes no arguments" >&2; exit 2; }; sw_vers -productVersion; softwareupdate -l 2>&1 | grep -E 'Title:' || true ;;
38
39
  *) echo "refused: $cmd" >&2; exit 2 ;;
39
40
  esac
@@ -12,6 +12,19 @@ rest=""
12
12
  [ "$cmd" != "$name" ] && rest="${cmd#* }"
13
13
  since_ok() { printf '%s' "$1" | grep -Eq '^[A-Za-z0-9 :+-]{1,40}$'; }
14
14
 
15
+ # A newer Proxmox kernel than the running one, or a package's reboot flag, means the host needs a reboot.
16
+ reboot_needed() {
17
+ running="$(uname -r)"
18
+ newest="$(ls /boot/vmlinuz-*-pve 2>/dev/null | sed 's#^/boot/vmlinuz-##' | sort -V | tail -n 1)"
19
+ if [ -n "$newest" ] && [ "$newest" != "$running" ]; then
20
+ echo "reboot-needed: kernel $running -> $newest"
21
+ elif [ -f /run/reboot-required ]; then
22
+ echo "reboot-needed: $(tr '\n' ' ' < /run/reboot-required.pkgs 2>/dev/null)"
23
+ else
24
+ echo "reboot-needed: no"
25
+ fi
26
+ }
27
+
15
28
  guest_cmd() {
16
29
  case "$1" in
17
30
  status) echo 'systemctl --failed --no-pager; uptime' ;;
@@ -28,9 +41,27 @@ case "$name" in
28
41
  uptime
29
42
  free -h
30
43
  df -h / /var/lib/vz
31
- sensors 2>/dev/null | grep -E 'Tctl|Composite|edge' || true ;;
44
+ sensors 2>/dev/null | grep -E 'Tctl|Composite|edge' || true
45
+ reboot_needed ;;
32
46
  host-journal) since_ok "$rest" || { echo "bad since" >&2; exit 2; }; journalctl -p warning --since "$rest" --no-pager -n 200 ;;
33
47
  updates) apt list --upgradable 2>/dev/null ;;
48
+ host-upgrade)
49
+ [ -z "$rest" ] || { echo "host-upgrade takes no arguments" >&2; exit 2; }
50
+ export DEBIAN_FRONTEND=noninteractive
51
+ apt-get update -q >/dev/null
52
+ # A new Proxmox VE major waits for the owner.
53
+ now="$(dpkg-query -W -f='${Version}' pve-manager)"
54
+ next="$(apt-get -s full-upgrade | sed -n 's/^Inst pve-manager \[[^]]*\] (\([^ ]*\) .*/\1/p')"
55
+ if [ -n "$next" ] && [ "${next%%.*}" != "${now%%.*}" ]; then
56
+ echo "held: Proxmox VE $now -> $next is a major upgrade"
57
+ exit 0
58
+ fi
59
+ apt-get -y -q -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold full-upgrade 2>&1 | grep -E '^[0-9]+ upgraded|^E:' || true
60
+ reboot_needed ;;
61
+ host-reboot)
62
+ [ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
63
+ shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
64
+ echo "reboot in 1 minute" ;;
34
65
  guest-exec)
35
66
  vmid="${rest%% *}"; action="${rest#* }"
36
67
  case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
@@ -12,6 +12,19 @@ rest=""
12
12
  [ "$cmd" != "$name" ] && rest="${cmd#* }"
13
13
  since_ok() { printf '%s' "$1" | grep -Eq '^[A-Za-z0-9 :+-]{1,40}$'; }
14
14
 
15
+ # A newer Proxmox kernel than the running one, or a package's reboot flag, means the host needs a reboot.
16
+ reboot_needed() {
17
+ running="$(uname -r)"
18
+ newest="$(ls /boot/vmlinuz-*-pve 2>/dev/null | sed 's#^/boot/vmlinuz-##' | sort -V | tail -n 1)"
19
+ if [ -n "$newest" ] && [ "$newest" != "$running" ]; then
20
+ echo "reboot-needed: kernel $running -> $newest"
21
+ elif [ -f /run/reboot-required ]; then
22
+ echo "reboot-needed: $(tr '\n' ' ' < /run/reboot-required.pkgs 2>/dev/null)"
23
+ else
24
+ echo "reboot-needed: no"
25
+ fi
26
+ }
27
+
15
28
  guest_cmd() {
16
29
  case "$1" in
17
30
  status) echo 'systemctl --failed --no-pager; uptime' ;;
@@ -28,9 +41,27 @@ case "$name" in
28
41
  uptime
29
42
  free -h
30
43
  df -h / /var/lib/vz
31
- sensors 2>/dev/null | grep -E 'Tctl|Composite|edge' || true ;;
44
+ sensors 2>/dev/null | grep -E 'Tctl|Composite|edge' || true
45
+ reboot_needed ;;
32
46
  host-journal) since_ok "$rest" || { echo "bad since" >&2; exit 2; }; journalctl -p warning --since "$rest" --no-pager -n 200 ;;
33
47
  updates) apt list --upgradable 2>/dev/null ;;
48
+ host-upgrade)
49
+ [ -z "$rest" ] || { echo "host-upgrade takes no arguments" >&2; exit 2; }
50
+ export DEBIAN_FRONTEND=noninteractive
51
+ apt-get update -q >/dev/null
52
+ # A new Proxmox VE major waits for the owner.
53
+ now="$(dpkg-query -W -f='${Version}' pve-manager)"
54
+ next="$(apt-get -s full-upgrade | sed -n 's/^Inst pve-manager \[[^]]*\] (\([^ ]*\) .*/\1/p')"
55
+ if [ -n "$next" ] && [ "${next%%.*}" != "${now%%.*}" ]; then
56
+ echo "held: Proxmox VE $now -> $next is a major upgrade"
57
+ exit 0
58
+ fi
59
+ apt-get -y -q -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold full-upgrade 2>&1 | grep -E '^[0-9]+ upgraded|^E:' || true
60
+ reboot_needed ;;
61
+ host-reboot)
62
+ [ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
63
+ shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
64
+ echo "reboot in 1 minute" ;;
34
65
  guest-exec)
35
66
  vmid="${rest%% *}"; action="${rest#* }"
36
67
  case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
@@ -39,6 +39,7 @@ export const INFERENCE_COMMANDS: Readonly<Record<string, RemoteSpec>> = {
39
39
  "nexus-stop": { args: 1, help: "nexus-stop <embed|rerank> --confirm launchctl bootout", timeoutMs: 2 * MINUTE, confirm: true },
40
40
  "nexus-log": { args: 2, help: "nexus-log <embed|rerank> <lines>", timeoutMs: MINUTE },
41
41
  "nexus-disk": { args: 0, help: "free space on the Nexus disk", timeoutMs: MINUTE },
42
+ "macos-updates": { args: 0, help: "installed macOS version and the updates Software Update offers", timeoutMs: 3 * MINUTE },
42
43
  };
43
44
 
44
45
  const NEXUS_SERVICES = ["embed", "rerank"];
@@ -85,9 +85,13 @@ export const HOST_COMMANDS: Readonly<Record<string, { args: number; help: string
85
85
  "host-status": { args: 0, help: "pveversion, uptime, load, memory, root disk, temperatures" },
86
86
  "host-journal": { args: 1, help: "host-journal <since> warnings and errors of the host journal" },
87
87
  updates: { args: 0, help: "upgradable packages on the host" },
88
+ "host-upgrade": { args: 0, help: "apt full-upgrade of the host; a new Proxmox VE major is held (prints held:); ends with reboot-needed" },
89
+ "host-reboot": { args: 1, help: "host-reboot --confirm reboot the host in one minute; every guest on it restarts" },
88
90
  "guest-exec": { args: 2, help: "guest-exec <vmid> <status|journal|df|updates> fixed read inside a guest (pct exec or qm guest exec)" },
89
91
  };
90
92
  export const GUEST_EXEC = ["status", "journal", "df", "updates"];
93
+ /** A full upgrade with a new kernel and pve-manager can take several minutes on the host. */
94
+ export const HOST_UPGRADE_TIMEOUT_MS = 20 * 60_000;
91
95
  const SINCE = /^[A-Za-z0-9 :\-+]{1,40}$/;
92
96
 
93
97
  const HELP = `proxmox specialist (Proxmox VE on Pulsar at 37 and SPACE-X at Mira)
@@ -328,6 +332,7 @@ export function buildHostRemote(args: string[]): string {
328
332
  if (!spec) throw new UsageError(`unknown host command: ${name ?? "(none)"}`);
329
333
  if (rest.length !== spec.args) throw new UsageError(`${name} takes ${spec.args} argument(s)`);
330
334
  if (name === "host-journal" && !SINCE.test(rest[0])) throw new UsageError("since: letters, digits, spaces, colon, plus, minus only");
335
+ if (name === "host-reboot" && rest[0] !== "--confirm") throw new UsageError("host-reboot restarts every guest on the host; add --confirm when the task asks for it explicitly");
331
336
  if (name === "guest-exec") {
332
337
  parseVmid(rest[0]);
333
338
  if (!GUEST_EXEC.includes(rest[1])) throw new UsageError(`guest-exec action must be one of ${GUEST_EXEC.join(", ")}`);
@@ -345,7 +350,7 @@ async function host(config: Target, args: string[]): Promise<number> {
345
350
  knownHostsFile: join(specialistsHome(), "config", "known_hosts"),
346
351
  },
347
352
  remote,
348
- 120_000,
353
+ args[0] === "host-upgrade" ? HOST_UPGRADE_TIMEOUT_MS : 120_000,
349
354
  );
350
355
  if (result.stdout.length > 0) printRaw(result.stdout);
351
356
  if (result.code !== 0) throw new ServiceError(result.stderr.trim() || `${args[0]} exited ${result.code}`);
@@ -216,6 +216,7 @@ describe("network wrapper", () => {
216
216
  describe("dns wrapper", () => {
217
217
  it("maps technitium commands to documented API paths", () => {
218
218
  expect(technitiumRequest("catalog.invalid", ["zones"])).toEqual({ path: "/api/zones/list", params: {}, primaryOnly: false });
219
+ expect(technitiumRequest("catalog.invalid", ["check-update"])).toEqual({ path: "/api/user/checkForUpdate", params: {}, primaryOnly: true });
219
220
  expect(technitiumRequest("catalog.invalid", ["zones", "*.37pla.net"])).toEqual({ path: "/api/zones/list", params: { filterName: "*.37pla.net" }, primaryOnly: false });
220
221
  expect(technitiumRequest("catalog.invalid", ["records", "37pla.net"])).toEqual({
221
222
  path: "/api/zones/records/get",
@@ -653,6 +654,10 @@ describe("proxmox wrapper", () => {
653
654
  expect(buildHostRemote(["guest-exec", "101", "df"])).toBe("guest-exec 101 df");
654
655
  expect(() => buildHostRemote(["guest-exec", "101", "rm"])).toThrow(/action/);
655
656
  expect(() => buildHostRemote(["host-journal", "1 hour ago; id"])).toThrow(/since/);
657
+ expect(buildHostRemote(["host-upgrade"])).toBe("host-upgrade");
658
+ expect(() => buildHostRemote(["host-upgrade", "now"])).toThrow(UsageError);
659
+ expect(buildHostRemote(["host-reboot", "--confirm"])).toBe("host-reboot --confirm");
660
+ expect(() => buildHostRemote(["host-reboot", "now"])).toThrow(/--confirm/);
656
661
  expect(() => buildHostRemote(["shell"])).toThrow(/unknown host command/);
657
662
  });
658
663