@astrofoundry/pi-astro 0.40.44 → 0.40.46

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,7 +1,7 @@
1
1
  import { describe, expect, it } from "vitest";
2
2
  import type { ArcaneReport } from "./daily-report.ts";
3
3
  import type { ToolsReport } from "./tools-report.ts";
4
- import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, pomeriumButtonId, wazuhRelease, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
4
+ import { agoSeconds, arcaneSummary, guestOwner, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, pomeriumButtonId, wazuhRelease, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
5
5
 
6
6
  const EDGE = `nginx active
7
7
  wg-quick@wg0 active
@@ -88,10 +88,14 @@ describe("morning status lines", () => {
88
88
  const wazuh = updatesStatus({ technitium2: held, synapse: "Listing..." });
89
89
  expect(wazuh.line).toBe("2 hosts up to date after the nightly run; Wazuh 4.14.10 is out (agents held at 4.14.9)");
90
90
  expect(wazuh.problem).toMatch(/^Updates: Wazuh 4\.14\.10 is out: ask me to upgrade the manager/);
91
- expect(updatesStatus({ arcane: vm, radius: null })).toMatchObject({
91
+ const left = updatesStatus({ arcane: vm, radius: null });
92
+ expect(left).toMatchObject({
92
93
  line: "still waiting after the nightly run: arcane 1; no update list from radius; Wazuh 4.14.9 is out (agents held at 4.14.7)",
93
94
  route: "admin",
94
95
  });
96
+ expect(left.items?.map((i) => i.owner)).toEqual(["Arcane", "Network", "Security"]);
97
+ expect(left.items?.slice(0, 2).map((i) => i.text)).toEqual(["arcane: docker-ce did not install overnight", "no update list from radius"]);
98
+ expect(["dmz", "observability", "backup", "tailscale-mira (Mira)", "technitium3 (Mira)", "synapse", "hermes", "Pulsar guests"].map(guestOwner)).toEqual(["Identity", "Security", "Backup", "Network", "DNS", "DNS", "Proxmox", "Proxmox"]);
95
99
  });
96
100
 
97
101
  it("checks RADIUS and the Tailscale router", () => {
@@ -167,32 +171,36 @@ describe("morning status lines", () => {
167
171
  });
168
172
 
169
173
  describe("morning summary", () => {
170
- it("lists each report and area, then what needs the owners, without a mention", () => {
174
+ it("gives each specialist and Cortex one line: all clear, or what needs the owners there, without a mention", () => {
171
175
  const text = summaryText({
172
176
  date: "Fri 24 Oct",
173
177
  guildId: "1",
174
178
  routes: { admin: "11", media: "12" },
175
179
  reports: [
176
180
  { area: "Security", line: "quiet day", needs: [], channelId: "21", messageId: "31" },
177
- { area: "Arcane", line: "3 stacks updated", needs: ["stirling-pdf major version"], channelId: "22", messageId: "32" },
181
+ { area: "Cortex tools", line: "1 updated", needs: [], channelId: "23", messageId: "33" },
182
+ { area: "Arcane", line: "3 stacks updated", needs: ["stirling-pdf waits for you (major version)"], channelId: "22", messageId: "32" },
178
183
  ],
179
184
  areas: [
180
185
  { area: "DNS", line: "both resolvers answer" },
181
186
  { area: "Backup", line: "no backup last night for identity", problem: "Backup: no Proxmox backup last night for identity", route: "admin" },
187
+ { area: "macOS", line: "Cortex 26.7.2" },
188
+ { area: "Updates", line: "x", problem: "Updates: arcane (docker-ce) did not install overnight", items: [{ owner: "Arcane", text: "arcane: docker-ce did not install overnight" }], route: "admin" },
182
189
  ],
183
190
  });
184
191
  expect(text).toBe(
185
192
  [
186
- "**Daily** · Fri 24 Oct · 🟠 2 items need you",
187
- "Security · quiet day · [report](https://discord.com/channels/1/21/31)",
188
- "Arcane · 3 stacks updated · [report](https://discord.com/channels/1/22/32)",
189
- "DNS · both resolvers answer",
190
- "Backup · no backup last night for identity",
191
- "Needs you · 1) stirling-pdf major version (<#22>) 2) Backup: no Proxmox backup last night for identity (<#11>)",
193
+ "**Daily** · Fri 24 Oct · 🟠 3 items need you",
194
+ "Security · All clear boss!",
195
+ "Cortex · All clear boss!",
196
+ "Arcane · stirling-pdf waits for you (major version) ([report](https://discord.com/channels/1/22/32)); arcane: docker-ce did not install overnight (<#11>)",
197
+ "Backup · no Proxmox backup last night for identity (<#11>)",
198
+ "Proxmox · All clear boss!",
199
+ "DNS · All clear boss!",
192
200
  ].join("\n"),
193
201
  );
194
202
  expect(text).not.toContain("<@");
195
- expect(summaryText({ date: "Fri 24 Oct", guildId: "1", routes: { admin: "11", media: "12" }, reports: [], areas: [] })).toBe("**Daily** · Fri 24 Oct · 🟢 nothing needs you");
203
+ expect(summaryText({ date: "Fri 24 Oct", guildId: "1", routes: { admin: "11", media: "12" }, reports: [], areas: [{ area: "Media", line: "downloads running" }] })).toBe("**Daily** · Fri 24 Oct · 🟢 All clear boss!\nMedia · All clear boss!");
196
204
  });
197
205
 
198
206
  it("tags the owners for an area's problem in its channel", () => {
@@ -215,11 +223,11 @@ describe("report summaries", () => {
215
223
  expect(toolsSummary(report, "2026-10-24", "5")).toEqual({
216
224
  area: "Cortex tools",
217
225
  line: "1 updated (jq 1.8.2), 1 failed (herdr), Pi 1.4.0, Node 26.1.0 wait for a button",
218
- needs: ["Node 26.1.0 (major version)", "herdr automatic update failed"],
226
+ needs: ["Node 26.1.0 waits for its button (major version)", "Pi 1.4.0 waits for its button", "herdr automatic update failed"],
219
227
  channelId: "5",
220
228
  messageId: "9",
221
229
  });
222
- expect(toolsSummary(report, "2026-10-25", "5").line).toBe("no report today; the bridge log has the reason");
230
+ expect(toolsSummary(report, "2026-10-25", "5")).toMatchObject({ line: "no report today; the bridge log has the reason", needs: ["the Cortex Tools Report did not run today; the bridge log has the reason"] });
223
231
  });
224
232
 
225
233
  it("says what the Arcane report updated, rolled back, and leaves to the owner", () => {
@@ -228,7 +236,7 @@ describe("report summaries", () => {
228
236
  expect(arcaneSummary(report, "2026-10-24", "6")).toEqual({
229
237
  area: "Arcane",
230
238
  line: "1 stack updated (media), mealie rolled back, stirling-pdf waits for you, supabase held",
231
- needs: ["stirling-pdf (Upgrade carefully)", "mealie automatic update rolled back"],
239
+ needs: ["stirling-pdf waits for you (Upgrade carefully)", "mealie automatic update rolled back"],
232
240
  channelId: "6",
233
241
  messageId: "8",
234
242
  });
@@ -12,11 +12,19 @@ export interface AreaButton {
12
12
  customId: string;
13
13
  }
14
14
 
15
+ /** One thing the owners must act on, under the summary line of the specialist that owns it. */
16
+ export interface SummaryItem {
17
+ owner: string;
18
+ text: string;
19
+ }
20
+
15
21
  /** One area of the morning summary: a line, and a problem that needs an owner, if any. */
16
22
  export interface AreaStatus {
17
23
  area: string;
18
24
  line: string;
19
25
  problem?: string;
26
+ /** The problem split by owner, when its parts belong to different specialists' lines. */
27
+ items?: SummaryItem[];
20
28
  route?: Route;
21
29
  buttons?: AreaButton[];
22
30
  }
@@ -151,6 +159,18 @@ export function wazuhRelease(hosts: Readonly<Record<string, string | null>>): {
151
159
  return found;
152
160
  }
153
161
 
162
+ /** The summary line for a guest's updates: the specialist that runs the guest, else Proxmox. */
163
+ export function guestOwner(name: string): string {
164
+ const guest = name.replace(/ \(Mira\)$/, "");
165
+ if (guest === "arcane") return "Arcane";
166
+ if (guest === "dmz" || guest === "identity") return "Identity";
167
+ if (guest === "observability") return "Security";
168
+ if (guest === "backup") return "Backup";
169
+ if (/^(radius|tailscale)/.test(guest)) return "Network";
170
+ if (/^(technitium|synapse)/.test(guest)) return "DNS";
171
+ return "Proxmox";
172
+ }
173
+
154
174
  /** Each guest's and Synapse's `updates` after the 04:30 run: anything still waiting did not install and needs a look. */
155
175
  export function updatesStatus(hosts: Readonly<Record<string, string | null>>): AreaStatus {
156
176
  const names = Object.keys(hosts);
@@ -167,12 +187,15 @@ export function updatesStatus(hosts: Readonly<Record<string, string | null>>): A
167
187
  unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null,
168
188
  wazuh ? `Wazuh ${wazuh.to} is out (agents held at ${wazuh.from})` : null,
169
189
  ].filter(Boolean);
170
- const problems = [
171
- left.length > 0 ? `${left.map((h) => `${h.name} (${h.packages.slice(0, 4).join(", ")}${h.packages.length > 4 ? ", ..." : ""})`).join("; ")} did not install overnight` : null,
172
- unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null,
173
- wazuh ? `Wazuh ${wazuh.to} is out: ask me to upgrade the manager (\`observability/compose.yaml\`, then \`deploy.sh\`), then the agents (\`scripts/wazuh-agents/deploy.sh --upgrade\`); an agent must never be newer than the manager` : null,
174
- ].filter(Boolean);
175
- return { area: "Updates", line: parts.join("; "), ...(problems.length > 0 ? { problem: `Updates: ${problems.join("; ")}`, route: "admin" as const } : {}) };
190
+ const packages = (h: { packages: string[] }) => `${h.packages.slice(0, 4).join(", ")}${h.packages.length > 4 ? ", ..." : ""}`;
191
+ const wazuhText = wazuh ? `Wazuh ${wazuh.to} is out: ask me to upgrade the manager (\`observability/compose.yaml\`, then \`deploy.sh\`), then the agents (\`scripts/wazuh-agents/deploy.sh --upgrade\`); an agent must never be newer than the manager` : null;
192
+ const problems = [left.length > 0 ? `${left.map((h) => `${h.name} (${packages(h)})`).join("; ")} did not install overnight` : null, unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null, wazuhText].filter(Boolean);
193
+ const items: SummaryItem[] = [
194
+ ...left.map((h) => ({ owner: guestOwner(h.name), text: `${h.name}: ${packages(h)} did not install overnight` })),
195
+ ...unknown.map((n) => ({ owner: guestOwner(n), text: `no update list from ${n}` })),
196
+ ...(wazuhText ? [{ owner: "Security", text: wazuhText }] : []),
197
+ ];
198
+ return { area: "Updates", line: parts.join("; "), ...(problems.length > 0 ? { problem: `Updates: ${problems.join("; ")}`, items, route: "admin" as const } : {}) };
176
199
  }
177
200
 
178
201
  /** `network pulsar radius-status` and `tailscale-status`: FreeRADIUS running, and every `tailscale*` subnet router (37 and Mira) online. */
@@ -358,7 +381,7 @@ function names(items: readonly string[]): string {
358
381
 
359
382
  /** The Cortex Tools Report after its automatic updates ran: what changed, what failed, and what waits for a button. */
360
383
  export function toolsSummary(report: ToolsReport | null, date: string, channelId: string): ReportLine {
361
- if (!report || report.date !== date) return { area: "Cortex tools", line: "no report today; the bridge log has the reason", needs: [], channelId };
384
+ if (!report || report.date !== date) return { area: "Cortex tools", line: "no report today; the bridge log has the reason", needs: ["the Cortex Tools Report did not run today; the bridge log has the reason"], channelId };
362
385
  const updated = report.lines.filter((l) => l.state === "updated");
363
386
  const failed = report.lines.filter((l) => l.state === "failed");
364
387
  const waiting = report.lines.filter((l) => (l.plan === "update" || l.plan === "button") && l.state === "offered");
@@ -370,7 +393,8 @@ export function toolsSummary(report: ToolsReport | null, date: string, channelId
370
393
  manual.length > 0 ? `${manual.length} by hand` : null,
371
394
  ].filter(Boolean);
372
395
  const needs = [
373
- ...report.lines.filter((l) => l.plan === "update" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} (${l.major ? "major version" : "release notes"})`),
396
+ ...report.lines.filter((l) => l.plan === "update" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} waits for its button (${l.major ? "major version" : "release notes"})`),
397
+ ...report.lines.filter((l) => l.plan === "button" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} waits for its button`),
374
398
  ...failed.filter((l) => l.plan === "auto").map((l) => `${escapeDiscord(l.label)} automatic update failed`),
375
399
  ];
376
400
  return { area: "Cortex tools", line: parts.join(", ") || "everything up to date", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
@@ -378,7 +402,7 @@ export function toolsSummary(report: ToolsReport | null, date: string, channelId
378
402
 
379
403
  /** The Arcane Daily Report after its automatic updates and checks: updated, rolled back, waiting for the owner, held. */
380
404
  export function arcaneSummary(report: ArcaneReport | null, date: string, channelId: string): ReportLine {
381
- if (!report || report.date !== date) return { area: "Arcane", line: "no report today; the bridge log has the reason", needs: [], channelId };
405
+ if (!report || report.date !== date) return { area: "Arcane", line: "no report today; the bridge log has the reason", needs: ["the Arcane Daily Report did not run today; the bridge log has the reason"], channelId };
382
406
  const by = (pick: (s: ArcaneReport["stacks"][number]) => boolean) => report.stacks.filter(pick).map((s) => s.stack);
383
407
  const updated = by((s) => s.state === "updated");
384
408
  const rolled = by((s) => s.state === "rolled back");
@@ -393,7 +417,7 @@ export function arcaneSummary(report: ArcaneReport | null, date: string, channel
393
417
  held.length > 0 ? `${names(held)} held` : null,
394
418
  ].filter(Boolean);
395
419
  const needs = [
396
- ...waiting.map((s) => `${escapeDiscord(s.stack)} (${s.plan === "careful" ? "Upgrade carefully" : "major version"})`),
420
+ ...waiting.map((s) => `${escapeDiscord(s.stack)} waits for you (${s.plan === "careful" ? "Upgrade carefully" : "major version"})`),
397
421
  ...report.stacks.filter((s) => s.plan === "auto" && (s.state === "rolled back" || s.state === "failed")).map((s) => `${escapeDiscord(s.stack)} automatic update ${s.state}`),
398
422
  ];
399
423
  return { area: "Arcane", line: parts.join(", ") || "no updates", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
@@ -412,16 +436,36 @@ function link(guildId: string, channelId: string, messageId: string): string {
412
436
  return `https://discord.com/channels/${guildId}/${channelId}/${messageId}`;
413
437
  }
414
438
 
415
- /** The #daily message: one line per report and area, then what needs the owners, with no mention. */
439
+ /** Reports and areas whose summary line is another specialist's, or Cortex for this host's own tools and macOS. */
440
+ const AREA_OWNER: Readonly<Record<string, string>> = { "Security (weekly)": "Security", "Cortex tools": "Cortex", macOS: "Cortex", Updates: "Proxmox" };
441
+
442
+ /** The summary lines in this order; any other owner follows them. */
443
+ const OWNER_ORDER = ["Security", "Cortex", "Arcane", "Backup", "Edge", "Proxmox", "Network", "Identity", "DNS", "Inference", "Media"];
444
+
445
+ const ALL_CLEAR = "All clear boss!";
446
+
447
+ function ownerOf(area: string): string {
448
+ return AREA_OWNER[area] ?? area;
449
+ }
450
+
451
+ /** The #daily message: one line per specialist and Cortex, either all clear or what needs the owners there, with no mention. */
416
452
  export function summaryText(s: Summary): string {
417
- const needs = [
418
- ...s.reports.flatMap((r) => r.needs.map((n) => `${n}${r.channelId ? ` (<#${r.channelId}>)` : ""}`)),
419
- ...s.areas.filter((a) => a.problem).map((a) => `${escapeDiscord(a.problem ?? "")} (<#${s.routes[a.route ?? "admin"]}>)`),
420
- ];
421
- const head = `**Daily** · ${s.date} · ${needs.length === 0 ? "🟢 nothing needs you" : `🟠 ${needs.length} item${needs.length === 1 ? " needs" : "s need"} you`}`;
422
- const reports = s.reports.map((r) => `${r.area} · ${r.line}${r.channelId && r.messageId ? ` · [report](${link(s.guildId, r.channelId, r.messageId)})` : ""}`);
423
- const areas = s.areas.map((a) => `${a.area} · ${a.line}`);
424
- return [head, ...reports, ...areas, ...(needs.length > 0 ? [`Needs you · ${needs.map((n, i) => `${i + 1}) ${n}`).join(" ")}`] : [])].join("\n");
453
+ const lines = new Map<string, string[]>();
454
+ const add = (owner: string, items: readonly string[]) => lines.set(owner, [...(lines.get(owner) ?? []), ...items]);
455
+ for (const r of s.reports) {
456
+ const where = r.channelId && r.messageId ? ` ([report](${link(s.guildId, r.channelId, r.messageId)}))` : r.channelId ? ` (<#${r.channelId}>)` : "";
457
+ add(ownerOf(r.area), r.needs.map((n) => `${n}${where}`));
458
+ }
459
+ for (const a of s.areas) {
460
+ const where = ` (<#${s.routes[a.route ?? "admin"]}>)`;
461
+ add(ownerOf(a.area), []);
462
+ for (const item of a.items ?? (a.problem ? [{ owner: ownerOf(a.area), text: a.problem.replace(`${a.area}: `, "") }] : [])) add(item.owner, [`${escapeDiscord(item.text)}${where}`]);
463
+ }
464
+ const count = [...lines.values()].reduce((n, items) => n + items.length, 0);
465
+ const head = `**Daily** · ${s.date} · ${count === 0 ? `🟢 ${ALL_CLEAR}` : `🟠 ${count} item${count === 1 ? " needs" : "s need"} you`}`;
466
+ const rank = (owner: string) => (OWNER_ORDER.includes(owner) ? OWNER_ORDER.indexOf(owner) : OWNER_ORDER.length);
467
+ const owners = [...lines.keys()].sort((x, y) => rank(x) - rank(y));
468
+ return [head, ...owners.map((o) => `${o} · ${lines.get(o)?.join("; ") || ALL_CLEAR}`)].join("\n");
425
469
  }
426
470
 
427
471
  /** Areas read by a specialist of another name, or by none: Cortex's macOS is this host's own. */
@@ -585,20 +585,21 @@ class Bridge {
585
585
  const name: ReportName = monday ? "security-weekly" : "security";
586
586
  const posted = done(name) ? undefined : await (monday ? this.security.weeklyReport(securitySchedule, date) : this.security.dailyReport(securitySchedule, date)).catch((err) => log(`daily ${date}: security report failed: ${err instanceof Error ? err.message : String(err)}`));
587
587
  const area = monday ? "Security (weekly)" : "Security";
588
- reports.push(posted ? { area, line: posted.headline, needs: posted.tagged ? [`the ${monday ? "weekly" : "daily"} security report`] : [], channelId: securitySchedule.channelId, messageId: posted.messageId } : { area, line: done(name) ? "already posted today" : "the report failed; the bridge log has the reason", needs: [], channelId: securitySchedule.channelId });
588
+ reports.push(posted ? { area, line: posted.headline, needs: posted.tagged ? [posted.headline] : [], channelId: securitySchedule.channelId, messageId: posted.messageId } : { area, line: done(name) ? "already posted today" : "the report failed; the bridge log has the reason", needs: done(name) ? [] : ["the security report failed; the bridge log has the reason"], channelId: securitySchedule.channelId });
589
589
  }
590
590
  const tools = reportSchedule(this.config, "tools");
591
591
  if (tools && !this.stopping) {
592
592
  if (!done("tools")) await this.toolsReport(tools, date).catch((err) => log(`daily ${date}: tools report failed: ${err instanceof Error ? err.message : String(err)}`));
593
- reports.push(toolsSummary(readPostedReport(), date, tools.channelId));
594
593
  }
595
594
  const arcane = reportSchedule(this.config, "arcane");
596
595
  if (arcane && !this.stopping) {
597
596
  if (!done("arcane")) await this.arcaneReport(arcane, date).catch((err) => log(`daily ${date}: arcane report failed: ${err instanceof Error ? err.message : String(err)}`));
598
- reports.push(arcaneSummary(readPostedArcane(), date, arcane.channelId));
599
597
  }
600
598
  if (this.stopping) return;
601
599
  const areas = await this.collectAreas();
600
+ // Read only now: a button pressed while the areas were checked has changed what still waits.
601
+ if (tools) reports.push(toolsSummary(readPostedReport(), date, tools.channelId));
602
+ if (arcane) reports.push(arcaneSummary(readPostedArcane(), date, arcane.channelId));
602
603
  const routes = this.routes();
603
604
  const owners = this.config.owners;
604
605
  for (const a of areas.filter((x) => x.problem)) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@astrofoundry/pi-astro",
3
- "version": "0.40.44",
3
+ "version": "0.40.46",
4
4
  "description": "Personal pi customizations (extensions, subagents, skills, prompts, themes) for the pi coding agent.",
5
5
  "keywords": [
6
6
  "pi-package"
@@ -44,6 +44,7 @@ Power actions and snapshot changes return `{ upid, exitstatus }`; `OK` means don
44
44
  | `host-upgrade` | apt full-upgrade of the host (risky, needs approval); a new Proxmox VE major is held and printed as `held:`; ends with `reboot-needed:`. The morning run does this on its own each day |
45
45
  | `host-reboot --confirm` | reboot the host in one minute (risky): every guest on it restarts. Only on the owner's explicit request; the morning run offers a Reboot button instead |
46
46
  | `guest-exec <vmid> <status|journal|df|updates>` | fixed read inside a guest: failed units and uptime, warning journal of 24 h, disk usage, upgradable packages |
47
+ | `nightly [days]` | start, end, duration (UTC) and result of each nightly job of the last 1..14 days (default 7): vzdump tasks from the API, then from the journals of the host and every running guest the host timers, `apt-daily-upgrade`, the backup and dump timers, and Docker restarts. Answer questions about nightly run times with it |
47
48
 
48
49
  ## Guest change procedure
49
50
 
@@ -11,7 +11,7 @@ A specialist is a Pi subagent that is the only way to operate one area of the ho
11
11
  | `astro.edge` | Frontdoor VPS (`frontdoor-1337`) over an IAP tunnel; `00-frontdoor-vps/` in the homelab repository | `probe` the public path; `route-probe` (status of the Supabase, Grimoire, and Mealie public routes from Cortex); `vps status|nginx-conf|nginx-test|journal|wg|bouncer|updates|package|changelog|auto-upgrades|deploy-nginx|reboot --confirm`; `git` on the homelab checkout, writes limited to `00-frontdoor-vps/` |
12
12
  | `astro.security` | Wazuh, CrowdSec, rsyslog archive on VM 106 | Read-only: `status`, `attention`, `agents`, `alerts`, `decisions`, `bouncers`, `metrics`, `wazuh-alerts`, `wazuh-vulns`, `wazuh-events`, `wazuh-vuln-events`, `wazuh-log`, `remote-hosts`, `remote-log`, `journal` |
13
13
  | `astro.backup` | vzdump, the shared Restic repository (Pulsar host config, Arcane app backups), GCS offsite from LXC 102 | Status, journals, and manual runs of every layer; `restic-snapshots|stats|ls`, `restic-restore` into `/var/tmp/spc-restore` on Pulsar, `restore-clean` |
14
- | `astro.proxmox` | Proxmox VE on Pulsar (37) and SPACE-X (Mira) | API token: `guests`, `guest`, `get <path>`, `tasks`; `start`, `shutdown|reboot|stop --confirm`; snapshots create, delete, rollback; `set` of CPU, memory, options, network. Host key: `host-status`, `host-journal`, `updates`, `host-upgrade` (apt full-upgrade; a new Proxmox VE major is held), `host-reboot --confirm`, `guest-exec <vmid> <status|journal|df|updates>`. Every command runs on Pulsar, or on SPACE-X with the `space-x` prefix |
14
+ | `astro.proxmox` | Proxmox VE on Pulsar (37) and SPACE-X (Mira) | API token: `guests`, `guest`, `get <path>`, `tasks`; `start`, `shutdown|reboot|stop --confirm`; snapshots create, delete, rollback; `set` of CPU, memory, options, network. Host key: `host-status`, `host-journal`, `updates`, `host-upgrade` (apt full-upgrade; a new Proxmox VE major is held), `host-reboot --confirm`, `guest-exec <vmid> <status|journal|df|updates>`, `nightly-journal <days>`; `nightly [days]` combines the vzdump tasks with that journal into a table of nightly run times. Every command runs on Pulsar, or on SPACE-X with the `space-x` prefix |
15
15
  | `astro.inference` | MLX embedding and reranking servers on Nexus, Europa health | `nexus-status|start|stop|restart|log|disk`, `macos-updates`; `europa-health` |
16
16
  | `astro.media` | trendarr over TMDB, OMDb, and the `media` project on VM 100 (Radarr, Sonarr, SABnzbd) | `trending`, `search`, `recommend`, `add-movies`, `add-series` (at most 20 IMDb IDs), `sabnzbd status|pause|resume`; always trendarr's JSON plus its warnings |
17
17
 
@@ -35,6 +35,14 @@ guest_cmd() {
35
35
  esac
36
36
  }
37
37
 
38
+ # The nightly jobs on the host and in the guests; a unit a machine does not have prints nothing.
39
+ NIGHTLY_UNITS="pulsar-config-backup mira-backup-pull apt-daily-upgrade docker arcane-backup pgdump-zitadel lego-renew gcs-backup"
40
+ nightly_cmd() {
41
+ units=""
42
+ for u in $NIGHTLY_UNITS; do units="$units -u $u.service"; done
43
+ echo "journalctl --utc -o short-iso --no-pager --since '$1 days ago' $units | grep -E 'systemd\[1\]: (Starting|Started|Finished|Failed|[^ ]+\.service: (Deactivated|Failed))' || true"
44
+ }
45
+
38
46
  case "$name" in
39
47
  host-status)
40
48
  pveversion
@@ -62,6 +70,15 @@ case "$name" in
62
70
  [ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
63
71
  shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
64
72
  echo "reboot in 1 minute" ;;
73
+ nightly-journal)
74
+ case "$rest" in [1-9]|1[0-4]) ;; *) echo "days must be 1..14" >&2; exit 2 ;; esac
75
+ inner="$(nightly_cmd "$rest")"
76
+ sh -c "$inner"
77
+ for vmid in $(pct list | awk 'NR > 1 && $2 == "running" { print $1 }'); do pct exec "$vmid" -- sh -c "$inner" || true; done
78
+ # The guest agent wraps a VM's output in JSON; only its out-data is journal lines.
79
+ for vmid in $(qm list | awk 'NR > 1 && $3 == "running" { print $1 }'); do
80
+ qm guest exec "$vmid" --timeout 60 -- sh -c "$inner" | perl -MJSON::PP -0777 -ne 'my $r = eval { decode_json($_) }; print $r->{"out-data"} // "" if $r' || true
81
+ done ;;
65
82
  guest-exec)
66
83
  vmid="${rest%% *}"; action="${rest#* }"
67
84
  case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
@@ -35,6 +35,14 @@ guest_cmd() {
35
35
  esac
36
36
  }
37
37
 
38
+ # The nightly jobs on the host and in the guests; a unit a machine does not have prints nothing.
39
+ NIGHTLY_UNITS="pulsar-config-backup mira-backup-pull apt-daily-upgrade docker arcane-backup pgdump-zitadel lego-renew gcs-backup"
40
+ nightly_cmd() {
41
+ units=""
42
+ for u in $NIGHTLY_UNITS; do units="$units -u $u.service"; done
43
+ echo "journalctl --utc -o short-iso --no-pager --since '$1 days ago' $units | grep -E 'systemd\[1\]: (Starting|Started|Finished|Failed|[^ ]+\.service: (Deactivated|Failed))' || true"
44
+ }
45
+
38
46
  case "$name" in
39
47
  host-status)
40
48
  pveversion
@@ -62,6 +70,15 @@ case "$name" in
62
70
  [ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
63
71
  shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
64
72
  echo "reboot in 1 minute" ;;
73
+ nightly-journal)
74
+ case "$rest" in [1-9]|1[0-4]) ;; *) echo "days must be 1..14" >&2; exit 2 ;; esac
75
+ inner="$(nightly_cmd "$rest")"
76
+ sh -c "$inner"
77
+ for vmid in $(pct list | awk 'NR > 1 && $2 == "running" { print $1 }'); do pct exec "$vmid" -- sh -c "$inner" || true; done
78
+ # The guest agent wraps a VM's output in JSON; only its out-data is journal lines.
79
+ for vmid in $(qm list | awk 'NR > 1 && $3 == "running" { print $1 }'); do
80
+ qm guest exec "$vmid" --timeout 60 -- sh -c "$inner" | perl -MJSON::PP -0777 -ne 'my $r = eval { decode_json($_) }; print $r->{"out-data"} // "" if $r' || true
81
+ done ;;
65
82
  guest-exec)
66
83
  vmid="${rest%% *}"; action="${rest#* }"
67
84
  case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
@@ -88,11 +88,14 @@ export const HOST_COMMANDS: Readonly<Record<string, { args: number; help: string
88
88
  "host-upgrade": { args: 0, help: "apt full-upgrade of the host; a new Proxmox VE major is held (prints held:); ends with reboot-needed" },
89
89
  "host-reboot": { args: 1, help: "host-reboot --confirm reboot the host in one minute; every guest on it restarts" },
90
90
  "guest-exec": { args: 2, help: "guest-exec <vmid> <status|journal|df|updates> fixed read inside a guest (pct exec or qm guest exec)" },
91
+ "nightly-journal": { args: 1, help: "nightly-journal <days> raw systemd start and end lines of the nightly jobs on the host and in every running guest" },
91
92
  };
92
93
  export const GUEST_EXEC = ["status", "journal", "df", "updates"];
93
94
  /** A full upgrade with a new kernel and pve-manager can take several minutes on the host. */
94
95
  export const HOST_UPGRADE_TIMEOUT_MS = 20 * 60_000;
95
96
  const SINCE = /^[A-Za-z0-9 :\-+]{1,40}$/;
97
+ /** Nights `nightly` reads; the entry scripts accept 1..14. */
98
+ export const NIGHTLY_DAYS = { default: 7, max: 14 };
96
99
 
97
100
  const HELP = `proxmox specialist (Proxmox VE on Pulsar at 37 and SPACE-X at Mira)
98
101
 
@@ -109,6 +112,7 @@ const HELP = `proxmox specialist (Proxmox VE on Pulsar at 37 and SPACE-X at Mira
109
112
  snapshot-delete <vmid> <name> --confirm
110
113
  snapshot-rollback <vmid> <name> --confirm
111
114
  set <vmid> key=value... VM: ${SET_KEYS.qemu.join(", ")}, netN; CT: ${SET_KEYS.lxc.join(", ")}, netN
115
+ nightly [days] start, end, and duration (UTC) of each nightly job of the last days (default ${NIGHTLY_DAYS.default}, max ${NIGHTLY_DAYS.max}): vzdump tasks, host timers, and each running guest's updates, backups, and Docker restarts
112
116
  ${Object.entries(HOST_COMMANDS)
113
117
  .map(([name, c]) => ` ${name.padEnd(35)} ${c.help}`)
114
118
  .join("\n")}
@@ -333,6 +337,7 @@ export function buildHostRemote(args: string[]): string {
333
337
  if (rest.length !== spec.args) throw new UsageError(`${name} takes ${spec.args} argument(s)`);
334
338
  if (name === "host-journal" && !SINCE.test(rest[0])) throw new UsageError("since: letters, digits, spaces, colon, plus, minus only");
335
339
  if (name === "host-reboot" && rest[0] !== "--confirm") throw new UsageError("host-reboot restarts every guest on the host; add --confirm when the task asks for it explicitly");
340
+ if (name === "nightly-journal") parseDays(rest[0]);
336
341
  if (name === "guest-exec") {
337
342
  parseVmid(rest[0]);
338
343
  if (!GUEST_EXEC.includes(rest[1])) throw new UsageError(`guest-exec action must be one of ${GUEST_EXEC.join(", ")}`);
@@ -340,7 +345,100 @@ export function buildHostRemote(args: string[]): string {
340
345
  return [name, ...rest].join(" ");
341
346
  }
342
347
 
343
- async function host(config: Target, args: string[]): Promise<number> {
348
+ export function parseDays(text: string | undefined): number {
349
+ const days = text === undefined ? NIGHTLY_DAYS.default : Number(text);
350
+ if (!Number.isInteger(days) || days < 1 || days > NIGHTLY_DAYS.max) throw new UsageError(`days must be 1..${NIGHTLY_DAYS.max}`);
351
+ return days;
352
+ }
353
+
354
+ /** One run of a nightly job on one machine; a Docker restart runs from the daemon's stop to its start. */
355
+ export interface NightlyRun {
356
+ machine: string;
357
+ job: string;
358
+ start: string;
359
+ end: string | null;
360
+ result: string;
361
+ }
362
+
363
+ const JOURNAL_LINE = /^(\S+) (\S+) systemd\[1\]: (.*)$/;
364
+
365
+ /** Runs from the `nightly-journal` lines: Starting to Finished or Failed, and Docker from Deactivated to Started. */
366
+ export function journalRuns(text: string): NightlyRun[] {
367
+ const runs: NightlyRun[] = [];
368
+ const open = new Map<string, NightlyRun>();
369
+ const close = (key: string, end: string, result: string) => {
370
+ const run = open.get(key);
371
+ if (!run) return;
372
+ run.end = end;
373
+ run.result = result;
374
+ open.delete(key);
375
+ };
376
+ for (const line of text.split("\n")) {
377
+ const m = JOURNAL_LINE.exec(line.trim());
378
+ if (!m) continue;
379
+ const [, time, machine, message] = m;
380
+ const at = new Date(time).toISOString();
381
+ const unit = /([\w@.-]+)\.service/.exec(message)?.[1];
382
+ if (!unit) continue;
383
+ const key = `${machine} ${unit}`;
384
+ if (unit === "docker") {
385
+ if (/: Deactivated successfully/.test(message)) {
386
+ const run = { machine, job: "docker restart", start: at, end: null, result: "running" };
387
+ runs.push(run);
388
+ open.set(key, run);
389
+ } else if (/^Started /.test(message)) close(key, at, "ok");
390
+ continue;
391
+ }
392
+ if (/^Starting /.test(message)) {
393
+ const run = { machine, job: unit, start: at, end: null, result: "running" };
394
+ runs.push(run);
395
+ open.set(key, run);
396
+ } else if (/^Finished /.test(message)) close(key, at, "ok");
397
+ else if (/: Failed with result '([^']+)'/.test(message)) close(key, at, `failed (${/'([^']+)'/.exec(message)?.[1] ?? "?"})`);
398
+ else if (/^Failed to start /.test(message)) close(key, at, "failed");
399
+ }
400
+ return runs;
401
+ }
402
+
403
+ /** Runs from `/nodes/<node>/tasks?typefilter=vzdump`: one per backup task, the whole job when it names no guest. */
404
+ export function vzdumpRuns(node: string, tasks: unknown): NightlyRun[] {
405
+ if (!Array.isArray(tasks)) return [];
406
+ return (tasks as { type?: unknown; id?: unknown; starttime?: unknown; endtime?: unknown; status?: unknown }[])
407
+ .filter((t) => t.type === "vzdump" && typeof t.starttime === "number")
408
+ .map((t) => ({
409
+ machine: node,
410
+ job: typeof t.id === "string" && t.id !== "" ? `vzdump ${t.id}` : "vzdump",
411
+ start: new Date((t.starttime as number) * 1000).toISOString(),
412
+ end: typeof t.endtime === "number" ? new Date(t.endtime * 1000).toISOString() : null,
413
+ result: typeof t.status === "string" ? t.status : "running",
414
+ }));
415
+ }
416
+
417
+ function durationText(start: string, end: string | null): string {
418
+ if (end === null) return "-";
419
+ const s = Math.round((Date.parse(end) - Date.parse(start)) / 1000);
420
+ return s >= 60 ? `${Math.floor(s / 60)}m${String(s % 60).padStart(2, "0")}s` : `${s}s`;
421
+ }
422
+
423
+ /** The runs as a fixed-width table, oldest first, times in UTC. */
424
+ export function nightlyTable(runs: readonly NightlyRun[]): string {
425
+ const rows = [...runs]
426
+ .sort((a, b) => a.start.localeCompare(b.start))
427
+ .map((r) => [r.start.slice(0, 10), r.start.slice(11, 19), r.end ? r.end.slice(11, 19) : "-", durationText(r.start, r.end), r.machine, r.job, r.result]);
428
+ const head = ["date", "start", "end", "duration", "machine", "job", "result"];
429
+ const widths = head.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)));
430
+ return [head, ...rows].map((r) => r.map((c, i) => c.padEnd(widths[i])).join(" ").trimEnd()).join("\n");
431
+ }
432
+
433
+ async function nightly(config: Target, days: number): Promise<number> {
434
+ const since = Math.floor(Date.now() / 1000) - days * 24 * 3600;
435
+ const tasks = await api(config, "GET", `/nodes/${config.node}/tasks?typefilter=vzdump&since=${since}&limit=500`);
436
+ const journal = await hostOutput(config, ["nightly-journal", String(days)]);
437
+ printRaw(`${nightlyTable([...vzdumpRuns(config.node, tasks), ...journalRuns(journal)])}\n(times in UTC)`);
438
+ return 0;
439
+ }
440
+
441
+ async function hostOutput(config: Target, args: string[]): Promise<string> {
344
442
  const remote = buildHostRemote(args);
345
443
  const result = await sshFixed(
346
444
  {
@@ -352,8 +450,16 @@ async function host(config: Target, args: string[]): Promise<number> {
352
450
  remote,
353
451
  args[0] === "host-upgrade" ? HOST_UPGRADE_TIMEOUT_MS : 120_000,
354
452
  );
355
- if (result.stdout.length > 0) printRaw(result.stdout);
356
- if (result.code !== 0) throw new ServiceError(result.stderr.trim() || `${args[0]} exited ${result.code}`);
453
+ if (result.code !== 0) {
454
+ if (result.stdout.length > 0) printRaw(result.stdout);
455
+ throw new ServiceError(result.stderr.trim() || `${args[0]} exited ${result.code}`);
456
+ }
457
+ return result.stdout;
458
+ }
459
+
460
+ async function host(config: Target, args: string[]): Promise<number> {
461
+ const out = await hostOutput(config, args);
462
+ if (out.length > 0) printRaw(out);
357
463
  return 0;
358
464
  }
359
465
 
@@ -373,6 +479,10 @@ export async function command(args: string[]): Promise<number> {
373
479
  printJson({ host: new URL(config.apiUrl).host, fingerprint256: probe.fingerprint });
374
480
  return 0;
375
481
  }
482
+ if (rest[0] === "nightly") {
483
+ if (rest.length > 2) throw new UsageError("nightly [days]");
484
+ return nightly(config, parseDays(rest[1]));
485
+ }
376
486
  if (rest[0] !== undefined && rest[0] in HOST_COMMANDS) return host(config, rest);
377
487
  return apiCommand(config, rest);
378
488
  }
@@ -12,7 +12,7 @@ import { MAX_ADD_IDS, MAX_EXCLUDE_IDS, buildTrendarrCall, interpretRun, parseImd
12
12
  import { ServiceError, UsageError } from "./lib/errors.ts";
13
13
  import { safeRepoPath, safeWritePath } from "./lib/repo.ts";
14
14
  import { appGetPath, appPolicyDelete, portForwardDelete, clientFixedIpWrite, syslogChanges, buildPulsarRemote, cloudPath, deviceNetworkWrite, NAT_PATH, natWrite, parseQuery, parseSiteArgs, redactUnifi, unifiPath, unifiWrite, command as networkCommand } from "./network/run.ts";
15
- import { buildHostRemote, splitHost, needsConfirm, parseSetPairs, parseVmid, validateApiPath as validatePvePath, command as proxmoxCommand } from "./proxmox/run.ts";
15
+ import { buildHostRemote, journalRuns, nightlyTable, parseDays, splitHost, needsConfirm, parseSetPairs, parseVmid, validateApiPath as validatePvePath, vzdumpRuns, command as proxmoxCommand } from "./proxmox/run.ts";
16
16
  import { banLength, buildObsRemote, compactDecisions, parseGoDuration, summariseAlerts, summariseVulns, summariseWazuh, vulnEvents, wazuhEvents, command as securityCommand } from "./security/run.ts";
17
17
 
18
18
  describe("arcane wrapper", () => {
@@ -680,6 +680,46 @@ describe("proxmox wrapper", () => {
680
680
  expect(buildHostRemote(["host-reboot", "--confirm"])).toBe("host-reboot --confirm");
681
681
  expect(() => buildHostRemote(["host-reboot", "now"])).toThrow(/--confirm/);
682
682
  expect(() => buildHostRemote(["shell"])).toThrow(/unknown host command/);
683
+ expect(buildHostRemote(["nightly-journal", "7"])).toBe("nightly-journal 7");
684
+ expect(() => buildHostRemote(["nightly-journal", "15"])).toThrow(/days/);
685
+ expect(() => buildHostRemote(["nightly-journal", "7; id"])).toThrow(/days/);
686
+ expect(parseDays(undefined)).toBe(7);
687
+ expect(() => parseDays("0")).toThrow(/days/);
688
+ });
689
+
690
+ it("turns the nightly journal and vzdump tasks into runs and a table", () => {
691
+ const journal = [
692
+ "2026-11-04T03:00:24+00:00 pulsar systemd[1]: Starting pulsar-config-backup.service - PVE host config backup to restic...",
693
+ "2026-11-04T03:00:42+00:00 pulsar systemd[1]: pulsar-config-backup.service: Deactivated successfully.",
694
+ "2026-11-04T03:00:42+00:00 pulsar systemd[1]: Finished pulsar-config-backup.service - PVE host config backup to restic.",
695
+ "2026-10-31T18:52:02+00:00 pulsar systemd[1]: Starting mira-backup-pull.service - Copy Mira guest backups from TESLA to the UNAS...",
696
+ "2026-10-31T18:52:05+00:00 pulsar systemd[1]: mira-backup-pull.service: Failed with result 'exit-code'.",
697
+ "2026-10-31T18:52:05+00:00 pulsar systemd[1]: Failed to start mira-backup-pull.service - Copy Mira guest backups from TESLA to the UNAS.",
698
+ "2026-11-03T03:30:01+00:00 arcane systemd[1]: Starting apt-daily-upgrade.service - Daily apt upgrade and clean activities...",
699
+ "2026-11-03T03:30:09+00:00 arcane systemd[1]: Finished apt-daily-upgrade.service - Daily apt upgrade and clean activities.",
700
+ "2026-11-03T03:30:22+00:00 arcane systemd[1]: docker.service: Deactivated successfully.",
701
+ "2026-11-03T03:30:32+00:00 arcane systemd[1]: Starting docker.service - Docker Application Container Engine...",
702
+ "2026-11-03T03:30:33+00:00 arcane systemd[1]: Started docker.service - Docker Application Container Engine.",
703
+ "2026-11-01T05:35:22+00:00 identity systemd[1]: Starting docker.service - Docker Application Container Engine...",
704
+ ].join("\n");
705
+ expect(journalRuns(journal)).toEqual([
706
+ { machine: "pulsar", job: "pulsar-config-backup", start: "2026-11-04T03:00:24.000Z", end: "2026-11-04T03:00:42.000Z", result: "ok" },
707
+ { machine: "pulsar", job: "mira-backup-pull", start: "2026-10-31T18:52:02.000Z", end: "2026-10-31T18:52:05.000Z", result: "failed (exit-code)" },
708
+ { machine: "arcane", job: "apt-daily-upgrade", start: "2026-11-03T03:30:01.000Z", end: "2026-11-03T03:30:09.000Z", result: "ok" },
709
+ { machine: "arcane", job: "docker restart", start: "2026-11-03T03:30:22.000Z", end: "2026-11-03T03:30:33.000Z", result: "ok" },
710
+ ]);
711
+ const tasks = [
712
+ { type: "vzdump", id: "", starttime: 1793755800, endtime: 1793756268, status: "OK" },
713
+ { type: "qmstart", id: "100", starttime: 1793755000, endtime: 1793755001, status: "OK" },
714
+ ];
715
+ const runs = [...vzdumpRuns("pulsar", tasks), ...journalRuns(journal)];
716
+ expect(runs[0]).toEqual({ machine: "pulsar", job: "vzdump", start: "2026-11-04T01:30:00.000Z", end: "2026-11-04T01:37:48.000Z", result: "OK" });
717
+ expect(vzdumpRuns("pulsar", null)).toEqual([]);
718
+ const table = nightlyTable(runs).split("\n");
719
+ expect(table[0]).toBe("date start end duration machine job result");
720
+ expect(table[1]).toBe("2026-10-31 18:52:02 18:52:05 3s pulsar mira-backup-pull failed (exit-code)");
721
+ expect(table.at(-1)).toBe("2026-11-04 03:00:24 03:00:42 18s pulsar pulsar-config-backup ok");
722
+ expect(table).toContain("2026-11-04 01:30:00 01:37:48 7m48s pulsar vzdump OK");
683
723
  });
684
724
 
685
725
  it("prints help without touching config", async () => {