@astrofoundry/pi-astro 0.40.44 → 0.40.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/astro-discord/daily-run.test.ts +22 -14
- package/extensions/astro-discord/daily-run.ts +63 -19
- package/extensions/astro-discord/index.ts +4 -3
- package/package.json +1 -1
- package/skills/proxmox/SKILL.md +1 -0
- package/specialists/README.md +1 -1
- package/specialists/entry/remote/spc-proxmox-pulsar-entry.sh +17 -0
- package/specialists/entry/remote/spc-proxmox-spacex-entry.sh +17 -0
- package/specialists/proxmox/run.ts +113 -3
- package/specialists/wrappers.test.ts +41 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
2
|
import type { ArcaneReport } from "./daily-report.ts";
|
|
3
3
|
import type { ToolsReport } from "./tools-report.ts";
|
|
4
|
-
import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, pomeriumButtonId, wazuhRelease, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
4
|
+
import { agoSeconds, arcaneSummary, guestOwner, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, pomeriumButtonId, wazuhRelease, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
5
5
|
|
|
6
6
|
const EDGE = `nginx active
|
|
7
7
|
wg-quick@wg0 active
|
|
@@ -88,10 +88,14 @@ describe("morning status lines", () => {
|
|
|
88
88
|
const wazuh = updatesStatus({ technitium2: held, synapse: "Listing..." });
|
|
89
89
|
expect(wazuh.line).toBe("2 hosts up to date after the nightly run; Wazuh 4.14.10 is out (agents held at 4.14.9)");
|
|
90
90
|
expect(wazuh.problem).toMatch(/^Updates: Wazuh 4\.14\.10 is out: ask me to upgrade the manager/);
|
|
91
|
-
|
|
91
|
+
const left = updatesStatus({ arcane: vm, radius: null });
|
|
92
|
+
expect(left).toMatchObject({
|
|
92
93
|
line: "still waiting after the nightly run: arcane 1; no update list from radius; Wazuh 4.14.9 is out (agents held at 4.14.7)",
|
|
93
94
|
route: "admin",
|
|
94
95
|
});
|
|
96
|
+
expect(left.items?.map((i) => i.owner)).toEqual(["Arcane", "Network", "Security"]);
|
|
97
|
+
expect(left.items?.slice(0, 2).map((i) => i.text)).toEqual(["arcane: docker-ce did not install overnight", "no update list from radius"]);
|
|
98
|
+
expect(["dmz", "observability", "backup", "tailscale-mira (Mira)", "technitium3 (Mira)", "synapse", "hermes", "Pulsar guests"].map(guestOwner)).toEqual(["Identity", "Security", "Backup", "Network", "DNS", "DNS", "Proxmox", "Proxmox"]);
|
|
95
99
|
});
|
|
96
100
|
|
|
97
101
|
it("checks RADIUS and the Tailscale router", () => {
|
|
@@ -167,32 +171,36 @@ describe("morning status lines", () => {
|
|
|
167
171
|
});
|
|
168
172
|
|
|
169
173
|
describe("morning summary", () => {
|
|
170
|
-
it("
|
|
174
|
+
it("gives each specialist and Cortex one line: all clear, or what needs the owners there, without a mention", () => {
|
|
171
175
|
const text = summaryText({
|
|
172
176
|
date: "Fri 24 Oct",
|
|
173
177
|
guildId: "1",
|
|
174
178
|
routes: { admin: "11", media: "12" },
|
|
175
179
|
reports: [
|
|
176
180
|
{ area: "Security", line: "quiet day", needs: [], channelId: "21", messageId: "31" },
|
|
177
|
-
{ area: "
|
|
181
|
+
{ area: "Cortex tools", line: "1 updated", needs: [], channelId: "23", messageId: "33" },
|
|
182
|
+
{ area: "Arcane", line: "3 stacks updated", needs: ["stirling-pdf waits for you (major version)"], channelId: "22", messageId: "32" },
|
|
178
183
|
],
|
|
179
184
|
areas: [
|
|
180
185
|
{ area: "DNS", line: "both resolvers answer" },
|
|
181
186
|
{ area: "Backup", line: "no backup last night for identity", problem: "Backup: no Proxmox backup last night for identity", route: "admin" },
|
|
187
|
+
{ area: "macOS", line: "Cortex 26.7.2" },
|
|
188
|
+
{ area: "Updates", line: "x", problem: "Updates: arcane (docker-ce) did not install overnight", items: [{ owner: "Arcane", text: "arcane: docker-ce did not install overnight" }], route: "admin" },
|
|
182
189
|
],
|
|
183
190
|
});
|
|
184
191
|
expect(text).toBe(
|
|
185
192
|
[
|
|
186
|
-
"**Daily** · Fri 24 Oct · 🟠
|
|
187
|
-
"Security ·
|
|
188
|
-
"
|
|
189
|
-
"
|
|
190
|
-
"Backup · no backup last night for identity",
|
|
191
|
-
"
|
|
193
|
+
"**Daily** · Fri 24 Oct · 🟠 3 items need you",
|
|
194
|
+
"Security · All clear boss!",
|
|
195
|
+
"Cortex · All clear boss!",
|
|
196
|
+
"Arcane · stirling-pdf waits for you (major version) ([report](https://discord.com/channels/1/22/32)); arcane: docker-ce did not install overnight (<#11>)",
|
|
197
|
+
"Backup · no Proxmox backup last night for identity (<#11>)",
|
|
198
|
+
"Proxmox · All clear boss!",
|
|
199
|
+
"DNS · All clear boss!",
|
|
192
200
|
].join("\n"),
|
|
193
201
|
);
|
|
194
202
|
expect(text).not.toContain("<@");
|
|
195
|
-
expect(summaryText({ date: "Fri 24 Oct", guildId: "1", routes: { admin: "11", media: "12" }, reports: [], areas: [] })).toBe("**Daily** · Fri 24 Oct · 🟢
|
|
203
|
+
expect(summaryText({ date: "Fri 24 Oct", guildId: "1", routes: { admin: "11", media: "12" }, reports: [], areas: [{ area: "Media", line: "downloads running" }] })).toBe("**Daily** · Fri 24 Oct · 🟢 All clear boss!\nMedia · All clear boss!");
|
|
196
204
|
});
|
|
197
205
|
|
|
198
206
|
it("tags the owners for an area's problem in its channel", () => {
|
|
@@ -215,11 +223,11 @@ describe("report summaries", () => {
|
|
|
215
223
|
expect(toolsSummary(report, "2026-10-24", "5")).toEqual({
|
|
216
224
|
area: "Cortex tools",
|
|
217
225
|
line: "1 updated (jq 1.8.2), 1 failed (herdr), Pi 1.4.0, Node 26.1.0 wait for a button",
|
|
218
|
-
needs: ["Node 26.1.0 (major version)", "herdr automatic update failed"],
|
|
226
|
+
needs: ["Node 26.1.0 waits for its button (major version)", "Pi 1.4.0 waits for its button", "herdr automatic update failed"],
|
|
219
227
|
channelId: "5",
|
|
220
228
|
messageId: "9",
|
|
221
229
|
});
|
|
222
|
-
expect(toolsSummary(report, "2026-10-25", "5")
|
|
230
|
+
expect(toolsSummary(report, "2026-10-25", "5")).toMatchObject({ line: "no report today; the bridge log has the reason", needs: ["the Cortex Tools Report did not run today; the bridge log has the reason"] });
|
|
223
231
|
});
|
|
224
232
|
|
|
225
233
|
it("says what the Arcane report updated, rolled back, and leaves to the owner", () => {
|
|
@@ -228,7 +236,7 @@ describe("report summaries", () => {
|
|
|
228
236
|
expect(arcaneSummary(report, "2026-10-24", "6")).toEqual({
|
|
229
237
|
area: "Arcane",
|
|
230
238
|
line: "1 stack updated (media), mealie rolled back, stirling-pdf waits for you, supabase held",
|
|
231
|
-
needs: ["stirling-pdf (Upgrade carefully)", "mealie automatic update rolled back"],
|
|
239
|
+
needs: ["stirling-pdf waits for you (Upgrade carefully)", "mealie automatic update rolled back"],
|
|
232
240
|
channelId: "6",
|
|
233
241
|
messageId: "8",
|
|
234
242
|
});
|
|
@@ -12,11 +12,19 @@ export interface AreaButton {
|
|
|
12
12
|
customId: string;
|
|
13
13
|
}
|
|
14
14
|
|
|
15
|
+
/** One thing the owners must act on, under the summary line of the specialist that owns it. */
|
|
16
|
+
export interface SummaryItem {
|
|
17
|
+
owner: string;
|
|
18
|
+
text: string;
|
|
19
|
+
}
|
|
20
|
+
|
|
15
21
|
/** One area of the morning summary: a line, and a problem that needs an owner, if any. */
|
|
16
22
|
export interface AreaStatus {
|
|
17
23
|
area: string;
|
|
18
24
|
line: string;
|
|
19
25
|
problem?: string;
|
|
26
|
+
/** The problem split by owner, when its parts belong to different specialists' lines. */
|
|
27
|
+
items?: SummaryItem[];
|
|
20
28
|
route?: Route;
|
|
21
29
|
buttons?: AreaButton[];
|
|
22
30
|
}
|
|
@@ -151,6 +159,18 @@ export function wazuhRelease(hosts: Readonly<Record<string, string | null>>): {
|
|
|
151
159
|
return found;
|
|
152
160
|
}
|
|
153
161
|
|
|
162
|
+
/** The summary line for a guest's updates: the specialist that runs the guest, else Proxmox. */
|
|
163
|
+
export function guestOwner(name: string): string {
|
|
164
|
+
const guest = name.replace(/ \(Mira\)$/, "");
|
|
165
|
+
if (guest === "arcane") return "Arcane";
|
|
166
|
+
if (guest === "dmz" || guest === "identity") return "Identity";
|
|
167
|
+
if (guest === "observability") return "Security";
|
|
168
|
+
if (guest === "backup") return "Backup";
|
|
169
|
+
if (/^(radius|tailscale)/.test(guest)) return "Network";
|
|
170
|
+
if (/^(technitium|synapse)/.test(guest)) return "DNS";
|
|
171
|
+
return "Proxmox";
|
|
172
|
+
}
|
|
173
|
+
|
|
154
174
|
/** Each guest's and Synapse's `updates` after the 04:30 run: anything still waiting did not install and needs a look. */
|
|
155
175
|
export function updatesStatus(hosts: Readonly<Record<string, string | null>>): AreaStatus {
|
|
156
176
|
const names = Object.keys(hosts);
|
|
@@ -167,12 +187,15 @@ export function updatesStatus(hosts: Readonly<Record<string, string | null>>): A
|
|
|
167
187
|
unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null,
|
|
168
188
|
wazuh ? `Wazuh ${wazuh.to} is out (agents held at ${wazuh.from})` : null,
|
|
169
189
|
].filter(Boolean);
|
|
170
|
-
const
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
190
|
+
const packages = (h: { packages: string[] }) => `${h.packages.slice(0, 4).join(", ")}${h.packages.length > 4 ? ", ..." : ""}`;
|
|
191
|
+
const wazuhText = wazuh ? `Wazuh ${wazuh.to} is out: ask me to upgrade the manager (\`observability/compose.yaml\`, then \`deploy.sh\`), then the agents (\`scripts/wazuh-agents/deploy.sh --upgrade\`); an agent must never be newer than the manager` : null;
|
|
192
|
+
const problems = [left.length > 0 ? `${left.map((h) => `${h.name} (${packages(h)})`).join("; ")} did not install overnight` : null, unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null, wazuhText].filter(Boolean);
|
|
193
|
+
const items: SummaryItem[] = [
|
|
194
|
+
...left.map((h) => ({ owner: guestOwner(h.name), text: `${h.name}: ${packages(h)} did not install overnight` })),
|
|
195
|
+
...unknown.map((n) => ({ owner: guestOwner(n), text: `no update list from ${n}` })),
|
|
196
|
+
...(wazuhText ? [{ owner: "Security", text: wazuhText }] : []),
|
|
197
|
+
];
|
|
198
|
+
return { area: "Updates", line: parts.join("; "), ...(problems.length > 0 ? { problem: `Updates: ${problems.join("; ")}`, items, route: "admin" as const } : {}) };
|
|
176
199
|
}
|
|
177
200
|
|
|
178
201
|
/** `network pulsar radius-status` and `tailscale-status`: FreeRADIUS running, and every `tailscale*` subnet router (37 and Mira) online. */
|
|
@@ -358,7 +381,7 @@ function names(items: readonly string[]): string {
|
|
|
358
381
|
|
|
359
382
|
/** The Cortex Tools Report after its automatic updates ran: what changed, what failed, and what waits for a button. */
|
|
360
383
|
export function toolsSummary(report: ToolsReport | null, date: string, channelId: string): ReportLine {
|
|
361
|
-
if (!report || report.date !== date) return { area: "Cortex tools", line: "no report today; the bridge log has the reason", needs: [], channelId };
|
|
384
|
+
if (!report || report.date !== date) return { area: "Cortex tools", line: "no report today; the bridge log has the reason", needs: ["the Cortex Tools Report did not run today; the bridge log has the reason"], channelId };
|
|
362
385
|
const updated = report.lines.filter((l) => l.state === "updated");
|
|
363
386
|
const failed = report.lines.filter((l) => l.state === "failed");
|
|
364
387
|
const waiting = report.lines.filter((l) => (l.plan === "update" || l.plan === "button") && l.state === "offered");
|
|
@@ -370,7 +393,8 @@ export function toolsSummary(report: ToolsReport | null, date: string, channelId
|
|
|
370
393
|
manual.length > 0 ? `${manual.length} by hand` : null,
|
|
371
394
|
].filter(Boolean);
|
|
372
395
|
const needs = [
|
|
373
|
-
...report.lines.filter((l) => l.plan === "update" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} (${l.major ? "major version" : "release notes"})`),
|
|
396
|
+
...report.lines.filter((l) => l.plan === "update" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} waits for its button (${l.major ? "major version" : "release notes"})`),
|
|
397
|
+
...report.lines.filter((l) => l.plan === "button" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} waits for its button`),
|
|
374
398
|
...failed.filter((l) => l.plan === "auto").map((l) => `${escapeDiscord(l.label)} automatic update failed`),
|
|
375
399
|
];
|
|
376
400
|
return { area: "Cortex tools", line: parts.join(", ") || "everything up to date", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
|
|
@@ -378,7 +402,7 @@ export function toolsSummary(report: ToolsReport | null, date: string, channelId
|
|
|
378
402
|
|
|
379
403
|
/** The Arcane Daily Report after its automatic updates and checks: updated, rolled back, waiting for the owner, held. */
|
|
380
404
|
export function arcaneSummary(report: ArcaneReport | null, date: string, channelId: string): ReportLine {
|
|
381
|
-
if (!report || report.date !== date) return { area: "Arcane", line: "no report today; the bridge log has the reason", needs: [], channelId };
|
|
405
|
+
if (!report || report.date !== date) return { area: "Arcane", line: "no report today; the bridge log has the reason", needs: ["the Arcane Daily Report did not run today; the bridge log has the reason"], channelId };
|
|
382
406
|
const by = (pick: (s: ArcaneReport["stacks"][number]) => boolean) => report.stacks.filter(pick).map((s) => s.stack);
|
|
383
407
|
const updated = by((s) => s.state === "updated");
|
|
384
408
|
const rolled = by((s) => s.state === "rolled back");
|
|
@@ -393,7 +417,7 @@ export function arcaneSummary(report: ArcaneReport | null, date: string, channel
|
|
|
393
417
|
held.length > 0 ? `${names(held)} held` : null,
|
|
394
418
|
].filter(Boolean);
|
|
395
419
|
const needs = [
|
|
396
|
-
...waiting.map((s) => `${escapeDiscord(s.stack)} (${s.plan === "careful" ? "Upgrade carefully" : "major version"})`),
|
|
420
|
+
...waiting.map((s) => `${escapeDiscord(s.stack)} waits for you (${s.plan === "careful" ? "Upgrade carefully" : "major version"})`),
|
|
397
421
|
...report.stacks.filter((s) => s.plan === "auto" && (s.state === "rolled back" || s.state === "failed")).map((s) => `${escapeDiscord(s.stack)} automatic update ${s.state}`),
|
|
398
422
|
];
|
|
399
423
|
return { area: "Arcane", line: parts.join(", ") || "no updates", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
|
|
@@ -412,16 +436,36 @@ function link(guildId: string, channelId: string, messageId: string): string {
|
|
|
412
436
|
return `https://discord.com/channels/${guildId}/${channelId}/${messageId}`;
|
|
413
437
|
}
|
|
414
438
|
|
|
415
|
-
/**
|
|
439
|
+
/** Reports and areas whose summary line is another specialist's, or Cortex for this host's own tools and macOS. */
|
|
440
|
+
const AREA_OWNER: Readonly<Record<string, string>> = { "Security (weekly)": "Security", "Cortex tools": "Cortex", macOS: "Cortex", Updates: "Proxmox" };
|
|
441
|
+
|
|
442
|
+
/** The summary lines in this order; any other owner follows them. */
|
|
443
|
+
const OWNER_ORDER = ["Security", "Cortex", "Arcane", "Backup", "Edge", "Proxmox", "Network", "Identity", "DNS", "Inference", "Media"];
|
|
444
|
+
|
|
445
|
+
const ALL_CLEAR = "All clear boss!";
|
|
446
|
+
|
|
447
|
+
function ownerOf(area: string): string {
|
|
448
|
+
return AREA_OWNER[area] ?? area;
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/** The #daily message: one line per specialist and Cortex, either all clear or what needs the owners there, with no mention. */
|
|
416
452
|
export function summaryText(s: Summary): string {
|
|
417
|
-
const
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
const
|
|
424
|
-
|
|
453
|
+
const lines = new Map<string, string[]>();
|
|
454
|
+
const add = (owner: string, items: readonly string[]) => lines.set(owner, [...(lines.get(owner) ?? []), ...items]);
|
|
455
|
+
for (const r of s.reports) {
|
|
456
|
+
const where = r.channelId && r.messageId ? ` ([report](${link(s.guildId, r.channelId, r.messageId)}))` : r.channelId ? ` (<#${r.channelId}>)` : "";
|
|
457
|
+
add(ownerOf(r.area), r.needs.map((n) => `${n}${where}`));
|
|
458
|
+
}
|
|
459
|
+
for (const a of s.areas) {
|
|
460
|
+
const where = ` (<#${s.routes[a.route ?? "admin"]}>)`;
|
|
461
|
+
add(ownerOf(a.area), []);
|
|
462
|
+
for (const item of a.items ?? (a.problem ? [{ owner: ownerOf(a.area), text: a.problem.replace(`${a.area}: `, "") }] : [])) add(item.owner, [`${escapeDiscord(item.text)}${where}`]);
|
|
463
|
+
}
|
|
464
|
+
const count = [...lines.values()].reduce((n, items) => n + items.length, 0);
|
|
465
|
+
const head = `**Daily** · ${s.date} · ${count === 0 ? `🟢 ${ALL_CLEAR}` : `🟠 ${count} item${count === 1 ? " needs" : "s need"} you`}`;
|
|
466
|
+
const rank = (owner: string) => (OWNER_ORDER.includes(owner) ? OWNER_ORDER.indexOf(owner) : OWNER_ORDER.length);
|
|
467
|
+
const owners = [...lines.keys()].sort((x, y) => rank(x) - rank(y));
|
|
468
|
+
return [head, ...owners.map((o) => `${o} · ${lines.get(o)?.join("; ") || ALL_CLEAR}`)].join("\n");
|
|
425
469
|
}
|
|
426
470
|
|
|
427
471
|
/** Areas read by a specialist of another name, or by none: Cortex's macOS is this host's own. */
|
|
@@ -585,20 +585,21 @@ class Bridge {
|
|
|
585
585
|
const name: ReportName = monday ? "security-weekly" : "security";
|
|
586
586
|
const posted = done(name) ? undefined : await (monday ? this.security.weeklyReport(securitySchedule, date) : this.security.dailyReport(securitySchedule, date)).catch((err) => log(`daily ${date}: security report failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
587
587
|
const area = monday ? "Security (weekly)" : "Security";
|
|
588
|
-
reports.push(posted ? { area, line: posted.headline, needs: posted.tagged ? [
|
|
588
|
+
reports.push(posted ? { area, line: posted.headline, needs: posted.tagged ? [posted.headline] : [], channelId: securitySchedule.channelId, messageId: posted.messageId } : { area, line: done(name) ? "already posted today" : "the report failed; the bridge log has the reason", needs: done(name) ? [] : ["the security report failed; the bridge log has the reason"], channelId: securitySchedule.channelId });
|
|
589
589
|
}
|
|
590
590
|
const tools = reportSchedule(this.config, "tools");
|
|
591
591
|
if (tools && !this.stopping) {
|
|
592
592
|
if (!done("tools")) await this.toolsReport(tools, date).catch((err) => log(`daily ${date}: tools report failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
593
|
-
reports.push(toolsSummary(readPostedReport(), date, tools.channelId));
|
|
594
593
|
}
|
|
595
594
|
const arcane = reportSchedule(this.config, "arcane");
|
|
596
595
|
if (arcane && !this.stopping) {
|
|
597
596
|
if (!done("arcane")) await this.arcaneReport(arcane, date).catch((err) => log(`daily ${date}: arcane report failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
598
|
-
reports.push(arcaneSummary(readPostedArcane(), date, arcane.channelId));
|
|
599
597
|
}
|
|
600
598
|
if (this.stopping) return;
|
|
601
599
|
const areas = await this.collectAreas();
|
|
600
|
+
// Read only now: a button pressed while the areas were checked has changed what still waits.
|
|
601
|
+
if (tools) reports.push(toolsSummary(readPostedReport(), date, tools.channelId));
|
|
602
|
+
if (arcane) reports.push(arcaneSummary(readPostedArcane(), date, arcane.channelId));
|
|
602
603
|
const routes = this.routes();
|
|
603
604
|
const owners = this.config.owners;
|
|
604
605
|
for (const a of areas.filter((x) => x.problem)) {
|
package/package.json
CHANGED
package/skills/proxmox/SKILL.md
CHANGED
|
@@ -44,6 +44,7 @@ Power actions and snapshot changes return `{ upid, exitstatus }`; `OK` means don
|
|
|
44
44
|
| `host-upgrade` | apt full-upgrade of the host (risky, needs approval); a new Proxmox VE major is held and printed as `held:`; ends with `reboot-needed:`. The morning run does this on its own each day |
|
|
45
45
|
| `host-reboot --confirm` | reboot the host in one minute (risky): every guest on it restarts. Only on the owner's explicit request; the morning run offers a Reboot button instead |
|
|
46
46
|
| `guest-exec <vmid> <status|journal|df|updates>` | fixed read inside a guest: failed units and uptime, warning journal of 24 h, disk usage, upgradable packages |
|
|
47
|
+
| `nightly [days]` | start, end, duration (UTC) and result of each nightly job of the last 1..14 days (default 7): vzdump tasks from the API, then from the journals of the host and every running guest the host timers, `apt-daily-upgrade`, the backup and dump timers, and Docker restarts. Answer questions about nightly run times with it |
|
|
47
48
|
|
|
48
49
|
## Guest change procedure
|
|
49
50
|
|
package/specialists/README.md
CHANGED
|
@@ -11,7 +11,7 @@ A specialist is a Pi subagent that is the only way to operate one area of the ho
|
|
|
11
11
|
| `astro.edge` | Frontdoor VPS (`frontdoor-1337`) over an IAP tunnel; `00-frontdoor-vps/` in the homelab repository | `probe` the public path; `route-probe` (status of the Supabase, Grimoire, and Mealie public routes from Cortex); `vps status|nginx-conf|nginx-test|journal|wg|bouncer|updates|package|changelog|auto-upgrades|deploy-nginx|reboot --confirm`; `git` on the homelab checkout, writes limited to `00-frontdoor-vps/` |
|
|
12
12
|
| `astro.security` | Wazuh, CrowdSec, rsyslog archive on VM 106 | Read-only: `status`, `attention`, `agents`, `alerts`, `decisions`, `bouncers`, `metrics`, `wazuh-alerts`, `wazuh-vulns`, `wazuh-events`, `wazuh-vuln-events`, `wazuh-log`, `remote-hosts`, `remote-log`, `journal` |
|
|
13
13
|
| `astro.backup` | vzdump, the shared Restic repository (Pulsar host config, Arcane app backups), GCS offsite from LXC 102 | Status, journals, and manual runs of every layer; `restic-snapshots|stats|ls`, `restic-restore` into `/var/tmp/spc-restore` on Pulsar, `restore-clean` |
|
|
14
|
-
| `astro.proxmox` | Proxmox VE on Pulsar (37) and SPACE-X (Mira) | API token: `guests`, `guest`, `get <path>`, `tasks`; `start`, `shutdown|reboot|stop --confirm`; snapshots create, delete, rollback; `set` of CPU, memory, options, network. Host key: `host-status`, `host-journal`, `updates`, `host-upgrade` (apt full-upgrade; a new Proxmox VE major is held), `host-reboot --confirm`, `guest-exec <vmid> <status|journal|df|updates
|
|
14
|
+
| `astro.proxmox` | Proxmox VE on Pulsar (37) and SPACE-X (Mira) | API token: `guests`, `guest`, `get <path>`, `tasks`; `start`, `shutdown|reboot|stop --confirm`; snapshots create, delete, rollback; `set` of CPU, memory, options, network. Host key: `host-status`, `host-journal`, `updates`, `host-upgrade` (apt full-upgrade; a new Proxmox VE major is held), `host-reboot --confirm`, `guest-exec <vmid> <status|journal|df|updates>`, `nightly-journal <days>`; `nightly [days]` combines the vzdump tasks with that journal into a table of nightly run times. Every command runs on Pulsar, or on SPACE-X with the `space-x` prefix |
|
|
15
15
|
| `astro.inference` | MLX embedding and reranking servers on Nexus, Europa health | `nexus-status|start|stop|restart|log|disk`, `macos-updates`; `europa-health` |
|
|
16
16
|
| `astro.media` | trendarr over TMDB, OMDb, and the `media` project on VM 100 (Radarr, Sonarr, SABnzbd) | `trending`, `search`, `recommend`, `add-movies`, `add-series` (at most 20 IMDb IDs), `sabnzbd status|pause|resume`; always trendarr's JSON plus its warnings |
|
|
17
17
|
|
|
@@ -35,6 +35,14 @@ guest_cmd() {
|
|
|
35
35
|
esac
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
# The nightly jobs on the host and in the guests; a unit a machine does not have prints nothing.
|
|
39
|
+
NIGHTLY_UNITS="pulsar-config-backup mira-backup-pull apt-daily-upgrade docker arcane-backup pgdump-zitadel lego-renew gcs-backup"
|
|
40
|
+
nightly_cmd() {
|
|
41
|
+
units=""
|
|
42
|
+
for u in $NIGHTLY_UNITS; do units="$units -u $u.service"; done
|
|
43
|
+
echo "journalctl --utc -o short-iso --no-pager --since '$1 days ago' $units | grep -E 'systemd\[1\]: (Starting|Started|Finished|Failed|[^ ]+\.service: (Deactivated|Failed))' || true"
|
|
44
|
+
}
|
|
45
|
+
|
|
38
46
|
case "$name" in
|
|
39
47
|
host-status)
|
|
40
48
|
pveversion
|
|
@@ -62,6 +70,15 @@ case "$name" in
|
|
|
62
70
|
[ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
|
|
63
71
|
shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
|
|
64
72
|
echo "reboot in 1 minute" ;;
|
|
73
|
+
nightly-journal)
|
|
74
|
+
case "$rest" in [1-9]|1[0-4]) ;; *) echo "days must be 1..14" >&2; exit 2 ;; esac
|
|
75
|
+
inner="$(nightly_cmd "$rest")"
|
|
76
|
+
sh -c "$inner"
|
|
77
|
+
for vmid in $(pct list | awk 'NR > 1 && $2 == "running" { print $1 }'); do pct exec "$vmid" -- sh -c "$inner" || true; done
|
|
78
|
+
# The guest agent wraps a VM's output in JSON; only its out-data is journal lines.
|
|
79
|
+
for vmid in $(qm list | awk 'NR > 1 && $3 == "running" { print $1 }'); do
|
|
80
|
+
qm guest exec "$vmid" --timeout 60 -- sh -c "$inner" | perl -MJSON::PP -0777 -ne 'my $r = eval { decode_json($_) }; print $r->{"out-data"} // "" if $r' || true
|
|
81
|
+
done ;;
|
|
65
82
|
guest-exec)
|
|
66
83
|
vmid="${rest%% *}"; action="${rest#* }"
|
|
67
84
|
case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
|
|
@@ -35,6 +35,14 @@ guest_cmd() {
|
|
|
35
35
|
esac
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
# The nightly jobs on the host and in the guests; a unit a machine does not have prints nothing.
|
|
39
|
+
NIGHTLY_UNITS="pulsar-config-backup mira-backup-pull apt-daily-upgrade docker arcane-backup pgdump-zitadel lego-renew gcs-backup"
|
|
40
|
+
nightly_cmd() {
|
|
41
|
+
units=""
|
|
42
|
+
for u in $NIGHTLY_UNITS; do units="$units -u $u.service"; done
|
|
43
|
+
echo "journalctl --utc -o short-iso --no-pager --since '$1 days ago' $units | grep -E 'systemd\[1\]: (Starting|Started|Finished|Failed|[^ ]+\.service: (Deactivated|Failed))' || true"
|
|
44
|
+
}
|
|
45
|
+
|
|
38
46
|
case "$name" in
|
|
39
47
|
host-status)
|
|
40
48
|
pveversion
|
|
@@ -62,6 +70,15 @@ case "$name" in
|
|
|
62
70
|
[ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
|
|
63
71
|
shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
|
|
64
72
|
echo "reboot in 1 minute" ;;
|
|
73
|
+
nightly-journal)
|
|
74
|
+
case "$rest" in [1-9]|1[0-4]) ;; *) echo "days must be 1..14" >&2; exit 2 ;; esac
|
|
75
|
+
inner="$(nightly_cmd "$rest")"
|
|
76
|
+
sh -c "$inner"
|
|
77
|
+
for vmid in $(pct list | awk 'NR > 1 && $2 == "running" { print $1 }'); do pct exec "$vmid" -- sh -c "$inner" || true; done
|
|
78
|
+
# The guest agent wraps a VM's output in JSON; only its out-data is journal lines.
|
|
79
|
+
for vmid in $(qm list | awk 'NR > 1 && $3 == "running" { print $1 }'); do
|
|
80
|
+
qm guest exec "$vmid" --timeout 60 -- sh -c "$inner" | perl -MJSON::PP -0777 -ne 'my $r = eval { decode_json($_) }; print $r->{"out-data"} // "" if $r' || true
|
|
81
|
+
done ;;
|
|
65
82
|
guest-exec)
|
|
66
83
|
vmid="${rest%% *}"; action="${rest#* }"
|
|
67
84
|
case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
|
|
@@ -88,11 +88,14 @@ export const HOST_COMMANDS: Readonly<Record<string, { args: number; help: string
|
|
|
88
88
|
"host-upgrade": { args: 0, help: "apt full-upgrade of the host; a new Proxmox VE major is held (prints held:); ends with reboot-needed" },
|
|
89
89
|
"host-reboot": { args: 1, help: "host-reboot --confirm reboot the host in one minute; every guest on it restarts" },
|
|
90
90
|
"guest-exec": { args: 2, help: "guest-exec <vmid> <status|journal|df|updates> fixed read inside a guest (pct exec or qm guest exec)" },
|
|
91
|
+
"nightly-journal": { args: 1, help: "nightly-journal <days> raw systemd start and end lines of the nightly jobs on the host and in every running guest" },
|
|
91
92
|
};
|
|
92
93
|
export const GUEST_EXEC = ["status", "journal", "df", "updates"];
|
|
93
94
|
/** A full upgrade with a new kernel and pve-manager can take several minutes on the host. */
|
|
94
95
|
export const HOST_UPGRADE_TIMEOUT_MS = 20 * 60_000;
|
|
95
96
|
const SINCE = /^[A-Za-z0-9 :\-+]{1,40}$/;
|
|
97
|
+
/** Nights `nightly` reads; the entry scripts accept 1..14. */
|
|
98
|
+
export const NIGHTLY_DAYS = { default: 7, max: 14 };
|
|
96
99
|
|
|
97
100
|
const HELP = `proxmox specialist (Proxmox VE on Pulsar at 37 and SPACE-X at Mira)
|
|
98
101
|
|
|
@@ -109,6 +112,7 @@ const HELP = `proxmox specialist (Proxmox VE on Pulsar at 37 and SPACE-X at Mira
|
|
|
109
112
|
snapshot-delete <vmid> <name> --confirm
|
|
110
113
|
snapshot-rollback <vmid> <name> --confirm
|
|
111
114
|
set <vmid> key=value... VM: ${SET_KEYS.qemu.join(", ")}, netN; CT: ${SET_KEYS.lxc.join(", ")}, netN
|
|
115
|
+
nightly [days] start, end, and duration (UTC) of each nightly job of the last days (default ${NIGHTLY_DAYS.default}, max ${NIGHTLY_DAYS.max}): vzdump tasks, host timers, and each running guest's updates, backups, and Docker restarts
|
|
112
116
|
${Object.entries(HOST_COMMANDS)
|
|
113
117
|
.map(([name, c]) => ` ${name.padEnd(35)} ${c.help}`)
|
|
114
118
|
.join("\n")}
|
|
@@ -333,6 +337,7 @@ export function buildHostRemote(args: string[]): string {
|
|
|
333
337
|
if (rest.length !== spec.args) throw new UsageError(`${name} takes ${spec.args} argument(s)`);
|
|
334
338
|
if (name === "host-journal" && !SINCE.test(rest[0])) throw new UsageError("since: letters, digits, spaces, colon, plus, minus only");
|
|
335
339
|
if (name === "host-reboot" && rest[0] !== "--confirm") throw new UsageError("host-reboot restarts every guest on the host; add --confirm when the task asks for it explicitly");
|
|
340
|
+
if (name === "nightly-journal") parseDays(rest[0]);
|
|
336
341
|
if (name === "guest-exec") {
|
|
337
342
|
parseVmid(rest[0]);
|
|
338
343
|
if (!GUEST_EXEC.includes(rest[1])) throw new UsageError(`guest-exec action must be one of ${GUEST_EXEC.join(", ")}`);
|
|
@@ -340,7 +345,100 @@ export function buildHostRemote(args: string[]): string {
|
|
|
340
345
|
return [name, ...rest].join(" ");
|
|
341
346
|
}
|
|
342
347
|
|
|
343
|
-
|
|
348
|
+
export function parseDays(text: string | undefined): number {
|
|
349
|
+
const days = text === undefined ? NIGHTLY_DAYS.default : Number(text);
|
|
350
|
+
if (!Number.isInteger(days) || days < 1 || days > NIGHTLY_DAYS.max) throw new UsageError(`days must be 1..${NIGHTLY_DAYS.max}`);
|
|
351
|
+
return days;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** One run of a nightly job on one machine; a Docker restart runs from the daemon's stop to its start. */
|
|
355
|
+
export interface NightlyRun {
|
|
356
|
+
machine: string;
|
|
357
|
+
job: string;
|
|
358
|
+
start: string;
|
|
359
|
+
end: string | null;
|
|
360
|
+
result: string;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
const JOURNAL_LINE = /^(\S+) (\S+) systemd\[1\]: (.*)$/;
|
|
364
|
+
|
|
365
|
+
/** Runs from the `nightly-journal` lines: Starting to Finished or Failed, and Docker from Deactivated to Started. */
|
|
366
|
+
export function journalRuns(text: string): NightlyRun[] {
|
|
367
|
+
const runs: NightlyRun[] = [];
|
|
368
|
+
const open = new Map<string, NightlyRun>();
|
|
369
|
+
const close = (key: string, end: string, result: string) => {
|
|
370
|
+
const run = open.get(key);
|
|
371
|
+
if (!run) return;
|
|
372
|
+
run.end = end;
|
|
373
|
+
run.result = result;
|
|
374
|
+
open.delete(key);
|
|
375
|
+
};
|
|
376
|
+
for (const line of text.split("\n")) {
|
|
377
|
+
const m = JOURNAL_LINE.exec(line.trim());
|
|
378
|
+
if (!m) continue;
|
|
379
|
+
const [, time, machine, message] = m;
|
|
380
|
+
const at = new Date(time).toISOString();
|
|
381
|
+
const unit = /([\w@.-]+)\.service/.exec(message)?.[1];
|
|
382
|
+
if (!unit) continue;
|
|
383
|
+
const key = `${machine} ${unit}`;
|
|
384
|
+
if (unit === "docker") {
|
|
385
|
+
if (/: Deactivated successfully/.test(message)) {
|
|
386
|
+
const run = { machine, job: "docker restart", start: at, end: null, result: "running" };
|
|
387
|
+
runs.push(run);
|
|
388
|
+
open.set(key, run);
|
|
389
|
+
} else if (/^Started /.test(message)) close(key, at, "ok");
|
|
390
|
+
continue;
|
|
391
|
+
}
|
|
392
|
+
if (/^Starting /.test(message)) {
|
|
393
|
+
const run = { machine, job: unit, start: at, end: null, result: "running" };
|
|
394
|
+
runs.push(run);
|
|
395
|
+
open.set(key, run);
|
|
396
|
+
} else if (/^Finished /.test(message)) close(key, at, "ok");
|
|
397
|
+
else if (/: Failed with result '([^']+)'/.test(message)) close(key, at, `failed (${/'([^']+)'/.exec(message)?.[1] ?? "?"})`);
|
|
398
|
+
else if (/^Failed to start /.test(message)) close(key, at, "failed");
|
|
399
|
+
}
|
|
400
|
+
return runs;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/** Runs from `/nodes/<node>/tasks?typefilter=vzdump`: one per backup task, the whole job when it names no guest. */
|
|
404
|
+
export function vzdumpRuns(node: string, tasks: unknown): NightlyRun[] {
|
|
405
|
+
if (!Array.isArray(tasks)) return [];
|
|
406
|
+
return (tasks as { type?: unknown; id?: unknown; starttime?: unknown; endtime?: unknown; status?: unknown }[])
|
|
407
|
+
.filter((t) => t.type === "vzdump" && typeof t.starttime === "number")
|
|
408
|
+
.map((t) => ({
|
|
409
|
+
machine: node,
|
|
410
|
+
job: typeof t.id === "string" && t.id !== "" ? `vzdump ${t.id}` : "vzdump",
|
|
411
|
+
start: new Date((t.starttime as number) * 1000).toISOString(),
|
|
412
|
+
end: typeof t.endtime === "number" ? new Date(t.endtime * 1000).toISOString() : null,
|
|
413
|
+
result: typeof t.status === "string" ? t.status : "running",
|
|
414
|
+
}));
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
function durationText(start: string, end: string | null): string {
|
|
418
|
+
if (end === null) return "-";
|
|
419
|
+
const s = Math.round((Date.parse(end) - Date.parse(start)) / 1000);
|
|
420
|
+
return s >= 60 ? `${Math.floor(s / 60)}m${String(s % 60).padStart(2, "0")}s` : `${s}s`;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/** The runs as a fixed-width table, oldest first, times in UTC. */
|
|
424
|
+
export function nightlyTable(runs: readonly NightlyRun[]): string {
|
|
425
|
+
const rows = [...runs]
|
|
426
|
+
.sort((a, b) => a.start.localeCompare(b.start))
|
|
427
|
+
.map((r) => [r.start.slice(0, 10), r.start.slice(11, 19), r.end ? r.end.slice(11, 19) : "-", durationText(r.start, r.end), r.machine, r.job, r.result]);
|
|
428
|
+
const head = ["date", "start", "end", "duration", "machine", "job", "result"];
|
|
429
|
+
const widths = head.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)));
|
|
430
|
+
return [head, ...rows].map((r) => r.map((c, i) => c.padEnd(widths[i])).join(" ").trimEnd()).join("\n");
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
async function nightly(config: Target, days: number): Promise<number> {
|
|
434
|
+
const since = Math.floor(Date.now() / 1000) - days * 24 * 3600;
|
|
435
|
+
const tasks = await api(config, "GET", `/nodes/${config.node}/tasks?typefilter=vzdump&since=${since}&limit=500`);
|
|
436
|
+
const journal = await hostOutput(config, ["nightly-journal", String(days)]);
|
|
437
|
+
printRaw(`${nightlyTable([...vzdumpRuns(config.node, tasks), ...journalRuns(journal)])}\n(times in UTC)`);
|
|
438
|
+
return 0;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
async function hostOutput(config: Target, args: string[]): Promise<string> {
|
|
344
442
|
const remote = buildHostRemote(args);
|
|
345
443
|
const result = await sshFixed(
|
|
346
444
|
{
|
|
@@ -352,8 +450,16 @@ async function host(config: Target, args: string[]): Promise<number> {
|
|
|
352
450
|
remote,
|
|
353
451
|
args[0] === "host-upgrade" ? HOST_UPGRADE_TIMEOUT_MS : 120_000,
|
|
354
452
|
);
|
|
355
|
-
if (result.
|
|
356
|
-
|
|
453
|
+
if (result.code !== 0) {
|
|
454
|
+
if (result.stdout.length > 0) printRaw(result.stdout);
|
|
455
|
+
throw new ServiceError(result.stderr.trim() || `${args[0]} exited ${result.code}`);
|
|
456
|
+
}
|
|
457
|
+
return result.stdout;
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
async function host(config: Target, args: string[]): Promise<number> {
|
|
461
|
+
const out = await hostOutput(config, args);
|
|
462
|
+
if (out.length > 0) printRaw(out);
|
|
357
463
|
return 0;
|
|
358
464
|
}
|
|
359
465
|
|
|
@@ -373,6 +479,10 @@ export async function command(args: string[]): Promise<number> {
|
|
|
373
479
|
printJson({ host: new URL(config.apiUrl).host, fingerprint256: probe.fingerprint });
|
|
374
480
|
return 0;
|
|
375
481
|
}
|
|
482
|
+
if (rest[0] === "nightly") {
|
|
483
|
+
if (rest.length > 2) throw new UsageError("nightly [days]");
|
|
484
|
+
return nightly(config, parseDays(rest[1]));
|
|
485
|
+
}
|
|
376
486
|
if (rest[0] !== undefined && rest[0] in HOST_COMMANDS) return host(config, rest);
|
|
377
487
|
return apiCommand(config, rest);
|
|
378
488
|
}
|
|
@@ -12,7 +12,7 @@ import { MAX_ADD_IDS, MAX_EXCLUDE_IDS, buildTrendarrCall, interpretRun, parseImd
|
|
|
12
12
|
import { ServiceError, UsageError } from "./lib/errors.ts";
|
|
13
13
|
import { safeRepoPath, safeWritePath } from "./lib/repo.ts";
|
|
14
14
|
import { appGetPath, appPolicyDelete, portForwardDelete, clientFixedIpWrite, syslogChanges, buildPulsarRemote, cloudPath, deviceNetworkWrite, NAT_PATH, natWrite, parseQuery, parseSiteArgs, redactUnifi, unifiPath, unifiWrite, command as networkCommand } from "./network/run.ts";
|
|
15
|
-
import { buildHostRemote, splitHost, needsConfirm, parseSetPairs, parseVmid, validateApiPath as validatePvePath, command as proxmoxCommand } from "./proxmox/run.ts";
|
|
15
|
+
import { buildHostRemote, journalRuns, nightlyTable, parseDays, splitHost, needsConfirm, parseSetPairs, parseVmid, validateApiPath as validatePvePath, vzdumpRuns, command as proxmoxCommand } from "./proxmox/run.ts";
|
|
16
16
|
import { banLength, buildObsRemote, compactDecisions, parseGoDuration, summariseAlerts, summariseVulns, summariseWazuh, vulnEvents, wazuhEvents, command as securityCommand } from "./security/run.ts";
|
|
17
17
|
|
|
18
18
|
describe("arcane wrapper", () => {
|
|
@@ -680,6 +680,46 @@ describe("proxmox wrapper", () => {
|
|
|
680
680
|
expect(buildHostRemote(["host-reboot", "--confirm"])).toBe("host-reboot --confirm");
|
|
681
681
|
expect(() => buildHostRemote(["host-reboot", "now"])).toThrow(/--confirm/);
|
|
682
682
|
expect(() => buildHostRemote(["shell"])).toThrow(/unknown host command/);
|
|
683
|
+
expect(buildHostRemote(["nightly-journal", "7"])).toBe("nightly-journal 7");
|
|
684
|
+
expect(() => buildHostRemote(["nightly-journal", "15"])).toThrow(/days/);
|
|
685
|
+
expect(() => buildHostRemote(["nightly-journal", "7; id"])).toThrow(/days/);
|
|
686
|
+
expect(parseDays(undefined)).toBe(7);
|
|
687
|
+
expect(() => parseDays("0")).toThrow(/days/);
|
|
688
|
+
});
|
|
689
|
+
|
|
690
|
+
it("turns the nightly journal and vzdump tasks into runs and a table", () => {
|
|
691
|
+
const journal = [
|
|
692
|
+
"2026-11-04T03:00:24+00:00 pulsar systemd[1]: Starting pulsar-config-backup.service - PVE host config backup to restic...",
|
|
693
|
+
"2026-11-04T03:00:42+00:00 pulsar systemd[1]: pulsar-config-backup.service: Deactivated successfully.",
|
|
694
|
+
"2026-11-04T03:00:42+00:00 pulsar systemd[1]: Finished pulsar-config-backup.service - PVE host config backup to restic.",
|
|
695
|
+
"2026-10-31T18:52:02+00:00 pulsar systemd[1]: Starting mira-backup-pull.service - Copy Mira guest backups from TESLA to the UNAS...",
|
|
696
|
+
"2026-10-31T18:52:05+00:00 pulsar systemd[1]: mira-backup-pull.service: Failed with result 'exit-code'.",
|
|
697
|
+
"2026-10-31T18:52:05+00:00 pulsar systemd[1]: Failed to start mira-backup-pull.service - Copy Mira guest backups from TESLA to the UNAS.",
|
|
698
|
+
"2026-11-03T03:30:01+00:00 arcane systemd[1]: Starting apt-daily-upgrade.service - Daily apt upgrade and clean activities...",
|
|
699
|
+
"2026-11-03T03:30:09+00:00 arcane systemd[1]: Finished apt-daily-upgrade.service - Daily apt upgrade and clean activities.",
|
|
700
|
+
"2026-11-03T03:30:22+00:00 arcane systemd[1]: docker.service: Deactivated successfully.",
|
|
701
|
+
"2026-11-03T03:30:32+00:00 arcane systemd[1]: Starting docker.service - Docker Application Container Engine...",
|
|
702
|
+
"2026-11-03T03:30:33+00:00 arcane systemd[1]: Started docker.service - Docker Application Container Engine.",
|
|
703
|
+
"2026-11-01T05:35:22+00:00 identity systemd[1]: Starting docker.service - Docker Application Container Engine...",
|
|
704
|
+
].join("\n");
|
|
705
|
+
expect(journalRuns(journal)).toEqual([
|
|
706
|
+
{ machine: "pulsar", job: "pulsar-config-backup", start: "2026-11-04T03:00:24.000Z", end: "2026-11-04T03:00:42.000Z", result: "ok" },
|
|
707
|
+
{ machine: "pulsar", job: "mira-backup-pull", start: "2026-10-31T18:52:02.000Z", end: "2026-10-31T18:52:05.000Z", result: "failed (exit-code)" },
|
|
708
|
+
{ machine: "arcane", job: "apt-daily-upgrade", start: "2026-11-03T03:30:01.000Z", end: "2026-11-03T03:30:09.000Z", result: "ok" },
|
|
709
|
+
{ machine: "arcane", job: "docker restart", start: "2026-11-03T03:30:22.000Z", end: "2026-11-03T03:30:33.000Z", result: "ok" },
|
|
710
|
+
]);
|
|
711
|
+
const tasks = [
|
|
712
|
+
{ type: "vzdump", id: "", starttime: 1793755800, endtime: 1793756268, status: "OK" },
|
|
713
|
+
{ type: "qmstart", id: "100", starttime: 1793755000, endtime: 1793755001, status: "OK" },
|
|
714
|
+
];
|
|
715
|
+
const runs = [...vzdumpRuns("pulsar", tasks), ...journalRuns(journal)];
|
|
716
|
+
expect(runs[0]).toEqual({ machine: "pulsar", job: "vzdump", start: "2026-11-04T01:30:00.000Z", end: "2026-11-04T01:37:48.000Z", result: "OK" });
|
|
717
|
+
expect(vzdumpRuns("pulsar", null)).toEqual([]);
|
|
718
|
+
const table = nightlyTable(runs).split("\n");
|
|
719
|
+
expect(table[0]).toBe("date start end duration machine job result");
|
|
720
|
+
expect(table[1]).toBe("2026-10-31 18:52:02 18:52:05 3s pulsar mira-backup-pull failed (exit-code)");
|
|
721
|
+
expect(table.at(-1)).toBe("2026-11-04 03:00:24 03:00:42 18s pulsar pulsar-config-backup ok");
|
|
722
|
+
expect(table).toContain("2026-11-04 01:30:00 01:37:48 7m48s pulsar vzdump OK");
|
|
683
723
|
});
|
|
684
724
|
|
|
685
725
|
it("prints help without touching config", async () => {
|