@astrofoundry/pi-astro 0.40.45 → 0.40.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/astro-discord/daily-report.test.ts +9 -2
- package/extensions/astro-discord/daily-report.ts +6 -2
- package/extensions/astro-discord/security-monitor.test.ts +2 -1
- package/extensions/astro-discord/security.test.ts +4 -1
- package/extensions/astro-discord/security.ts +4 -3
- package/extensions/astro-discord/update-policy.test.ts +1 -0
- package/extensions/astro-discord/update-policy.ts +5 -3
- package/package.json +1 -1
- package/skills/arcane/SKILL.md +2 -2
- package/skills/proxmox/SKILL.md +1 -0
- package/specialists/README.md +1 -1
- package/specialists/entry/remote/spc-proxmox-pulsar-entry.sh +17 -0
- package/specialists/entry/remote/spc-proxmox-spacex-entry.sh +17 -0
- package/specialists/lib/image-updates.test.ts +13 -8
- package/specialists/lib/image-updates.ts +5 -4
- package/specialists/proxmox/run.ts +113 -3
- package/specialists/wrappers.test.ts +41 -1
|
@@ -94,13 +94,20 @@ function entry(r: ArcaneReport, stack: string): StackEntry {
|
|
|
94
94
|
}
|
|
95
95
|
|
|
96
96
|
describe("Arcane Daily Report layout", () => {
|
|
97
|
+
it("marks the images our own CI builds", () => {
|
|
98
|
+
const own = { stack: "bigcapital", service: "server", ref: "ghcr.io/astrofoundry/bigcapital-server:astrofoundry", running: "?", latest: "new build", status: "update", action: "pull", note: "versions unavailable: private image" };
|
|
99
|
+
const built = buildStacks({ ...updateReport, rows: [rows[0], own] });
|
|
100
|
+
expect(built?.stacks.find((s) => s.stack === "bigcapital")?.lines[0].own).toBe(true);
|
|
101
|
+
expect(built?.stacks.find((s) => s.stack === "vpn")?.lines[0].own).toBeUndefined();
|
|
102
|
+
});
|
|
103
|
+
|
|
97
104
|
it("draws one section per stack with its services and the kind of change", () => {
|
|
98
105
|
const r = report();
|
|
99
106
|
entry(r, "stirling-pdf").lines[0].review = { verdict: "check", reason: "the config key changed" };
|
|
100
107
|
const [page] = layoutMessages(r);
|
|
101
108
|
expect(page).toEqual({ head: true, stacks: ["arcane", "bigcapital", "media", "postgres", "stirling-pdf", "supabase", "vpn", "x"] });
|
|
102
109
|
const components = messageComponents(r, page);
|
|
103
|
-
expect(components[0]).toEqual({ type: 10, content: `**${REPORT_PREFIX}** 6 stacks with updates (44 images, 2 private excluded, checked 06:00 CEST).` });
|
|
110
|
+
expect(components[0]).toEqual({ type: 10, content: `**${REPORT_PREFIX}** 6 stacks with updates (44 images, 2 pinned private excluded, checked 06:00 CEST).` });
|
|
104
111
|
const text = (c: (typeof components)[number]) => (c.type === 9 ? c.components?.[0].content : c.content);
|
|
105
112
|
expect(components.slice(1).map(text)).toEqual([
|
|
106
113
|
"**arcane**\n• arcane: v2.16.1 → v2.17.0 · new version (digest pin)",
|
|
@@ -142,7 +149,7 @@ describe("Arcane Daily Report layout", () => {
|
|
|
142
149
|
it("says so when nothing needs updating, warns about an unreadable holds file, and refuses a bad answer", () => {
|
|
143
150
|
const empty = buildStacks({ ...updateReport, rows: [], holds: {}, holdsError: "update-holds.json is not JSON" });
|
|
144
151
|
if (!empty) throw new Error("no report");
|
|
145
|
-
expect(headText({ ...empty, timeZone: "Europe/Madrid" }).content).toBe(`**${REPORT_PREFIX}** no updates (44 images, 2 private excluded, checked 06:00 CEST).\n-# ⚠️ update-holds.json is not JSON; Update still refuses held stacks`);
|
|
152
|
+
expect(headText({ ...empty, timeZone: "Europe/Madrid" }).content).toBe(`**${REPORT_PREFIX}** no updates (44 images, 2 pinned private excluded, checked 06:00 CEST).\n-# ⚠️ update-holds.json is not JSON; Update still refuses held stacks`);
|
|
146
153
|
expect(buildStacks({ nonsense: true })).toBeNull();
|
|
147
154
|
expect(failureMessage().startsWith(REPORT_PREFIX)).toBe(true);
|
|
148
155
|
});
|
|
@@ -4,6 +4,7 @@ import { parseImageRef } from "../../specialists/lib/registry.ts";
|
|
|
4
4
|
import { configDir } from "./config.ts";
|
|
5
5
|
import { escapeDiscord } from "./media.ts";
|
|
6
6
|
import type { MessageComponent } from "./rest.ts";
|
|
7
|
+
import { isOwnImage } from "./security.ts";
|
|
7
8
|
import type { ReleaseNote } from "../../specialists/lib/releases.ts";
|
|
8
9
|
import type { Review, Verdict } from "./tools-report.ts";
|
|
9
10
|
import type { Plan } from "./update-policy.ts";
|
|
@@ -79,6 +80,8 @@ export interface ServiceLine {
|
|
|
79
80
|
major: boolean;
|
|
80
81
|
note: string;
|
|
81
82
|
review?: Review;
|
|
83
|
+
/** The image is one our own CI builds, so its update goes through Upgrade carefully. */
|
|
84
|
+
own?: boolean;
|
|
82
85
|
}
|
|
83
86
|
|
|
84
87
|
export type StackState = "offered" | "running" | "updated" | "rolled back" | "failed" | "current" | "planning";
|
|
@@ -160,7 +163,8 @@ export function buildStacks(report: unknown): ReportStacks | null {
|
|
|
160
163
|
.map((r): ServiceLine => {
|
|
161
164
|
const running = releaseTag(r.running);
|
|
162
165
|
const latest = releaseTag(r.latest);
|
|
163
|
-
|
|
166
|
+
const own = r.ref ? isOwnImage(r.ref) : false;
|
|
167
|
+
return { service: r.service, status: r.status, action: r.action ?? "", running, latest, tag: tagOf(r.ref ?? ""), ...(r.ref ? { ref: r.ref } : {}), major: PENDING.has(r.status) && crossesMajor(running, latest), note: r.note ?? "", ...(own ? { own } : {}) };
|
|
164
168
|
});
|
|
165
169
|
const pending = lines.some((l) => PENDING.has(l.status));
|
|
166
170
|
const held = holds[stack];
|
|
@@ -287,7 +291,7 @@ type ReportView = Omit<ArcaneReport, "messages" | "date" | "channelId">;
|
|
|
287
291
|
|
|
288
292
|
export function headText(report: ReportView): MessageComponent {
|
|
289
293
|
const withUpdates = report.stacks.filter((s) => s.lines.some((l) => PENDING.has(l.status))).length;
|
|
290
|
-
const scope = `${report.images} images${report.privateCount > 0 ? `, ${report.privateCount} private excluded` : ""}, checked ${clock(report.checkedAt, report.timeZone)}`;
|
|
294
|
+
const scope = `${report.images} images${report.privateCount > 0 ? `, ${report.privateCount} pinned private excluded` : ""}, checked ${clock(report.checkedAt, report.timeZone)}`;
|
|
291
295
|
const head =
|
|
292
296
|
report.stacks.length === 0
|
|
293
297
|
? `**${REPORT_PREFIX}** no updates (${scope}).`
|
|
@@ -300,7 +300,8 @@ describe("security monitor", () => {
|
|
|
300
300
|
const summary = await h.monitor.dailyReport(SCHEDULE, "2026-10-21");
|
|
301
301
|
expect(h.rest.messages).toEqual([]);
|
|
302
302
|
const text = JSON.stringify(h.rest.components[0].components);
|
|
303
|
-
expect(text).toContain("🐳 **Arcane images** · 1 critical CVE whose package has a fix, in 1 image
|
|
303
|
+
expect(text).toContain("🐳 **Arcane images** · 1 critical CVE whose package has a fix, in 1 image\"");
|
|
304
|
+
expect(text).not.toContain("waiting on the publisher");
|
|
304
305
|
expect(h.calls).toContainEqual(["arcane", "images", "list", "--all"]);
|
|
305
306
|
expect(h.calls).toContainEqual(["arcane", "images", "vulnerabilities", "list", "--severity", "critical", "--all"]);
|
|
306
307
|
expect(summary).toEqual({ headline: "quiet day (24 h to 06:05 CEST)", tagged: false, messageId: "100" });
|
|
@@ -433,7 +433,10 @@ describe("Arcane image scans", () => {
|
|
|
433
433
|
expect(offeredImages(null, offers)).toEqual([]);
|
|
434
434
|
expect(imageVulnLine(null, offers)).toBe("Arcane images · the check failed");
|
|
435
435
|
expect(imageVulnLine({ scanned: 0, total: 57, critical: [] }, offers)).toBe("Arcane images · no Trivy scan yet (57 images)");
|
|
436
|
-
expect(imageVulnLine({ ...facts, critical: facts.critical.slice(4) }, new Map(), undefined,
|
|
436
|
+
expect(imageVulnLine({ ...facts, critical: facts.critical.slice(4) }, new Map(), undefined, 1)).toBe("Arcane images · 1 critical CVE whose package has a fix, in 1 image · waiting on the publisher: nginx 1");
|
|
437
|
+
expect(imageVulnLine(facts, offers, undefined, 0)).toBe(
|
|
438
|
+
"Arcane images · 5 critical CVEs whose package has a fix, in 3 images · update offered in the Arcane Daily Report: mealie 3 (mealie) · held: postgres:15.8.1.085 1 (supabase: coordinated release: review first)",
|
|
439
|
+
);
|
|
437
440
|
});
|
|
438
441
|
|
|
439
442
|
it("names our own images with their build date and a newer build waiting in the registry", () => {
|
|
@@ -661,7 +661,7 @@ export function offeredImages(facts: ImageVulnFacts | null, offers: ReadonlyMap<
|
|
|
661
661
|
return [...stacks].sort();
|
|
662
662
|
}
|
|
663
663
|
|
|
664
|
-
function isOwnImage(image: string): boolean {
|
|
664
|
+
export function isOwnImage(image: string): boolean {
|
|
665
665
|
const repo = repositoryOf(image);
|
|
666
666
|
return repo !== null && OWN_IMAGE_PREFIXES.some((p) => repo.startsWith(p));
|
|
667
667
|
}
|
|
@@ -686,6 +686,7 @@ function buildText(build: ImageBuild | undefined): string {
|
|
|
686
686
|
* Critical CVEs whose package has a fixed version, grouped by who can act:
|
|
687
687
|
* our own images (with their build date), images an update in the Arcane
|
|
688
688
|
* Daily Report replaces, held stacks, and images waiting on their publisher.
|
|
689
|
+
* A `maxWaiting` of 0 leaves the publisher group out: nobody here can act on it.
|
|
689
690
|
*/
|
|
690
691
|
export function imageVulnLine(facts: ImageVulnFacts | null, offers: ReadonlyMap<string, ImageOffer>, arcaneChannelId?: string, maxWaiting = 5): string {
|
|
691
692
|
if (!facts) return "Arcane images · the check failed";
|
|
@@ -712,7 +713,7 @@ export function imageVulnLine(facts: ImageVulnFacts | null, offers: ReadonlyMap<
|
|
|
712
713
|
if (offered.length > 0) parts.push(`update offered${arcaneChannelId ? ` in <#${arcaneChannelId}>` : " in the Arcane Daily Report"}: ${offered.join(", ")}`);
|
|
713
714
|
if (held.length > 0) parts.push(`held: ${held.join(", ")}`);
|
|
714
715
|
const rest = waiting.length - Math.min(waiting.length, maxWaiting);
|
|
715
|
-
if (waiting.length > 0) parts.push(`waiting on the publisher: ${[...waiting.slice(0, maxWaiting), ...(rest > 0 ? [`+${rest} more`] : [])].join(", ")}`);
|
|
716
|
+
if (waiting.length > 0 && maxWaiting > 0) parts.push(`waiting on the publisher: ${[...waiting.slice(0, maxWaiting), ...(rest > 0 ? [`+${rest} more`] : [])].join(", ")}`);
|
|
716
717
|
return parts.join(" · ");
|
|
717
718
|
}
|
|
718
719
|
|
|
@@ -783,7 +784,7 @@ export function dailyCard(d: DailyData, owners: readonly string[]): ReportCard {
|
|
|
783
784
|
section("👁️", d.wazuh.line),
|
|
784
785
|
section("📦", vulnLine(d.found)),
|
|
785
786
|
section("🩹", patchLine(d.closed)),
|
|
786
|
-
section("🐳", imageVulnLine(d.images, d.offers, d.arcaneChannelId)),
|
|
787
|
+
section("🐳", imageVulnLine(d.images, d.offers, d.arcaneChannelId, 0)),
|
|
787
788
|
section("🩺", healthLine(d.attention)),
|
|
788
789
|
];
|
|
789
790
|
return {
|
|
@@ -13,6 +13,7 @@ describe("update policy", () => {
|
|
|
13
13
|
|
|
14
14
|
it("plans carefully when notes need attention, even for a minor update", () => {
|
|
15
15
|
expect(stackPlan([{ major: false, review: check }])).toBe("careful");
|
|
16
|
+
expect(stackPlan([{ major: false, review: none }, { major: false, own: true }])).toBe("careful");
|
|
16
17
|
expect(stackPlan([{ major: true, review: none }, { major: false, review: { verdict: "breaking", reason: "x" } }])).toBe("careful");
|
|
17
18
|
});
|
|
18
19
|
|
|
@@ -4,7 +4,7 @@ import type { Review } from "./tools-report.ts";
|
|
|
4
4
|
* What the morning run does with an offered update (approved 2026-10-23):
|
|
5
5
|
* - `auto`: applied without anyone and checked again minutes later;
|
|
6
6
|
* - `update`: the owners are tagged, and Update runs it on its own (a major whose notes raise nothing);
|
|
7
|
-
* - `careful`: the owners are tagged, and Upgrade carefully plans it in a thread first (the notes need attention,
|
|
7
|
+
* - `careful`: the owners are tagged, and Upgrade carefully plans it in a thread first (the notes need attention, there are none for a major, or the image is one we build);
|
|
8
8
|
* - `button`: offered without a tag (Pi, the bridge's own runtime, never updates itself).
|
|
9
9
|
*/
|
|
10
10
|
export type Plan = "auto" | "update" | "careful" | "button";
|
|
@@ -16,12 +16,14 @@ export function needsAttention(review: Review | undefined): boolean {
|
|
|
16
16
|
interface PlannedLine {
|
|
17
17
|
major: boolean;
|
|
18
18
|
review?: Review;
|
|
19
|
+
/** An image our own CI builds: it has no release notes, and its stack's README sets the rollout (a queue drain, a database backup). */
|
|
20
|
+
own?: boolean;
|
|
19
21
|
}
|
|
20
22
|
|
|
21
|
-
/** One stack's pending updates:
|
|
23
|
+
/** One stack's pending updates: an image of ours or a line whose notes need attention makes it careful; a major is careful unless its notes say nothing relevant. */
|
|
22
24
|
export function stackPlan(pending: readonly PlannedLine[]): Plan | null {
|
|
23
25
|
if (pending.length === 0) return null;
|
|
24
|
-
if (pending.some((l) => needsAttention(l.review))) return "careful";
|
|
26
|
+
if (pending.some((l) => l.own || needsAttention(l.review))) return "careful";
|
|
25
27
|
const majors = pending.filter((l) => l.major);
|
|
26
28
|
if (majors.length === 0) return "auto";
|
|
27
29
|
return majors.every((l) => l.review?.verdict === "none") ? "update" : "careful";
|
package/package.json
CHANGED
package/skills/arcane/SKILL.md
CHANGED
|
@@ -36,9 +36,9 @@ The output is JSON with `counts`, `rows`, and `table`. Answer with the `table` f
|
|
|
36
36
|
- `Running` and `Latest` are release tags. `older build`: the running build carries no tag any more. `new build`: no release tag names the newer build. `17 (older build)`: the running build of that major series is no longer tagged.
|
|
37
37
|
- `Update`: `pull` means redeploying the stack with an image pull applies it; `edit tag` means changing the tag in the GitOps repository; `edit digest` means changing the pinned digest in the repository (`latestDigest` in the row holds it); `unknown` has its reason in the notes under the table.
|
|
38
38
|
- A note "major version change" or "newer major ... available" means the update crosses a major version. Read its release notes yourself before asking anything (next section).
|
|
39
|
-
- Private images (the registry refuses anonymous access, such as
|
|
39
|
+
- Private images (the registry refuses anonymous access, such as graphile-worker) come from Arcane's own check, which uses its registry login: their rows show `new build` without version names. Digest-pinned private images (the Grimoire builds, moved only by their release workflow) are left out of the table and the counts; the summary line counts them and `privateImages` lists them. Mention them only when the caller asks.
|
|
40
40
|
|
|
41
|
-
Messages that start with "Arcane Daily Report:" are the bridge's scheduled report, built from `update-report` in the morning run without you: one section per stack with updates and a release-notes verdict on every pending update. Stacks without a major version and without notes that need attention (including an Arcane manager release) are updated by the bridge right after the post, without you, and checked again five minutes later; a stack that fails then is rolled back where possible and handed to `astro.foreman`. A major whose notes say nothing relevant gets an Update button that owners press, which runs `stack-update` without you; anything else gets Upgrade carefully (next sections). When the caller replies to one, its list is the context ("update these", "why is this a new build"); run `update-report` again only when the caller wants fresh data.
|
|
41
|
+
Messages that start with "Arcane Daily Report:" are the bridge's scheduled report, built from `update-report` in the morning run without you: one section per stack with updates and a release-notes verdict on every pending update. Stacks without a major version and without notes that need attention (including an Arcane manager release) are updated by the bridge right after the post, without you, and checked again five minutes later; a stack that fails then is rolled back where possible and handed to `astro.foreman`. A major whose notes say nothing relevant gets an Update button that owners press, which runs `stack-update` without you; anything else, and every update of an image our own CI builds (ghcr.io astronaute77, astrofoundry, monadeo: the stack README sets its rollout), gets Upgrade carefully (next sections). When the caller replies to one, its list is the context ("update these", "why is this a new build"); run `update-report` again only when the caller wants fresh data.
|
|
42
42
|
|
|
43
43
|
## Applying updates
|
|
44
44
|
|
package/skills/proxmox/SKILL.md
CHANGED
|
@@ -44,6 +44,7 @@ Power actions and snapshot changes return `{ upid, exitstatus }`; `OK` means don
|
|
|
44
44
|
| `host-upgrade` | apt full-upgrade of the host (risky, needs approval); a new Proxmox VE major is held and printed as `held:`; ends with `reboot-needed:`. The morning run does this on its own each day |
|
|
45
45
|
| `host-reboot --confirm` | reboot the host in one minute (risky): every guest on it restarts. Only on the owner's explicit request; the morning run offers a Reboot button instead |
|
|
46
46
|
| `guest-exec <vmid> <status|journal|df|updates>` | fixed read inside a guest: failed units and uptime, warning journal of 24 h, disk usage, upgradable packages |
|
|
47
|
+
| `nightly [days]` | start, end, duration (UTC) and result of each nightly job of the last 1..14 days (default 7): vzdump tasks from the API, then from the journals of the host and every running guest the host timers, `apt-daily-upgrade`, the backup and dump timers, and Docker restarts. Answer questions about nightly run times with it |
|
|
47
48
|
|
|
48
49
|
## Guest change procedure
|
|
49
50
|
|
package/specialists/README.md
CHANGED
|
@@ -11,7 +11,7 @@ A specialist is a Pi subagent that is the only way to operate one area of the ho
|
|
|
11
11
|
| `astro.edge` | Frontdoor VPS (`frontdoor-1337`) over an IAP tunnel; `00-frontdoor-vps/` in the homelab repository | `probe` the public path; `route-probe` (status of the Supabase, Grimoire, and Mealie public routes from Cortex); `vps status|nginx-conf|nginx-test|journal|wg|bouncer|updates|package|changelog|auto-upgrades|deploy-nginx|reboot --confirm`; `git` on the homelab checkout, writes limited to `00-frontdoor-vps/` |
|
|
12
12
|
| `astro.security` | Wazuh, CrowdSec, rsyslog archive on VM 106 | Read-only: `status`, `attention`, `agents`, `alerts`, `decisions`, `bouncers`, `metrics`, `wazuh-alerts`, `wazuh-vulns`, `wazuh-events`, `wazuh-vuln-events`, `wazuh-log`, `remote-hosts`, `remote-log`, `journal` |
|
|
13
13
|
| `astro.backup` | vzdump, the shared Restic repository (Pulsar host config, Arcane app backups), GCS offsite from LXC 102 | Status, journals, and manual runs of every layer; `restic-snapshots|stats|ls`, `restic-restore` into `/var/tmp/spc-restore` on Pulsar, `restore-clean` |
|
|
14
|
-
| `astro.proxmox` | Proxmox VE on Pulsar (37) and SPACE-X (Mira) | API token: `guests`, `guest`, `get <path>`, `tasks`; `start`, `shutdown|reboot|stop --confirm`; snapshots create, delete, rollback; `set` of CPU, memory, options, network. Host key: `host-status`, `host-journal`, `updates`, `host-upgrade` (apt full-upgrade; a new Proxmox VE major is held), `host-reboot --confirm`, `guest-exec <vmid> <status|journal|df|updates
|
|
14
|
+
| `astro.proxmox` | Proxmox VE on Pulsar (37) and SPACE-X (Mira) | API token: `guests`, `guest`, `get <path>`, `tasks`; `start`, `shutdown|reboot|stop --confirm`; snapshots create, delete, rollback; `set` of CPU, memory, options, network. Host key: `host-status`, `host-journal`, `updates`, `host-upgrade` (apt full-upgrade; a new Proxmox VE major is held), `host-reboot --confirm`, `guest-exec <vmid> <status|journal|df|updates>`, `nightly-journal <days>`; `nightly [days]` combines the vzdump tasks with that journal into a table of nightly run times. Every command runs on Pulsar, or on SPACE-X with the `space-x` prefix |
|
|
15
15
|
| `astro.inference` | MLX embedding and reranking servers on Nexus, Europa health | `nexus-status|start|stop|restart|log|disk`, `macos-updates`; `europa-health` |
|
|
16
16
|
| `astro.media` | trendarr over TMDB, OMDb, and the `media` project on VM 100 (Radarr, Sonarr, SABnzbd) | `trending`, `search`, `recommend`, `add-movies`, `add-series` (at most 20 IMDb IDs), `sabnzbd status|pause|resume`; always trendarr's JSON plus its warnings |
|
|
17
17
|
|
|
@@ -35,6 +35,14 @@ guest_cmd() {
|
|
|
35
35
|
esac
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
# The nightly jobs on the host and in the guests; a unit a machine does not have prints nothing.
|
|
39
|
+
NIGHTLY_UNITS="pulsar-config-backup mira-backup-pull apt-daily-upgrade docker arcane-backup pgdump-zitadel lego-renew gcs-backup"
|
|
40
|
+
nightly_cmd() {
|
|
41
|
+
units=""
|
|
42
|
+
for u in $NIGHTLY_UNITS; do units="$units -u $u.service"; done
|
|
43
|
+
echo "journalctl --utc -o short-iso --no-pager --since '$1 days ago' $units | grep -E 'systemd\[1\]: (Starting|Started|Finished|Failed|[^ ]+\.service: (Deactivated|Failed))' || true"
|
|
44
|
+
}
|
|
45
|
+
|
|
38
46
|
case "$name" in
|
|
39
47
|
host-status)
|
|
40
48
|
pveversion
|
|
@@ -62,6 +70,15 @@ case "$name" in
|
|
|
62
70
|
[ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
|
|
63
71
|
shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
|
|
64
72
|
echo "reboot in 1 minute" ;;
|
|
73
|
+
nightly-journal)
|
|
74
|
+
case "$rest" in [1-9]|1[0-4]) ;; *) echo "days must be 1..14" >&2; exit 2 ;; esac
|
|
75
|
+
inner="$(nightly_cmd "$rest")"
|
|
76
|
+
sh -c "$inner"
|
|
77
|
+
for vmid in $(pct list | awk 'NR > 1 && $2 == "running" { print $1 }'); do pct exec "$vmid" -- sh -c "$inner" || true; done
|
|
78
|
+
# The guest agent wraps a VM's output in JSON; only its out-data is journal lines.
|
|
79
|
+
for vmid in $(qm list | awk 'NR > 1 && $3 == "running" { print $1 }'); do
|
|
80
|
+
qm guest exec "$vmid" --timeout 60 -- sh -c "$inner" | perl -MJSON::PP -0777 -ne 'my $r = eval { decode_json($_) }; print $r->{"out-data"} // "" if $r' || true
|
|
81
|
+
done ;;
|
|
65
82
|
guest-exec)
|
|
66
83
|
vmid="${rest%% *}"; action="${rest#* }"
|
|
67
84
|
case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
|
|
@@ -35,6 +35,14 @@ guest_cmd() {
|
|
|
35
35
|
esac
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
# The nightly jobs on the host and in the guests; a unit a machine does not have prints nothing.
|
|
39
|
+
NIGHTLY_UNITS="pulsar-config-backup mira-backup-pull apt-daily-upgrade docker arcane-backup pgdump-zitadel lego-renew gcs-backup"
|
|
40
|
+
nightly_cmd() {
|
|
41
|
+
units=""
|
|
42
|
+
for u in $NIGHTLY_UNITS; do units="$units -u $u.service"; done
|
|
43
|
+
echo "journalctl --utc -o short-iso --no-pager --since '$1 days ago' $units | grep -E 'systemd\[1\]: (Starting|Started|Finished|Failed|[^ ]+\.service: (Deactivated|Failed))' || true"
|
|
44
|
+
}
|
|
45
|
+
|
|
38
46
|
case "$name" in
|
|
39
47
|
host-status)
|
|
40
48
|
pveversion
|
|
@@ -62,6 +70,15 @@ case "$name" in
|
|
|
62
70
|
[ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
|
|
63
71
|
shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
|
|
64
72
|
echo "reboot in 1 minute" ;;
|
|
73
|
+
nightly-journal)
|
|
74
|
+
case "$rest" in [1-9]|1[0-4]) ;; *) echo "days must be 1..14" >&2; exit 2 ;; esac
|
|
75
|
+
inner="$(nightly_cmd "$rest")"
|
|
76
|
+
sh -c "$inner"
|
|
77
|
+
for vmid in $(pct list | awk 'NR > 1 && $2 == "running" { print $1 }'); do pct exec "$vmid" -- sh -c "$inner" || true; done
|
|
78
|
+
# The guest agent wraps a VM's output in JSON; only its out-data is journal lines.
|
|
79
|
+
for vmid in $(qm list | awk 'NR > 1 && $3 == "running" { print $1 }'); do
|
|
80
|
+
qm guest exec "$vmid" --timeout 60 -- sh -c "$inner" | perl -MJSON::PP -0777 -ne 'my $r = eval { decode_json($_) }; print $r->{"out-data"} // "" if $r' || true
|
|
81
|
+
done ;;
|
|
65
82
|
guest-exec)
|
|
66
83
|
vmid="${rest%% *}"; action="${rest#* }"
|
|
67
84
|
case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
|
|
@@ -69,6 +69,7 @@ describe("resolveRef", () => {
|
|
|
69
69
|
"library/nginx": { tags: ["1.31.4-alpine"], digests: {} },
|
|
70
70
|
"getarcaneapp/manager": { tags: ["v1.2.0", "v1.3.0", "latest"], digests: { "v1.3.0": "sha256:new", "v1.2.0": "sha256:old" } },
|
|
71
71
|
"astrofoundry/app": "private",
|
|
72
|
+
"monadeo/server": "private",
|
|
72
73
|
});
|
|
73
74
|
|
|
74
75
|
it("resolves a floating tag to the release versions behind both digests", async () => {
|
|
@@ -99,8 +100,11 @@ describe("resolveRef", () => {
|
|
|
99
100
|
expect(await resolveRef("ghcr.io/getarcaneapp/manager@sha256:new", undefined, registry, NOW)).toMatchObject({ running: "v1.3.0", status: "up-to-date" });
|
|
100
101
|
});
|
|
101
102
|
|
|
102
|
-
it("falls back to
|
|
103
|
-
|
|
103
|
+
it("falls back to Arcane's digest check for private images, leaves out private digest pins, and marks bad checks unknown", async () => {
|
|
104
|
+
const own = await resolveRef("ghcr.io/astrofoundry/app:main", entry("A1", "A2"), registry, NOW);
|
|
105
|
+
expect(own).toMatchObject({ status: "update", latest: "new build", action: "pull", note: "versions unavailable: private image" });
|
|
106
|
+
expect(own.private).toBeUndefined();
|
|
107
|
+
expect(await resolveRef("ghcr.io/monadeo/server:v1.0.0@sha256:abc", undefined, registry, NOW)).toMatchObject({ status: "unknown", private: true });
|
|
104
108
|
expect(await resolveRef("nginx:1.31.4-alpine", undefined, registry, NOW)).toMatchObject({ status: "unknown", note: "not in the fresh check" });
|
|
105
109
|
expect(await resolveRef("nginx:1.31.4-alpine", entry("N", "N", { checkTime: "2026-09-24T00:00:00Z" }), registry, NOW)).toMatchObject({ status: "unknown", note: expect.stringContaining("stale") });
|
|
106
110
|
expect(await resolveRef("nginx:1.31.4-alpine", entry("N", "N", { error: "timeout" }), registry, NOW)).toMatchObject({ status: "unknown", note: "check failed: timeout" });
|
|
@@ -121,6 +125,7 @@ describe("buildUpdateReport", () => {
|
|
|
121
125
|
{ name: "canary", status: "stopped", updateInfo: { serviceUpdates: { canary: { imageRef: "busybox:latest" } } } },
|
|
122
126
|
{ name: "empty", status: "running" },
|
|
123
127
|
{ name: "ours", status: "running", updateInfo: { serviceUpdates: { api: { imageRef: "ghcr.io/astrofoundry/app:main" }, worker: { imageRef: "ghcr.io/astrofoundry/app:main" } } } },
|
|
128
|
+
{ name: "pinned", status: "running", updateInfo: { serviceUpdates: { server: { imageRef: "ghcr.io/monadeo/server:v1.0.0@sha256:abc" } } } },
|
|
124
129
|
],
|
|
125
130
|
};
|
|
126
131
|
const check = {
|
|
@@ -130,15 +135,15 @@ describe("buildUpdateReport", () => {
|
|
|
130
135
|
"ghcr.io/astrofoundry/app:main": entry("A1", "A2"),
|
|
131
136
|
};
|
|
132
137
|
const report = await buildUpdateReport({ check, projects, registry, startedAt: NOW, all: false });
|
|
133
|
-
expect(report.counts).toEqual({ update:
|
|
134
|
-
expect(report.rows.map((r) => `${r.stack}/${r.service}:${r.status}`)).toEqual(["media/prowlarr:update", "empty/empty:unknown"]);
|
|
135
|
-
expect(report.table).toMatch(/^Fresh check 2026-09-25T18:30:00Z:
|
|
136
|
-
expect(report.privateImages).toEqual(["ghcr.io/
|
|
137
|
-
expect(report.rows.some((r) => r.stack === "
|
|
138
|
+
expect(report.counts).toEqual({ update: 3, release: 0, rebuild: 0, unknown: 1, "up-to-date": 2 });
|
|
139
|
+
expect(report.rows.map((r) => `${r.stack}/${r.service}:${r.status}`)).toEqual(["media/prowlarr:update", "ours/api:update", "ours/worker:update", "empty/empty:unknown"]);
|
|
140
|
+
expect(report.table).toMatch(/^Fresh check 2026-09-25T18:30:00Z: 6 services in 6 stacks, 4 images\. 1 pinned private images \(1 services\) excluded\./);
|
|
141
|
+
expect(report.privateImages).toEqual(["ghcr.io/monadeo/server:v1.0.0@sha256:abc"]);
|
|
142
|
+
expect(report.rows.some((r) => r.stack === "pinned")).toBe(false);
|
|
138
143
|
expect(report.table).toContain("media/prowlarr prowlarr 2.0.5.5160-ls123 2.1.0.5190-ls124 pull");
|
|
139
144
|
expect(report.table).toContain("* empty: Arcane lists no images for this stack");
|
|
140
145
|
const all = await buildUpdateReport({ check, projects, registry, startedAt: NOW, all: true });
|
|
141
|
-
expect(all.rows).toHaveLength(
|
|
146
|
+
expect(all.rows).toHaveLength(6);
|
|
142
147
|
expect(all.table).toContain("* canary: stack is stopped");
|
|
143
148
|
});
|
|
144
149
|
|
|
@@ -34,7 +34,7 @@ export interface UpdateReport {
|
|
|
34
34
|
checkedAt: string;
|
|
35
35
|
stacks: number;
|
|
36
36
|
images: number;
|
|
37
|
-
/**
|
|
37
|
+
/** Digest-pinned private image references, excluded from `rows`, `counts`, and the table: only their own release workflow moves them. */
|
|
38
38
|
privateImages: string[];
|
|
39
39
|
counts: Record<UpdateStatus, number>;
|
|
40
40
|
rows: UpdateRow[];
|
|
@@ -152,7 +152,7 @@ interface Resolution {
|
|
|
152
152
|
action: string;
|
|
153
153
|
note: string;
|
|
154
154
|
latestDigest?: string;
|
|
155
|
-
/** The registry refused anonymous access: an image of our own, left out of the report. */
|
|
155
|
+
/** The registry refused anonymous access to a digest pin: an image of our own that only its release workflow moves, left out of the report. */
|
|
156
156
|
private?: boolean;
|
|
157
157
|
}
|
|
158
158
|
|
|
@@ -179,7 +179,8 @@ export async function resolveRef(refText: string, entry: CheckEntry | undefined,
|
|
|
179
179
|
tags = (await registry.tags(ref)).map(parseTag).filter((t): t is TagVersion => t !== null);
|
|
180
180
|
} catch (err) {
|
|
181
181
|
const status = err instanceof RegistryError ? err.status : 0;
|
|
182
|
-
|
|
182
|
+
// Arcane checks a private image with its own registry login, so only the version names are missing.
|
|
183
|
+
if (status === 401 || status === 403) return kind === "pinned" ? { ...unknown("pinned private image"), private: true } : digestOnly("private image");
|
|
183
184
|
return digestOnly(err instanceof Error ? err.message : String(err));
|
|
184
185
|
}
|
|
185
186
|
|
|
@@ -377,7 +378,7 @@ export async function buildUpdateReport(input: { check: unknown; projects: unkno
|
|
|
377
378
|
const shown = input.all ? rows : rows.filter((r) => r.status !== "up-to-date" || r.note.includes("newer major"));
|
|
378
379
|
const checkedAt = new Date(input.startedAt).toISOString().replace(/\.\d+Z$/, "Z");
|
|
379
380
|
const stacks = new Set(services.map((s) => s.stack)).size;
|
|
380
|
-
const excluded = privateImages.length === 0 ? "" : ` ${privateImages.length} private images (${services.length - listed.length} services) excluded.`;
|
|
381
|
+
const excluded = privateImages.length === 0 ? "" : ` ${privateImages.length} pinned private images (${services.length - listed.length} services) excluded.`;
|
|
381
382
|
const summary =
|
|
382
383
|
`Fresh check ${checkedAt}: ${listed.length} services in ${stacks} stacks, ${refs.length - privateImages.length} images.${excluded} ` +
|
|
383
384
|
`${counts.update} new builds, ${counts.release} new releases, ${counts.rebuild} rebuilds, ${counts.unknown} unknown, ${counts["up-to-date"]} up to date` +
|
|
@@ -88,11 +88,14 @@ export const HOST_COMMANDS: Readonly<Record<string, { args: number; help: string
|
|
|
88
88
|
"host-upgrade": { args: 0, help: "apt full-upgrade of the host; a new Proxmox VE major is held (prints held:); ends with reboot-needed" },
|
|
89
89
|
"host-reboot": { args: 1, help: "host-reboot --confirm reboot the host in one minute; every guest on it restarts" },
|
|
90
90
|
"guest-exec": { args: 2, help: "guest-exec <vmid> <status|journal|df|updates> fixed read inside a guest (pct exec or qm guest exec)" },
|
|
91
|
+
"nightly-journal": { args: 1, help: "nightly-journal <days> raw systemd start and end lines of the nightly jobs on the host and in every running guest" },
|
|
91
92
|
};
|
|
92
93
|
export const GUEST_EXEC = ["status", "journal", "df", "updates"];
|
|
93
94
|
/** A full upgrade with a new kernel and pve-manager can take several minutes on the host. */
|
|
94
95
|
export const HOST_UPGRADE_TIMEOUT_MS = 20 * 60_000;
|
|
95
96
|
const SINCE = /^[A-Za-z0-9 :\-+]{1,40}$/;
|
|
97
|
+
/** Nights `nightly` reads; the entry scripts accept 1..14. */
|
|
98
|
+
export const NIGHTLY_DAYS = { default: 7, max: 14 };
|
|
96
99
|
|
|
97
100
|
const HELP = `proxmox specialist (Proxmox VE on Pulsar at 37 and SPACE-X at Mira)
|
|
98
101
|
|
|
@@ -109,6 +112,7 @@ const HELP = `proxmox specialist (Proxmox VE on Pulsar at 37 and SPACE-X at Mira
|
|
|
109
112
|
snapshot-delete <vmid> <name> --confirm
|
|
110
113
|
snapshot-rollback <vmid> <name> --confirm
|
|
111
114
|
set <vmid> key=value... VM: ${SET_KEYS.qemu.join(", ")}, netN; CT: ${SET_KEYS.lxc.join(", ")}, netN
|
|
115
|
+
nightly [days] start, end, and duration (UTC) of each nightly job of the last days (default ${NIGHTLY_DAYS.default}, max ${NIGHTLY_DAYS.max}): vzdump tasks, host timers, and each running guest's updates, backups, and Docker restarts
|
|
112
116
|
${Object.entries(HOST_COMMANDS)
|
|
113
117
|
.map(([name, c]) => ` ${name.padEnd(35)} ${c.help}`)
|
|
114
118
|
.join("\n")}
|
|
@@ -333,6 +337,7 @@ export function buildHostRemote(args: string[]): string {
|
|
|
333
337
|
if (rest.length !== spec.args) throw new UsageError(`${name} takes ${spec.args} argument(s)`);
|
|
334
338
|
if (name === "host-journal" && !SINCE.test(rest[0])) throw new UsageError("since: letters, digits, spaces, colon, plus, minus only");
|
|
335
339
|
if (name === "host-reboot" && rest[0] !== "--confirm") throw new UsageError("host-reboot restarts every guest on the host; add --confirm when the task asks for it explicitly");
|
|
340
|
+
if (name === "nightly-journal") parseDays(rest[0]);
|
|
336
341
|
if (name === "guest-exec") {
|
|
337
342
|
parseVmid(rest[0]);
|
|
338
343
|
if (!GUEST_EXEC.includes(rest[1])) throw new UsageError(`guest-exec action must be one of ${GUEST_EXEC.join(", ")}`);
|
|
@@ -340,7 +345,100 @@ export function buildHostRemote(args: string[]): string {
|
|
|
340
345
|
return [name, ...rest].join(" ");
|
|
341
346
|
}
|
|
342
347
|
|
|
343
|
-
|
|
348
|
+
export function parseDays(text: string | undefined): number {
|
|
349
|
+
const days = text === undefined ? NIGHTLY_DAYS.default : Number(text);
|
|
350
|
+
if (!Number.isInteger(days) || days < 1 || days > NIGHTLY_DAYS.max) throw new UsageError(`days must be 1..${NIGHTLY_DAYS.max}`);
|
|
351
|
+
return days;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** One run of a nightly job on one machine; a Docker restart runs from the daemon's stop to its start. */
|
|
355
|
+
export interface NightlyRun {
|
|
356
|
+
machine: string;
|
|
357
|
+
job: string;
|
|
358
|
+
start: string;
|
|
359
|
+
end: string | null;
|
|
360
|
+
result: string;
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
const JOURNAL_LINE = /^(\S+) (\S+) systemd\[1\]: (.*)$/;
|
|
364
|
+
|
|
365
|
+
/** Runs from the `nightly-journal` lines: Starting to Finished or Failed, and Docker from Deactivated to Started. */
|
|
366
|
+
export function journalRuns(text: string): NightlyRun[] {
|
|
367
|
+
const runs: NightlyRun[] = [];
|
|
368
|
+
const open = new Map<string, NightlyRun>();
|
|
369
|
+
const close = (key: string, end: string, result: string) => {
|
|
370
|
+
const run = open.get(key);
|
|
371
|
+
if (!run) return;
|
|
372
|
+
run.end = end;
|
|
373
|
+
run.result = result;
|
|
374
|
+
open.delete(key);
|
|
375
|
+
};
|
|
376
|
+
for (const line of text.split("\n")) {
|
|
377
|
+
const m = JOURNAL_LINE.exec(line.trim());
|
|
378
|
+
if (!m) continue;
|
|
379
|
+
const [, time, machine, message] = m;
|
|
380
|
+
const at = new Date(time).toISOString();
|
|
381
|
+
const unit = /([\w@.-]+)\.service/.exec(message)?.[1];
|
|
382
|
+
if (!unit) continue;
|
|
383
|
+
const key = `${machine} ${unit}`;
|
|
384
|
+
if (unit === "docker") {
|
|
385
|
+
if (/: Deactivated successfully/.test(message)) {
|
|
386
|
+
const run = { machine, job: "docker restart", start: at, end: null, result: "running" };
|
|
387
|
+
runs.push(run);
|
|
388
|
+
open.set(key, run);
|
|
389
|
+
} else if (/^Started /.test(message)) close(key, at, "ok");
|
|
390
|
+
continue;
|
|
391
|
+
}
|
|
392
|
+
if (/^Starting /.test(message)) {
|
|
393
|
+
const run = { machine, job: unit, start: at, end: null, result: "running" };
|
|
394
|
+
runs.push(run);
|
|
395
|
+
open.set(key, run);
|
|
396
|
+
} else if (/^Finished /.test(message)) close(key, at, "ok");
|
|
397
|
+
else if (/: Failed with result '([^']+)'/.test(message)) close(key, at, `failed (${/'([^']+)'/.exec(message)?.[1] ?? "?"})`);
|
|
398
|
+
else if (/^Failed to start /.test(message)) close(key, at, "failed");
|
|
399
|
+
}
|
|
400
|
+
return runs;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/** Runs from `/nodes/<node>/tasks?typefilter=vzdump`: one per backup task, the whole job when it names no guest. */
|
|
404
|
+
export function vzdumpRuns(node: string, tasks: unknown): NightlyRun[] {
|
|
405
|
+
if (!Array.isArray(tasks)) return [];
|
|
406
|
+
return (tasks as { type?: unknown; id?: unknown; starttime?: unknown; endtime?: unknown; status?: unknown }[])
|
|
407
|
+
.filter((t) => t.type === "vzdump" && typeof t.starttime === "number")
|
|
408
|
+
.map((t) => ({
|
|
409
|
+
machine: node,
|
|
410
|
+
job: typeof t.id === "string" && t.id !== "" ? `vzdump ${t.id}` : "vzdump",
|
|
411
|
+
start: new Date((t.starttime as number) * 1000).toISOString(),
|
|
412
|
+
end: typeof t.endtime === "number" ? new Date(t.endtime * 1000).toISOString() : null,
|
|
413
|
+
result: typeof t.status === "string" ? t.status : "running",
|
|
414
|
+
}));
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
function durationText(start: string, end: string | null): string {
|
|
418
|
+
if (end === null) return "-";
|
|
419
|
+
const s = Math.round((Date.parse(end) - Date.parse(start)) / 1000);
|
|
420
|
+
return s >= 60 ? `${Math.floor(s / 60)}m${String(s % 60).padStart(2, "0")}s` : `${s}s`;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/** The runs as a fixed-width table, oldest first, times in UTC. */
|
|
424
|
+
export function nightlyTable(runs: readonly NightlyRun[]): string {
|
|
425
|
+
const rows = [...runs]
|
|
426
|
+
.sort((a, b) => a.start.localeCompare(b.start))
|
|
427
|
+
.map((r) => [r.start.slice(0, 10), r.start.slice(11, 19), r.end ? r.end.slice(11, 19) : "-", durationText(r.start, r.end), r.machine, r.job, r.result]);
|
|
428
|
+
const head = ["date", "start", "end", "duration", "machine", "job", "result"];
|
|
429
|
+
const widths = head.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)));
|
|
430
|
+
return [head, ...rows].map((r) => r.map((c, i) => c.padEnd(widths[i])).join(" ").trimEnd()).join("\n");
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
async function nightly(config: Target, days: number): Promise<number> {
|
|
434
|
+
const since = Math.floor(Date.now() / 1000) - days * 24 * 3600;
|
|
435
|
+
const tasks = await api(config, "GET", `/nodes/${config.node}/tasks?typefilter=vzdump&since=${since}&limit=500`);
|
|
436
|
+
const journal = await hostOutput(config, ["nightly-journal", String(days)]);
|
|
437
|
+
printRaw(`${nightlyTable([...vzdumpRuns(config.node, tasks), ...journalRuns(journal)])}\n(times in UTC)`);
|
|
438
|
+
return 0;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
async function hostOutput(config: Target, args: string[]): Promise<string> {
|
|
344
442
|
const remote = buildHostRemote(args);
|
|
345
443
|
const result = await sshFixed(
|
|
346
444
|
{
|
|
@@ -352,8 +450,16 @@ async function host(config: Target, args: string[]): Promise<number> {
|
|
|
352
450
|
remote,
|
|
353
451
|
args[0] === "host-upgrade" ? HOST_UPGRADE_TIMEOUT_MS : 120_000,
|
|
354
452
|
);
|
|
355
|
-
if (result.
|
|
356
|
-
|
|
453
|
+
if (result.code !== 0) {
|
|
454
|
+
if (result.stdout.length > 0) printRaw(result.stdout);
|
|
455
|
+
throw new ServiceError(result.stderr.trim() || `${args[0]} exited ${result.code}`);
|
|
456
|
+
}
|
|
457
|
+
return result.stdout;
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
async function host(config: Target, args: string[]): Promise<number> {
|
|
461
|
+
const out = await hostOutput(config, args);
|
|
462
|
+
if (out.length > 0) printRaw(out);
|
|
357
463
|
return 0;
|
|
358
464
|
}
|
|
359
465
|
|
|
@@ -373,6 +479,10 @@ export async function command(args: string[]): Promise<number> {
|
|
|
373
479
|
printJson({ host: new URL(config.apiUrl).host, fingerprint256: probe.fingerprint });
|
|
374
480
|
return 0;
|
|
375
481
|
}
|
|
482
|
+
if (rest[0] === "nightly") {
|
|
483
|
+
if (rest.length > 2) throw new UsageError("nightly [days]");
|
|
484
|
+
return nightly(config, parseDays(rest[1]));
|
|
485
|
+
}
|
|
376
486
|
if (rest[0] !== undefined && rest[0] in HOST_COMMANDS) return host(config, rest);
|
|
377
487
|
return apiCommand(config, rest);
|
|
378
488
|
}
|
|
@@ -12,7 +12,7 @@ import { MAX_ADD_IDS, MAX_EXCLUDE_IDS, buildTrendarrCall, interpretRun, parseImd
|
|
|
12
12
|
import { ServiceError, UsageError } from "./lib/errors.ts";
|
|
13
13
|
import { safeRepoPath, safeWritePath } from "./lib/repo.ts";
|
|
14
14
|
import { appGetPath, appPolicyDelete, portForwardDelete, clientFixedIpWrite, syslogChanges, buildPulsarRemote, cloudPath, deviceNetworkWrite, NAT_PATH, natWrite, parseQuery, parseSiteArgs, redactUnifi, unifiPath, unifiWrite, command as networkCommand } from "./network/run.ts";
|
|
15
|
-
import { buildHostRemote, splitHost, needsConfirm, parseSetPairs, parseVmid, validateApiPath as validatePvePath, command as proxmoxCommand } from "./proxmox/run.ts";
|
|
15
|
+
import { buildHostRemote, journalRuns, nightlyTable, parseDays, splitHost, needsConfirm, parseSetPairs, parseVmid, validateApiPath as validatePvePath, vzdumpRuns, command as proxmoxCommand } from "./proxmox/run.ts";
|
|
16
16
|
import { banLength, buildObsRemote, compactDecisions, parseGoDuration, summariseAlerts, summariseVulns, summariseWazuh, vulnEvents, wazuhEvents, command as securityCommand } from "./security/run.ts";
|
|
17
17
|
|
|
18
18
|
describe("arcane wrapper", () => {
|
|
@@ -680,6 +680,46 @@ describe("proxmox wrapper", () => {
|
|
|
680
680
|
expect(buildHostRemote(["host-reboot", "--confirm"])).toBe("host-reboot --confirm");
|
|
681
681
|
expect(() => buildHostRemote(["host-reboot", "now"])).toThrow(/--confirm/);
|
|
682
682
|
expect(() => buildHostRemote(["shell"])).toThrow(/unknown host command/);
|
|
683
|
+
expect(buildHostRemote(["nightly-journal", "7"])).toBe("nightly-journal 7");
|
|
684
|
+
expect(() => buildHostRemote(["nightly-journal", "15"])).toThrow(/days/);
|
|
685
|
+
expect(() => buildHostRemote(["nightly-journal", "7; id"])).toThrow(/days/);
|
|
686
|
+
expect(parseDays(undefined)).toBe(7);
|
|
687
|
+
expect(() => parseDays("0")).toThrow(/days/);
|
|
688
|
+
});
|
|
689
|
+
|
|
690
|
+
it("turns the nightly journal and vzdump tasks into runs and a table", () => {
|
|
691
|
+
const journal = [
|
|
692
|
+
"2026-11-04T03:00:24+00:00 pulsar systemd[1]: Starting pulsar-config-backup.service - PVE host config backup to restic...",
|
|
693
|
+
"2026-11-04T03:00:42+00:00 pulsar systemd[1]: pulsar-config-backup.service: Deactivated successfully.",
|
|
694
|
+
"2026-11-04T03:00:42+00:00 pulsar systemd[1]: Finished pulsar-config-backup.service - PVE host config backup to restic.",
|
|
695
|
+
"2026-10-31T18:52:02+00:00 pulsar systemd[1]: Starting mira-backup-pull.service - Copy Mira guest backups from TESLA to the UNAS...",
|
|
696
|
+
"2026-10-31T18:52:05+00:00 pulsar systemd[1]: mira-backup-pull.service: Failed with result 'exit-code'.",
|
|
697
|
+
"2026-10-31T18:52:05+00:00 pulsar systemd[1]: Failed to start mira-backup-pull.service - Copy Mira guest backups from TESLA to the UNAS.",
|
|
698
|
+
"2026-11-03T03:30:01+00:00 arcane systemd[1]: Starting apt-daily-upgrade.service - Daily apt upgrade and clean activities...",
|
|
699
|
+
"2026-11-03T03:30:09+00:00 arcane systemd[1]: Finished apt-daily-upgrade.service - Daily apt upgrade and clean activities.",
|
|
700
|
+
"2026-11-03T03:30:22+00:00 arcane systemd[1]: docker.service: Deactivated successfully.",
|
|
701
|
+
"2026-11-03T03:30:32+00:00 arcane systemd[1]: Starting docker.service - Docker Application Container Engine...",
|
|
702
|
+
"2026-11-03T03:30:33+00:00 arcane systemd[1]: Started docker.service - Docker Application Container Engine.",
|
|
703
|
+
"2026-11-01T05:35:22+00:00 identity systemd[1]: Starting docker.service - Docker Application Container Engine...",
|
|
704
|
+
].join("\n");
|
|
705
|
+
expect(journalRuns(journal)).toEqual([
|
|
706
|
+
{ machine: "pulsar", job: "pulsar-config-backup", start: "2026-11-04T03:00:24.000Z", end: "2026-11-04T03:00:42.000Z", result: "ok" },
|
|
707
|
+
{ machine: "pulsar", job: "mira-backup-pull", start: "2026-10-31T18:52:02.000Z", end: "2026-10-31T18:52:05.000Z", result: "failed (exit-code)" },
|
|
708
|
+
{ machine: "arcane", job: "apt-daily-upgrade", start: "2026-11-03T03:30:01.000Z", end: "2026-11-03T03:30:09.000Z", result: "ok" },
|
|
709
|
+
{ machine: "arcane", job: "docker restart", start: "2026-11-03T03:30:22.000Z", end: "2026-11-03T03:30:33.000Z", result: "ok" },
|
|
710
|
+
]);
|
|
711
|
+
const tasks = [
|
|
712
|
+
{ type: "vzdump", id: "", starttime: 1793755800, endtime: 1793756268, status: "OK" },
|
|
713
|
+
{ type: "qmstart", id: "100", starttime: 1793755000, endtime: 1793755001, status: "OK" },
|
|
714
|
+
];
|
|
715
|
+
const runs = [...vzdumpRuns("pulsar", tasks), ...journalRuns(journal)];
|
|
716
|
+
expect(runs[0]).toEqual({ machine: "pulsar", job: "vzdump", start: "2026-11-04T01:30:00.000Z", end: "2026-11-04T01:37:48.000Z", result: "OK" });
|
|
717
|
+
expect(vzdumpRuns("pulsar", null)).toEqual([]);
|
|
718
|
+
const table = nightlyTable(runs).split("\n");
|
|
719
|
+
expect(table[0]).toBe("date start end duration machine job result");
|
|
720
|
+
expect(table[1]).toBe("2026-10-31 18:52:02 18:52:05 3s pulsar mira-backup-pull failed (exit-code)");
|
|
721
|
+
expect(table.at(-1)).toBe("2026-11-04 03:00:24 03:00:42 18s pulsar pulsar-config-backup ok");
|
|
722
|
+
expect(table).toContain("2026-11-04 01:30:00 01:37:48 7m48s pulsar vzdump OK");
|
|
683
723
|
});
|
|
684
724
|
|
|
685
725
|
it("prints help without touching config", async () => {
|