@astrofoundry/pi-astro 0.40.45 → 0.40.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -94,13 +94,20 @@ function entry(r: ArcaneReport, stack: string): StackEntry {
94
94
  }
95
95
 
96
96
  describe("Arcane Daily Report layout", () => {
97
+ it("marks the images our own CI builds", () => {
98
+ const own = { stack: "bigcapital", service: "server", ref: "ghcr.io/astrofoundry/bigcapital-server:astrofoundry", running: "?", latest: "new build", status: "update", action: "pull", note: "versions unavailable: private image" };
99
+ const built = buildStacks({ ...updateReport, rows: [rows[0], own] });
100
+ expect(built?.stacks.find((s) => s.stack === "bigcapital")?.lines[0].own).toBe(true);
101
+ expect(built?.stacks.find((s) => s.stack === "vpn")?.lines[0].own).toBeUndefined();
102
+ });
103
+
97
104
  it("draws one section per stack with its services and the kind of change", () => {
98
105
  const r = report();
99
106
  entry(r, "stirling-pdf").lines[0].review = { verdict: "check", reason: "the config key changed" };
100
107
  const [page] = layoutMessages(r);
101
108
  expect(page).toEqual({ head: true, stacks: ["arcane", "bigcapital", "media", "postgres", "stirling-pdf", "supabase", "vpn", "x"] });
102
109
  const components = messageComponents(r, page);
103
- expect(components[0]).toEqual({ type: 10, content: `**${REPORT_PREFIX}** 6 stacks with updates (44 images, 2 private excluded, checked 06:00 CEST).` });
110
+ expect(components[0]).toEqual({ type: 10, content: `**${REPORT_PREFIX}** 6 stacks with updates (44 images, 2 pinned private excluded, checked 06:00 CEST).` });
104
111
  const text = (c: (typeof components)[number]) => (c.type === 9 ? c.components?.[0].content : c.content);
105
112
  expect(components.slice(1).map(text)).toEqual([
106
113
  "**arcane**\n• arcane: v2.16.1 → v2.17.0 · new version (digest pin)",
@@ -142,7 +149,7 @@ describe("Arcane Daily Report layout", () => {
142
149
  it("says so when nothing needs updating, warns about an unreadable holds file, and refuses a bad answer", () => {
143
150
  const empty = buildStacks({ ...updateReport, rows: [], holds: {}, holdsError: "update-holds.json is not JSON" });
144
151
  if (!empty) throw new Error("no report");
145
- expect(headText({ ...empty, timeZone: "Europe/Madrid" }).content).toBe(`**${REPORT_PREFIX}** no updates (44 images, 2 private excluded, checked 06:00 CEST).\n-# ⚠️ update-holds.json is not JSON; Update still refuses held stacks`);
152
+ expect(headText({ ...empty, timeZone: "Europe/Madrid" }).content).toBe(`**${REPORT_PREFIX}** no updates (44 images, 2 pinned private excluded, checked 06:00 CEST).\n-# ⚠️ update-holds.json is not JSON; Update still refuses held stacks`);
146
153
  expect(buildStacks({ nonsense: true })).toBeNull();
147
154
  expect(failureMessage().startsWith(REPORT_PREFIX)).toBe(true);
148
155
  });
@@ -4,6 +4,7 @@ import { parseImageRef } from "../../specialists/lib/registry.ts";
4
4
  import { configDir } from "./config.ts";
5
5
  import { escapeDiscord } from "./media.ts";
6
6
  import type { MessageComponent } from "./rest.ts";
7
+ import { isOwnImage } from "./security.ts";
7
8
  import type { ReleaseNote } from "../../specialists/lib/releases.ts";
8
9
  import type { Review, Verdict } from "./tools-report.ts";
9
10
  import type { Plan } from "./update-policy.ts";
@@ -79,6 +80,8 @@ export interface ServiceLine {
79
80
  major: boolean;
80
81
  note: string;
81
82
  review?: Review;
83
+ /** The image is one our own CI builds, so its update goes through Upgrade carefully. */
84
+ own?: boolean;
82
85
  }
83
86
 
84
87
  export type StackState = "offered" | "running" | "updated" | "rolled back" | "failed" | "current" | "planning";
@@ -160,7 +163,8 @@ export function buildStacks(report: unknown): ReportStacks | null {
160
163
  .map((r): ServiceLine => {
161
164
  const running = releaseTag(r.running);
162
165
  const latest = releaseTag(r.latest);
163
- return { service: r.service, status: r.status, action: r.action ?? "", running, latest, tag: tagOf(r.ref ?? ""), ...(r.ref ? { ref: r.ref } : {}), major: PENDING.has(r.status) && crossesMajor(running, latest), note: r.note ?? "" };
166
+ const own = r.ref ? isOwnImage(r.ref) : false;
167
+ return { service: r.service, status: r.status, action: r.action ?? "", running, latest, tag: tagOf(r.ref ?? ""), ...(r.ref ? { ref: r.ref } : {}), major: PENDING.has(r.status) && crossesMajor(running, latest), note: r.note ?? "", ...(own ? { own } : {}) };
164
168
  });
165
169
  const pending = lines.some((l) => PENDING.has(l.status));
166
170
  const held = holds[stack];
@@ -287,7 +291,7 @@ type ReportView = Omit<ArcaneReport, "messages" | "date" | "channelId">;
287
291
 
288
292
  export function headText(report: ReportView): MessageComponent {
289
293
  const withUpdates = report.stacks.filter((s) => s.lines.some((l) => PENDING.has(l.status))).length;
290
- const scope = `${report.images} images${report.privateCount > 0 ? `, ${report.privateCount} private excluded` : ""}, checked ${clock(report.checkedAt, report.timeZone)}`;
294
+ const scope = `${report.images} images${report.privateCount > 0 ? `, ${report.privateCount} pinned private excluded` : ""}, checked ${clock(report.checkedAt, report.timeZone)}`;
291
295
  const head =
292
296
  report.stacks.length === 0
293
297
  ? `**${REPORT_PREFIX}** no updates (${scope}).`
@@ -300,7 +300,8 @@ describe("security monitor", () => {
300
300
  const summary = await h.monitor.dailyReport(SCHEDULE, "2026-10-21");
301
301
  expect(h.rest.messages).toEqual([]);
302
302
  const text = JSON.stringify(h.rest.components[0].components);
303
- expect(text).toContain("🐳 **Arcane images** · 1 critical CVE whose package has a fix, in 1 image\\n- waiting on the publisher: nginx 1");
303
+ expect(text).toContain("🐳 **Arcane images** · 1 critical CVE whose package has a fix, in 1 image\"");
304
+ expect(text).not.toContain("waiting on the publisher");
304
305
  expect(h.calls).toContainEqual(["arcane", "images", "list", "--all"]);
305
306
  expect(h.calls).toContainEqual(["arcane", "images", "vulnerabilities", "list", "--severity", "critical", "--all"]);
306
307
  expect(summary).toEqual({ headline: "quiet day (24 h to 06:05 CEST)", tagged: false, messageId: "100" });
@@ -433,7 +433,10 @@ describe("Arcane image scans", () => {
433
433
  expect(offeredImages(null, offers)).toEqual([]);
434
434
  expect(imageVulnLine(null, offers)).toBe("Arcane images · the check failed");
435
435
  expect(imageVulnLine({ scanned: 0, total: 57, critical: [] }, offers)).toBe("Arcane images · no Trivy scan yet (57 images)");
436
- expect(imageVulnLine({ ...facts, critical: facts.critical.slice(4) }, new Map(), undefined, 0)).toBe("Arcane images · 1 critical CVE whose package has a fix, in 1 image · waiting on the publisher: +1 more");
436
+ expect(imageVulnLine({ ...facts, critical: facts.critical.slice(4) }, new Map(), undefined, 1)).toBe("Arcane images · 1 critical CVE whose package has a fix, in 1 image · waiting on the publisher: nginx 1");
437
+ expect(imageVulnLine(facts, offers, undefined, 0)).toBe(
438
+ "Arcane images · 5 critical CVEs whose package has a fix, in 3 images · update offered in the Arcane Daily Report: mealie 3 (mealie) · held: postgres:15.8.1.085 1 (supabase: coordinated release: review first)",
439
+ );
437
440
  });
438
441
 
439
442
  it("names our own images with their build date and a newer build waiting in the registry", () => {
@@ -661,7 +661,7 @@ export function offeredImages(facts: ImageVulnFacts | null, offers: ReadonlyMap<
661
661
  return [...stacks].sort();
662
662
  }
663
663
 
664
- function isOwnImage(image: string): boolean {
664
+ export function isOwnImage(image: string): boolean {
665
665
  const repo = repositoryOf(image);
666
666
  return repo !== null && OWN_IMAGE_PREFIXES.some((p) => repo.startsWith(p));
667
667
  }
@@ -686,6 +686,7 @@ function buildText(build: ImageBuild | undefined): string {
686
686
  * Critical CVEs whose package has a fixed version, grouped by who can act:
687
687
  * our own images (with their build date), images an update in the Arcane
688
688
  * Daily Report replaces, held stacks, and images waiting on their publisher.
689
+ * A `maxWaiting` of 0 leaves the publisher group out: nobody here can act on it.
689
690
  */
690
691
  export function imageVulnLine(facts: ImageVulnFacts | null, offers: ReadonlyMap<string, ImageOffer>, arcaneChannelId?: string, maxWaiting = 5): string {
691
692
  if (!facts) return "Arcane images · the check failed";
@@ -712,7 +713,7 @@ export function imageVulnLine(facts: ImageVulnFacts | null, offers: ReadonlyMap<
712
713
  if (offered.length > 0) parts.push(`update offered${arcaneChannelId ? ` in <#${arcaneChannelId}>` : " in the Arcane Daily Report"}: ${offered.join(", ")}`);
713
714
  if (held.length > 0) parts.push(`held: ${held.join(", ")}`);
714
715
  const rest = waiting.length - Math.min(waiting.length, maxWaiting);
715
- if (waiting.length > 0) parts.push(`waiting on the publisher: ${[...waiting.slice(0, maxWaiting), ...(rest > 0 ? [`+${rest} more`] : [])].join(", ")}`);
716
+ if (waiting.length > 0 && maxWaiting > 0) parts.push(`waiting on the publisher: ${[...waiting.slice(0, maxWaiting), ...(rest > 0 ? [`+${rest} more`] : [])].join(", ")}`);
716
717
  return parts.join(" · ");
717
718
  }
718
719
 
@@ -783,7 +784,7 @@ export function dailyCard(d: DailyData, owners: readonly string[]): ReportCard {
783
784
  section("👁️", d.wazuh.line),
784
785
  section("📦", vulnLine(d.found)),
785
786
  section("🩹", patchLine(d.closed)),
786
- section("🐳", imageVulnLine(d.images, d.offers, d.arcaneChannelId)),
787
+ section("🐳", imageVulnLine(d.images, d.offers, d.arcaneChannelId, 0)),
787
788
  section("🩺", healthLine(d.attention)),
788
789
  ];
789
790
  return {
@@ -13,6 +13,7 @@ describe("update policy", () => {
13
13
 
14
14
  it("plans carefully when notes need attention, even for a minor update", () => {
15
15
  expect(stackPlan([{ major: false, review: check }])).toBe("careful");
16
+ expect(stackPlan([{ major: false, review: none }, { major: false, own: true }])).toBe("careful");
16
17
  expect(stackPlan([{ major: true, review: none }, { major: false, review: { verdict: "breaking", reason: "x" } }])).toBe("careful");
17
18
  });
18
19
 
@@ -4,7 +4,7 @@ import type { Review } from "./tools-report.ts";
4
4
  * What the morning run does with an offered update (approved 2026-10-23):
5
5
  * - `auto`: applied without anyone and checked again minutes later;
6
6
  * - `update`: the owners are tagged, and Update runs it on its own (a major whose notes raise nothing);
7
- * - `careful`: the owners are tagged, and Upgrade carefully plans it in a thread first (the notes need attention, or there are none for a major);
7
+ * - `careful`: the owners are tagged, and Upgrade carefully plans it in a thread first (the notes need attention, there are none for a major, or the image is one we build);
8
8
  * - `button`: offered without a tag (Pi, the bridge's own runtime, never updates itself).
9
9
  */
10
10
  export type Plan = "auto" | "update" | "careful" | "button";
@@ -16,12 +16,14 @@ export function needsAttention(review: Review | undefined): boolean {
16
16
  interface PlannedLine {
17
17
  major: boolean;
18
18
  review?: Review;
19
+ /** An image our own CI builds: it has no release notes, and its stack's README sets the rollout (a queue drain, a database backup). */
20
+ own?: boolean;
19
21
  }
20
22
 
21
- /** One stack's pending updates: any line whose notes need attention makes it careful; a major is careful unless its notes say nothing relevant. */
23
+ /** One stack's pending updates: an image of ours or a line whose notes need attention makes it careful; a major is careful unless its notes say nothing relevant. */
22
24
  export function stackPlan(pending: readonly PlannedLine[]): Plan | null {
23
25
  if (pending.length === 0) return null;
24
- if (pending.some((l) => needsAttention(l.review))) return "careful";
26
+ if (pending.some((l) => l.own || needsAttention(l.review))) return "careful";
25
27
  const majors = pending.filter((l) => l.major);
26
28
  if (majors.length === 0) return "auto";
27
29
  return majors.every((l) => l.review?.verdict === "none") ? "update" : "careful";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@astrofoundry/pi-astro",
3
- "version": "0.40.45",
3
+ "version": "0.40.47",
4
4
  "description": "Personal pi customizations (extensions, subagents, skills, prompts, themes) for the pi coding agent.",
5
5
  "keywords": [
6
6
  "pi-package"
@@ -36,9 +36,9 @@ The output is JSON with `counts`, `rows`, and `table`. Answer with the `table` f
36
36
  - `Running` and `Latest` are release tags. `older build`: the running build carries no tag any more. `new build`: no release tag names the newer build. `17 (older build)`: the running build of that major series is no longer tagged.
37
37
  - `Update`: `pull` means redeploying the stack with an image pull applies it; `edit tag` means changing the tag in the GitOps repository; `edit digest` means changing the pinned digest in the repository (`latestDigest` in the row holds it); `unknown` has its reason in the notes under the table.
38
38
  - A note "major version change" or "newer major ... available" means the update crosses a major version. Read its release notes yourself before asking anything (next section).
39
- - Private images (the registry refuses anonymous access, such as our own Grimoire builds) are left out of the table and the counts; the summary line counts them and `privateImages` lists them. Mention them only when the caller asks.
39
+ - Private images (the registry refuses anonymous access, such as graphile-worker) come from Arcane's own check, which uses its registry login: their rows show `new build` without version names. Digest-pinned private images (the Grimoire builds, moved only by their release workflow) are left out of the table and the counts; the summary line counts them and `privateImages` lists them. Mention them only when the caller asks.
40
40
 
41
- Messages that start with "Arcane Daily Report:" are the bridge's scheduled report, built from `update-report` in the morning run without you: one section per stack with updates and a release-notes verdict on every pending update. Stacks without a major version and without notes that need attention (including an Arcane manager release) are updated by the bridge right after the post, without you, and checked again five minutes later; a stack that fails then is rolled back where possible and handed to `astro.foreman`. A major whose notes say nothing relevant gets an Update button that owners press, which runs `stack-update` without you; anything else gets Upgrade carefully (next sections). When the caller replies to one, its list is the context ("update these", "why is this a new build"); run `update-report` again only when the caller wants fresh data.
41
+ Messages that start with "Arcane Daily Report:" are the bridge's scheduled report, built from `update-report` in the morning run without you: one section per stack with updates and a release-notes verdict on every pending update. Stacks without a major version and without notes that need attention (including an Arcane manager release) are updated by the bridge right after the post, without you, and checked again five minutes later; a stack that fails then is rolled back where possible and handed to `astro.foreman`. A major whose notes say nothing relevant gets an Update button that owners press, which runs `stack-update` without you; anything else, and every update of an image our own CI builds (ghcr.io astronaute77, astrofoundry, monadeo: the stack README sets its rollout), gets Upgrade carefully (next sections). When the caller replies to one, its list is the context ("update these", "why is this a new build"); run `update-report` again only when the caller wants fresh data.
42
42
 
43
43
  ## Applying updates
44
44
 
@@ -44,6 +44,7 @@ Power actions and snapshot changes return `{ upid, exitstatus }`; `OK` means don
44
44
  | `host-upgrade` | apt full-upgrade of the host (risky, needs approval); a new Proxmox VE major is held and printed as `held:`; ends with `reboot-needed:`. The morning run does this on its own each day |
45
45
  | `host-reboot --confirm` | reboot the host in one minute (risky): every guest on it restarts. Only on the owner's explicit request; the morning run offers a Reboot button instead |
46
46
  | `guest-exec <vmid> <status|journal|df|updates>` | fixed read inside a guest: failed units and uptime, warning journal of 24 h, disk usage, upgradable packages |
47
+ | `nightly [days]` | start, end, duration (UTC) and result of each nightly job of the last 1..14 days (default 7): vzdump tasks from the API, then from the journals of the host and every running guest the host timers, `apt-daily-upgrade`, the backup and dump timers, and Docker restarts. Answer questions about nightly run times with it |
47
48
 
48
49
  ## Guest change procedure
49
50
 
@@ -11,7 +11,7 @@ A specialist is a Pi subagent that is the only way to operate one area of the ho
11
11
  | `astro.edge` | Frontdoor VPS (`frontdoor-1337`) over an IAP tunnel; `00-frontdoor-vps/` in the homelab repository | `probe` the public path; `route-probe` (status of the Supabase, Grimoire, and Mealie public routes from Cortex); `vps status|nginx-conf|nginx-test|journal|wg|bouncer|updates|package|changelog|auto-upgrades|deploy-nginx|reboot --confirm`; `git` on the homelab checkout, writes limited to `00-frontdoor-vps/` |
12
12
  | `astro.security` | Wazuh, CrowdSec, rsyslog archive on VM 106 | Read-only: `status`, `attention`, `agents`, `alerts`, `decisions`, `bouncers`, `metrics`, `wazuh-alerts`, `wazuh-vulns`, `wazuh-events`, `wazuh-vuln-events`, `wazuh-log`, `remote-hosts`, `remote-log`, `journal` |
13
13
  | `astro.backup` | vzdump, the shared Restic repository (Pulsar host config, Arcane app backups), GCS offsite from LXC 102 | Status, journals, and manual runs of every layer; `restic-snapshots|stats|ls`, `restic-restore` into `/var/tmp/spc-restore` on Pulsar, `restore-clean` |
14
- | `astro.proxmox` | Proxmox VE on Pulsar (37) and SPACE-X (Mira) | API token: `guests`, `guest`, `get <path>`, `tasks`; `start`, `shutdown|reboot|stop --confirm`; snapshots create, delete, rollback; `set` of CPU, memory, options, network. Host key: `host-status`, `host-journal`, `updates`, `host-upgrade` (apt full-upgrade; a new Proxmox VE major is held), `host-reboot --confirm`, `guest-exec <vmid> <status|journal|df|updates>`. Every command runs on Pulsar, or on SPACE-X with the `space-x` prefix |
14
+ | `astro.proxmox` | Proxmox VE on Pulsar (37) and SPACE-X (Mira) | API token: `guests`, `guest`, `get <path>`, `tasks`; `start`, `shutdown|reboot|stop --confirm`; snapshots create, delete, rollback; `set` of CPU, memory, options, network. Host key: `host-status`, `host-journal`, `updates`, `host-upgrade` (apt full-upgrade; a new Proxmox VE major is held), `host-reboot --confirm`, `guest-exec <vmid> <status|journal|df|updates>`, `nightly-journal <days>`; `nightly [days]` combines the vzdump tasks with that journal into a table of nightly run times. Every command runs on Pulsar, or on SPACE-X with the `space-x` prefix |
15
15
  | `astro.inference` | MLX embedding and reranking servers on Nexus, Europa health | `nexus-status|start|stop|restart|log|disk`, `macos-updates`; `europa-health` |
16
16
  | `astro.media` | trendarr over TMDB, OMDb, and the `media` project on VM 100 (Radarr, Sonarr, SABnzbd) | `trending`, `search`, `recommend`, `add-movies`, `add-series` (at most 20 IMDb IDs), `sabnzbd status|pause|resume`; always trendarr's JSON plus its warnings |
17
17
 
@@ -35,6 +35,14 @@ guest_cmd() {
35
35
  esac
36
36
  }
37
37
 
38
+ # The nightly jobs on the host and in the guests; a unit a machine does not have prints nothing.
39
+ NIGHTLY_UNITS="pulsar-config-backup mira-backup-pull apt-daily-upgrade docker arcane-backup pgdump-zitadel lego-renew gcs-backup"
40
+ nightly_cmd() {
41
+ units=""
42
+ for u in $NIGHTLY_UNITS; do units="$units -u $u.service"; done
43
+ echo "journalctl --utc -o short-iso --no-pager --since '$1 days ago' $units | grep -E 'systemd\[1\]: (Starting|Started|Finished|Failed|[^ ]+\.service: (Deactivated|Failed))' || true"
44
+ }
45
+
38
46
  case "$name" in
39
47
  host-status)
40
48
  pveversion
@@ -62,6 +70,15 @@ case "$name" in
62
70
  [ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
63
71
  shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
64
72
  echo "reboot in 1 minute" ;;
73
+ nightly-journal)
74
+ case "$rest" in [1-9]|1[0-4]) ;; *) echo "days must be 1..14" >&2; exit 2 ;; esac
75
+ inner="$(nightly_cmd "$rest")"
76
+ sh -c "$inner"
77
+ for vmid in $(pct list | awk 'NR > 1 && $2 == "running" { print $1 }'); do pct exec "$vmid" -- sh -c "$inner" || true; done
78
+ # The guest agent wraps a VM's output in JSON; only its out-data is journal lines.
79
+ for vmid in $(qm list | awk 'NR > 1 && $3 == "running" { print $1 }'); do
80
+ qm guest exec "$vmid" --timeout 60 -- sh -c "$inner" | perl -MJSON::PP -0777 -ne 'my $r = eval { decode_json($_) }; print $r->{"out-data"} // "" if $r' || true
81
+ done ;;
65
82
  guest-exec)
66
83
  vmid="${rest%% *}"; action="${rest#* }"
67
84
  case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
@@ -35,6 +35,14 @@ guest_cmd() {
35
35
  esac
36
36
  }
37
37
 
38
+ # The nightly jobs on the host and in the guests; a unit a machine does not have prints nothing.
39
+ NIGHTLY_UNITS="pulsar-config-backup mira-backup-pull apt-daily-upgrade docker arcane-backup pgdump-zitadel lego-renew gcs-backup"
40
+ nightly_cmd() {
41
+ units=""
42
+ for u in $NIGHTLY_UNITS; do units="$units -u $u.service"; done
43
+ echo "journalctl --utc -o short-iso --no-pager --since '$1 days ago' $units | grep -E 'systemd\[1\]: (Starting|Started|Finished|Failed|[^ ]+\.service: (Deactivated|Failed))' || true"
44
+ }
45
+
38
46
  case "$name" in
39
47
  host-status)
40
48
  pveversion
@@ -62,6 +70,15 @@ case "$name" in
62
70
  [ "$rest" = "--confirm" ] || { echo "host-reboot needs --confirm" >&2; exit 2; }
63
71
  shutdown -r +1 "reboot requested through the proxmox specialist" 2>&1
64
72
  echo "reboot in 1 minute" ;;
73
+ nightly-journal)
74
+ case "$rest" in [1-9]|1[0-4]) ;; *) echo "days must be 1..14" >&2; exit 2 ;; esac
75
+ inner="$(nightly_cmd "$rest")"
76
+ sh -c "$inner"
77
+ for vmid in $(pct list | awk 'NR > 1 && $2 == "running" { print $1 }'); do pct exec "$vmid" -- sh -c "$inner" || true; done
78
+ # The guest agent wraps a VM's output in JSON; only its out-data is journal lines.
79
+ for vmid in $(qm list | awk 'NR > 1 && $3 == "running" { print $1 }'); do
80
+ qm guest exec "$vmid" --timeout 60 -- sh -c "$inner" | perl -MJSON::PP -0777 -ne 'my $r = eval { decode_json($_) }; print $r->{"out-data"} // "" if $r' || true
81
+ done ;;
65
82
  guest-exec)
66
83
  vmid="${rest%% *}"; action="${rest#* }"
67
84
  case "$vmid" in 1[0-9][0-9]) ;; *) echo "bad vmid" >&2; exit 2 ;; esac
@@ -69,6 +69,7 @@ describe("resolveRef", () => {
69
69
  "library/nginx": { tags: ["1.31.4-alpine"], digests: {} },
70
70
  "getarcaneapp/manager": { tags: ["v1.2.0", "v1.3.0", "latest"], digests: { "v1.3.0": "sha256:new", "v1.2.0": "sha256:old" } },
71
71
  "astrofoundry/app": "private",
72
+ "monadeo/server": "private",
72
73
  });
73
74
 
74
75
  it("resolves a floating tag to the release versions behind both digests", async () => {
@@ -99,8 +100,11 @@ describe("resolveRef", () => {
99
100
  expect(await resolveRef("ghcr.io/getarcaneapp/manager@sha256:new", undefined, registry, NOW)).toMatchObject({ running: "v1.3.0", status: "up-to-date" });
100
101
  });
101
102
 
102
- it("falls back to the digest result for private images and marks bad checks unknown", async () => {
103
- expect(await resolveRef("ghcr.io/astrofoundry/app:main", entry("A1", "A2"), registry, NOW)).toMatchObject({ status: "update", latest: "new build", private: true });
103
+ it("falls back to Arcane's digest check for private images, leaves out private digest pins, and marks bad checks unknown", async () => {
104
+ const own = await resolveRef("ghcr.io/astrofoundry/app:main", entry("A1", "A2"), registry, NOW);
105
+ expect(own).toMatchObject({ status: "update", latest: "new build", action: "pull", note: "versions unavailable: private image" });
106
+ expect(own.private).toBeUndefined();
107
+ expect(await resolveRef("ghcr.io/monadeo/server:v1.0.0@sha256:abc", undefined, registry, NOW)).toMatchObject({ status: "unknown", private: true });
104
108
  expect(await resolveRef("nginx:1.31.4-alpine", undefined, registry, NOW)).toMatchObject({ status: "unknown", note: "not in the fresh check" });
105
109
  expect(await resolveRef("nginx:1.31.4-alpine", entry("N", "N", { checkTime: "2026-09-24T00:00:00Z" }), registry, NOW)).toMatchObject({ status: "unknown", note: expect.stringContaining("stale") });
106
110
  expect(await resolveRef("nginx:1.31.4-alpine", entry("N", "N", { error: "timeout" }), registry, NOW)).toMatchObject({ status: "unknown", note: "check failed: timeout" });
@@ -121,6 +125,7 @@ describe("buildUpdateReport", () => {
121
125
  { name: "canary", status: "stopped", updateInfo: { serviceUpdates: { canary: { imageRef: "busybox:latest" } } } },
122
126
  { name: "empty", status: "running" },
123
127
  { name: "ours", status: "running", updateInfo: { serviceUpdates: { api: { imageRef: "ghcr.io/astrofoundry/app:main" }, worker: { imageRef: "ghcr.io/astrofoundry/app:main" } } } },
128
+ { name: "pinned", status: "running", updateInfo: { serviceUpdates: { server: { imageRef: "ghcr.io/monadeo/server:v1.0.0@sha256:abc" } } } },
124
129
  ],
125
130
  };
126
131
  const check = {
@@ -130,15 +135,15 @@ describe("buildUpdateReport", () => {
130
135
  "ghcr.io/astrofoundry/app:main": entry("A1", "A2"),
131
136
  };
132
137
  const report = await buildUpdateReport({ check, projects, registry, startedAt: NOW, all: false });
133
- expect(report.counts).toEqual({ update: 1, release: 0, rebuild: 0, unknown: 1, "up-to-date": 2 });
134
- expect(report.rows.map((r) => `${r.stack}/${r.service}:${r.status}`)).toEqual(["media/prowlarr:update", "empty/empty:unknown"]);
135
- expect(report.table).toMatch(/^Fresh check 2026-09-25T18:30:00Z: 4 services in 5 stacks, 3 images\. 1 private images \(2 services\) excluded\./);
136
- expect(report.privateImages).toEqual(["ghcr.io/astrofoundry/app:main"]);
137
- expect(report.rows.some((r) => r.stack === "ours")).toBe(false);
138
+ expect(report.counts).toEqual({ update: 3, release: 0, rebuild: 0, unknown: 1, "up-to-date": 2 });
139
+ expect(report.rows.map((r) => `${r.stack}/${r.service}:${r.status}`)).toEqual(["media/prowlarr:update", "ours/api:update", "ours/worker:update", "empty/empty:unknown"]);
140
+ expect(report.table).toMatch(/^Fresh check 2026-09-25T18:30:00Z: 6 services in 6 stacks, 4 images\. 1 pinned private images \(1 services\) excluded\./);
141
+ expect(report.privateImages).toEqual(["ghcr.io/monadeo/server:v1.0.0@sha256:abc"]);
142
+ expect(report.rows.some((r) => r.stack === "pinned")).toBe(false);
138
143
  expect(report.table).toContain("media/prowlarr prowlarr 2.0.5.5160-ls123 2.1.0.5190-ls124 pull");
139
144
  expect(report.table).toContain("* empty: Arcane lists no images for this stack");
140
145
  const all = await buildUpdateReport({ check, projects, registry, startedAt: NOW, all: true });
141
- expect(all.rows).toHaveLength(4);
146
+ expect(all.rows).toHaveLength(6);
142
147
  expect(all.table).toContain("* canary: stack is stopped");
143
148
  });
144
149
 
@@ -34,7 +34,7 @@ export interface UpdateReport {
34
34
  checkedAt: string;
35
35
  stacks: number;
36
36
  images: number;
37
- /** Private image references, excluded from `rows`, `counts`, and the table. */
37
+ /** Digest-pinned private image references, excluded from `rows`, `counts`, and the table: only their own release workflow moves them. */
38
38
  privateImages: string[];
39
39
  counts: Record<UpdateStatus, number>;
40
40
  rows: UpdateRow[];
@@ -152,7 +152,7 @@ interface Resolution {
152
152
  action: string;
153
153
  note: string;
154
154
  latestDigest?: string;
155
- /** The registry refused anonymous access: an image of our own, left out of the report. */
155
+ /** The registry refused anonymous access to a digest pin: an image of our own that only its release workflow moves, left out of the report. */
156
156
  private?: boolean;
157
157
  }
158
158
 
@@ -179,7 +179,8 @@ export async function resolveRef(refText: string, entry: CheckEntry | undefined,
179
179
  tags = (await registry.tags(ref)).map(parseTag).filter((t): t is TagVersion => t !== null);
180
180
  } catch (err) {
181
181
  const status = err instanceof RegistryError ? err.status : 0;
182
- if (status === 401 || status === 403) return { ...digestOnly("private image, the registry needs credentials"), private: true };
182
+ // Arcane checks a private image with its own registry login, so only the version names are missing.
183
+ if (status === 401 || status === 403) return kind === "pinned" ? { ...unknown("pinned private image"), private: true } : digestOnly("private image");
183
184
  return digestOnly(err instanceof Error ? err.message : String(err));
184
185
  }
185
186
 
@@ -377,7 +378,7 @@ export async function buildUpdateReport(input: { check: unknown; projects: unkno
377
378
  const shown = input.all ? rows : rows.filter((r) => r.status !== "up-to-date" || r.note.includes("newer major"));
378
379
  const checkedAt = new Date(input.startedAt).toISOString().replace(/\.\d+Z$/, "Z");
379
380
  const stacks = new Set(services.map((s) => s.stack)).size;
380
- const excluded = privateImages.length === 0 ? "" : ` ${privateImages.length} private images (${services.length - listed.length} services) excluded.`;
381
+ const excluded = privateImages.length === 0 ? "" : ` ${privateImages.length} pinned private images (${services.length - listed.length} services) excluded.`;
381
382
  const summary =
382
383
  `Fresh check ${checkedAt}: ${listed.length} services in ${stacks} stacks, ${refs.length - privateImages.length} images.${excluded} ` +
383
384
  `${counts.update} new builds, ${counts.release} new releases, ${counts.rebuild} rebuilds, ${counts.unknown} unknown, ${counts["up-to-date"]} up to date` +
@@ -88,11 +88,14 @@ export const HOST_COMMANDS: Readonly<Record<string, { args: number; help: string
88
88
  "host-upgrade": { args: 0, help: "apt full-upgrade of the host; a new Proxmox VE major is held (prints held:); ends with reboot-needed" },
89
89
  "host-reboot": { args: 1, help: "host-reboot --confirm reboot the host in one minute; every guest on it restarts" },
90
90
  "guest-exec": { args: 2, help: "guest-exec <vmid> <status|journal|df|updates> fixed read inside a guest (pct exec or qm guest exec)" },
91
+ "nightly-journal": { args: 1, help: "nightly-journal <days> raw systemd start and end lines of the nightly jobs on the host and in every running guest" },
91
92
  };
92
93
  export const GUEST_EXEC = ["status", "journal", "df", "updates"];
93
94
  /** A full upgrade with a new kernel and pve-manager can take several minutes on the host. */
94
95
  export const HOST_UPGRADE_TIMEOUT_MS = 20 * 60_000;
95
96
  const SINCE = /^[A-Za-z0-9 :\-+]{1,40}$/;
97
+ /** Nights `nightly` reads; the entry scripts accept 1..14. */
98
+ export const NIGHTLY_DAYS = { default: 7, max: 14 };
96
99
 
97
100
  const HELP = `proxmox specialist (Proxmox VE on Pulsar at 37 and SPACE-X at Mira)
98
101
 
@@ -109,6 +112,7 @@ const HELP = `proxmox specialist (Proxmox VE on Pulsar at 37 and SPACE-X at Mira
109
112
  snapshot-delete <vmid> <name> --confirm
110
113
  snapshot-rollback <vmid> <name> --confirm
111
114
  set <vmid> key=value... VM: ${SET_KEYS.qemu.join(", ")}, netN; CT: ${SET_KEYS.lxc.join(", ")}, netN
115
+ nightly [days] start, end, and duration (UTC) of each nightly job of the last days (default ${NIGHTLY_DAYS.default}, max ${NIGHTLY_DAYS.max}): vzdump tasks, host timers, and each running guest's updates, backups, and Docker restarts
112
116
  ${Object.entries(HOST_COMMANDS)
113
117
  .map(([name, c]) => ` ${name.padEnd(35)} ${c.help}`)
114
118
  .join("\n")}
@@ -333,6 +337,7 @@ export function buildHostRemote(args: string[]): string {
333
337
  if (rest.length !== spec.args) throw new UsageError(`${name} takes ${spec.args} argument(s)`);
334
338
  if (name === "host-journal" && !SINCE.test(rest[0])) throw new UsageError("since: letters, digits, spaces, colon, plus, minus only");
335
339
  if (name === "host-reboot" && rest[0] !== "--confirm") throw new UsageError("host-reboot restarts every guest on the host; add --confirm when the task asks for it explicitly");
340
+ if (name === "nightly-journal") parseDays(rest[0]);
336
341
  if (name === "guest-exec") {
337
342
  parseVmid(rest[0]);
338
343
  if (!GUEST_EXEC.includes(rest[1])) throw new UsageError(`guest-exec action must be one of ${GUEST_EXEC.join(", ")}`);
@@ -340,7 +345,100 @@ export function buildHostRemote(args: string[]): string {
340
345
  return [name, ...rest].join(" ");
341
346
  }
342
347
 
343
- async function host(config: Target, args: string[]): Promise<number> {
348
+ export function parseDays(text: string | undefined): number {
349
+ const days = text === undefined ? NIGHTLY_DAYS.default : Number(text);
350
+ if (!Number.isInteger(days) || days < 1 || days > NIGHTLY_DAYS.max) throw new UsageError(`days must be 1..${NIGHTLY_DAYS.max}`);
351
+ return days;
352
+ }
353
+
354
+ /** One run of a nightly job on one machine; a Docker restart runs from the daemon's stop to its start. */
355
+ export interface NightlyRun {
356
+ machine: string;
357
+ job: string;
358
+ start: string;
359
+ end: string | null;
360
+ result: string;
361
+ }
362
+
363
+ const JOURNAL_LINE = /^(\S+) (\S+) systemd\[1\]: (.*)$/;
364
+
365
+ /** Runs from the `nightly-journal` lines: Starting to Finished or Failed, and Docker from Deactivated to Started. */
366
+ export function journalRuns(text: string): NightlyRun[] {
367
+ const runs: NightlyRun[] = [];
368
+ const open = new Map<string, NightlyRun>();
369
+ const close = (key: string, end: string, result: string) => {
370
+ const run = open.get(key);
371
+ if (!run) return;
372
+ run.end = end;
373
+ run.result = result;
374
+ open.delete(key);
375
+ };
376
+ for (const line of text.split("\n")) {
377
+ const m = JOURNAL_LINE.exec(line.trim());
378
+ if (!m) continue;
379
+ const [, time, machine, message] = m;
380
+ const at = new Date(time).toISOString();
381
+ const unit = /([\w@.-]+)\.service/.exec(message)?.[1];
382
+ if (!unit) continue;
383
+ const key = `${machine} ${unit}`;
384
+ if (unit === "docker") {
385
+ if (/: Deactivated successfully/.test(message)) {
386
+ const run = { machine, job: "docker restart", start: at, end: null, result: "running" };
387
+ runs.push(run);
388
+ open.set(key, run);
389
+ } else if (/^Started /.test(message)) close(key, at, "ok");
390
+ continue;
391
+ }
392
+ if (/^Starting /.test(message)) {
393
+ const run = { machine, job: unit, start: at, end: null, result: "running" };
394
+ runs.push(run);
395
+ open.set(key, run);
396
+ } else if (/^Finished /.test(message)) close(key, at, "ok");
397
+ else if (/: Failed with result '([^']+)'/.test(message)) close(key, at, `failed (${/'([^']+)'/.exec(message)?.[1] ?? "?"})`);
398
+ else if (/^Failed to start /.test(message)) close(key, at, "failed");
399
+ }
400
+ return runs;
401
+ }
402
+
403
+ /** Runs from `/nodes/<node>/tasks?typefilter=vzdump`: one per backup task, the whole job when it names no guest. */
404
+ export function vzdumpRuns(node: string, tasks: unknown): NightlyRun[] {
405
+ if (!Array.isArray(tasks)) return [];
406
+ return (tasks as { type?: unknown; id?: unknown; starttime?: unknown; endtime?: unknown; status?: unknown }[])
407
+ .filter((t) => t.type === "vzdump" && typeof t.starttime === "number")
408
+ .map((t) => ({
409
+ machine: node,
410
+ job: typeof t.id === "string" && t.id !== "" ? `vzdump ${t.id}` : "vzdump",
411
+ start: new Date((t.starttime as number) * 1000).toISOString(),
412
+ end: typeof t.endtime === "number" ? new Date(t.endtime * 1000).toISOString() : null,
413
+ result: typeof t.status === "string" ? t.status : "running",
414
+ }));
415
+ }
416
+
417
+ function durationText(start: string, end: string | null): string {
418
+ if (end === null) return "-";
419
+ const s = Math.round((Date.parse(end) - Date.parse(start)) / 1000);
420
+ return s >= 60 ? `${Math.floor(s / 60)}m${String(s % 60).padStart(2, "0")}s` : `${s}s`;
421
+ }
422
+
423
+ /** The runs as a fixed-width table, oldest first, times in UTC. */
424
+ export function nightlyTable(runs: readonly NightlyRun[]): string {
425
+ const rows = [...runs]
426
+ .sort((a, b) => a.start.localeCompare(b.start))
427
+ .map((r) => [r.start.slice(0, 10), r.start.slice(11, 19), r.end ? r.end.slice(11, 19) : "-", durationText(r.start, r.end), r.machine, r.job, r.result]);
428
+ const head = ["date", "start", "end", "duration", "machine", "job", "result"];
429
+ const widths = head.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)));
430
+ return [head, ...rows].map((r) => r.map((c, i) => c.padEnd(widths[i])).join(" ").trimEnd()).join("\n");
431
+ }
432
+
433
+ async function nightly(config: Target, days: number): Promise<number> {
434
+ const since = Math.floor(Date.now() / 1000) - days * 24 * 3600;
435
+ const tasks = await api(config, "GET", `/nodes/${config.node}/tasks?typefilter=vzdump&since=${since}&limit=500`);
436
+ const journal = await hostOutput(config, ["nightly-journal", String(days)]);
437
+ printRaw(`${nightlyTable([...vzdumpRuns(config.node, tasks), ...journalRuns(journal)])}\n(times in UTC)`);
438
+ return 0;
439
+ }
440
+
441
+ async function hostOutput(config: Target, args: string[]): Promise<string> {
344
442
  const remote = buildHostRemote(args);
345
443
  const result = await sshFixed(
346
444
  {
@@ -352,8 +450,16 @@ async function host(config: Target, args: string[]): Promise<number> {
352
450
  remote,
353
451
  args[0] === "host-upgrade" ? HOST_UPGRADE_TIMEOUT_MS : 120_000,
354
452
  );
355
- if (result.stdout.length > 0) printRaw(result.stdout);
356
- if (result.code !== 0) throw new ServiceError(result.stderr.trim() || `${args[0]} exited ${result.code}`);
453
+ if (result.code !== 0) {
454
+ if (result.stdout.length > 0) printRaw(result.stdout);
455
+ throw new ServiceError(result.stderr.trim() || `${args[0]} exited ${result.code}`);
456
+ }
457
+ return result.stdout;
458
+ }
459
+
460
+ async function host(config: Target, args: string[]): Promise<number> {
461
+ const out = await hostOutput(config, args);
462
+ if (out.length > 0) printRaw(out);
357
463
  return 0;
358
464
  }
359
465
 
@@ -373,6 +479,10 @@ export async function command(args: string[]): Promise<number> {
373
479
  printJson({ host: new URL(config.apiUrl).host, fingerprint256: probe.fingerprint });
374
480
  return 0;
375
481
  }
482
+ if (rest[0] === "nightly") {
483
+ if (rest.length > 2) throw new UsageError("nightly [days]");
484
+ return nightly(config, parseDays(rest[1]));
485
+ }
376
486
  if (rest[0] !== undefined && rest[0] in HOST_COMMANDS) return host(config, rest);
377
487
  return apiCommand(config, rest);
378
488
  }
@@ -12,7 +12,7 @@ import { MAX_ADD_IDS, MAX_EXCLUDE_IDS, buildTrendarrCall, interpretRun, parseImd
12
12
  import { ServiceError, UsageError } from "./lib/errors.ts";
13
13
  import { safeRepoPath, safeWritePath } from "./lib/repo.ts";
14
14
  import { appGetPath, appPolicyDelete, portForwardDelete, clientFixedIpWrite, syslogChanges, buildPulsarRemote, cloudPath, deviceNetworkWrite, NAT_PATH, natWrite, parseQuery, parseSiteArgs, redactUnifi, unifiPath, unifiWrite, command as networkCommand } from "./network/run.ts";
15
- import { buildHostRemote, splitHost, needsConfirm, parseSetPairs, parseVmid, validateApiPath as validatePvePath, command as proxmoxCommand } from "./proxmox/run.ts";
15
+ import { buildHostRemote, journalRuns, nightlyTable, parseDays, splitHost, needsConfirm, parseSetPairs, parseVmid, validateApiPath as validatePvePath, vzdumpRuns, command as proxmoxCommand } from "./proxmox/run.ts";
16
16
  import { banLength, buildObsRemote, compactDecisions, parseGoDuration, summariseAlerts, summariseVulns, summariseWazuh, vulnEvents, wazuhEvents, command as securityCommand } from "./security/run.ts";
17
17
 
18
18
  describe("arcane wrapper", () => {
@@ -680,6 +680,46 @@ describe("proxmox wrapper", () => {
680
680
  expect(buildHostRemote(["host-reboot", "--confirm"])).toBe("host-reboot --confirm");
681
681
  expect(() => buildHostRemote(["host-reboot", "now"])).toThrow(/--confirm/);
682
682
  expect(() => buildHostRemote(["shell"])).toThrow(/unknown host command/);
683
+ expect(buildHostRemote(["nightly-journal", "7"])).toBe("nightly-journal 7");
684
+ expect(() => buildHostRemote(["nightly-journal", "15"])).toThrow(/days/);
685
+ expect(() => buildHostRemote(["nightly-journal", "7; id"])).toThrow(/days/);
686
+ expect(parseDays(undefined)).toBe(7);
687
+ expect(() => parseDays("0")).toThrow(/days/);
688
+ });
689
+
690
+ it("turns the nightly journal and vzdump tasks into runs and a table", () => {
691
+ const journal = [
692
+ "2026-11-04T03:00:24+00:00 pulsar systemd[1]: Starting pulsar-config-backup.service - PVE host config backup to restic...",
693
+ "2026-11-04T03:00:42+00:00 pulsar systemd[1]: pulsar-config-backup.service: Deactivated successfully.",
694
+ "2026-11-04T03:00:42+00:00 pulsar systemd[1]: Finished pulsar-config-backup.service - PVE host config backup to restic.",
695
+ "2026-10-31T18:52:02+00:00 pulsar systemd[1]: Starting mira-backup-pull.service - Copy Mira guest backups from TESLA to the UNAS...",
696
+ "2026-10-31T18:52:05+00:00 pulsar systemd[1]: mira-backup-pull.service: Failed with result 'exit-code'.",
697
+ "2026-10-31T18:52:05+00:00 pulsar systemd[1]: Failed to start mira-backup-pull.service - Copy Mira guest backups from TESLA to the UNAS.",
698
+ "2026-11-03T03:30:01+00:00 arcane systemd[1]: Starting apt-daily-upgrade.service - Daily apt upgrade and clean activities...",
699
+ "2026-11-03T03:30:09+00:00 arcane systemd[1]: Finished apt-daily-upgrade.service - Daily apt upgrade and clean activities.",
700
+ "2026-11-03T03:30:22+00:00 arcane systemd[1]: docker.service: Deactivated successfully.",
701
+ "2026-11-03T03:30:32+00:00 arcane systemd[1]: Starting docker.service - Docker Application Container Engine...",
702
+ "2026-11-03T03:30:33+00:00 arcane systemd[1]: Started docker.service - Docker Application Container Engine.",
703
+ "2026-11-01T05:35:22+00:00 identity systemd[1]: Starting docker.service - Docker Application Container Engine...",
704
+ ].join("\n");
705
+ expect(journalRuns(journal)).toEqual([
706
+ { machine: "pulsar", job: "pulsar-config-backup", start: "2026-11-04T03:00:24.000Z", end: "2026-11-04T03:00:42.000Z", result: "ok" },
707
+ { machine: "pulsar", job: "mira-backup-pull", start: "2026-10-31T18:52:02.000Z", end: "2026-10-31T18:52:05.000Z", result: "failed (exit-code)" },
708
+ { machine: "arcane", job: "apt-daily-upgrade", start: "2026-11-03T03:30:01.000Z", end: "2026-11-03T03:30:09.000Z", result: "ok" },
709
+ { machine: "arcane", job: "docker restart", start: "2026-11-03T03:30:22.000Z", end: "2026-11-03T03:30:33.000Z", result: "ok" },
710
+ ]);
711
+ const tasks = [
712
+ { type: "vzdump", id: "", starttime: 1793755800, endtime: 1793756268, status: "OK" },
713
+ { type: "qmstart", id: "100", starttime: 1793755000, endtime: 1793755001, status: "OK" },
714
+ ];
715
+ const runs = [...vzdumpRuns("pulsar", tasks), ...journalRuns(journal)];
716
+ expect(runs[0]).toEqual({ machine: "pulsar", job: "vzdump", start: "2026-11-04T01:30:00.000Z", end: "2026-11-04T01:37:48.000Z", result: "OK" });
717
+ expect(vzdumpRuns("pulsar", null)).toEqual([]);
718
+ const table = nightlyTable(runs).split("\n");
719
+ expect(table[0]).toBe("date start end duration machine job result");
720
+ expect(table[1]).toBe("2026-10-31 18:52:02 18:52:05 3s pulsar mira-backup-pull failed (exit-code)");
721
+ expect(table.at(-1)).toBe("2026-11-04 03:00:24 03:00:42 18s pulsar pulsar-config-backup ok");
722
+ expect(table).toContain("2026-11-04 01:30:00 01:37:48 7m48s pulsar vzdump OK");
683
723
  });
684
724
 
685
725
  it("prints help without touching config", async () => {