@astrofoundry/pi-astro 0.40.40 โ†’ 0.40.42

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  import {
2
2
  escalationText,
3
+ hostRestartText,
3
4
  observe,
4
5
  parseSnapshot,
5
6
  readWatchState,
@@ -98,6 +99,10 @@ export class ArcaneWatch {
98
99
  const now = this.deps.now();
99
100
  const seen = observe(state, snapshot, now, this.deps.nasOutage(now));
100
101
  for (const note of seen.notes) this.deps.log(`arcane watch: ${note}`);
102
+ for (const e of seen.hostRestarts) {
103
+ this.deps.log(`arcane watch: ${e.count} of ${e.total} containers restarted between ${e.from} and ${e.to}`);
104
+ await this.post(s.channelId, { content: hostRestartText(e, s.timeZone) });
105
+ }
101
106
  for (const u of seen.unhealthy) {
102
107
  const posted = await this.post(s.channelId, { content: unhealthyText(u, owners, s.timeZone, now), mentionUsers: owners });
103
108
  if (posted) this.deps.log(`arcane watch: ${u.name} ${u.problem} after a redeploy`);
@@ -4,7 +4,7 @@ import { join } from "node:path";
4
4
  import { afterEach, beforeEach, describe, expect, it } from "vitest";
5
5
  import type { WatchedContainer } from "../../specialists/lib/container-watch.ts";
6
6
  import { ArcaneWatch } from "./arcane-watch-monitor.ts";
7
- import { emptyWatchState, escalationText, healthProblem, observe, parseSnapshot, readWatchState, restartInvestigationTask, unhealthyText, type WatchState } from "./arcane-watch.ts";
7
+ import { emptyWatchState, escalationText, healthProblem, hostRestartText, observe, parseSnapshot, readWatchState, restartInvestigationTask, unhealthyText, type WatchState } from "./arcane-watch.ts";
8
8
  import type { DiscordConfig } from "./config.ts";
9
9
  import type { CreateMessage } from "./rest.ts";
10
10
  import type { InvestigationRequest } from "./security-monitor.ts";
@@ -34,7 +34,7 @@ describe("container watch rules", () => {
34
34
  it("only records a baseline the first time", () => {
35
35
  const state = emptyWatchState();
36
36
  const out = observe(state, [ct({ startedAt: iso(T0 - MIN) })], T0, false);
37
- expect(out).toEqual({ escalations: [], unhealthy: [], notes: [] });
37
+ expect(out).toEqual({ escalations: [], unhealthy: [], hostRestarts: [], notes: [] });
38
38
  expect(state.containers.syncthing).toEqual({ id: "a", imageId: "sha256:1", startedAt: iso(T0 - MIN), seenAt: iso(T0) });
39
39
  });
40
40
 
@@ -70,6 +70,33 @@ describe("container watch rules", () => {
70
70
  expect(state.restarts).toEqual({});
71
71
  });
72
72
 
73
+ it("counts a restart of most containers as one host event, also when two checks split it", () => {
74
+ const four = ["a", "b", "c", "d"].map((name) => ct({ name, id: name }));
75
+ const state = seenAt(four, T0);
76
+ const early = iso(T0 + 4 * MIN + 50_000);
77
+ observe(state, [ct({ name: "a", id: "a", startedAt: early }), ...four.slice(1)], T0 + 5 * MIN, false);
78
+ expect(state.restarts).toEqual({ a: [early] });
79
+ const together = iso(T0 + 5 * MIN + 10_000);
80
+ const out = observe(state, [ct({ name: "a", id: "a", startedAt: early }), ct({ name: "b", id: "b", startedAt: together }), ct({ name: "c", id: "c", startedAt: together }), four[3]], T0 + 5 * MIN + 40_000, false);
81
+ expect(out.hostRestarts).toEqual([{ count: 2, total: 4, from: together, to: together }]);
82
+ expect(state.restarts).toEqual({});
83
+ const late = iso(T0 + 5 * MIN + 45_000);
84
+ const after = observe(state, [ct({ name: "a", id: "a", startedAt: early }), ct({ name: "b", id: "b", startedAt: together }), ct({ name: "c", id: "c", startedAt: together }), ct({ name: "d", id: "d", startedAt: late })], T0 + 10 * MIN, false);
85
+ expect(after.hostRestarts).toEqual([]);
86
+ expect(after.notes).toEqual(["d restarted with a host-wide restart"]);
87
+ expect(state.restarts).toEqual({});
88
+ const alone = iso(T0 + 19 * MIN);
89
+ observe(state, [ct({ name: "a", id: "a", startedAt: alone }), ct({ name: "b", id: "b", startedAt: together }), ct({ name: "c", id: "c", startedAt: together }), ct({ name: "d", id: "d", startedAt: late })], T0 + 20 * MIN, false);
90
+ expect(state.restarts).toEqual({ a: [alone] });
91
+ });
92
+
93
+ it("forgets host events after a day", () => {
94
+ const state = seenAt([ct()], T0);
95
+ state.hostRestarts = [{ from: iso(T0 - 25 * 60 * MIN), to: iso(T0 - 25 * 60 * MIN) }];
96
+ observe(state, [ct()], T0 + 5 * MIN, false);
97
+ expect(state.hostRestarts).toEqual([]);
98
+ });
99
+
73
100
  it("checks a redeployed container after five minutes and reports what is wrong", () => {
74
101
  const state = seenAt([ct({ name: "mealie", id: "old", imageId: "sha256:1", project: "mealie" })], T0);
75
102
  const started = T0 + MIN;
@@ -125,6 +152,12 @@ describe("container watch texts", () => {
125
152
  );
126
153
  });
127
154
 
155
+ it("names a host-wide restart without tagging anyone", () => {
156
+ expect(hostRestartText({ count: 46, total: 52, from: iso(T0), to: iso(T0 + 8_000) }, TZ)).toBe(
157
+ "๐Ÿ” **Host-wide restart** ยท 46 of 52 containers restarted at 22 Oct 10:00 CEST, so Docker or the Arcane VM restarted. They do not count as container restarts.",
158
+ );
159
+ });
160
+
128
161
  it("asks the specialist to read only, within the host's one-day window", () => {
129
162
  const now = T0 + 24 * 60 * MIN;
130
163
  const task = restartInvestigationTask({ name: "syncthing", restarts: [iso(T0 + MIN), iso(now - MIN)] }, now);
@@ -219,6 +252,22 @@ describe("container watch monitor", () => {
219
252
  expect(h.investigations[0].task).toContain("restarted 3 times");
220
253
  });
221
254
 
255
+ it("posts a host-wide restart once, untagged, and investigates nothing", async () => {
256
+ const h = harness();
257
+ const three = ["a", "b", "c"].map((name) => ct({ name, id: name }));
258
+ h.answer.next = () => JSON.stringify({ containers: three });
259
+ await h.watch.poll(SCHEDULE);
260
+ h.clock.now = T0 + 5 * MIN;
261
+ const started = iso(T0 + 4 * MIN);
262
+ h.answer.next = () => JSON.stringify({ containers: three.map((c) => ({ ...c, startedAt: started })) });
263
+ await h.watch.poll(SCHEDULE);
264
+ expect(h.messages).toHaveLength(1);
265
+ expect(h.messages[0].message.content).toContain("๐Ÿ” **Host-wide restart** ยท 3 of 3 containers");
266
+ expect(h.messages[0].message.mentionUsers).toBeUndefined();
267
+ expect(h.investigations).toEqual([]);
268
+ expect(readWatchState().restarts).toEqual({});
269
+ });
270
+
222
271
  it("warns once after 30 minutes of failed checks and says when they work again", async () => {
223
272
  const h = harness();
224
273
  h.answer.next = () => "not json";
@@ -17,6 +17,10 @@ export const HEALTH_GRACE_MS = 5 * MINUTE;
17
17
  const STARTING_LIMIT_MS = 15 * MINUTE;
18
18
  /** The only stack that mounts the NAS share (`nas_media` in the arcane repository's media/compose.yaml). */
19
19
  export const NAS_STACKS: readonly string[] = ["media"];
20
+ /** When at least this share of the containers restarted since the last check, Docker or the VM restarted them all: one host event, no container's count. */
21
+ export const HOST_RESTART_SHARE = 0.5;
22
+ /** Restarts this close to a host event belong to it, so an event that two checks split counts nowhere. */
23
+ const HOST_RESTART_MARGIN_MS = 2 * MINUTE;
20
24
 
21
25
  export interface WatchState {
22
26
  /** The last observation of each container by name. */
@@ -27,9 +31,18 @@ export interface WatchState {
27
31
  escalated: Record<string, string>;
28
32
  /** Redeployed containers whose health is checked once the grace time has passed. */
29
33
  pending: PendingHealth[];
34
+ /** Host events of the last 24 hours, from the first to the last restart they covered. */
35
+ hostRestarts: { from: string; to: string }[];
30
36
  poll: { failures: number; warned: boolean };
31
37
  }
32
38
 
39
+ export interface HostRestart {
40
+ count: number;
41
+ total: number;
42
+ from: string;
43
+ to: string;
44
+ }
45
+
33
46
  export interface PendingHealth {
34
47
  name: string;
35
48
  id: string;
@@ -56,12 +69,13 @@ export interface Unhealthy {
56
69
  export interface Observation {
57
70
  escalations: Escalation[];
58
71
  unhealthy: Unhealthy[];
59
- /** Log lines for what was seen but is expected: our operations and NAS outages. */
72
+ hostRestarts: HostRestart[];
73
+ /** Log lines for what was seen but is expected: our operations, NAS outages, and host events. */
60
74
  notes: string[];
61
75
  }
62
76
 
63
77
  export function emptyWatchState(): WatchState {
64
- return { containers: {}, restarts: {}, escalated: {}, pending: [], poll: { failures: 0, warned: false } };
78
+ return { containers: {}, restarts: {}, escalated: {}, pending: [], hostRestarts: [], poll: { failures: 0, warned: false } };
65
79
  }
66
80
 
67
81
  function statePath(): string {
@@ -109,14 +123,17 @@ export function healthProblem(c: WatchedContainer, sinceMs: number): string | nu
109
123
  /**
110
124
  * Compares a fresh snapshot with the last one: a new id is a redeploy, whose
111
125
  * health is checked after the grace time; a newer start time with the same id
112
- * is a restart. The first snapshot only records the baseline.
126
+ * is a restart. The first snapshot only records the baseline. When most
127
+ * containers restarted at once, that is one host event and no container's
128
+ * restart.
113
129
  */
114
130
  export function observe(state: WatchState, snapshot: readonly WatchedContainer[], now: number, nasOutage: boolean): Observation {
115
- const out: Observation = { escalations: [], unhealthy: [], notes: [] };
131
+ const out: Observation = { escalations: [], unhealthy: [], hostRestarts: [], notes: [] };
116
132
  const baseline = Object.keys(state.containers).length === 0;
117
133
  const at = new Date(now).toISOString();
118
134
  const nasExpected = (c: { project?: string }) => nasOutage && c.project !== undefined && NAS_STACKS.includes(c.project);
119
135
  const current = new Map(snapshot.map((c) => [c.name, c]));
136
+ const restarted: { name: string; startedAt: string }[] = [];
120
137
  for (const c of snapshot) {
121
138
  const prev = state.containers[c.name];
122
139
  if (prev && !baseline) {
@@ -127,13 +144,28 @@ export function observe(state: WatchState, snapshot: readonly WatchedContainer[]
127
144
  } else if (c.startedAt && c.startedAt !== prev.startedAt && Date.parse(c.startedAt) > Date.parse(prev.seenAt)) {
128
145
  if (c.operation) out.notes.push(`${c.name} restarted during ${operationText(c.operation)}`);
129
146
  else if (nasExpected(c)) out.notes.push(`${c.name} restarted during a NAS outage`);
130
- else state.restarts[c.name] = [...(state.restarts[c.name] ?? []), c.startedAt];
147
+ else restarted.push({ name: c.name, startedAt: c.startedAt });
131
148
  }
132
149
  }
133
150
  const startedAt = c.startedAt ?? (prev?.id === c.id ? prev.startedAt : null);
134
151
  state.containers[c.name] = { id: c.id, imageId: c.imageId, startedAt, seenAt: at };
135
152
  }
136
153
  for (const name of Object.keys(state.containers)) if (!current.has(name)) delete state.containers[name];
154
+ state.hostRestarts = state.hostRestarts.filter((e) => Date.parse(e.to) >= now - DAY);
155
+ if (restarted.length >= Math.max(2, Math.ceil(snapshot.length * HOST_RESTART_SHARE))) {
156
+ const times = restarted.map((r) => Date.parse(r.startedAt));
157
+ const event = { from: new Date(Math.min(...times)).toISOString(), to: new Date(Math.max(...times)).toISOString() };
158
+ state.hostRestarts.push(event);
159
+ out.hostRestarts.push({ count: restarted.length, total: snapshot.length, ...event });
160
+ restarted.length = 0;
161
+ }
162
+ const inHostRestart = (t: string) => state.hostRestarts.some((e) => Date.parse(t) >= Date.parse(e.from) - HOST_RESTART_MARGIN_MS && Date.parse(t) <= Date.parse(e.to) + HOST_RESTART_MARGIN_MS);
163
+ for (const r of restarted) {
164
+ if (inHostRestart(r.startedAt)) out.notes.push(`${r.name} restarted with a host-wide restart`);
165
+ else state.restarts[r.name] = [...(state.restarts[r.name] ?? []), r.startedAt];
166
+ }
167
+ // A check that saw only the first restarts of an event counted them before the event was known.
168
+ for (const [name, times] of Object.entries(state.restarts)) state.restarts[name] = times.filter((t) => !inHostRestart(t));
137
169
  for (const [name, times] of Object.entries(state.restarts)) {
138
170
  const recent = times.filter((t) => Date.parse(t) >= now - DAY);
139
171
  if (recent.length === 0) delete state.restarts[name];
@@ -186,6 +218,11 @@ export function escalationText(e: Escalation, owners: readonly string[], timeZon
186
218
  ].join("\n");
187
219
  }
188
220
 
221
+ /** Untagged: Docker or the VM restarting is planned maintenance far more often than a fault. */
222
+ export function hostRestartText(e: HostRestart, timeZone: string): string {
223
+ return `๐Ÿ” **Host-wide restart** ยท ${e.count} of ${e.total} containers restarted at ${clockText(e.from, timeZone)}, so Docker or the Arcane VM restarted. They do not count as container restarts.`;
224
+ }
225
+
189
226
  export function unhealthyText(u: Unhealthy, owners: readonly string[], timeZone: string, now: number): string {
190
227
  const minutes = Math.round((now - Date.parse(u.since)) / MINUTE);
191
228
  return [
@@ -1,7 +1,7 @@
1
1
  import { describe, expect, it } from "vitest";
2
2
  import type { ArcaneReport } from "./daily-report.ts";
3
3
  import type { ToolsReport } from "./tools-report.ts";
4
- import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
4
+ import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, pomeriumButtonId, wazuhRelease, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
5
5
 
6
6
  const EDGE = `nginx active
7
7
  wg-quick@wg0 active
@@ -78,15 +78,18 @@ describe("morning status lines", () => {
78
78
  expect(proxmoxStatus([{ name: "Pulsar", id: "pulsar", status: PULSAR, updates: PULSAR_UPDATES, upgrade: null }]).problem).toBe("Proxmox: the host upgrade on Pulsar failed");
79
79
  });
80
80
 
81
- it("flags updates the nightly run left behind, reading guest-agent JSON too, and ignores the held Wazuh agent", () => {
81
+ it("flags updates the nightly run left behind, reading guest-agent JSON too, and announces a Wazuh release from the held agent", () => {
82
82
  const vm = JSON.stringify({ exitcode: 0, exited: 1, "out-data": "Listing...\ndocker-ce/trixie 5:29.9.2-1 amd64 [upgradable from: 5:29.7.2-1]\nwazuh-agent/stable 4.14.9-1 amd64 [upgradable from: 4.14.7-1]\n" });
83
83
  expect(pendingPackageLines(vm)).toEqual(["docker-ce/trixie 5:29.9.2-1 amd64 [upgradable from: 5:29.7.2-1]"]);
84
- const held = "Listing...\nwazuh-agent/stable 4.14.9-1 amd64 [upgradable from: 4.14.7-1]";
85
- expect(updatesStatus({ technitium2: held, synapse: "Listing..." })).toEqual({ area: "Updates", line: "2 hosts up to date after the nightly run" });
86
- expect(updatesStatus({ arcane: vm, "technitium3 (Mira)": held, radius: null })).toEqual({
87
- area: "Updates",
88
- line: "still waiting after the nightly run: arcane 1; no update list from radius",
89
- problem: "Updates: arcane (docker-ce) did not install overnight; no update list from radius",
84
+ const held = "Listing...\nwazuh-agent/stable 4.14.10-1 amd64 [upgradable from: 4.14.9-1]";
85
+ expect(wazuhRelease({ arcane: vm, synapse: held, radius: null })).toEqual({ from: "4.14.9", to: "4.14.10" });
86
+ expect(wazuhRelease({ synapse: "Listing..." })).toBeNull();
87
+ expect(updatesStatus({ synapse: "Listing..." })).toEqual({ area: "Updates", line: "1 host up to date after the nightly run" });
88
+ const wazuh = updatesStatus({ technitium2: held, synapse: "Listing..." });
89
+ expect(wazuh.line).toBe("2 hosts up to date after the nightly run; Wazuh 4.14.10 is out (agents held at 4.14.9)");
90
+ expect(wazuh.problem).toMatch(/^Updates: Wazuh 4\.14\.10 is out: ask me to upgrade the manager/);
91
+ expect(updatesStatus({ arcane: vm, radius: null })).toMatchObject({
92
+ line: "still waiting after the nightly run: arcane 1; no update list from radius; Wazuh 4.14.9 is out (agents held at 4.14.7)",
90
93
  route: "admin",
91
94
  });
92
95
  });
@@ -105,6 +108,16 @@ describe("morning status lines", () => {
105
108
  expect(identityStatus(DMZ.replace("Active: active (running) since Thu 2026-09-24 17:21:24 CEST; 4 weeks 0 days ago\nโ— nginx", "Active: failed (Result: signal)\nโ— nginx")).problem).toBe("Identity: pomerium.service failed on the DMZ VM");
106
109
  });
107
110
 
111
+ it("offers an owner button when a newer Pomerium release is out", () => {
112
+ const notes = "https://github.com/pomerium/pomerium/releases/tag/v0.33.6";
113
+ expect(identityStatus(DMZ, { installed: "v0.33.6", newest: "v0.33.6", update: false, notes })).toEqual({ area: "Identity", line: "pomerium, nginx, lego-renew.timer active; Pomerium v0.33.6" });
114
+ const newer = identityStatus(DMZ, { installed: "v0.33.1", newest: "v0.33.6", update: true, notes });
115
+ expect(newer.line).toBe("pomerium, nginx, lego-renew.timer active; Pomerium v0.33.1, v0.33.6 available");
116
+ expect(newer.problem).toContain(`Pomerium v0.33.6 is out (running v0.33.1; notes: <${notes}>)`);
117
+ expect(newer.buttons).toEqual([{ label: "Update Pomerium to v0.33.6", customId: pomeriumButtonId("v0.33.6") }]);
118
+ expect(identityStatus(DMZ, null).line).toBe("pomerium, nginx, lego-renew.timer active; Pomerium version unknown");
119
+ });
120
+
108
121
  it("checks both resolvers, Nexus, and SABnzbd", () => {
109
122
  expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: { status: "ok" }, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "technitium1, technitium2, technitium3 answer" });
110
123
  expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: null, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "no answer from technitium2", problem: "DNS: technitium2 does not answer", route: "admin" });
@@ -121,8 +121,8 @@ export function proxmoxStatus(hosts: readonly ProxmoxHost[]): AreaStatus {
121
121
  /** Packages the nightly run leaves alone on purpose: a Wazuh agent must never be newer than the manager. */
122
122
  const HELD_PACKAGES = new Set(["wazuh-agent"]);
123
123
 
124
- /** The `[upgradable from` lines of `apt list --upgradable`, also when a VM's guest agent wraps them in JSON (`out-data`); held packages left out. */
125
- export function pendingPackageLines(output: string): string[] {
124
+ /** The `[upgradable from` lines of `apt list --upgradable`, also when a VM's guest agent wraps them in JSON (`out-data`). */
125
+ function upgradableLines(output: string): string[] {
126
126
  let text = output;
127
127
  try {
128
128
  const wrapped = (JSON.parse(output) as { "out-data"?: unknown } | null)?.["out-data"];
@@ -130,15 +130,31 @@ export function pendingPackageLines(output: string): string[] {
130
130
  } catch {
131
131
  /* plain apt output */
132
132
  }
133
- return text
134
- .split("\n")
135
- .filter((l) => l.includes("[upgradable from"))
136
- .filter((l) => !HELD_PACKAGES.has(l.split("/")[0]));
133
+ return text.split("\n").filter((l) => l.includes("[upgradable from"));
134
+ }
135
+
136
+ /** The upgradable packages of one host, held packages left out. */
137
+ export function pendingPackageLines(output: string): string[] {
138
+ return upgradableLines(output).filter((l) => !HELD_PACKAGES.has(l.split("/")[0]));
139
+ }
140
+
141
+ /** The newest `wazuh-agent` that apt offers on any host: held until the manager runs it, so it announces a Wazuh release. */
142
+ export function wazuhRelease(hosts: Readonly<Record<string, string | null>>): { from: string; to: string } | null {
143
+ let found: { from: string; to: string } | null = null;
144
+ for (const out of Object.values(hosts)) {
145
+ if (out === null) continue;
146
+ const m = upgradableLines(out)
147
+ .map((l) => /^wazuh-agent\/\S+\s+(\d+\.\d+\.\d+)\S*\s.*\[upgradable from: (\d+\.\d+\.\d+)/.exec(l))
148
+ .find((x) => x !== null);
149
+ if (m && (found === null || m[1].localeCompare(found.to, undefined, { numeric: true }) > 0)) found = { from: m[2], to: m[1] };
150
+ }
151
+ return found;
137
152
  }
138
153
 
139
154
  /** Each guest's and Synapse's `updates` after the 04:30 run: anything still waiting did not install and needs a look. */
140
155
  export function updatesStatus(hosts: Readonly<Record<string, string | null>>): AreaStatus {
141
156
  const names = Object.keys(hosts);
157
+ const wazuh = wazuhRelease(hosts);
142
158
  const unknown = names.filter((n) => hosts[n] === null);
143
159
  const left = names.flatMap((n) => {
144
160
  const out = hosts[n];
@@ -149,10 +165,12 @@ export function updatesStatus(hosts: Readonly<Record<string, string | null>>): A
149
165
  const parts = [
150
166
  left.length === 0 ? `${checked} host${checked === 1 ? "" : "s"} up to date after the nightly run` : `still waiting after the nightly run: ${left.map((h) => `${escapeDiscord(h.name)} ${h.packages.length}`).join(", ")}`,
151
167
  unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null,
168
+ wazuh ? `Wazuh ${wazuh.to} is out (agents held at ${wazuh.from})` : null,
152
169
  ].filter(Boolean);
153
170
  const problems = [
154
171
  left.length > 0 ? `${left.map((h) => `${h.name} (${h.packages.slice(0, 4).join(", ")}${h.packages.length > 4 ? ", ..." : ""})`).join("; ")} did not install overnight` : null,
155
172
  unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null,
173
+ wazuh ? `Wazuh ${wazuh.to} is out: ask me to upgrade the manager (\`observability/compose.yaml\`, then \`deploy.sh\`), then the agents (\`scripts/wazuh-agents/deploy.sh --upgrade\`); an agent must never be newer than the manager` : null,
156
174
  ].filter(Boolean);
157
175
  return { area: "Updates", line: parts.join("; "), ...(problems.length > 0 ? { problem: `Updates: ${problems.join("; ")}`, route: "admin" as const } : {}) };
158
176
  }
@@ -178,13 +196,43 @@ export function networkStatus(radius: string | null, tailscale: string | null):
178
196
  return { area: "Network", line: parts.join(", "), ...(problems.length > 0 ? { problem: `Network: ${problems.join("; ")}`, route: "admin" as const } : {}) };
179
197
  }
180
198
 
181
- /** `identity dmz status`: Pomerium, nginx, and the certificate renewal timer on the DMZ VM. */
182
- export function identityStatus(status: string | null): AreaStatus {
199
+ /** The custom id of the button that upgrades Pomerium to one release. */
200
+ export function pomeriumButtonId(tag: string): string {
201
+ return `identity:pomerium:${tag}`;
202
+ }
203
+
204
+ /** `identity dmz pomerium-check`: the installed and the newest release. */
205
+ interface PomeriumCheck {
206
+ installed: string;
207
+ newest: string;
208
+ notes: string;
209
+ }
210
+
211
+ function pomeriumCheck(value: unknown): PomeriumCheck | null {
212
+ const v = value as { installed?: unknown; newest?: unknown; notes?: unknown } | null;
213
+ if (typeof v?.installed !== "string" || typeof v.newest !== "string" || typeof v.notes !== "string") return null;
214
+ return { installed: v.installed, newest: v.newest, notes: v.notes };
215
+ }
216
+
217
+ /** `identity dmz status` and `dmz pomerium-check`: Pomerium, nginx, and the certificate renewal timer on the DMZ VM, plus a newer Pomerium release with its owner button. */
218
+ export function identityStatus(status: string | null, pomerium?: unknown): AreaStatus {
183
219
  if (status === null) return failed("Identity", "admin", "dmz status");
184
220
  const units = [...status.matchAll(/^[โ—โ—‹ร—]\s+(\S+)\s+-[^\n]*\n(?:.*\n)*?\s+Active:\s+(\S+)(?:\s+\((\w+)\))?/gm)].map((m) => ({ unit: m[1], active: m[2], sub: m[3] ?? "" }));
185
221
  const bad = units.filter((u) => u.active !== "active").map((u) => `${u.unit} ${u.active}`);
186
- const line = units.length === 0 ? "no unit status read" : bad.length === 0 ? `${units.map((u) => u.unit.replace(/\.service$/, "")).join(", ")} active` : `not active: ${bad.join(", ")}`;
187
- return { area: "Identity", line, ...(bad.length > 0 || units.length === 0 ? { problem: `Identity: ${bad.length > 0 ? bad.join(", ") : "no unit status read"} on the DMZ VM`, route: "admin" as const } : {}) };
222
+ const release = pomeriumCheck(pomerium);
223
+ const newer = release !== null && release.installed !== release.newest ? release : null;
224
+ const unitLine = units.length === 0 ? "no unit status read" : bad.length === 0 ? `${units.map((u) => u.unit.replace(/\.service$/, "")).join(", ")} active` : `not active: ${bad.join(", ")}`;
225
+ const releaseLine = pomerium === undefined ? null : release === null ? "Pomerium version unknown" : newer ? `Pomerium ${newer.installed}, ${newer.newest} available` : `Pomerium ${release.installed}`;
226
+ const problems = [
227
+ bad.length > 0 || units.length === 0 ? `${bad.length > 0 ? bad.join(", ") : "no unit status read"} on the DMZ VM` : null,
228
+ newer ? `Pomerium ${newer.newest} is out (running ${newer.installed}; notes: <${newer.notes}>). The button installs the official release after checking its sha256, restarts Pomerium for a few seconds, and puts ${newer.installed} back if the login route fails` : null,
229
+ ].filter(Boolean);
230
+ return {
231
+ area: "Identity",
232
+ line: [unitLine, releaseLine].filter(Boolean).join("; "),
233
+ ...(problems.length > 0 ? { problem: `Identity: ${problems.join("; ")}`, route: "admin" as const } : {}),
234
+ ...(newer ? { buttons: [{ label: `Update Pomerium to ${newer.newest}`, customId: pomeriumButtonId(newer.newest) }] } : {}),
235
+ };
188
236
  }
189
237
 
190
238
  /** Upgrade order: Mira's node first, the primary last, one node per morning. */
@@ -34,7 +34,7 @@ import {
34
34
  writePostedArcane,
35
35
  } from "./daily-report.ts";
36
36
  import { GatewayClient } from "./gateway.ts";
37
- import { type CommandOption, IS_COMPONENTS_V2, KeyedQueue, MEDIA, MEDIA_COMMANDS, type MediaKind, type Outcome, PAUSED_TEXT, RESUME_ID, addOutcome, addedWhilePaused, applyOutcome, commandArgs, hasActiveButton, listMessage, mediaAccess, mediaPointer, parseAddButton, parseAddReport, parseMediaList, parseQueueState, queueLine, resumedText, unpauseRow } from "./media.ts";
37
+ import { type CommandOption, IS_COMPONENTS_V2, KeyedQueue, MEDIA, MEDIA_COMMANDS, type MediaKind, type Outcome, PAUSED_TEXT, RESUME_ID, addOutcome, addedWhilePaused, applyOutcome, commandArgs, escapeDiscord, hasActiveButton, listMessage, mediaAccess, mediaPointer, parseAddButton, parseAddReport, parseMediaList, parseQueueState, queueLine, resumedText, unpauseRow } from "./media.ts";
38
38
  import { cancelledApprovalText, interruptedTaskText, readOpenWork, type OpenWork, type StopReason, writeOpenWork } from "./open-work.ts";
39
39
  import { DISMISSED, type Picks, parseNoButton, parsePicks, picksArgs, picksFailureText, picksMessage, readPicksState, withDismissed, withSuggested, writePicksState } from "./media-picks.ts";
40
40
  import { FRIDAY, isDue, MONDAY, readReportState, writeReportState, zonedWeekday } from "./reports.ts";
@@ -183,6 +183,8 @@ const PROXMOX_HOSTS: readonly { name: string; id: string; prefix: string[] }[] =
183
183
  const DNS_UPGRADE_MS = 11 * 60_000;
184
184
  /** The wrapper allows a host upgrade 20 minutes; the bridge waits a little longer for its answer. */
185
185
  const HOST_UPGRADE_MS = 21 * 60_000;
186
+ /** The download plus the DMZ's own upgrade, health wait, and possible rollback: the wrapper allows 10 minutes for the DMZ part. */
187
+ const POMERIUM_UPGRADE_MS = 16 * 60_000;
186
188
 
187
189
  function pause(ms: number): Promise<void> {
188
190
  return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)).unref());
@@ -241,6 +243,7 @@ class Bridge {
241
243
  private readonly toolRuns = new Map<string, { channelId: string; label: string }>();
242
244
  /** Proxmox hosts that an owner's Reboot click is restarting. */
243
245
  private readonly rebooting = new Set<string>();
246
+ private pomeriumUpgrading = false;
244
247
  private readonly toolClicks = new KeyedQueue();
245
248
  private gateway: GatewayClient | undefined;
246
249
  private stopping = false;
@@ -669,7 +672,7 @@ class Bridge {
669
672
  );
670
673
  }
671
674
  if (present.has("network")) areas.push((async () => networkStatus(await text("network", ["pulsar", "radius-status"]), await text("network", ["pulsar", "tailscale-status"])))());
672
- if (present.has("identity")) areas.push((async () => identityStatus(await text("identity", ["dmz", "status"])))());
675
+ if (present.has("identity")) areas.push((async () => identityStatus(await text("identity", ["dmz", "status"]), await json("identity", ["dmz", "pomerium-check"])))());
673
676
  if (present.has("dns")) {
674
677
  areas.push(
675
678
  (async () => {
@@ -1448,6 +1451,45 @@ class Bridge {
1448
1451
  }
1449
1452
  }
1450
1453
 
1454
+ /**
1455
+ * Upgrades Pomerium to the release on the button for an owner, when the DMZ still runs an older one.
1456
+ * The wrapper checks the published sha256 and the DMZ puts the old binary back if the login route fails.
1457
+ */
1458
+ private async onPomeriumButton(interaction: Interaction, tag: string): Promise<void> {
1459
+ const userId = interaction.member?.user?.id ?? interaction.user?.id ?? "";
1460
+ this.refreshConfig();
1461
+ if (!isOwner(this.config, userId)) {
1462
+ await this.rest.ephemeralReply(interaction.id, interaction.token, "Only owners upgrade Pomerium.");
1463
+ return;
1464
+ }
1465
+ if (this.pomeriumUpgrading) {
1466
+ await this.rest.ephemeralReply(interaction.id, interaction.token, "A Pomerium upgrade is already running.");
1467
+ return;
1468
+ }
1469
+ const check = (await this.runEntry("identity", userId, ["dmz", "pomerium-check"], 2 * 60_000).then((out) => JSON.parse(out) as { installed?: unknown }).catch(() => null)) ?? {};
1470
+ if (check.installed === tag) {
1471
+ await this.rest.ephemeralReply(interaction.id, interaction.token, `Pomerium already runs ${tag}.`);
1472
+ return;
1473
+ }
1474
+ await this.rest.deferUpdate(interaction.id, interaction.token);
1475
+ const channelId = interaction.channel_id ?? this.config.adminChannelId;
1476
+ const messageId = interaction.message?.id;
1477
+ this.pomeriumUpgrading = true;
1478
+ try {
1479
+ log(`pomerium upgrade to ${tag} by ${userId}`);
1480
+ if (messageId) await this.rest.editMessage(channelId, messageId, `${interaction.message?.content ?? ""}\nโณ Pomerium ${tag} is installing, started by <@${userId}>.`.slice(0, 1900), []).catch(() => undefined);
1481
+ const out = await this.runEntry("identity", userId, ["dmz", "pomerium-upgrade", tag], POMERIUM_UPGRADE_MS);
1482
+ await this.rest.createMessage(channelId, { content: `โœ… ${escapeDiscord(out.trim().split("\n").pop() ?? `Pomerium ${tag} installed`)}` });
1483
+ } catch (err) {
1484
+ await this.rest.createMessage(channelId, {
1485
+ content: `โŒ The Pomerium upgrade to ${tag} failed: ${escapeDiscord((err instanceof Error ? err.message : String(err)).slice(0, 600))} <@${userId}>`,
1486
+ mentionUsers: [userId],
1487
+ });
1488
+ } finally {
1489
+ this.pomeriumUpgrading = false;
1490
+ }
1491
+ }
1492
+
1451
1493
  /** Runs one specialist wrapper through the sudo entry point, without a model; rejects with the wrapper's stderr. */
1452
1494
  private runEntry(service: string, caller: string, args: string[], timeoutMs: number): Promise<string> {
1453
1495
  const gate = loadSpecialists().config;
@@ -1852,6 +1894,10 @@ class Bridge {
1852
1894
  await this.onArcaneButton(interaction, customId);
1853
1895
  return;
1854
1896
  }
1897
+ if (customId.startsWith("identity:pomerium:")) {
1898
+ await this.onPomeriumButton(interaction, customId.slice("identity:pomerium:".length));
1899
+ return;
1900
+ }
1855
1901
  if (customId.startsWith("proxmox:reboot:")) {
1856
1902
  await this.onRebootButton(interaction, customId.slice("proxmox:reboot:".length));
1857
1903
  return;
@@ -16,6 +16,8 @@ describe("approvals", () => {
16
16
  expect(isRisky("identity", ["zitadel", "GET", "/v2/users"])).toBe(false);
17
17
  expect(isRisky("identity", ["zitadel", "DELETE", "/v2/users/1"])).toBe(true);
18
18
  expect(isRisky("identity", ["dmz", "deploy-config"])).toBe(true);
19
+ expect(isRisky("identity", ["dmz", "pomerium-upgrade", "v0.33.6"])).toBe(true);
20
+ expect(isRisky("identity", ["dmz", "pomerium-check"])).toBe(false);
19
21
  expect(isRisky("network", ["unifi", "mira", "networks", "x"])).toBe(false);
20
22
  expect(isRisky("network", ["unifi", "37", "write", "DELETE", "/v1/sites/x/acl-rules/y"])).toBe(true);
21
23
  expect(isRisky("network", ["unifi", "mira", "--local", "write", "PUT", "/v1/sites/x/networks/y", "{}"])).toBe(true);
@@ -31,7 +31,7 @@ export const RISKY: Readonly<Record<string, readonly RegExp[]>> = {
31
31
  /^manager-(upgrade|rollback)\b/,
32
32
  /^host\s+(db-restore|radarr-movie|radarr-interface)\b/,
33
33
  ],
34
- identity: [/^zitadel\s+(DELETE|POST|PUT|PATCH)\b/, /^dmz\s+(deploy-config|pomerium-restart|render|lego-renew)\b/],
34
+ identity: [/^zitadel\s+(DELETE|POST|PUT|PATCH)\b/, /^dmz\s+(deploy-config|pomerium-restart|pomerium-upgrade|render|lego-renew)\b/],
35
35
  // Every UniFi write, on either console.
36
36
  network: [/^unifi\s+\S+\s+(write|nat-write|device-network|app-policy-delete|portforward-delete|client-fixed-ip|syslog)\b/],
37
37
  dns: [
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@astrofoundry/pi-astro",
3
- "version": "0.40.40",
3
+ "version": "0.40.42",
4
4
  "description": "Personal pi customizations (extensions, subagents, skills, prompts, themes) for the pi coding agent.",
5
5
  "keywords": [
6
6
  "pi-package"
@@ -35,6 +35,11 @@ Fixed operations on VM 104 through a forced-command SSH key. No other command re
35
35
  | `nginx-reload` | `nginx -t` then reload |
36
36
  | `lego-renew` | run the certificate renewal now |
37
37
  | `deploy-config` | install the pushed `config.yaml` as `/etc/pomerium/config.yaml.template`, render with the Vaultwarden values, restart Pomerium, check it is active; on failure restores the previous template and restarts |
38
+ | `pomerium-version` | the installed Pomerium version |
39
+ | `pomerium-check` | JSON: installed and newest Pomerium release, with the release notes link (runs on Cortex plus `pomerium-version`) |
40
+ | `pomerium-upgrade <vX.Y.Z>` | download that official release on Cortex, check it against the release's published sha256, send it to the DMZ, swap the binary, restart, and check the login route; the DMZ puts the old binary back when the check fails |
41
+
42
+ The morning run reports a newer Pomerium release with an owner button that runs `pomerium-upgrade`. Run it yourself only when an owner asks; it restarts Pomerium, so every route drops for a few seconds.
38
43
 
39
44
  ## `git <command>` (homelab repository)
40
45
 
@@ -53,7 +53,7 @@ Wrappers ship as TypeScript and run under Node 24 type stripping: erasable synta
53
53
  - `inference` reaches Nexus through one forced-command key. The Europa check is HTTP only until Windows OpenSSH forced commands are verified.
54
54
  - `REFUSED_PREFIXES` protects only the wrapper itself (`config`, `auth`, `self-update`, `completion`). The GitOps-first rule for managed projects lives in the skill and the agent prompt, not in code, because unmanaged projects and emergencies need the direct CLI.
55
55
  - The wrapper's repository commands are the `git` group; `gitops` must stay free because it is an `arcane-cli` command group (server-side syncs).
56
- - Every passthrough call writes a private per-call copy of `arcanecli.yml` with `jwt_token` set, because some `arcane-cli` commands validate the config file before reading `ARCANE_TOKEN`. The copy is removed after the call.
56
+ - Every passthrough call writes a private per-call copy of `arcanecli.yml` with `jwt_token` set, because some `arcane-cli` commands validate the config file before reading `ARCANE_TOKEN`. The copy is removed after the call. arcane-cli prints a warning on stderr that `jwt_token` and `refresh_token` in the config stop working in v3 (in the bridge log since 2026-10-23); `cliFailure` therefore names the exit code and the filtered stdout as well, because a failure can carry nothing else on stderr (the media update on 2026-11-02).
57
57
  - Every wrapper answers `--help` without reading config, so agents can discover subcommands before anything else is set up.
58
58
  - `entry/specialist-install.sh` runs as root, so it runs only `/usr/bin`, `/bin`, `/usr/sbin`, `/sbin` and the Node it installed, never Homebrew or anything else the main user can write; it checks that `/usr/local/libexec/specialist-node` (and each `specialist-<tool>` folder) and its parents belong to root and are writable by nobody else. Arguments are checked before the root test (tests exercise that part). Downloads go to a root-only temporary directory and are verified before use: Node against `SHASUMS256.txt` and the LTS flag in `index.json`, npm tarballs against the registry's `sha512` integrity, arcane-cli against the release's checksums file. Inside `/Users/specialist` it works only as `specialist` (`sudo -u specialist -H`), from a staging directory root hands over, so a link the specialist plants cannot redirect a root write. Each component switches by rename after a check (every installed wrapper's `--help`, `--version` of the new binary) and restores the previous copy when the check fails. Older versions are refused; `rollback` swaps current and previous. `python`, `gcloud`, `bw`, and `jq` follow the same rules: version folders under `/usr/local/libexec/specialist-<tool>/` behind the links `current` and `previous`, sealed `root:wheel` and not writable by others, smoke-tested as `specialist` before `current` moves; `python` refuses a switch that breaks the current gcloud. Consumers use the `current` paths: `provision.sh` for `bw` and `jq`, `edge.json` for `gcloud` and `gcloudPython`. Nothing a wrapper runs with secrets may come from Homebrew.
59
59
  - `main()` waits for stdout and stderr to flush before `process.exit`: on a pipe, exiting right after a write drops everything past the 64 KiB buffer, which cut large answers such as `projects list --all` until 2026-09-25 (`lib/flush.test.ts`).
@@ -26,7 +26,7 @@ Call one with `/run astro.<name> -- <task>` or through the `subagent` tool (both
26
26
  - Every call is logged in `/Users/specialist/.specialists/audit.log`.
27
27
  - The `specialist-gate` extension registers the tool only inside the matching specialist child and blocks other agents from calling the CLIs or reading `/Users/specialist`. `security-guard` carries the same blocks.
28
28
 
29
- Limits: `network` writes through the UniFi Network API plus a few narrow commands on the Network application's own paths (`nat-write`, `device-network`, `client-fixed-ip`, `syslog`, `app-policy-delete`, `portforward-delete`); it cannot change Site Magic. `security` is read-only. `identity` deploys only `config.yaml` to the DMZ. `edge` changes only `nginx.conf` on the VPS and can reboot it. `dns` writes zones, records, and the cluster only on the Technitium primary; on another node it only sets the server domain, resyncs a zone, or deletes a stale Secondary zone, and `technitium host` restarts or upgrades one node at a time. `proxmox` cannot create, clone, or destroy guests, change disks, or change the host configuration; on a host it only upgrades packages (`host-upgrade`) and reboots (`host-reboot --confirm`). `backup` cannot change retention or schedules. `inference` cannot start Europa or change models. `media` adds titles and toggles the download queue; it cannot delete anything or change Radarr, Sonarr, or SABnzbd settings.
29
+ Limits: `network` writes through the UniFi Network API plus a few narrow commands on the Network application's own paths (`nat-write`, `device-network`, `client-fixed-ip`, `syslog`, `app-policy-delete`, `portforward-delete`); it cannot change Site Magic. `security` is read-only. `identity` deploys only `config.yaml` to the DMZ and upgrades Pomerium to an official release whose sha256 matches (`dmz pomerium-upgrade`). `edge` changes only `nginx.conf` on the VPS and can reboot it. `dns` writes zones, records, and the cluster only on the Technitium primary; on another node it only sets the server domain, resyncs a zone, or deletes a stale Secondary zone, and `technitium host` restarts or upgrades one node at a time. `proxmox` cannot create, clone, or destroy guests, change disks, or change the host configuration; on a host it only upgrades packages (`host-upgrade`) and reboots (`host-reboot --confirm`). `backup` cannot change retention or schedules. `inference` cannot start Europa or change models. `media` adds titles and toggles the download queue; it cannot delete anything or change Radarr, Sonarr, or SABnzbd settings.
30
30
 
31
31
  ## Enable specialists on a host
32
32
 
@@ -212,9 +212,15 @@ async function passthrough(config: ArcaneConfig, args: string[]): Promise<number
212
212
  });
213
213
  }
214
214
 
215
+ /** arcane-cli can fail with only a deprecation warning on stderr; the exit code and stdout then carry the cause. */
216
+ export function cliFailure(args: readonly string[], result: RunResult): string {
217
+ const stdout = filterJsonText(result.stdout, (v) => stripKeys(v, ARCANE_DENIED_KEYS)).replace(/\s+/g, " ").trim().slice(0, 500);
218
+ return [`${args.join(" ")}: exited ${result.code}`, result.stderr.trim(), stdout && `stdout: ${stdout}`].filter((part) => part.length > 0).join("; ");
219
+ }
220
+
215
221
  async function cliJson(cli: Cli, args: string[]): Promise<unknown> {
216
222
  const result = await cli(["--output", "json", ...args]);
217
- if (result.code !== 0) throw new ServiceError(`${args.join(" ")}: ${result.stderr.trim() || `exited ${result.code}`}`);
223
+ if (result.code !== 0) throw new ServiceError(cliFailure(args, result));
218
224
  try {
219
225
  return JSON.parse(result.stdout);
220
226
  } catch {
@@ -1,3 +1,4 @@
1
+ import { createHash } from "node:crypto";
1
2
  import { join } from "node:path";
2
3
  import { readConfig } from "../lib/config.ts";
3
4
  import { ServiceError, UsageError } from "../lib/errors.ts";
@@ -49,8 +50,14 @@ export const DMZ_COMMANDS: Readonly<Record<string, { args: number; help: string;
49
50
  "nginx-reload": { args: 0, help: "test nginx config and reload" },
50
51
  "lego-renew": { args: 0, help: "run lego-renew.service now" },
51
52
  "deploy-config": { args: 0, help: "install the tracked Pomerium config.yaml from the pushed homelab checkout as the template, render, restart, check; rolls back on failure", stdin: true },
53
+ "pomerium-version": { args: 0, help: "the installed Pomerium version" },
52
54
  };
53
55
 
56
+ const RELEASES = "https://github.com/pomerium/pomerium/releases";
57
+ const TARBALL = "pomerium-linux-amd64.tar.gz";
58
+ const RELEASE_TAG = /^v\d+\.\d+\.\d+$/;
59
+ const UPGRADE_TIMEOUT_MS = 10 * 60_000;
60
+
54
61
  const DMZ_UNITS = ["pomerium", "nginx", "lego-renew"];
55
62
  const SINCE = /^[A-Za-z0-9 :\-+]{1,40}$/;
56
63
 
@@ -62,6 +69,9 @@ const HELP = `identity specialist
62
69
  ${Object.entries(DMZ_COMMANDS)
63
70
  .map(([name, c]) => ` ${name.padEnd(18)} ${c.help}`)
64
71
  .join("\n")}
72
+ pomerium-check installed and newest Pomerium release (JSON)
73
+ pomerium-upgrade pomerium-upgrade <vX.Y.Z>: install that official release after checking its published sha256,
74
+ restart, check the login route; the old binary comes back when the check fails
65
75
  ${REPO_HELP}
66
76
  (homelab repository; writes are limited to the configured area)
67
77
 
@@ -127,9 +137,37 @@ function repoSpec(config: IdentityConfig): RepoSpec {
127
137
  };
128
138
  }
129
139
 
130
- async function dmz(config: IdentityConfig, args: string[]): Promise<number> {
131
- const remote = buildDmzRemote(args);
132
- const input = DMZ_COMMANDS[args[0]]?.stdin ? await trackedFile(repoSpec(config), config.pomeriumConfig) : undefined;
140
+ /** The sha256 that a release's `pomerium_checksums.txt` publishes for the Linux amd64 tarball. */
141
+ export function publishedSha256(checksums: string): string {
142
+ const sum = checksums
143
+ .split("\n")
144
+ .map((l) => l.trim().split(/\s+/))
145
+ .find((f) => f[1] === TARBALL)?.[0];
146
+ if (sum === undefined || !/^[0-9a-f]{64}$/.test(sum)) throw new ServiceError(`the release checksums list no sha256 for ${TARBALL}`);
147
+ return sum;
148
+ }
149
+
150
+ /** `pomerium: 0.33.1+01af7466f` as `v0.33.1`. */
151
+ export function installedRelease(output: string): string | null {
152
+ const m = /pomerium:\s*(\d+\.\d+\.\d+)/.exec(output);
153
+ return m ? `v${m[1]}` : null;
154
+ }
155
+
156
+ async function download(url: string, timeoutMs: number): Promise<Buffer> {
157
+ const response = await fetch(url, { headers: { "User-Agent": "pi-astro-identity" }, signal: AbortSignal.timeout(timeoutMs) }).catch((err: unknown) => {
158
+ throw new ServiceError(`${url}: ${err instanceof Error ? err.message : String(err)}`);
159
+ });
160
+ if (!response.ok) throw new ServiceError(`${url} answered ${response.status}`);
161
+ return Buffer.from(await response.arrayBuffer());
162
+ }
163
+
164
+ async function newestRelease(): Promise<string> {
165
+ const body = JSON.parse((await download("https://api.github.com/repos/pomerium/pomerium/releases/latest", 30_000)).toString("utf8")) as { tag_name?: unknown };
166
+ if (typeof body.tag_name !== "string" || !RELEASE_TAG.test(body.tag_name)) throw new ServiceError("GitHub returned no Pomerium release tag");
167
+ return body.tag_name;
168
+ }
169
+
170
+ async function sshDmz(config: IdentityConfig, remote: string, timeoutMs: number, input?: string): Promise<string> {
133
171
  const result = await sshFixed(
134
172
  {
135
173
  host: config.dmzHost,
@@ -138,11 +176,40 @@ async function dmz(config: IdentityConfig, args: string[]): Promise<number> {
138
176
  knownHostsFile: join(specialistsHome(), "config", "known_hosts"),
139
177
  },
140
178
  remote,
141
- 180_000,
179
+ timeoutMs,
142
180
  input,
143
181
  );
144
- if (result.stdout.length > 0) printRaw(result.stdout);
145
- if (result.code !== 0) throw new ServiceError(result.stderr.trim() || `dmz ${args[0]} exited ${result.code}`);
182
+ if (result.code !== 0) {
183
+ if (result.stdout.length > 0) printRaw(result.stdout);
184
+ throw new ServiceError(result.stderr.trim() || `dmz ${remote.split(" ")[0]} exited ${result.code}`);
185
+ }
186
+ return result.stdout;
187
+ }
188
+
189
+ /** Downloads the release on Cortex, checks it against the published sha256, and sends it to the DMZ as base64 on stdin. */
190
+ async function pomeriumUpgrade(config: IdentityConfig, args: string[]): Promise<number> {
191
+ const [tag, ...extra] = args;
192
+ if (tag === undefined || !RELEASE_TAG.test(tag) || extra.length > 0) throw new UsageError("dmz pomerium-upgrade <vX.Y.Z>");
193
+ const sum = publishedSha256((await download(`${RELEASES}/download/${tag}/pomerium_checksums.txt`, 30_000)).toString("utf8"));
194
+ const tarball = await download(`${RELEASES}/download/${tag}/${TARBALL}`, 5 * 60_000);
195
+ if (createHash("sha256").update(tarball).digest("hex") !== sum) throw new ServiceError(`the downloaded ${TARBALL} does not match the published sha256; nothing installed`);
196
+ printRaw(await sshDmz(config, `pomerium-upgrade ${tag} ${sum}`, UPGRADE_TIMEOUT_MS, tarball.toString("base64")));
197
+ return 0;
198
+ }
199
+
200
+ async function dmz(config: IdentityConfig, args: string[]): Promise<number> {
201
+ if (args[0] === "pomerium-upgrade") return pomeriumUpgrade(config, args.slice(1));
202
+ if (args[0] === "pomerium-check") {
203
+ if (args.length !== 1) throw new UsageError("dmz pomerium-check takes no arguments");
204
+ const installed = installedRelease(await sshDmz(config, "pomerium-version", 60_000));
205
+ const newest = await newestRelease();
206
+ printJson({ installed, newest, update: installed !== null && installed !== newest, notes: `${RELEASES}/tag/${newest}` });
207
+ return 0;
208
+ }
209
+ const remote = buildDmzRemote(args);
210
+ const input = DMZ_COMMANDS[args[0]]?.stdin ? await trackedFile(repoSpec(config), config.pomeriumConfig) : undefined;
211
+ const stdout = await sshDmz(config, remote, 180_000, input);
212
+ if (stdout.length > 0) printRaw(stdout);
146
213
  return 0;
147
214
  }
148
215
 
@@ -2,11 +2,11 @@ import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync
2
2
  import { tmpdir } from "node:os";
3
3
  import { join } from "node:path";
4
4
  import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
5
- import { refusedPrefix, command as arcaneCommand } from "./arcane/run.ts";
5
+ import { cliFailure, refusedPrefix, command as arcaneCommand } from "./arcane/run.ts";
6
6
  import { buildBackupRemote, command as backupCommand } from "./backup/run.ts";
7
7
  import { buildNodeRemote, parseJoinArgs, parseParams, parseRecordFilters, technitiumRequest, technitiumTarget, command as dnsCommand } from "./dns/run.ts";
8
8
  import { buildVpsRemote, command as edgeCommand, isTransientSshFailure, PUBLIC_ROUTES, routeProbe } from "./edge/run.ts";
9
- import { API_PREFIXES, buildDmzRemote, validateApiPath, command as identityCommand } from "./identity/run.ts";
9
+ import { API_PREFIXES, buildDmzRemote, installedRelease, publishedSha256, validateApiPath, command as identityCommand } from "./identity/run.ts";
10
10
  import { buildInferenceRemote, command as inferenceCommand } from "./inference/run.ts";
11
11
  import { MAX_ADD_IDS, MAX_EXCLUDE_IDS, buildTrendarrCall, interpretRun, parseImdbIds, trendarrConfig, trendarrError, trendarrWarnings, command as mediaCommand } from "./media/run.ts";
12
12
  import { ServiceError, UsageError } from "./lib/errors.ts";
@@ -27,6 +27,14 @@ describe("arcane wrapper", () => {
27
27
  expect(refusedPrefix(["containers", "logs", "c1"])).toBeNull();
28
28
  });
29
29
 
30
+ it("names the exit code and stdout when arcane-cli fails with only a warning", () => {
31
+ const warning = "jwt_token and refresh_token in /tmp/x/arcanecli.yml are deprecated and stop working in v3.";
32
+ expect(cliFailure(["images", "updates", "check", "lscr.io/linuxserver/bazarr:latest"], { code: 1, stderr: `${warning}\n`, stdout: '{"success":false,"error":"too many requests","env":["SECRET=1"]}' })).toBe(
33
+ `images updates check lscr.io/linuxserver/bazarr:latest: exited 1; ${warning}; stdout: { "success": false, "error": "too many requests" }`,
34
+ );
35
+ expect(cliFailure(["version"], { code: 2, stderr: "", stdout: "" })).toBe("version: exited 2");
36
+ });
37
+
30
38
  it("keeps gitops writes inside the checkout", () => {
31
39
  const dir = "/repo";
32
40
  expect(safeRepoPath(dir, "mealie/compose.yaml")).toBe("/repo/mealie/compose.yaml");
@@ -83,6 +91,16 @@ describe("identity wrapper", () => {
83
91
  expect(() => buildDmzRemote(["render", "extra"])).toThrow(/0 argument/);
84
92
  expect(() => buildDmzRemote(["reboot"])).toThrow(/unknown dmz command/);
85
93
  expect(buildDmzRemote(["deploy-config"])).toBe("deploy-config");
94
+ expect(buildDmzRemote(["pomerium-version"])).toBe("pomerium-version");
95
+ expect(() => buildDmzRemote(["pomerium-upgrade", "v0.33.6"])).toThrow(/unknown dmz command/);
96
+ });
97
+
98
+ it("reads the published sha256 and the installed Pomerium release", () => {
99
+ const sum = "f466fa4f55127f917bd33cbe5aba10966e535b16cc26780a277750f752afb76f";
100
+ expect(publishedSha256(`${"0".repeat(64)} pomerium-darwin-arm64.tar.gz\n${sum} pomerium-linux-amd64.tar.gz\n`)).toBe(sum);
101
+ expect(() => publishedSha256(`${"0".repeat(64)} pomerium-darwin-arm64.tar.gz\n`)).toThrow(/no sha256/);
102
+ expect(installedRelease("pomerium: 0.33.1+01af7466f\nenvoy: 20260618180819-16ede7a56f2f\n")).toBe("v0.33.1");
103
+ expect(installedRelease("command not found")).toBeNull();
86
104
  });
87
105
 
88
106
  it("prints help without touching config", async () => {