@astrofoundry/pi-astro 0.40.40 โ 0.40.42
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/astro-discord/arcane-watch-monitor.ts +5 -0
- package/extensions/astro-discord/arcane-watch.test.ts +51 -2
- package/extensions/astro-discord/arcane-watch.ts +42 -5
- package/extensions/astro-discord/daily-run.test.ts +21 -8
- package/extensions/astro-discord/daily-run.ts +58 -10
- package/extensions/astro-discord/index.ts +48 -2
- package/extensions/specialist-gate/approvals.test.ts +2 -0
- package/extensions/specialist-gate/index.ts +1 -1
- package/package.json +1 -1
- package/skills/identity/SKILL.md +5 -0
- package/specialists/AGENTS.md +1 -1
- package/specialists/README.md +1 -1
- package/specialists/arcane/run.ts +7 -1
- package/specialists/identity/run.ts +73 -6
- package/specialists/wrappers.test.ts +20 -2
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
escalationText,
|
|
3
|
+
hostRestartText,
|
|
3
4
|
observe,
|
|
4
5
|
parseSnapshot,
|
|
5
6
|
readWatchState,
|
|
@@ -98,6 +99,10 @@ export class ArcaneWatch {
|
|
|
98
99
|
const now = this.deps.now();
|
|
99
100
|
const seen = observe(state, snapshot, now, this.deps.nasOutage(now));
|
|
100
101
|
for (const note of seen.notes) this.deps.log(`arcane watch: ${note}`);
|
|
102
|
+
for (const e of seen.hostRestarts) {
|
|
103
|
+
this.deps.log(`arcane watch: ${e.count} of ${e.total} containers restarted between ${e.from} and ${e.to}`);
|
|
104
|
+
await this.post(s.channelId, { content: hostRestartText(e, s.timeZone) });
|
|
105
|
+
}
|
|
101
106
|
for (const u of seen.unhealthy) {
|
|
102
107
|
const posted = await this.post(s.channelId, { content: unhealthyText(u, owners, s.timeZone, now), mentionUsers: owners });
|
|
103
108
|
if (posted) this.deps.log(`arcane watch: ${u.name} ${u.problem} after a redeploy`);
|
|
@@ -4,7 +4,7 @@ import { join } from "node:path";
|
|
|
4
4
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
5
5
|
import type { WatchedContainer } from "../../specialists/lib/container-watch.ts";
|
|
6
6
|
import { ArcaneWatch } from "./arcane-watch-monitor.ts";
|
|
7
|
-
import { emptyWatchState, escalationText, healthProblem, observe, parseSnapshot, readWatchState, restartInvestigationTask, unhealthyText, type WatchState } from "./arcane-watch.ts";
|
|
7
|
+
import { emptyWatchState, escalationText, healthProblem, hostRestartText, observe, parseSnapshot, readWatchState, restartInvestigationTask, unhealthyText, type WatchState } from "./arcane-watch.ts";
|
|
8
8
|
import type { DiscordConfig } from "./config.ts";
|
|
9
9
|
import type { CreateMessage } from "./rest.ts";
|
|
10
10
|
import type { InvestigationRequest } from "./security-monitor.ts";
|
|
@@ -34,7 +34,7 @@ describe("container watch rules", () => {
|
|
|
34
34
|
it("only records a baseline the first time", () => {
|
|
35
35
|
const state = emptyWatchState();
|
|
36
36
|
const out = observe(state, [ct({ startedAt: iso(T0 - MIN) })], T0, false);
|
|
37
|
-
expect(out).toEqual({ escalations: [], unhealthy: [], notes: [] });
|
|
37
|
+
expect(out).toEqual({ escalations: [], unhealthy: [], hostRestarts: [], notes: [] });
|
|
38
38
|
expect(state.containers.syncthing).toEqual({ id: "a", imageId: "sha256:1", startedAt: iso(T0 - MIN), seenAt: iso(T0) });
|
|
39
39
|
});
|
|
40
40
|
|
|
@@ -70,6 +70,33 @@ describe("container watch rules", () => {
|
|
|
70
70
|
expect(state.restarts).toEqual({});
|
|
71
71
|
});
|
|
72
72
|
|
|
73
|
+
it("counts a restart of most containers as one host event, also when two checks split it", () => {
|
|
74
|
+
const four = ["a", "b", "c", "d"].map((name) => ct({ name, id: name }));
|
|
75
|
+
const state = seenAt(four, T0);
|
|
76
|
+
const early = iso(T0 + 4 * MIN + 50_000);
|
|
77
|
+
observe(state, [ct({ name: "a", id: "a", startedAt: early }), ...four.slice(1)], T0 + 5 * MIN, false);
|
|
78
|
+
expect(state.restarts).toEqual({ a: [early] });
|
|
79
|
+
const together = iso(T0 + 5 * MIN + 10_000);
|
|
80
|
+
const out = observe(state, [ct({ name: "a", id: "a", startedAt: early }), ct({ name: "b", id: "b", startedAt: together }), ct({ name: "c", id: "c", startedAt: together }), four[3]], T0 + 5 * MIN + 40_000, false);
|
|
81
|
+
expect(out.hostRestarts).toEqual([{ count: 2, total: 4, from: together, to: together }]);
|
|
82
|
+
expect(state.restarts).toEqual({});
|
|
83
|
+
const late = iso(T0 + 5 * MIN + 45_000);
|
|
84
|
+
const after = observe(state, [ct({ name: "a", id: "a", startedAt: early }), ct({ name: "b", id: "b", startedAt: together }), ct({ name: "c", id: "c", startedAt: together }), ct({ name: "d", id: "d", startedAt: late })], T0 + 10 * MIN, false);
|
|
85
|
+
expect(after.hostRestarts).toEqual([]);
|
|
86
|
+
expect(after.notes).toEqual(["d restarted with a host-wide restart"]);
|
|
87
|
+
expect(state.restarts).toEqual({});
|
|
88
|
+
const alone = iso(T0 + 19 * MIN);
|
|
89
|
+
observe(state, [ct({ name: "a", id: "a", startedAt: alone }), ct({ name: "b", id: "b", startedAt: together }), ct({ name: "c", id: "c", startedAt: together }), ct({ name: "d", id: "d", startedAt: late })], T0 + 20 * MIN, false);
|
|
90
|
+
expect(state.restarts).toEqual({ a: [alone] });
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it("forgets host events after a day", () => {
|
|
94
|
+
const state = seenAt([ct()], T0);
|
|
95
|
+
state.hostRestarts = [{ from: iso(T0 - 25 * 60 * MIN), to: iso(T0 - 25 * 60 * MIN) }];
|
|
96
|
+
observe(state, [ct()], T0 + 5 * MIN, false);
|
|
97
|
+
expect(state.hostRestarts).toEqual([]);
|
|
98
|
+
});
|
|
99
|
+
|
|
73
100
|
it("checks a redeployed container after five minutes and reports what is wrong", () => {
|
|
74
101
|
const state = seenAt([ct({ name: "mealie", id: "old", imageId: "sha256:1", project: "mealie" })], T0);
|
|
75
102
|
const started = T0 + MIN;
|
|
@@ -125,6 +152,12 @@ describe("container watch texts", () => {
|
|
|
125
152
|
);
|
|
126
153
|
});
|
|
127
154
|
|
|
155
|
+
it("names a host-wide restart without tagging anyone", () => {
|
|
156
|
+
expect(hostRestartText({ count: 46, total: 52, from: iso(T0), to: iso(T0 + 8_000) }, TZ)).toBe(
|
|
157
|
+
"๐ **Host-wide restart** ยท 46 of 52 containers restarted at 22 Oct 10:00 CEST, so Docker or the Arcane VM restarted. They do not count as container restarts.",
|
|
158
|
+
);
|
|
159
|
+
});
|
|
160
|
+
|
|
128
161
|
it("asks the specialist to read only, within the host's one-day window", () => {
|
|
129
162
|
const now = T0 + 24 * 60 * MIN;
|
|
130
163
|
const task = restartInvestigationTask({ name: "syncthing", restarts: [iso(T0 + MIN), iso(now - MIN)] }, now);
|
|
@@ -219,6 +252,22 @@ describe("container watch monitor", () => {
|
|
|
219
252
|
expect(h.investigations[0].task).toContain("restarted 3 times");
|
|
220
253
|
});
|
|
221
254
|
|
|
255
|
+
it("posts a host-wide restart once, untagged, and investigates nothing", async () => {
|
|
256
|
+
const h = harness();
|
|
257
|
+
const three = ["a", "b", "c"].map((name) => ct({ name, id: name }));
|
|
258
|
+
h.answer.next = () => JSON.stringify({ containers: three });
|
|
259
|
+
await h.watch.poll(SCHEDULE);
|
|
260
|
+
h.clock.now = T0 + 5 * MIN;
|
|
261
|
+
const started = iso(T0 + 4 * MIN);
|
|
262
|
+
h.answer.next = () => JSON.stringify({ containers: three.map((c) => ({ ...c, startedAt: started })) });
|
|
263
|
+
await h.watch.poll(SCHEDULE);
|
|
264
|
+
expect(h.messages).toHaveLength(1);
|
|
265
|
+
expect(h.messages[0].message.content).toContain("๐ **Host-wide restart** ยท 3 of 3 containers");
|
|
266
|
+
expect(h.messages[0].message.mentionUsers).toBeUndefined();
|
|
267
|
+
expect(h.investigations).toEqual([]);
|
|
268
|
+
expect(readWatchState().restarts).toEqual({});
|
|
269
|
+
});
|
|
270
|
+
|
|
222
271
|
it("warns once after 30 minutes of failed checks and says when they work again", async () => {
|
|
223
272
|
const h = harness();
|
|
224
273
|
h.answer.next = () => "not json";
|
|
@@ -17,6 +17,10 @@ export const HEALTH_GRACE_MS = 5 * MINUTE;
|
|
|
17
17
|
const STARTING_LIMIT_MS = 15 * MINUTE;
|
|
18
18
|
/** The only stack that mounts the NAS share (`nas_media` in the arcane repository's media/compose.yaml). */
|
|
19
19
|
export const NAS_STACKS: readonly string[] = ["media"];
|
|
20
|
+
/** When at least this share of the containers restarted since the last check, Docker or the VM restarted them all: one host event, no container's count. */
|
|
21
|
+
export const HOST_RESTART_SHARE = 0.5;
|
|
22
|
+
/** Restarts this close to a host event belong to it, so an event that two checks split counts nowhere. */
|
|
23
|
+
const HOST_RESTART_MARGIN_MS = 2 * MINUTE;
|
|
20
24
|
|
|
21
25
|
export interface WatchState {
|
|
22
26
|
/** The last observation of each container by name. */
|
|
@@ -27,9 +31,18 @@ export interface WatchState {
|
|
|
27
31
|
escalated: Record<string, string>;
|
|
28
32
|
/** Redeployed containers whose health is checked once the grace time has passed. */
|
|
29
33
|
pending: PendingHealth[];
|
|
34
|
+
/** Host events of the last 24 hours, from the first to the last restart they covered. */
|
|
35
|
+
hostRestarts: { from: string; to: string }[];
|
|
30
36
|
poll: { failures: number; warned: boolean };
|
|
31
37
|
}
|
|
32
38
|
|
|
39
|
+
export interface HostRestart {
|
|
40
|
+
count: number;
|
|
41
|
+
total: number;
|
|
42
|
+
from: string;
|
|
43
|
+
to: string;
|
|
44
|
+
}
|
|
45
|
+
|
|
33
46
|
export interface PendingHealth {
|
|
34
47
|
name: string;
|
|
35
48
|
id: string;
|
|
@@ -56,12 +69,13 @@ export interface Unhealthy {
|
|
|
56
69
|
export interface Observation {
|
|
57
70
|
escalations: Escalation[];
|
|
58
71
|
unhealthy: Unhealthy[];
|
|
59
|
-
|
|
72
|
+
hostRestarts: HostRestart[];
|
|
73
|
+
/** Log lines for what was seen but is expected: our operations, NAS outages, and host events. */
|
|
60
74
|
notes: string[];
|
|
61
75
|
}
|
|
62
76
|
|
|
63
77
|
export function emptyWatchState(): WatchState {
|
|
64
|
-
return { containers: {}, restarts: {}, escalated: {}, pending: [], poll: { failures: 0, warned: false } };
|
|
78
|
+
return { containers: {}, restarts: {}, escalated: {}, pending: [], hostRestarts: [], poll: { failures: 0, warned: false } };
|
|
65
79
|
}
|
|
66
80
|
|
|
67
81
|
function statePath(): string {
|
|
@@ -109,14 +123,17 @@ export function healthProblem(c: WatchedContainer, sinceMs: number): string | nu
|
|
|
109
123
|
/**
|
|
110
124
|
* Compares a fresh snapshot with the last one: a new id is a redeploy, whose
|
|
111
125
|
* health is checked after the grace time; a newer start time with the same id
|
|
112
|
-
* is a restart. The first snapshot only records the baseline.
|
|
126
|
+
* is a restart. The first snapshot only records the baseline. When most
|
|
127
|
+
* containers restarted at once, that is one host event and no container's
|
|
128
|
+
* restart.
|
|
113
129
|
*/
|
|
114
130
|
export function observe(state: WatchState, snapshot: readonly WatchedContainer[], now: number, nasOutage: boolean): Observation {
|
|
115
|
-
const out: Observation = { escalations: [], unhealthy: [], notes: [] };
|
|
131
|
+
const out: Observation = { escalations: [], unhealthy: [], hostRestarts: [], notes: [] };
|
|
116
132
|
const baseline = Object.keys(state.containers).length === 0;
|
|
117
133
|
const at = new Date(now).toISOString();
|
|
118
134
|
const nasExpected = (c: { project?: string }) => nasOutage && c.project !== undefined && NAS_STACKS.includes(c.project);
|
|
119
135
|
const current = new Map(snapshot.map((c) => [c.name, c]));
|
|
136
|
+
const restarted: { name: string; startedAt: string }[] = [];
|
|
120
137
|
for (const c of snapshot) {
|
|
121
138
|
const prev = state.containers[c.name];
|
|
122
139
|
if (prev && !baseline) {
|
|
@@ -127,13 +144,28 @@ export function observe(state: WatchState, snapshot: readonly WatchedContainer[]
|
|
|
127
144
|
} else if (c.startedAt && c.startedAt !== prev.startedAt && Date.parse(c.startedAt) > Date.parse(prev.seenAt)) {
|
|
128
145
|
if (c.operation) out.notes.push(`${c.name} restarted during ${operationText(c.operation)}`);
|
|
129
146
|
else if (nasExpected(c)) out.notes.push(`${c.name} restarted during a NAS outage`);
|
|
130
|
-
else
|
|
147
|
+
else restarted.push({ name: c.name, startedAt: c.startedAt });
|
|
131
148
|
}
|
|
132
149
|
}
|
|
133
150
|
const startedAt = c.startedAt ?? (prev?.id === c.id ? prev.startedAt : null);
|
|
134
151
|
state.containers[c.name] = { id: c.id, imageId: c.imageId, startedAt, seenAt: at };
|
|
135
152
|
}
|
|
136
153
|
for (const name of Object.keys(state.containers)) if (!current.has(name)) delete state.containers[name];
|
|
154
|
+
state.hostRestarts = state.hostRestarts.filter((e) => Date.parse(e.to) >= now - DAY);
|
|
155
|
+
if (restarted.length >= Math.max(2, Math.ceil(snapshot.length * HOST_RESTART_SHARE))) {
|
|
156
|
+
const times = restarted.map((r) => Date.parse(r.startedAt));
|
|
157
|
+
const event = { from: new Date(Math.min(...times)).toISOString(), to: new Date(Math.max(...times)).toISOString() };
|
|
158
|
+
state.hostRestarts.push(event);
|
|
159
|
+
out.hostRestarts.push({ count: restarted.length, total: snapshot.length, ...event });
|
|
160
|
+
restarted.length = 0;
|
|
161
|
+
}
|
|
162
|
+
const inHostRestart = (t: string) => state.hostRestarts.some((e) => Date.parse(t) >= Date.parse(e.from) - HOST_RESTART_MARGIN_MS && Date.parse(t) <= Date.parse(e.to) + HOST_RESTART_MARGIN_MS);
|
|
163
|
+
for (const r of restarted) {
|
|
164
|
+
if (inHostRestart(r.startedAt)) out.notes.push(`${r.name} restarted with a host-wide restart`);
|
|
165
|
+
else state.restarts[r.name] = [...(state.restarts[r.name] ?? []), r.startedAt];
|
|
166
|
+
}
|
|
167
|
+
// A check that saw only the first restarts of an event counted them before the event was known.
|
|
168
|
+
for (const [name, times] of Object.entries(state.restarts)) state.restarts[name] = times.filter((t) => !inHostRestart(t));
|
|
137
169
|
for (const [name, times] of Object.entries(state.restarts)) {
|
|
138
170
|
const recent = times.filter((t) => Date.parse(t) >= now - DAY);
|
|
139
171
|
if (recent.length === 0) delete state.restarts[name];
|
|
@@ -186,6 +218,11 @@ export function escalationText(e: Escalation, owners: readonly string[], timeZon
|
|
|
186
218
|
].join("\n");
|
|
187
219
|
}
|
|
188
220
|
|
|
221
|
+
/** Untagged: Docker or the VM restarting is planned maintenance far more often than a fault. */
|
|
222
|
+
export function hostRestartText(e: HostRestart, timeZone: string): string {
|
|
223
|
+
return `๐ **Host-wide restart** ยท ${e.count} of ${e.total} containers restarted at ${clockText(e.from, timeZone)}, so Docker or the Arcane VM restarted. They do not count as container restarts.`;
|
|
224
|
+
}
|
|
225
|
+
|
|
189
226
|
export function unhealthyText(u: Unhealthy, owners: readonly string[], timeZone: string, now: number): string {
|
|
190
227
|
const minutes = Math.round((now - Date.parse(u.since)) / MINUTE);
|
|
191
228
|
return [
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
2
|
import type { ArcaneReport } from "./daily-report.ts";
|
|
3
3
|
import type { ToolsReport } from "./tools-report.ts";
|
|
4
|
-
import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
4
|
+
import { agoSeconds, arcaneSummary, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, pomeriumButtonId, wazuhRelease, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
5
5
|
|
|
6
6
|
const EDGE = `nginx active
|
|
7
7
|
wg-quick@wg0 active
|
|
@@ -78,15 +78,18 @@ describe("morning status lines", () => {
|
|
|
78
78
|
expect(proxmoxStatus([{ name: "Pulsar", id: "pulsar", status: PULSAR, updates: PULSAR_UPDATES, upgrade: null }]).problem).toBe("Proxmox: the host upgrade on Pulsar failed");
|
|
79
79
|
});
|
|
80
80
|
|
|
81
|
-
it("flags updates the nightly run left behind, reading guest-agent JSON too, and
|
|
81
|
+
it("flags updates the nightly run left behind, reading guest-agent JSON too, and announces a Wazuh release from the held agent", () => {
|
|
82
82
|
const vm = JSON.stringify({ exitcode: 0, exited: 1, "out-data": "Listing...\ndocker-ce/trixie 5:29.9.2-1 amd64 [upgradable from: 5:29.7.2-1]\nwazuh-agent/stable 4.14.9-1 amd64 [upgradable from: 4.14.7-1]\n" });
|
|
83
83
|
expect(pendingPackageLines(vm)).toEqual(["docker-ce/trixie 5:29.9.2-1 amd64 [upgradable from: 5:29.7.2-1]"]);
|
|
84
|
-
const held = "Listing...\nwazuh-agent/stable 4.14.
|
|
85
|
-
expect(
|
|
86
|
-
expect(
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
84
|
+
const held = "Listing...\nwazuh-agent/stable 4.14.10-1 amd64 [upgradable from: 4.14.9-1]";
|
|
85
|
+
expect(wazuhRelease({ arcane: vm, synapse: held, radius: null })).toEqual({ from: "4.14.9", to: "4.14.10" });
|
|
86
|
+
expect(wazuhRelease({ synapse: "Listing..." })).toBeNull();
|
|
87
|
+
expect(updatesStatus({ synapse: "Listing..." })).toEqual({ area: "Updates", line: "1 host up to date after the nightly run" });
|
|
88
|
+
const wazuh = updatesStatus({ technitium2: held, synapse: "Listing..." });
|
|
89
|
+
expect(wazuh.line).toBe("2 hosts up to date after the nightly run; Wazuh 4.14.10 is out (agents held at 4.14.9)");
|
|
90
|
+
expect(wazuh.problem).toMatch(/^Updates: Wazuh 4\.14\.10 is out: ask me to upgrade the manager/);
|
|
91
|
+
expect(updatesStatus({ arcane: vm, radius: null })).toMatchObject({
|
|
92
|
+
line: "still waiting after the nightly run: arcane 1; no update list from radius; Wazuh 4.14.9 is out (agents held at 4.14.7)",
|
|
90
93
|
route: "admin",
|
|
91
94
|
});
|
|
92
95
|
});
|
|
@@ -105,6 +108,16 @@ describe("morning status lines", () => {
|
|
|
105
108
|
expect(identityStatus(DMZ.replace("Active: active (running) since Thu 2026-09-24 17:21:24 CEST; 4 weeks 0 days ago\nโ nginx", "Active: failed (Result: signal)\nโ nginx")).problem).toBe("Identity: pomerium.service failed on the DMZ VM");
|
|
106
109
|
});
|
|
107
110
|
|
|
111
|
+
it("offers an owner button when a newer Pomerium release is out", () => {
|
|
112
|
+
const notes = "https://github.com/pomerium/pomerium/releases/tag/v0.33.6";
|
|
113
|
+
expect(identityStatus(DMZ, { installed: "v0.33.6", newest: "v0.33.6", update: false, notes })).toEqual({ area: "Identity", line: "pomerium, nginx, lego-renew.timer active; Pomerium v0.33.6" });
|
|
114
|
+
const newer = identityStatus(DMZ, { installed: "v0.33.1", newest: "v0.33.6", update: true, notes });
|
|
115
|
+
expect(newer.line).toBe("pomerium, nginx, lego-renew.timer active; Pomerium v0.33.1, v0.33.6 available");
|
|
116
|
+
expect(newer.problem).toContain(`Pomerium v0.33.6 is out (running v0.33.1; notes: <${notes}>)`);
|
|
117
|
+
expect(newer.buttons).toEqual([{ label: "Update Pomerium to v0.33.6", customId: pomeriumButtonId("v0.33.6") }]);
|
|
118
|
+
expect(identityStatus(DMZ, null).line).toBe("pomerium, nginx, lego-renew.timer active; Pomerium version unknown");
|
|
119
|
+
});
|
|
120
|
+
|
|
108
121
|
it("checks both resolvers, Nexus, and SABnzbd", () => {
|
|
109
122
|
expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: { status: "ok" }, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "technitium1, technitium2, technitium3 answer" });
|
|
110
123
|
expect(dnsStatus({ technitium1: { status: "ok" }, technitium2: null, technitium3: { status: "ok" } })).toEqual({ area: "DNS", line: "no answer from technitium2", problem: "DNS: technitium2 does not answer", route: "admin" });
|
|
@@ -121,8 +121,8 @@ export function proxmoxStatus(hosts: readonly ProxmoxHost[]): AreaStatus {
|
|
|
121
121
|
/** Packages the nightly run leaves alone on purpose: a Wazuh agent must never be newer than the manager. */
|
|
122
122
|
const HELD_PACKAGES = new Set(["wazuh-agent"]);
|
|
123
123
|
|
|
124
|
-
/** The `[upgradable from` lines of `apt list --upgradable`, also when a VM's guest agent wraps them in JSON (`out-data`)
|
|
125
|
-
|
|
124
|
+
/** The `[upgradable from` lines of `apt list --upgradable`, also when a VM's guest agent wraps them in JSON (`out-data`). */
|
|
125
|
+
function upgradableLines(output: string): string[] {
|
|
126
126
|
let text = output;
|
|
127
127
|
try {
|
|
128
128
|
const wrapped = (JSON.parse(output) as { "out-data"?: unknown } | null)?.["out-data"];
|
|
@@ -130,15 +130,31 @@ export function pendingPackageLines(output: string): string[] {
|
|
|
130
130
|
} catch {
|
|
131
131
|
/* plain apt output */
|
|
132
132
|
}
|
|
133
|
-
return text
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
133
|
+
return text.split("\n").filter((l) => l.includes("[upgradable from"));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** The upgradable packages of one host, held packages left out. */
|
|
137
|
+
export function pendingPackageLines(output: string): string[] {
|
|
138
|
+
return upgradableLines(output).filter((l) => !HELD_PACKAGES.has(l.split("/")[0]));
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
/** The newest `wazuh-agent` that apt offers on any host: held until the manager runs it, so it announces a Wazuh release. */
|
|
142
|
+
export function wazuhRelease(hosts: Readonly<Record<string, string | null>>): { from: string; to: string } | null {
|
|
143
|
+
let found: { from: string; to: string } | null = null;
|
|
144
|
+
for (const out of Object.values(hosts)) {
|
|
145
|
+
if (out === null) continue;
|
|
146
|
+
const m = upgradableLines(out)
|
|
147
|
+
.map((l) => /^wazuh-agent\/\S+\s+(\d+\.\d+\.\d+)\S*\s.*\[upgradable from: (\d+\.\d+\.\d+)/.exec(l))
|
|
148
|
+
.find((x) => x !== null);
|
|
149
|
+
if (m && (found === null || m[1].localeCompare(found.to, undefined, { numeric: true }) > 0)) found = { from: m[2], to: m[1] };
|
|
150
|
+
}
|
|
151
|
+
return found;
|
|
137
152
|
}
|
|
138
153
|
|
|
139
154
|
/** Each guest's and Synapse's `updates` after the 04:30 run: anything still waiting did not install and needs a look. */
|
|
140
155
|
export function updatesStatus(hosts: Readonly<Record<string, string | null>>): AreaStatus {
|
|
141
156
|
const names = Object.keys(hosts);
|
|
157
|
+
const wazuh = wazuhRelease(hosts);
|
|
142
158
|
const unknown = names.filter((n) => hosts[n] === null);
|
|
143
159
|
const left = names.flatMap((n) => {
|
|
144
160
|
const out = hosts[n];
|
|
@@ -149,10 +165,12 @@ export function updatesStatus(hosts: Readonly<Record<string, string | null>>): A
|
|
|
149
165
|
const parts = [
|
|
150
166
|
left.length === 0 ? `${checked} host${checked === 1 ? "" : "s"} up to date after the nightly run` : `still waiting after the nightly run: ${left.map((h) => `${escapeDiscord(h.name)} ${h.packages.length}`).join(", ")}`,
|
|
151
167
|
unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null,
|
|
168
|
+
wazuh ? `Wazuh ${wazuh.to} is out (agents held at ${wazuh.from})` : null,
|
|
152
169
|
].filter(Boolean);
|
|
153
170
|
const problems = [
|
|
154
171
|
left.length > 0 ? `${left.map((h) => `${h.name} (${h.packages.slice(0, 4).join(", ")}${h.packages.length > 4 ? ", ..." : ""})`).join("; ")} did not install overnight` : null,
|
|
155
172
|
unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null,
|
|
173
|
+
wazuh ? `Wazuh ${wazuh.to} is out: ask me to upgrade the manager (\`observability/compose.yaml\`, then \`deploy.sh\`), then the agents (\`scripts/wazuh-agents/deploy.sh --upgrade\`); an agent must never be newer than the manager` : null,
|
|
156
174
|
].filter(Boolean);
|
|
157
175
|
return { area: "Updates", line: parts.join("; "), ...(problems.length > 0 ? { problem: `Updates: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
158
176
|
}
|
|
@@ -178,13 +196,43 @@ export function networkStatus(radius: string | null, tailscale: string | null):
|
|
|
178
196
|
return { area: "Network", line: parts.join(", "), ...(problems.length > 0 ? { problem: `Network: ${problems.join("; ")}`, route: "admin" as const } : {}) };
|
|
179
197
|
}
|
|
180
198
|
|
|
181
|
-
/**
|
|
182
|
-
export function
|
|
199
|
+
/** The custom id of the button that upgrades Pomerium to one release. */
|
|
200
|
+
export function pomeriumButtonId(tag: string): string {
|
|
201
|
+
return `identity:pomerium:${tag}`;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/** `identity dmz pomerium-check`: the installed and the newest release. */
|
|
205
|
+
interface PomeriumCheck {
|
|
206
|
+
installed: string;
|
|
207
|
+
newest: string;
|
|
208
|
+
notes: string;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function pomeriumCheck(value: unknown): PomeriumCheck | null {
|
|
212
|
+
const v = value as { installed?: unknown; newest?: unknown; notes?: unknown } | null;
|
|
213
|
+
if (typeof v?.installed !== "string" || typeof v.newest !== "string" || typeof v.notes !== "string") return null;
|
|
214
|
+
return { installed: v.installed, newest: v.newest, notes: v.notes };
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** `identity dmz status` and `dmz pomerium-check`: Pomerium, nginx, and the certificate renewal timer on the DMZ VM, plus a newer Pomerium release with its owner button. */
|
|
218
|
+
export function identityStatus(status: string | null, pomerium?: unknown): AreaStatus {
|
|
183
219
|
if (status === null) return failed("Identity", "admin", "dmz status");
|
|
184
220
|
const units = [...status.matchAll(/^[โโร]\s+(\S+)\s+-[^\n]*\n(?:.*\n)*?\s+Active:\s+(\S+)(?:\s+\((\w+)\))?/gm)].map((m) => ({ unit: m[1], active: m[2], sub: m[3] ?? "" }));
|
|
185
221
|
const bad = units.filter((u) => u.active !== "active").map((u) => `${u.unit} ${u.active}`);
|
|
186
|
-
const
|
|
187
|
-
|
|
222
|
+
const release = pomeriumCheck(pomerium);
|
|
223
|
+
const newer = release !== null && release.installed !== release.newest ? release : null;
|
|
224
|
+
const unitLine = units.length === 0 ? "no unit status read" : bad.length === 0 ? `${units.map((u) => u.unit.replace(/\.service$/, "")).join(", ")} active` : `not active: ${bad.join(", ")}`;
|
|
225
|
+
const releaseLine = pomerium === undefined ? null : release === null ? "Pomerium version unknown" : newer ? `Pomerium ${newer.installed}, ${newer.newest} available` : `Pomerium ${release.installed}`;
|
|
226
|
+
const problems = [
|
|
227
|
+
bad.length > 0 || units.length === 0 ? `${bad.length > 0 ? bad.join(", ") : "no unit status read"} on the DMZ VM` : null,
|
|
228
|
+
newer ? `Pomerium ${newer.newest} is out (running ${newer.installed}; notes: <${newer.notes}>). The button installs the official release after checking its sha256, restarts Pomerium for a few seconds, and puts ${newer.installed} back if the login route fails` : null,
|
|
229
|
+
].filter(Boolean);
|
|
230
|
+
return {
|
|
231
|
+
area: "Identity",
|
|
232
|
+
line: [unitLine, releaseLine].filter(Boolean).join("; "),
|
|
233
|
+
...(problems.length > 0 ? { problem: `Identity: ${problems.join("; ")}`, route: "admin" as const } : {}),
|
|
234
|
+
...(newer ? { buttons: [{ label: `Update Pomerium to ${newer.newest}`, customId: pomeriumButtonId(newer.newest) }] } : {}),
|
|
235
|
+
};
|
|
188
236
|
}
|
|
189
237
|
|
|
190
238
|
/** Upgrade order: Mira's node first, the primary last, one node per morning. */
|
|
@@ -34,7 +34,7 @@ import {
|
|
|
34
34
|
writePostedArcane,
|
|
35
35
|
} from "./daily-report.ts";
|
|
36
36
|
import { GatewayClient } from "./gateway.ts";
|
|
37
|
-
import { type CommandOption, IS_COMPONENTS_V2, KeyedQueue, MEDIA, MEDIA_COMMANDS, type MediaKind, type Outcome, PAUSED_TEXT, RESUME_ID, addOutcome, addedWhilePaused, applyOutcome, commandArgs, hasActiveButton, listMessage, mediaAccess, mediaPointer, parseAddButton, parseAddReport, parseMediaList, parseQueueState, queueLine, resumedText, unpauseRow } from "./media.ts";
|
|
37
|
+
import { type CommandOption, IS_COMPONENTS_V2, KeyedQueue, MEDIA, MEDIA_COMMANDS, type MediaKind, type Outcome, PAUSED_TEXT, RESUME_ID, addOutcome, addedWhilePaused, applyOutcome, commandArgs, escapeDiscord, hasActiveButton, listMessage, mediaAccess, mediaPointer, parseAddButton, parseAddReport, parseMediaList, parseQueueState, queueLine, resumedText, unpauseRow } from "./media.ts";
|
|
38
38
|
import { cancelledApprovalText, interruptedTaskText, readOpenWork, type OpenWork, type StopReason, writeOpenWork } from "./open-work.ts";
|
|
39
39
|
import { DISMISSED, type Picks, parseNoButton, parsePicks, picksArgs, picksFailureText, picksMessage, readPicksState, withDismissed, withSuggested, writePicksState } from "./media-picks.ts";
|
|
40
40
|
import { FRIDAY, isDue, MONDAY, readReportState, writeReportState, zonedWeekday } from "./reports.ts";
|
|
@@ -183,6 +183,8 @@ const PROXMOX_HOSTS: readonly { name: string; id: string; prefix: string[] }[] =
|
|
|
183
183
|
const DNS_UPGRADE_MS = 11 * 60_000;
|
|
184
184
|
/** The wrapper allows a host upgrade 20 minutes; the bridge waits a little longer for its answer. */
|
|
185
185
|
const HOST_UPGRADE_MS = 21 * 60_000;
|
|
186
|
+
/** The download plus the DMZ's own upgrade, health wait, and possible rollback: the wrapper allows 10 minutes for the DMZ part. */
|
|
187
|
+
const POMERIUM_UPGRADE_MS = 16 * 60_000;
|
|
186
188
|
|
|
187
189
|
function pause(ms: number): Promise<void> {
|
|
188
190
|
return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)).unref());
|
|
@@ -241,6 +243,7 @@ class Bridge {
|
|
|
241
243
|
private readonly toolRuns = new Map<string, { channelId: string; label: string }>();
|
|
242
244
|
/** Proxmox hosts that an owner's Reboot click is restarting. */
|
|
243
245
|
private readonly rebooting = new Set<string>();
|
|
246
|
+
private pomeriumUpgrading = false;
|
|
244
247
|
private readonly toolClicks = new KeyedQueue();
|
|
245
248
|
private gateway: GatewayClient | undefined;
|
|
246
249
|
private stopping = false;
|
|
@@ -669,7 +672,7 @@ class Bridge {
|
|
|
669
672
|
);
|
|
670
673
|
}
|
|
671
674
|
if (present.has("network")) areas.push((async () => networkStatus(await text("network", ["pulsar", "radius-status"]), await text("network", ["pulsar", "tailscale-status"])))());
|
|
672
|
-
if (present.has("identity")) areas.push((async () => identityStatus(await text("identity", ["dmz", "status"])))());
|
|
675
|
+
if (present.has("identity")) areas.push((async () => identityStatus(await text("identity", ["dmz", "status"]), await json("identity", ["dmz", "pomerium-check"])))());
|
|
673
676
|
if (present.has("dns")) {
|
|
674
677
|
areas.push(
|
|
675
678
|
(async () => {
|
|
@@ -1448,6 +1451,45 @@ class Bridge {
|
|
|
1448
1451
|
}
|
|
1449
1452
|
}
|
|
1450
1453
|
|
|
1454
|
+
/**
|
|
1455
|
+
* Upgrades Pomerium to the release on the button for an owner, when the DMZ still runs an older one.
|
|
1456
|
+
* The wrapper checks the published sha256 and the DMZ puts the old binary back if the login route fails.
|
|
1457
|
+
*/
|
|
1458
|
+
private async onPomeriumButton(interaction: Interaction, tag: string): Promise<void> {
|
|
1459
|
+
const userId = interaction.member?.user?.id ?? interaction.user?.id ?? "";
|
|
1460
|
+
this.refreshConfig();
|
|
1461
|
+
if (!isOwner(this.config, userId)) {
|
|
1462
|
+
await this.rest.ephemeralReply(interaction.id, interaction.token, "Only owners upgrade Pomerium.");
|
|
1463
|
+
return;
|
|
1464
|
+
}
|
|
1465
|
+
if (this.pomeriumUpgrading) {
|
|
1466
|
+
await this.rest.ephemeralReply(interaction.id, interaction.token, "A Pomerium upgrade is already running.");
|
|
1467
|
+
return;
|
|
1468
|
+
}
|
|
1469
|
+
const check = (await this.runEntry("identity", userId, ["dmz", "pomerium-check"], 2 * 60_000).then((out) => JSON.parse(out) as { installed?: unknown }).catch(() => null)) ?? {};
|
|
1470
|
+
if (check.installed === tag) {
|
|
1471
|
+
await this.rest.ephemeralReply(interaction.id, interaction.token, `Pomerium already runs ${tag}.`);
|
|
1472
|
+
return;
|
|
1473
|
+
}
|
|
1474
|
+
await this.rest.deferUpdate(interaction.id, interaction.token);
|
|
1475
|
+
const channelId = interaction.channel_id ?? this.config.adminChannelId;
|
|
1476
|
+
const messageId = interaction.message?.id;
|
|
1477
|
+
this.pomeriumUpgrading = true;
|
|
1478
|
+
try {
|
|
1479
|
+
log(`pomerium upgrade to ${tag} by ${userId}`);
|
|
1480
|
+
if (messageId) await this.rest.editMessage(channelId, messageId, `${interaction.message?.content ?? ""}\nโณ Pomerium ${tag} is installing, started by <@${userId}>.`.slice(0, 1900), []).catch(() => undefined);
|
|
1481
|
+
const out = await this.runEntry("identity", userId, ["dmz", "pomerium-upgrade", tag], POMERIUM_UPGRADE_MS);
|
|
1482
|
+
await this.rest.createMessage(channelId, { content: `โ
${escapeDiscord(out.trim().split("\n").pop() ?? `Pomerium ${tag} installed`)}` });
|
|
1483
|
+
} catch (err) {
|
|
1484
|
+
await this.rest.createMessage(channelId, {
|
|
1485
|
+
content: `โ The Pomerium upgrade to ${tag} failed: ${escapeDiscord((err instanceof Error ? err.message : String(err)).slice(0, 600))} <@${userId}>`,
|
|
1486
|
+
mentionUsers: [userId],
|
|
1487
|
+
});
|
|
1488
|
+
} finally {
|
|
1489
|
+
this.pomeriumUpgrading = false;
|
|
1490
|
+
}
|
|
1491
|
+
}
|
|
1492
|
+
|
|
1451
1493
|
/** Runs one specialist wrapper through the sudo entry point, without a model; rejects with the wrapper's stderr. */
|
|
1452
1494
|
private runEntry(service: string, caller: string, args: string[], timeoutMs: number): Promise<string> {
|
|
1453
1495
|
const gate = loadSpecialists().config;
|
|
@@ -1852,6 +1894,10 @@ class Bridge {
|
|
|
1852
1894
|
await this.onArcaneButton(interaction, customId);
|
|
1853
1895
|
return;
|
|
1854
1896
|
}
|
|
1897
|
+
if (customId.startsWith("identity:pomerium:")) {
|
|
1898
|
+
await this.onPomeriumButton(interaction, customId.slice("identity:pomerium:".length));
|
|
1899
|
+
return;
|
|
1900
|
+
}
|
|
1855
1901
|
if (customId.startsWith("proxmox:reboot:")) {
|
|
1856
1902
|
await this.onRebootButton(interaction, customId.slice("proxmox:reboot:".length));
|
|
1857
1903
|
return;
|
|
@@ -16,6 +16,8 @@ describe("approvals", () => {
|
|
|
16
16
|
expect(isRisky("identity", ["zitadel", "GET", "/v2/users"])).toBe(false);
|
|
17
17
|
expect(isRisky("identity", ["zitadel", "DELETE", "/v2/users/1"])).toBe(true);
|
|
18
18
|
expect(isRisky("identity", ["dmz", "deploy-config"])).toBe(true);
|
|
19
|
+
expect(isRisky("identity", ["dmz", "pomerium-upgrade", "v0.33.6"])).toBe(true);
|
|
20
|
+
expect(isRisky("identity", ["dmz", "pomerium-check"])).toBe(false);
|
|
19
21
|
expect(isRisky("network", ["unifi", "mira", "networks", "x"])).toBe(false);
|
|
20
22
|
expect(isRisky("network", ["unifi", "37", "write", "DELETE", "/v1/sites/x/acl-rules/y"])).toBe(true);
|
|
21
23
|
expect(isRisky("network", ["unifi", "mira", "--local", "write", "PUT", "/v1/sites/x/networks/y", "{}"])).toBe(true);
|
|
@@ -31,7 +31,7 @@ export const RISKY: Readonly<Record<string, readonly RegExp[]>> = {
|
|
|
31
31
|
/^manager-(upgrade|rollback)\b/,
|
|
32
32
|
/^host\s+(db-restore|radarr-movie|radarr-interface)\b/,
|
|
33
33
|
],
|
|
34
|
-
identity: [/^zitadel\s+(DELETE|POST|PUT|PATCH)\b/, /^dmz\s+(deploy-config|pomerium-restart|render|lego-renew)\b/],
|
|
34
|
+
identity: [/^zitadel\s+(DELETE|POST|PUT|PATCH)\b/, /^dmz\s+(deploy-config|pomerium-restart|pomerium-upgrade|render|lego-renew)\b/],
|
|
35
35
|
// Every UniFi write, on either console.
|
|
36
36
|
network: [/^unifi\s+\S+\s+(write|nat-write|device-network|app-policy-delete|portforward-delete|client-fixed-ip|syslog)\b/],
|
|
37
37
|
dns: [
|
package/package.json
CHANGED
package/skills/identity/SKILL.md
CHANGED
|
@@ -35,6 +35,11 @@ Fixed operations on VM 104 through a forced-command SSH key. No other command re
|
|
|
35
35
|
| `nginx-reload` | `nginx -t` then reload |
|
|
36
36
|
| `lego-renew` | run the certificate renewal now |
|
|
37
37
|
| `deploy-config` | install the pushed `config.yaml` as `/etc/pomerium/config.yaml.template`, render with the Vaultwarden values, restart Pomerium, check it is active; on failure restores the previous template and restarts |
|
|
38
|
+
| `pomerium-version` | the installed Pomerium version |
|
|
39
|
+
| `pomerium-check` | JSON: installed and newest Pomerium release, with the release notes link (runs on Cortex plus `pomerium-version`) |
|
|
40
|
+
| `pomerium-upgrade <vX.Y.Z>` | download that official release on Cortex, check it against the release's published sha256, send it to the DMZ, swap the binary, restart, and check the login route; the DMZ puts the old binary back when the check fails |
|
|
41
|
+
|
|
42
|
+
The morning run reports a newer Pomerium release with an owner button that runs `pomerium-upgrade`. Run it yourself only when an owner asks; it restarts Pomerium, so every route drops for a few seconds.
|
|
38
43
|
|
|
39
44
|
## `git <command>` (homelab repository)
|
|
40
45
|
|
package/specialists/AGENTS.md
CHANGED
|
@@ -53,7 +53,7 @@ Wrappers ship as TypeScript and run under Node 24 type stripping: erasable synta
|
|
|
53
53
|
- `inference` reaches Nexus through one forced-command key. The Europa check is HTTP only until Windows OpenSSH forced commands are verified.
|
|
54
54
|
- `REFUSED_PREFIXES` protects only the wrapper itself (`config`, `auth`, `self-update`, `completion`). The GitOps-first rule for managed projects lives in the skill and the agent prompt, not in code, because unmanaged projects and emergencies need the direct CLI.
|
|
55
55
|
- The wrapper's repository commands are the `git` group; `gitops` must stay free because it is an `arcane-cli` command group (server-side syncs).
|
|
56
|
-
- Every passthrough call writes a private per-call copy of `arcanecli.yml` with `jwt_token` set, because some `arcane-cli` commands validate the config file before reading `ARCANE_TOKEN`. The copy is removed after the call.
|
|
56
|
+
- Every passthrough call writes a private per-call copy of `arcanecli.yml` with `jwt_token` set, because some `arcane-cli` commands validate the config file before reading `ARCANE_TOKEN`. The copy is removed after the call. arcane-cli prints a warning on stderr that `jwt_token` and `refresh_token` in the config stop working in v3 (in the bridge log since 2026-10-23); `cliFailure` therefore names the exit code and the filtered stdout as well, because a failure can carry nothing else on stderr (the media update on 2026-11-02).
|
|
57
57
|
- Every wrapper answers `--help` without reading config, so agents can discover subcommands before anything else is set up.
|
|
58
58
|
- `entry/specialist-install.sh` runs as root, so it runs only `/usr/bin`, `/bin`, `/usr/sbin`, `/sbin` and the Node it installed, never Homebrew or anything else the main user can write; it checks that `/usr/local/libexec/specialist-node` (and each `specialist-<tool>` folder) and its parents belong to root and are writable by nobody else. Arguments are checked before the root test (tests exercise that part). Downloads go to a root-only temporary directory and are verified before use: Node against `SHASUMS256.txt` and the LTS flag in `index.json`, npm tarballs against the registry's `sha512` integrity, arcane-cli against the release's checksums file. Inside `/Users/specialist` it works only as `specialist` (`sudo -u specialist -H`), from a staging directory root hands over, so a link the specialist plants cannot redirect a root write. Each component switches by rename after a check (every installed wrapper's `--help`, `--version` of the new binary) and restores the previous copy when the check fails. Older versions are refused; `rollback` swaps current and previous. `python`, `gcloud`, `bw`, and `jq` follow the same rules: version folders under `/usr/local/libexec/specialist-<tool>/` behind the links `current` and `previous`, sealed `root:wheel` and not writable by others, smoke-tested as `specialist` before `current` moves; `python` refuses a switch that breaks the current gcloud. Consumers use the `current` paths: `provision.sh` for `bw` and `jq`, `edge.json` for `gcloud` and `gcloudPython`. Nothing a wrapper runs with secrets may come from Homebrew.
|
|
59
59
|
- `main()` waits for stdout and stderr to flush before `process.exit`: on a pipe, exiting right after a write drops everything past the 64 KiB buffer, which cut large answers such as `projects list --all` until 2026-09-25 (`lib/flush.test.ts`).
|
package/specialists/README.md
CHANGED
|
@@ -26,7 +26,7 @@ Call one with `/run astro.<name> -- <task>` or through the `subagent` tool (both
|
|
|
26
26
|
- Every call is logged in `/Users/specialist/.specialists/audit.log`.
|
|
27
27
|
- The `specialist-gate` extension registers the tool only inside the matching specialist child and blocks other agents from calling the CLIs or reading `/Users/specialist`. `security-guard` carries the same blocks.
|
|
28
28
|
|
|
29
|
-
Limits: `network` writes through the UniFi Network API plus a few narrow commands on the Network application's own paths (`nat-write`, `device-network`, `client-fixed-ip`, `syslog`, `app-policy-delete`, `portforward-delete`); it cannot change Site Magic. `security` is read-only. `identity` deploys only `config.yaml` to the DMZ. `edge` changes only `nginx.conf` on the VPS and can reboot it. `dns` writes zones, records, and the cluster only on the Technitium primary; on another node it only sets the server domain, resyncs a zone, or deletes a stale Secondary zone, and `technitium host` restarts or upgrades one node at a time. `proxmox` cannot create, clone, or destroy guests, change disks, or change the host configuration; on a host it only upgrades packages (`host-upgrade`) and reboots (`host-reboot --confirm`). `backup` cannot change retention or schedules. `inference` cannot start Europa or change models. `media` adds titles and toggles the download queue; it cannot delete anything or change Radarr, Sonarr, or SABnzbd settings.
|
|
29
|
+
Limits: `network` writes through the UniFi Network API plus a few narrow commands on the Network application's own paths (`nat-write`, `device-network`, `client-fixed-ip`, `syslog`, `app-policy-delete`, `portforward-delete`); it cannot change Site Magic. `security` is read-only. `identity` deploys only `config.yaml` to the DMZ and upgrades Pomerium to an official release whose sha256 matches (`dmz pomerium-upgrade`). `edge` changes only `nginx.conf` on the VPS and can reboot it. `dns` writes zones, records, and the cluster only on the Technitium primary; on another node it only sets the server domain, resyncs a zone, or deletes a stale Secondary zone, and `technitium host` restarts or upgrades one node at a time. `proxmox` cannot create, clone, or destroy guests, change disks, or change the host configuration; on a host it only upgrades packages (`host-upgrade`) and reboots (`host-reboot --confirm`). `backup` cannot change retention or schedules. `inference` cannot start Europa or change models. `media` adds titles and toggles the download queue; it cannot delete anything or change Radarr, Sonarr, or SABnzbd settings.
|
|
30
30
|
|
|
31
31
|
## Enable specialists on a host
|
|
32
32
|
|
|
@@ -212,9 +212,15 @@ async function passthrough(config: ArcaneConfig, args: string[]): Promise<number
|
|
|
212
212
|
});
|
|
213
213
|
}
|
|
214
214
|
|
|
215
|
+
/** arcane-cli can fail with only a deprecation warning on stderr; the exit code and stdout then carry the cause. */
|
|
216
|
+
export function cliFailure(args: readonly string[], result: RunResult): string {
|
|
217
|
+
const stdout = filterJsonText(result.stdout, (v) => stripKeys(v, ARCANE_DENIED_KEYS)).replace(/\s+/g, " ").trim().slice(0, 500);
|
|
218
|
+
return [`${args.join(" ")}: exited ${result.code}`, result.stderr.trim(), stdout && `stdout: ${stdout}`].filter((part) => part.length > 0).join("; ");
|
|
219
|
+
}
|
|
220
|
+
|
|
215
221
|
async function cliJson(cli: Cli, args: string[]): Promise<unknown> {
|
|
216
222
|
const result = await cli(["--output", "json", ...args]);
|
|
217
|
-
if (result.code !== 0) throw new ServiceError(
|
|
223
|
+
if (result.code !== 0) throw new ServiceError(cliFailure(args, result));
|
|
218
224
|
try {
|
|
219
225
|
return JSON.parse(result.stdout);
|
|
220
226
|
} catch {
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import { join } from "node:path";
|
|
2
3
|
import { readConfig } from "../lib/config.ts";
|
|
3
4
|
import { ServiceError, UsageError } from "../lib/errors.ts";
|
|
@@ -49,8 +50,14 @@ export const DMZ_COMMANDS: Readonly<Record<string, { args: number; help: string;
|
|
|
49
50
|
"nginx-reload": { args: 0, help: "test nginx config and reload" },
|
|
50
51
|
"lego-renew": { args: 0, help: "run lego-renew.service now" },
|
|
51
52
|
"deploy-config": { args: 0, help: "install the tracked Pomerium config.yaml from the pushed homelab checkout as the template, render, restart, check; rolls back on failure", stdin: true },
|
|
53
|
+
"pomerium-version": { args: 0, help: "the installed Pomerium version" },
|
|
52
54
|
};
|
|
53
55
|
|
|
56
|
+
const RELEASES = "https://github.com/pomerium/pomerium/releases";
|
|
57
|
+
const TARBALL = "pomerium-linux-amd64.tar.gz";
|
|
58
|
+
const RELEASE_TAG = /^v\d+\.\d+\.\d+$/;
|
|
59
|
+
const UPGRADE_TIMEOUT_MS = 10 * 60_000;
|
|
60
|
+
|
|
54
61
|
const DMZ_UNITS = ["pomerium", "nginx", "lego-renew"];
|
|
55
62
|
const SINCE = /^[A-Za-z0-9 :\-+]{1,40}$/;
|
|
56
63
|
|
|
@@ -62,6 +69,9 @@ const HELP = `identity specialist
|
|
|
62
69
|
${Object.entries(DMZ_COMMANDS)
|
|
63
70
|
.map(([name, c]) => ` ${name.padEnd(18)} ${c.help}`)
|
|
64
71
|
.join("\n")}
|
|
72
|
+
pomerium-check installed and newest Pomerium release (JSON)
|
|
73
|
+
pomerium-upgrade pomerium-upgrade <vX.Y.Z>: install that official release after checking its published sha256,
|
|
74
|
+
restart, check the login route; the old binary comes back when the check fails
|
|
65
75
|
${REPO_HELP}
|
|
66
76
|
(homelab repository; writes are limited to the configured area)
|
|
67
77
|
|
|
@@ -127,9 +137,37 @@ function repoSpec(config: IdentityConfig): RepoSpec {
|
|
|
127
137
|
};
|
|
128
138
|
}
|
|
129
139
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
const
|
|
140
|
+
/** The sha256 that a release's `pomerium_checksums.txt` publishes for the Linux amd64 tarball. */
|
|
141
|
+
export function publishedSha256(checksums: string): string {
|
|
142
|
+
const sum = checksums
|
|
143
|
+
.split("\n")
|
|
144
|
+
.map((l) => l.trim().split(/\s+/))
|
|
145
|
+
.find((f) => f[1] === TARBALL)?.[0];
|
|
146
|
+
if (sum === undefined || !/^[0-9a-f]{64}$/.test(sum)) throw new ServiceError(`the release checksums list no sha256 for ${TARBALL}`);
|
|
147
|
+
return sum;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** `pomerium: 0.33.1+01af7466f` as `v0.33.1`. */
|
|
151
|
+
export function installedRelease(output: string): string | null {
|
|
152
|
+
const m = /pomerium:\s*(\d+\.\d+\.\d+)/.exec(output);
|
|
153
|
+
return m ? `v${m[1]}` : null;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
async function download(url: string, timeoutMs: number): Promise<Buffer> {
|
|
157
|
+
const response = await fetch(url, { headers: { "User-Agent": "pi-astro-identity" }, signal: AbortSignal.timeout(timeoutMs) }).catch((err: unknown) => {
|
|
158
|
+
throw new ServiceError(`${url}: ${err instanceof Error ? err.message : String(err)}`);
|
|
159
|
+
});
|
|
160
|
+
if (!response.ok) throw new ServiceError(`${url} answered ${response.status}`);
|
|
161
|
+
return Buffer.from(await response.arrayBuffer());
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
async function newestRelease(): Promise<string> {
|
|
165
|
+
const body = JSON.parse((await download("https://api.github.com/repos/pomerium/pomerium/releases/latest", 30_000)).toString("utf8")) as { tag_name?: unknown };
|
|
166
|
+
if (typeof body.tag_name !== "string" || !RELEASE_TAG.test(body.tag_name)) throw new ServiceError("GitHub returned no Pomerium release tag");
|
|
167
|
+
return body.tag_name;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
async function sshDmz(config: IdentityConfig, remote: string, timeoutMs: number, input?: string): Promise<string> {
|
|
133
171
|
const result = await sshFixed(
|
|
134
172
|
{
|
|
135
173
|
host: config.dmzHost,
|
|
@@ -138,11 +176,40 @@ async function dmz(config: IdentityConfig, args: string[]): Promise<number> {
|
|
|
138
176
|
knownHostsFile: join(specialistsHome(), "config", "known_hosts"),
|
|
139
177
|
},
|
|
140
178
|
remote,
|
|
141
|
-
|
|
179
|
+
timeoutMs,
|
|
142
180
|
input,
|
|
143
181
|
);
|
|
144
|
-
if (result.
|
|
145
|
-
|
|
182
|
+
if (result.code !== 0) {
|
|
183
|
+
if (result.stdout.length > 0) printRaw(result.stdout);
|
|
184
|
+
throw new ServiceError(result.stderr.trim() || `dmz ${remote.split(" ")[0]} exited ${result.code}`);
|
|
185
|
+
}
|
|
186
|
+
return result.stdout;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** Downloads the release on Cortex, checks it against the published sha256, and sends it to the DMZ as base64 on stdin. */
|
|
190
|
+
async function pomeriumUpgrade(config: IdentityConfig, args: string[]): Promise<number> {
|
|
191
|
+
const [tag, ...extra] = args;
|
|
192
|
+
if (tag === undefined || !RELEASE_TAG.test(tag) || extra.length > 0) throw new UsageError("dmz pomerium-upgrade <vX.Y.Z>");
|
|
193
|
+
const sum = publishedSha256((await download(`${RELEASES}/download/${tag}/pomerium_checksums.txt`, 30_000)).toString("utf8"));
|
|
194
|
+
const tarball = await download(`${RELEASES}/download/${tag}/${TARBALL}`, 5 * 60_000);
|
|
195
|
+
if (createHash("sha256").update(tarball).digest("hex") !== sum) throw new ServiceError(`the downloaded ${TARBALL} does not match the published sha256; nothing installed`);
|
|
196
|
+
printRaw(await sshDmz(config, `pomerium-upgrade ${tag} ${sum}`, UPGRADE_TIMEOUT_MS, tarball.toString("base64")));
|
|
197
|
+
return 0;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
async function dmz(config: IdentityConfig, args: string[]): Promise<number> {
|
|
201
|
+
if (args[0] === "pomerium-upgrade") return pomeriumUpgrade(config, args.slice(1));
|
|
202
|
+
if (args[0] === "pomerium-check") {
|
|
203
|
+
if (args.length !== 1) throw new UsageError("dmz pomerium-check takes no arguments");
|
|
204
|
+
const installed = installedRelease(await sshDmz(config, "pomerium-version", 60_000));
|
|
205
|
+
const newest = await newestRelease();
|
|
206
|
+
printJson({ installed, newest, update: installed !== null && installed !== newest, notes: `${RELEASES}/tag/${newest}` });
|
|
207
|
+
return 0;
|
|
208
|
+
}
|
|
209
|
+
const remote = buildDmzRemote(args);
|
|
210
|
+
const input = DMZ_COMMANDS[args[0]]?.stdin ? await trackedFile(repoSpec(config), config.pomeriumConfig) : undefined;
|
|
211
|
+
const stdout = await sshDmz(config, remote, 180_000, input);
|
|
212
|
+
if (stdout.length > 0) printRaw(stdout);
|
|
146
213
|
return 0;
|
|
147
214
|
}
|
|
148
215
|
|
|
@@ -2,11 +2,11 @@ import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync
|
|
|
2
2
|
import { tmpdir } from "node:os";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
5
|
-
import { refusedPrefix, command as arcaneCommand } from "./arcane/run.ts";
|
|
5
|
+
import { cliFailure, refusedPrefix, command as arcaneCommand } from "./arcane/run.ts";
|
|
6
6
|
import { buildBackupRemote, command as backupCommand } from "./backup/run.ts";
|
|
7
7
|
import { buildNodeRemote, parseJoinArgs, parseParams, parseRecordFilters, technitiumRequest, technitiumTarget, command as dnsCommand } from "./dns/run.ts";
|
|
8
8
|
import { buildVpsRemote, command as edgeCommand, isTransientSshFailure, PUBLIC_ROUTES, routeProbe } from "./edge/run.ts";
|
|
9
|
-
import { API_PREFIXES, buildDmzRemote, validateApiPath, command as identityCommand } from "./identity/run.ts";
|
|
9
|
+
import { API_PREFIXES, buildDmzRemote, installedRelease, publishedSha256, validateApiPath, command as identityCommand } from "./identity/run.ts";
|
|
10
10
|
import { buildInferenceRemote, command as inferenceCommand } from "./inference/run.ts";
|
|
11
11
|
import { MAX_ADD_IDS, MAX_EXCLUDE_IDS, buildTrendarrCall, interpretRun, parseImdbIds, trendarrConfig, trendarrError, trendarrWarnings, command as mediaCommand } from "./media/run.ts";
|
|
12
12
|
import { ServiceError, UsageError } from "./lib/errors.ts";
|
|
@@ -27,6 +27,14 @@ describe("arcane wrapper", () => {
|
|
|
27
27
|
expect(refusedPrefix(["containers", "logs", "c1"])).toBeNull();
|
|
28
28
|
});
|
|
29
29
|
|
|
30
|
+
it("names the exit code and stdout when arcane-cli fails with only a warning", () => {
|
|
31
|
+
const warning = "jwt_token and refresh_token in /tmp/x/arcanecli.yml are deprecated and stop working in v3.";
|
|
32
|
+
expect(cliFailure(["images", "updates", "check", "lscr.io/linuxserver/bazarr:latest"], { code: 1, stderr: `${warning}\n`, stdout: '{"success":false,"error":"too many requests","env":["SECRET=1"]}' })).toBe(
|
|
33
|
+
`images updates check lscr.io/linuxserver/bazarr:latest: exited 1; ${warning}; stdout: { "success": false, "error": "too many requests" }`,
|
|
34
|
+
);
|
|
35
|
+
expect(cliFailure(["version"], { code: 2, stderr: "", stdout: "" })).toBe("version: exited 2");
|
|
36
|
+
});
|
|
37
|
+
|
|
30
38
|
it("keeps gitops writes inside the checkout", () => {
|
|
31
39
|
const dir = "/repo";
|
|
32
40
|
expect(safeRepoPath(dir, "mealie/compose.yaml")).toBe("/repo/mealie/compose.yaml");
|
|
@@ -83,6 +91,16 @@ describe("identity wrapper", () => {
|
|
|
83
91
|
expect(() => buildDmzRemote(["render", "extra"])).toThrow(/0 argument/);
|
|
84
92
|
expect(() => buildDmzRemote(["reboot"])).toThrow(/unknown dmz command/);
|
|
85
93
|
expect(buildDmzRemote(["deploy-config"])).toBe("deploy-config");
|
|
94
|
+
expect(buildDmzRemote(["pomerium-version"])).toBe("pomerium-version");
|
|
95
|
+
expect(() => buildDmzRemote(["pomerium-upgrade", "v0.33.6"])).toThrow(/unknown dmz command/);
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it("reads the published sha256 and the installed Pomerium release", () => {
|
|
99
|
+
const sum = "f466fa4f55127f917bd33cbe5aba10966e535b16cc26780a277750f752afb76f";
|
|
100
|
+
expect(publishedSha256(`${"0".repeat(64)} pomerium-darwin-arm64.tar.gz\n${sum} pomerium-linux-amd64.tar.gz\n`)).toBe(sum);
|
|
101
|
+
expect(() => publishedSha256(`${"0".repeat(64)} pomerium-darwin-arm64.tar.gz\n`)).toThrow(/no sha256/);
|
|
102
|
+
expect(installedRelease("pomerium: 0.33.1+01af7466f\nenvoy: 20260618180819-16ede7a56f2f\n")).toBe("v0.33.1");
|
|
103
|
+
expect(installedRelease("command not found")).toBeNull();
|
|
86
104
|
});
|
|
87
105
|
|
|
88
106
|
it("prints help without touching config", async () => {
|