@astrofoundry/pi-astro 0.37.2 → 0.37.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,133 @@
1
+ import {
2
+ escalationText,
3
+ observe,
4
+ parseSnapshot,
5
+ readWatchState,
6
+ restartInvestigationTask,
7
+ unhealthyText,
8
+ WATCH_POLL_MS,
9
+ WATCH_RECOVERED_TEXT,
10
+ watchFailingText,
11
+ writeWatchState,
12
+ } from "./arcane-watch.ts";
13
+ import { type DiscordConfig, type ReportSchedule, reportSchedule } from "./config.ts";
14
+ import type { CreateMessage, DiscordMessage } from "./rest.ts";
15
+ import type { InvestigationRequest } from "./security-monitor.ts";
16
+
17
+ export interface WatchRest {
18
+ createMessage(channelId: string, message: CreateMessage): Promise<DiscordMessage>;
19
+ startThread(channelId: string, messageId: string, name: string): Promise<{ id: string }>;
20
+ }
21
+
22
+ export interface WatchDeps {
23
+ rest: WatchRest;
24
+ runEntry: (service: string, caller: string, args: string[], timeoutMs: number) => Promise<string>;
25
+ config: () => DiscordConfig;
26
+ log: (line: string) => void;
27
+ /** Runs astro.arcane on the task in the thread; resolves true when it answered. */
28
+ investigate: (request: InvestigationRequest) => Promise<boolean>;
29
+ /** Whether the NAS share is lost or just came back (the security monitor's record of rules 100012 and 100013). */
30
+ nasOutage: (now: number) => boolean;
31
+ stopping: () => boolean;
32
+ now: () => number;
33
+ }
34
+
35
+ const MINUTE = 60_000;
36
+ /** Six failed polls (30 minutes) tag the owners once. */
37
+ const FAILS_TO_WARN = 6;
38
+ const CALLER = "arcane-watch";
39
+
40
+ function reason(err: unknown): string {
41
+ return (err instanceof Error ? err.message : String(err)).split("\n")[0].slice(0, 300);
42
+ }
43
+
44
+ /**
45
+ * Restarts and redeploys on the Arcane VM, every five minutes through the
46
+ * arcane wrapper's `container-watch`, without a model. An `arcane` report entry
47
+ * turns it on in that report's channel.
48
+ */
49
+ export class ArcaneWatch {
50
+ private readonly deps: WatchDeps;
51
+ private running = false;
52
+ private last = 0;
53
+
54
+ constructor(deps: WatchDeps) {
55
+ this.deps = deps;
56
+ }
57
+
58
+ tick(): void {
59
+ const s = reportSchedule(this.deps.config(), "arcane");
60
+ if (!s || this.deps.stopping() || this.running) return;
61
+ const now = this.deps.now();
62
+ if (now - this.last < WATCH_POLL_MS - 5000) return;
63
+ this.running = true;
64
+ this.last = now;
65
+ void this.poll(s)
66
+ .catch((err) => this.deps.log(`arcane watch: check failed: ${reason(err)}`))
67
+ .finally(() => (this.running = false));
68
+ }
69
+
70
+ private async post(channelId: string, message: CreateMessage): Promise<DiscordMessage | null> {
71
+ try {
72
+ return await this.deps.rest.createMessage(channelId, message);
73
+ } catch (err) {
74
+ this.deps.log(`arcane watch: message failed: ${reason(err)}`);
75
+ return null;
76
+ }
77
+ }
78
+
79
+ async poll(s: ReportSchedule): Promise<void> {
80
+ const owners = this.deps.config().owners;
81
+ const state = readWatchState();
82
+ let snapshot;
83
+ try {
84
+ snapshot = parseSnapshot(JSON.parse(await this.deps.runEntry("arcane", CALLER, ["container-watch"], 3 * MINUTE)));
85
+ if (!snapshot) throw new Error("container-watch returned an unexpected answer");
86
+ } catch (err) {
87
+ const why = reason(err);
88
+ this.deps.log(`arcane watch: container-watch failed: ${why}`);
89
+ state.poll.failures++;
90
+ if (state.poll.failures >= FAILS_TO_WARN && !state.poll.warned) {
91
+ state.poll.warned = (await this.post(s.channelId, { content: watchFailingText((state.poll.failures * WATCH_POLL_MS) / MINUTE, why, owners), mentionUsers: owners })) !== null;
92
+ }
93
+ writeWatchState(state);
94
+ return;
95
+ }
96
+ if (state.poll.warned) await this.post(s.channelId, { content: WATCH_RECOVERED_TEXT });
97
+ state.poll = { failures: 0, warned: false };
98
+ const now = this.deps.now();
99
+ const seen = observe(state, snapshot, now, this.deps.nasOutage(now));
100
+ for (const note of seen.notes) this.deps.log(`arcane watch: ${note}`);
101
+ for (const u of seen.unhealthy) {
102
+ const posted = await this.post(s.channelId, { content: unhealthyText(u, owners, s.timeZone, now), mentionUsers: owners });
103
+ if (posted) this.deps.log(`arcane watch: ${u.name} ${u.problem} after a redeploy`);
104
+ else state.pending.push({ name: u.name, id: state.containers[u.name]?.id ?? "", since: u.since, newImage: u.newImage, ...(u.project ? { project: u.project } : {}) });
105
+ }
106
+ const started: Promise<void>[] = [];
107
+ for (const e of seen.escalations) {
108
+ const posted = await this.post(s.channelId, { content: escalationText(e, owners, s.timeZone), mentionUsers: owners });
109
+ if (!posted) {
110
+ delete state.escalated[e.name];
111
+ continue;
112
+ }
113
+ this.deps.log(`arcane watch: ${e.name} restarted ${e.restarts.length} times in 24 h`);
114
+ started.push(this.investigate(s.channelId, posted.id, e.name, restartInvestigationTask(e, now)));
115
+ }
116
+ writeWatchState(state);
117
+ // Investigations take minutes; the next poll does not wait for them.
118
+ void Promise.all(started);
119
+ }
120
+
121
+ private async investigate(channelId: string, messageId: string, name: string, task: string): Promise<void> {
122
+ try {
123
+ const thread = await this.deps.rest.startThread(channelId, messageId, `Restarts of ${name}`.slice(0, 100));
124
+ const starter = await this.deps.rest.createMessage(thread.id, { content: "🔎 The arcane specialist reads only; a risky read still asks an owner here. Write in this thread to continue." });
125
+ const owner = this.deps.config().owners[0];
126
+ const ok = await this.deps.investigate({ threadId: thread.id, starterId: starter.id, userId: owner, task });
127
+ if (!ok) this.deps.log(`arcane watch: the investigation of ${name} did not answer`);
128
+ } catch (err) {
129
+ this.deps.log(`arcane watch: investigation of ${name} failed: ${reason(err)}`);
130
+ await this.post(channelId, { content: `❌ Could not start the investigation of ${name} (${reason(err)}); ask the arcane specialist in the channel.`, replyTo: messageId });
131
+ }
132
+ }
133
+ }
@@ -0,0 +1,242 @@
1
+ import { mkdtempSync, rmSync } from "node:fs";
2
+ import { tmpdir } from "node:os";
3
+ import { join } from "node:path";
4
+ import { afterEach, beforeEach, describe, expect, it } from "vitest";
5
+ import type { WatchedContainer } from "../../specialists/lib/container-watch.ts";
6
+ import { ArcaneWatch } from "./arcane-watch-monitor.ts";
7
+ import { emptyWatchState, escalationText, healthProblem, observe, parseSnapshot, readWatchState, restartInvestigationTask, unhealthyText, type WatchState } from "./arcane-watch.ts";
8
+ import type { DiscordConfig } from "./config.ts";
9
+ import type { CreateMessage } from "./rest.ts";
10
+ import type { InvestigationRequest } from "./security-monitor.ts";
11
+
12
+ const OWNER = "178096425509453824";
13
+ const CHANNEL = "1549873938140499999";
14
+ const TZ = "Europe/Madrid";
15
+ const T0 = Date.parse("2026-10-22T08:00:00Z");
16
+ const MIN = 60_000;
17
+
18
+ function ct(over: Partial<WatchedContainer> = {}): WatchedContainer {
19
+ return { name: "syncthing", id: "a", imageId: "sha256:1", state: "running", health: "healthy", exitCode: null, uptimeSeconds: 86_400, created: "2026-10-07T21:39:38Z", ...over };
20
+ }
21
+
22
+ function iso(ms: number): string {
23
+ return new Date(ms).toISOString();
24
+ }
25
+
26
+ /** A state that has already seen `list` at `at`. */
27
+ function seenAt(list: WatchedContainer[], at: number): WatchState {
28
+ const state = emptyWatchState();
29
+ observe(state, list, at, false);
30
+ return state;
31
+ }
32
+
33
+ describe("container watch rules", () => {
34
+ it("only records a baseline the first time", () => {
35
+ const state = emptyWatchState();
36
+ const out = observe(state, [ct({ startedAt: iso(T0 - MIN) })], T0, false);
37
+ expect(out).toEqual({ escalations: [], unhealthy: [], notes: [] });
38
+ expect(state.containers.syncthing).toEqual({ id: "a", imageId: "sha256:1", startedAt: iso(T0 - MIN), seenAt: iso(T0) });
39
+ });
40
+
41
+ it("escalates the third restart within 24 hours once a day", () => {
42
+ const state = seenAt([ct(), ct({ name: "mealie", id: "m", project: "mealie" })], T0);
43
+ for (let i = 1; i <= 2; i++) {
44
+ const now = T0 + i * 5 * MIN;
45
+ expect(observe(state, [ct({ startedAt: iso(now - MIN), project: "syncthing" })], now, false).escalations).toEqual([]);
46
+ }
47
+ const now = T0 + 15 * MIN;
48
+ const out = observe(state, [ct({ startedAt: iso(now - MIN), project: "syncthing" })], now, false);
49
+ expect(out.escalations).toEqual([{ name: "syncthing", project: "syncthing", restarts: [iso(T0 + 4 * MIN), iso(T0 + 9 * MIN), iso(T0 + 14 * MIN)] }]);
50
+ expect(state.containers.mealie).toBeUndefined();
51
+ const later = T0 + 20 * MIN;
52
+ expect(observe(state, [ct({ startedAt: iso(later - MIN) })], later, false).escalations).toEqual([]);
53
+ expect(state.restarts.syncthing).toHaveLength(4);
54
+ });
55
+
56
+ it("does not count a start it already saw, or one before the last observation", () => {
57
+ const state = seenAt([ct({ startedAt: iso(T0 - 2 * MIN) })], T0);
58
+ observe(state, [ct({ startedAt: iso(T0 - 2 * MIN) })], T0 + 5 * MIN, false);
59
+ observe(state, [ct()], T0 + 10 * MIN, false);
60
+ expect(state.restarts).toEqual({});
61
+ expect(state.containers.syncthing.startedAt).toBe(iso(T0 - 2 * MIN));
62
+ });
63
+
64
+ it("leaves out restarts our operations or a NAS outage explain", () => {
65
+ const state = seenAt([ct({ name: "slskd", id: "s", project: "media" }), ct({ name: "mealie", id: "m", project: "mealie" })], T0);
66
+ const now = T0 + 5 * MIN;
67
+ const operation = { command: "host db-restore mealie 20261022T0700Z", caller: "discord-1", from: iso(now - 3 * MIN), to: iso(now) };
68
+ const out = observe(state, [ct({ name: "slskd", id: "s", project: "media", startedAt: iso(now - MIN) }), ct({ name: "mealie", id: "m", project: "mealie", startedAt: iso(now - MIN), operation })], now, true);
69
+ expect(out.notes).toEqual(["slskd restarted during a NAS outage", "mealie restarted during host db-restore mealie 20261022T0700Z (discord-1)"]);
70
+ expect(state.restarts).toEqual({});
71
+ });
72
+
73
+ it("checks a redeployed container after five minutes and reports what is wrong", () => {
74
+ const state = seenAt([ct({ name: "mealie", id: "old", imageId: "sha256:1", project: "mealie" })], T0);
75
+ const started = T0 + MIN;
76
+ const fresh = ct({ name: "mealie", id: "new", imageId: "sha256:2", project: "mealie", startedAt: iso(started), health: "starting", uptimeSeconds: 60 });
77
+ expect(observe(state, [fresh], T0 + 2 * MIN, false).unhealthy).toEqual([]);
78
+ expect(state.pending).toEqual([{ name: "mealie", id: "new", since: iso(started), newImage: true, project: "mealie" }]);
79
+ expect(observe(state, [{ ...fresh, health: "starting" }], T0 + 7 * MIN, false).unhealthy).toEqual([]);
80
+ const out = observe(state, [{ ...fresh, health: "unhealthy" }], T0 + 12 * MIN, false);
81
+ expect(out.unhealthy).toEqual([{ name: "mealie", project: "mealie", since: iso(started), newImage: true, problem: "unhealthy" }]);
82
+ expect(state.pending).toEqual([]);
83
+ });
84
+
85
+ it("drops a redeploy that came up fine, was replaced, or that our operation reports", () => {
86
+ const state = seenAt([ct({ name: "a", id: "1" }), ct({ name: "b", id: "1b" }), ct({ name: "c", id: "1c", project: "c" })], T0);
87
+ const operation = { command: "stack-update c", caller: "discord-1", from: iso(T0), to: iso(T0 + MIN) };
88
+ const created = iso(T0);
89
+ observe(state, [ct({ name: "a", id: "2", created }), ct({ name: "b", id: "2b", created }), ct({ name: "c", id: "2c", project: "c", operation, state: "exited", exitCode: 1, created })], T0 + MIN, false);
90
+ expect(state.pending.map((p) => p.id)).toEqual(["2", "2b", "2c"]);
91
+ const out = observe(state, [ct({ name: "a", id: "2", created }), ct({ name: "b", id: "3b", created: iso(T0 + 6 * MIN) }), ct({ name: "c", id: "2c", project: "c", state: "exited", exitCode: 1, created })], T0 + 7 * MIN, false);
92
+ expect(out.unhealthy).toEqual([]);
93
+ expect(out.notes).toEqual(["c exited with code 1 after stack-update c (discord-1), which reports it"]);
94
+ expect(state.pending.map((p) => p.id)).toEqual(["3b"]);
95
+ });
96
+
97
+ it("names what is wrong with a container", () => {
98
+ expect(healthProblem(ct({ state: "exited", exitCode: 0 }), 0)).toBeNull();
99
+ expect(healthProblem(ct({ state: "exited", exitCode: 137 }), 0)).toBe("exited with code 137");
100
+ expect(healthProblem(ct({ state: "restarting" }), 0)).toBe("restarting");
101
+ expect(healthProblem(ct({ state: "created" }), 0)).toBe("never started");
102
+ expect(healthProblem(ct({ health: "starting" }), 16 * MIN)).toBe("health check still starting");
103
+ expect(healthProblem(ct({ health: "none" }), 16 * MIN)).toBeNull();
104
+ });
105
+
106
+ it("forgets restarts and escalations after a day", () => {
107
+ const state = emptyWatchState();
108
+ state.containers.x = { id: "1", imageId: null, startedAt: null, seenAt: iso(T0) };
109
+ state.restarts.x = [iso(T0 - 25 * 60 * MIN)];
110
+ state.escalated.x = iso(T0 - 25 * 60 * MIN);
111
+ observe(state, [ct({ name: "x", id: "1" })], T0, false);
112
+ expect(state.restarts).toEqual({});
113
+ expect(state.escalated).toEqual({});
114
+ });
115
+ });
116
+
117
+ describe("container watch texts", () => {
118
+ it("tags the owners and says what happened", () => {
119
+ const e = { name: "syncthing", project: "syncthing", restarts: [iso(T0), iso(T0 + 5 * MIN), iso(T0 + 10 * MIN)] };
120
+ expect(escalationText(e, [OWNER], TZ)).toBe(
121
+ `🔁 **Container restarting** · syncthing · 3 restarts in 24 h, last 22 Oct 10:10 CEST\nAuto-heal or Docker restarted it each time. The arcane specialist investigates in the thread and changes nothing.\n<@${OWNER}>`,
122
+ );
123
+ expect(unhealthyText({ name: "worker", project: "grimoire", since: iso(T0), newImage: true, problem: "unhealthy" }, [OWNER], TZ, T0 + 7 * MIN)).toBe(
124
+ `⚠️ **Container unhealthy after a redeploy** · worker (stack grimoire) · recreated 22 Oct 10:00 CEST with a new image, still unhealthy 7 min later\nNo stack-update or other specialist operation explains the redeploy. Ask the arcane specialist here to investigate.\n<@${OWNER}>`,
125
+ );
126
+ });
127
+
128
+ it("asks the specialist to read only, within the host's one-day window", () => {
129
+ const now = T0 + 24 * 60 * MIN;
130
+ const task = restartInvestigationTask({ name: "syncthing", restarts: [iso(T0 + MIN), iso(now - MIN)] }, now);
131
+ expect(task).toContain("Read only: change nothing");
132
+ expect(task).toContain(`host container-logs syncthing ${iso(T0 + MIN)} ${iso(now)}`);
133
+ });
134
+
135
+ it("accepts only a well-formed snapshot", () => {
136
+ expect(parseSnapshot({ containers: [ct()] })).toEqual([ct()]);
137
+ expect(parseSnapshot({ containers: [{ name: "x" }] })).toBeNull();
138
+ expect(parseSnapshot({})).toBeNull();
139
+ });
140
+ });
141
+
142
+ function config(): DiscordConfig {
143
+ return {
144
+ applicationId: "1",
145
+ guildId: "2",
146
+ channels: { [CHANNEL]: { trigger: "always", specialists: ["arcane"] } },
147
+ adminChannelId: CHANNEL,
148
+ owners: [OWNER],
149
+ access: {},
150
+ approvalTimeoutMinutes: 10,
151
+ reports: [{ report: "arcane", channelId: CHANNEL, time: "06:00", timeZone: TZ }],
152
+ };
153
+ }
154
+
155
+ describe("container watch monitor", () => {
156
+ let dir: string;
157
+ beforeEach(() => {
158
+ dir = mkdtempSync(join(tmpdir(), "astro-watch-"));
159
+ process.env.ASTRO_DISCORD_DIR = dir;
160
+ });
161
+ afterEach(() => {
162
+ delete process.env.ASTRO_DISCORD_DIR;
163
+ rmSync(dir, { recursive: true, force: true });
164
+ });
165
+
166
+ function harness() {
167
+ const messages: { channelId: string; message: CreateMessage }[] = [];
168
+ const threads: string[] = [];
169
+ const investigations: InvestigationRequest[] = [];
170
+ const logs: string[] = [];
171
+ const clock = { now: T0 };
172
+ const answer: { next: () => string } = { next: () => JSON.stringify({ checkedAt: iso(clock.now), containers: [ct()] }) };
173
+ let id = 500;
174
+ const watch = new ArcaneWatch({
175
+ rest: {
176
+ createMessage: async (channelId, message) => {
177
+ messages.push({ channelId, message });
178
+ return { id: String(id++), channel_id: channelId };
179
+ },
180
+ startThread: async (_channelId, messageId) => {
181
+ threads.push(messageId);
182
+ return { id: `t${messageId}` };
183
+ },
184
+ },
185
+ runEntry: async (service, caller, args) => {
186
+ expect([service, caller, ...args]).toEqual(["arcane", "arcane-watch", "container-watch"]);
187
+ return answer.next();
188
+ },
189
+ config,
190
+ log: (line) => logs.push(line),
191
+ investigate: async (request) => {
192
+ investigations.push(request);
193
+ return true;
194
+ },
195
+ nasOutage: () => false,
196
+ stopping: () => false,
197
+ now: () => clock.now,
198
+ });
199
+ return { watch, messages, threads, investigations, logs, clock, answer };
200
+ }
201
+
202
+ const SCHEDULE = config().reports[0];
203
+
204
+ it("posts the escalation with a mention, opens a thread, and investigates there", async () => {
205
+ const h = harness();
206
+ await h.watch.poll(SCHEDULE);
207
+ for (let i = 1; i <= 3; i++) {
208
+ h.clock.now = T0 + i * 5 * MIN;
209
+ const started = iso(h.clock.now - MIN);
210
+ h.answer.next = () => JSON.stringify({ containers: [ct({ startedAt: started, project: "syncthing" })] });
211
+ await h.watch.poll(SCHEDULE);
212
+ }
213
+ await new Promise((resolve) => setTimeout(resolve, 0));
214
+ expect(h.messages[0].message.content).toContain("🔁 **Container restarting** · syncthing · 3 restarts in 24 h");
215
+ expect(h.messages[0].message.mentionUsers).toEqual([OWNER]);
216
+ expect(h.threads).toEqual(["500"]);
217
+ expect(h.investigations).toHaveLength(1);
218
+ expect(h.investigations[0]).toMatchObject({ threadId: "t500", starterId: "501", userId: OWNER });
219
+ expect(h.investigations[0].task).toContain("restarted 3 times");
220
+ });
221
+
222
+ it("warns once after 30 minutes of failed checks and says when they work again", async () => {
223
+ const h = harness();
224
+ h.answer.next = () => "not json";
225
+ for (let i = 0; i < 7; i++) await h.watch.poll(SCHEDULE);
226
+ expect(h.messages).toHaveLength(1);
227
+ expect(h.messages[0].message.content).toContain("⚠️ **Container checks failing:** the bridge could not read Arcane's containers for 30 minutes");
228
+ h.answer.next = () => JSON.stringify({ containers: [ct()] });
229
+ await h.watch.poll(SCHEDULE);
230
+ expect(h.messages[1].message.content).toBe("✅ Container checks work again.");
231
+ expect(readWatchState().poll).toEqual({ failures: 0, warned: false });
232
+ });
233
+
234
+ it("polls only with an arcane report entry and every five minutes", () => {
235
+ const h = harness();
236
+ h.watch.tick();
237
+ h.watch.tick();
238
+ h.clock.now = T0 + 5 * MIN;
239
+ h.watch.tick();
240
+ expect(h.logs).toEqual([]);
241
+ });
242
+ });
@@ -0,0 +1,215 @@
1
+ import { readFileSync, renameSync, writeFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import type { Operation } from "../../specialists/lib/change-origin.ts";
4
+ import type { WatchedContainer } from "../../specialists/lib/container-watch.ts";
5
+ import { configDir } from "./config.ts";
6
+ import { escapeDiscord } from "./media.ts";
7
+ import { clockText } from "./security.ts";
8
+
9
+ const MINUTE = 60_000;
10
+ const DAY = 86_400_000;
11
+ export const WATCH_POLL_MS = 5 * MINUTE;
12
+ /** Three restarts of one container within 24 hours start an investigation, at most once a day per container. */
13
+ export const RESTARTS_TO_ESCALATE = 3;
14
+ /** A redeployed container gets five minutes to come up before its health counts. */
15
+ export const HEALTH_GRACE_MS = 5 * MINUTE;
16
+ /** A health check still starting after this long counts as failing. */
17
+ const STARTING_LIMIT_MS = 15 * MINUTE;
18
+ /** The only stack that mounts the NAS share (`nas_media` in the arcane repository's media/compose.yaml). */
19
+ export const NAS_STACKS: readonly string[] = ["media"];
20
+
21
+ export interface WatchState {
22
+ /** The last observation of each container by name. */
23
+ containers: Record<string, { id: string; imageId: string | null; startedAt: string | null; seenAt: string }>;
24
+ /** Restart times per container within the last 24 hours, ours and NAS-related ones left out. */
25
+ restarts: Record<string, string[]>;
26
+ /** When each container was last escalated. */
27
+ escalated: Record<string, string>;
28
+ /** Redeployed containers whose health is checked once the grace time has passed. */
29
+ pending: PendingHealth[];
30
+ poll: { failures: number; warned: boolean };
31
+ }
32
+
33
+ export interface PendingHealth {
34
+ name: string;
35
+ id: string;
36
+ since: string;
37
+ project?: string;
38
+ newImage: boolean;
39
+ operation?: string;
40
+ }
41
+
42
+ export interface Escalation {
43
+ name: string;
44
+ project?: string;
45
+ restarts: string[];
46
+ }
47
+
48
+ export interface Unhealthy {
49
+ name: string;
50
+ project?: string;
51
+ since: string;
52
+ newImage: boolean;
53
+ problem: string;
54
+ }
55
+
56
+ export interface Observation {
57
+ escalations: Escalation[];
58
+ unhealthy: Unhealthy[];
59
+ /** Log lines for what was seen but is expected: our operations and NAS outages. */
60
+ notes: string[];
61
+ }
62
+
63
+ export function emptyWatchState(): WatchState {
64
+ return { containers: {}, restarts: {}, escalated: {}, pending: [], poll: { failures: 0, warned: false } };
65
+ }
66
+
67
+ function statePath(): string {
68
+ return join(configDir(), "arcane-watch-state.json");
69
+ }
70
+
71
+ export function readWatchState(): WatchState {
72
+ try {
73
+ return { ...emptyWatchState(), ...(JSON.parse(readFileSync(statePath(), "utf-8")) as Partial<WatchState>) };
74
+ } catch {
75
+ return emptyWatchState();
76
+ }
77
+ }
78
+
79
+ export function writeWatchState(state: WatchState): void {
80
+ const path = statePath();
81
+ writeFileSync(`${path}.tmp`, `${JSON.stringify(state)}\n`, { mode: 0o600 });
82
+ renameSync(`${path}.tmp`, path);
83
+ }
84
+
85
+ function isContainer(value: unknown): value is WatchedContainer {
86
+ const c = value as Partial<WatchedContainer> | null;
87
+ return !!c && typeof c.name === "string" && typeof c.id === "string" && typeof c.state === "string" && typeof c.health === "string";
88
+ }
89
+
90
+ /** The `container-watch` answer of the arcane wrapper. */
91
+ export function parseSnapshot(value: unknown): WatchedContainer[] | null {
92
+ const r = value as { containers?: unknown } | null;
93
+ return r && Array.isArray(r.containers) && r.containers.every(isContainer) ? r.containers : null;
94
+ }
95
+
96
+ function operationText(o: Operation): string {
97
+ return `${o.command} (${o.caller})`;
98
+ }
99
+
100
+ /** What is wrong with a container that should be up, or null. */
101
+ export function healthProblem(c: WatchedContainer, sinceMs: number): string | null {
102
+ if (c.state === "exited") return c.exitCode === 0 ? null : `exited with code ${c.exitCode ?? "?"}`;
103
+ if (c.state === "restarting" || c.state === "dead" || c.state === "created") return c.state === "created" ? "never started" : c.state;
104
+ if (c.health === "unhealthy") return "unhealthy";
105
+ if (c.health === "starting" && sinceMs >= STARTING_LIMIT_MS) return "health check still starting";
106
+ return null;
107
+ }
108
+
109
+ /**
110
+ * Compares a fresh snapshot with the last one: a new id is a redeploy, whose
111
+ * health is checked after the grace time; a newer start time with the same id
112
+ * is a restart. The first snapshot only records the baseline.
113
+ */
114
+ export function observe(state: WatchState, snapshot: readonly WatchedContainer[], now: number, nasOutage: boolean): Observation {
115
+ const out: Observation = { escalations: [], unhealthy: [], notes: [] };
116
+ const baseline = Object.keys(state.containers).length === 0;
117
+ const at = new Date(now).toISOString();
118
+ const nasExpected = (c: { project?: string }) => nasOutage && c.project !== undefined && NAS_STACKS.includes(c.project);
119
+ const current = new Map(snapshot.map((c) => [c.name, c]));
120
+ for (const c of snapshot) {
121
+ const prev = state.containers[c.name];
122
+ if (prev && !baseline) {
123
+ if (prev.id !== c.id) {
124
+ if (!state.pending.some((p) => p.id === c.id)) {
125
+ state.pending.push({ name: c.name, id: c.id, since: c.startedAt ?? c.created ?? at, newImage: prev.imageId !== c.imageId, ...(c.project ? { project: c.project } : {}), ...(c.operation ? { operation: operationText(c.operation) } : {}) });
126
+ }
127
+ } else if (c.startedAt && c.startedAt !== prev.startedAt && Date.parse(c.startedAt) > Date.parse(prev.seenAt)) {
128
+ if (c.operation) out.notes.push(`${c.name} restarted during ${operationText(c.operation)}`);
129
+ else if (nasExpected(c)) out.notes.push(`${c.name} restarted during a NAS outage`);
130
+ else state.restarts[c.name] = [...(state.restarts[c.name] ?? []), c.startedAt];
131
+ }
132
+ }
133
+ const startedAt = c.startedAt ?? (prev?.id === c.id ? prev.startedAt : null);
134
+ state.containers[c.name] = { id: c.id, imageId: c.imageId, startedAt, seenAt: at };
135
+ }
136
+ for (const name of Object.keys(state.containers)) if (!current.has(name)) delete state.containers[name];
137
+ for (const [name, times] of Object.entries(state.restarts)) {
138
+ const recent = times.filter((t) => Date.parse(t) >= now - DAY);
139
+ if (recent.length === 0) delete state.restarts[name];
140
+ else state.restarts[name] = recent;
141
+ }
142
+ state.escalated = Object.fromEntries(Object.entries(state.escalated).filter(([, t]) => Date.parse(t) >= now - DAY));
143
+ for (const [name, times] of Object.entries(state.restarts)) {
144
+ if (times.length < RESTARTS_TO_ESCALATE || state.escalated[name]) continue;
145
+ state.escalated[name] = at;
146
+ const project = current.get(name)?.project;
147
+ out.escalations.push({ name, ...(project ? { project } : {}), restarts: [...times] });
148
+ }
149
+ const waiting: PendingHealth[] = [];
150
+ for (const p of state.pending) {
151
+ const c = current.get(p.name);
152
+ if (!c || c.id !== p.id) continue;
153
+ const age = now - Date.parse(p.since);
154
+ if (age < HEALTH_GRACE_MS) {
155
+ waiting.push(p);
156
+ continue;
157
+ }
158
+ const problem = healthProblem(c, age);
159
+ if (problem === null && c.health === "starting") {
160
+ waiting.push(p);
161
+ continue;
162
+ }
163
+ if (problem === null) continue;
164
+ if (p.operation) out.notes.push(`${p.name} ${problem} after ${p.operation}, which reports it`);
165
+ else if (nasExpected(p)) out.notes.push(`${p.name} ${problem} during a NAS outage`);
166
+ else out.unhealthy.push({ name: p.name, ...(p.project ? { project: p.project } : {}), since: p.since, newImage: p.newImage, problem });
167
+ }
168
+ state.pending = waiting;
169
+ return out;
170
+ }
171
+
172
+ function mention(owners: readonly string[]): string {
173
+ return owners.map((id) => `<@${id}>`).join(" ");
174
+ }
175
+
176
+ function where(name: string, project?: string): string {
177
+ return `${escapeDiscord(name)}${project && project !== name ? ` (stack ${escapeDiscord(project)})` : ""}`;
178
+ }
179
+
180
+ export function escalationText(e: Escalation, owners: readonly string[], timeZone: string): string {
181
+ const last = e.restarts[e.restarts.length - 1];
182
+ return [
183
+ `🔁 **Container restarting** · ${where(e.name, e.project)} · ${e.restarts.length} restarts in 24 h, last ${clockText(last, timeZone)}`,
184
+ "Auto-heal or Docker restarted it each time. The arcane specialist investigates in the thread and changes nothing.",
185
+ mention(owners),
186
+ ].join("\n");
187
+ }
188
+
189
+ export function unhealthyText(u: Unhealthy, owners: readonly string[], timeZone: string, now: number): string {
190
+ const minutes = Math.round((now - Date.parse(u.since)) / MINUTE);
191
+ return [
192
+ `⚠️ **Container ${u.problem} after a redeploy** · ${where(u.name, u.project)} · recreated ${clockText(u.since, timeZone)}${u.newImage ? " with a new image" : ""}, still ${u.problem} ${minutes} min later`,
193
+ "No stack-update or other specialist operation explains the redeploy. Ask the arcane specialist here to investigate.",
194
+ mention(owners),
195
+ ].join("\n");
196
+ }
197
+
198
+ export function watchFailingText(minutes: number, reason: string, owners: readonly string[]): string {
199
+ return `⚠️ **Container checks failing:** the bridge could not read Arcane's containers for ${minutes} minutes (${escapeDiscord(reason.slice(0, 200))}). Restarts and redeploys are not checked meanwhile; the bridge log on Cortex has details.\n${mention(owners)}`;
200
+ }
201
+
202
+ export const WATCH_RECOVERED_TEXT = "✅ Container checks work again.";
203
+
204
+ /** The read-only task astro.arcane gets in the escalation's thread. */
205
+ export function restartInvestigationTask(e: Escalation, now: number): string {
206
+ // The host's log and OOM windows span at most one day.
207
+ const since = new Date(Math.max(Date.parse(e.restarts[0]) - 10 * MINUTE, now - DAY + MINUTE)).toISOString();
208
+ const until = new Date(now).toISOString();
209
+ return [
210
+ `Investigate why the container ${e.name}${e.project ? ` (stack ${e.project})` : ""} restarted ${e.restarts.length} times in the last 24 hours, at ${e.restarts.join(", ")} (UTC).`,
211
+ "Read only: change nothing, restart nothing, and run no stack-update, manager-upgrade, git write, or other write command. A fix goes through GitOps and needs an owner's approval, so describe it instead.",
212
+ `Collect evidence with \`containers get ${e.name}\`, \`host container-logs ${e.name} ${since} ${until}\`, \`host container-oom ${e.name} ${since} ${until}\`, \`host kernel-oom ${since} ${until}\`, and \`host container-memory ${e.name}\`. Name each command whose output you rely on.`,
213
+ "Answer with the likely cause, the evidence for it, and the change you would propose.",
214
+ ].join("\n");
215
+ }
@@ -71,7 +71,9 @@ import {
71
71
  } from "./tools-report.ts";
72
72
  import { type Quote, type QuotedMessage, withQuotes } from "./quote.ts";
73
73
  import { DiscordRest, type MessageComponent } from "./rest.ts";
74
+ import { ArcaneWatch } from "./arcane-watch-monitor.ts";
74
75
  import { type InvestigationRequest, SecurityMonitor } from "./security-monitor.ts";
76
+ import { nasOutage, readSecurityState } from "./security.ts";
75
77
 
76
78
  const REACTION = { running: "⏳", waiting: "🔒", done: "✅", failed: "❌" } as const;
77
79
  /** Answers longer than this go as a file attachment with a short summary. */
@@ -222,6 +224,7 @@ class Bridge {
222
224
  private readonly ctx: ExtensionContext;
223
225
  private readonly token: string;
224
226
  private readonly security: SecurityMonitor;
227
+ private readonly arcaneWatch: ArcaneWatch;
225
228
 
226
229
  constructor(ctx: ExtensionContext, config: DiscordConfig, token: string) {
227
230
  this.ctx = ctx;
@@ -235,7 +238,17 @@ class Bridge {
235
238
  runEntry: (service, caller, args, timeoutMs) => this.runEntry(service, caller, args, timeoutMs),
236
239
  config: () => this.config,
237
240
  log,
238
- investigate: (request) => this.investigateSecurity(request),
241
+ investigate: (request) => this.investigateIn("security", request),
242
+ stopping: () => this.stopping,
243
+ now: () => Date.now(),
244
+ });
245
+ this.arcaneWatch = new ArcaneWatch({
246
+ rest: this.rest,
247
+ runEntry: (service, caller, args, timeoutMs) => this.runEntry(service, caller, args, timeoutMs),
248
+ config: () => this.config,
249
+ log,
250
+ investigate: (request) => this.investigateIn("arcane", request),
251
+ nasOutage: (now) => nasOutage(readSecurityState().nas, now),
239
252
  stopping: () => this.stopping,
240
253
  now: () => Date.now(),
241
254
  });
@@ -346,10 +359,12 @@ class Bridge {
346
359
  void this.maybeReports();
347
360
  void this.processRestarts();
348
361
  this.security.tick();
362
+ this.arcaneWatch.tick();
349
363
  }, 60_000);
350
364
  this.reportTimer.unref();
351
365
  void this.maybeReports();
352
366
  this.security.tick();
367
+ this.arcaneWatch.tick();
353
368
  log(`connected as ${me.username} (${me.id}); channels ${Object.keys(this.config.channels).join(", ")}; specialists ${this.specialists().names.join(", ")}`);
354
369
  }
355
370
 
@@ -1118,29 +1133,30 @@ class Bridge {
1118
1133
  }
1119
1134
 
1120
1135
  /**
1121
- * Investigate on a security alert: astro.security in the alert's thread,
1122
- * queued behind any other security task, as a fresh conversation that the
1136
+ * An investigation in a thread: astro.security on a security alert, or
1137
+ * astro.arcane on a container that keeps restarting. It waits behind any
1138
+ * other task of that specialist and starts a fresh conversation that the
1123
1139
  * owner can continue by writing in the thread.
1124
1140
  */
1125
- private async investigateSecurity(request: InvestigationRequest): Promise<boolean> {
1126
- const agent = this.specialists().agents.find((a) => a.name === "astro.security");
1141
+ private async investigateIn(specialist: "security" | "arcane", request: InvestigationRequest): Promise<boolean> {
1142
+ const agent = this.specialists().agents.find((a) => a.name === `astro.${specialist}`);
1127
1143
  if (!agent) {
1128
- await this.rest.createMessage(request.threadId, { content: "❌ astro.security is not configured on this host." }).catch(() => undefined);
1144
+ await this.rest.createMessage(request.threadId, { content: `❌ astro.${specialist} is not configured on this host.` }).catch(() => undefined);
1129
1145
  return false;
1130
1146
  }
1131
1147
  const now = Date.now();
1132
- const conversation = resolveConversation(this.conversations, { channelId: request.threadId, specialist: "security", fresh: true, now }, () => newConversationId("security", now));
1148
+ const conversation = resolveConversation(this.conversations, { channelId: request.threadId, specialist, fresh: true, now }, () => newConversationId(specialist, now));
1133
1149
  this.conversations = startConversation(this.conversations, conversation.key, conversation.id, now);
1134
1150
  saveState(this.conversations);
1135
- const task: ActiveTask = { specialist: "security", channelId: request.threadId, messageId: request.starterId, userId: request.userId, startedAt: now, conversation, images: [] };
1151
+ const task: ActiveTask = { specialist, channelId: request.threadId, messageId: request.starterId, userId: request.userId, startedAt: now, conversation, images: [] };
1136
1152
  let failed = true;
1137
- const previous = this.queues.get("security") ?? Promise.resolve();
1153
+ const previous = this.queues.get(specialist) ?? Promise.resolve();
1138
1154
  const next = previous
1139
1155
  .then(async () => {
1140
1156
  failed = await this.runTask(agent, request.task, task);
1141
1157
  })
1142
- .catch((err) => log(`security investigation failed: ${err instanceof Error ? err.message : String(err)}`));
1143
- this.queues.set("security", next);
1158
+ .catch((err) => log(`${specialist} investigation failed: ${err instanceof Error ? err.message : String(err)}`));
1159
+ this.queues.set(specialist, next);
1144
1160
  await next;
1145
1161
  return !failed;
1146
1162
  }
@@ -27,6 +27,7 @@ import {
27
27
  isAcked,
28
28
  MAX_POSTS_PER_CHECK,
29
29
  needsOrigin,
30
+ noteNas,
30
31
  noticeText,
31
32
  ORIGIN_RETRY_MS,
32
33
  type OriginVerdict,
@@ -277,6 +278,7 @@ export class SecurityMonitor {
277
278
  for (const e of fresh) {
278
279
  if (state.seen[e.id] !== undefined) continue;
279
280
  state.seen[e.id] = e.timestamp;
281
+ noteNas(state, e);
280
282
  if (needsOrigin(e)) settle(e, new Date(now).toISOString());
281
283
  else ready.push(e);
282
284
  }
@@ -39,6 +39,8 @@ import {
39
39
  type ImageVulnFacts,
40
40
  offeredImages,
41
41
  parseImageVulns,
42
+ nasOutage,
43
+ noteNas,
42
44
  repeatText,
43
45
  resticStatus,
44
46
  type VulnEvent,
@@ -363,6 +365,21 @@ describe("vulnerabilities", () => {
363
365
  });
364
366
  });
365
367
 
368
+ describe("NAS outage record", () => {
369
+ it("keeps the newest loss and return and calls the share out until half an hour after it returns", () => {
370
+ const state = emptyState();
371
+ noteNas(state, event({ rule: "100012", timestamp: "2026-10-17T22:47:00.000+0000" }));
372
+ noteNas(state, event({ rule: "100012", timestamp: "2026-10-17T20:00:00.000+0000" }));
373
+ noteNas(state, event({ rule: "5712", timestamp: "2026-10-18T09:00:00.000+0000" }));
374
+ expect(state.nas).toEqual({ lostAt: "2026-10-17T22:47:00.000+0000", restoredAt: null });
375
+ expect(nasOutage(state.nas, Date.parse("2026-10-18T03:00:00Z"))).toBe(true);
376
+ noteNas(state, event({ rule: "100013", timestamp: "2026-10-18T06:25:00.000+0000" }));
377
+ expect(nasOutage(state.nas, Date.parse("2026-10-18T06:50:00Z"))).toBe(true);
378
+ expect(nasOutage(state.nas, Date.parse("2026-10-18T07:00:00Z"))).toBe(false);
379
+ expect(nasOutage(emptyState().nas, Date.parse("2026-10-18T07:00:00Z"))).toBe(false);
380
+ });
381
+ });
382
+
366
383
  describe("Arcane image scans", () => {
367
384
  const item = (over: Record<string, unknown> = {}) => ({ vulnerabilityId: "CVE-2026-1", pkgName: "openssl", installedVersion: "3.0.1", fixedVersion: "3.0.2", severity: "CRITICAL", ignored: false, imageName: "ghcr.io/mealie-recipes/mealie:latest", ...over });
368
385
  const stacks = [
@@ -125,6 +125,8 @@ export interface SecurityState {
125
125
  vulns: { reported: Record<string, ReportedVuln>; processedUntil: string | null; since: string | null };
126
126
  /** Ban totals of earlier weekly reports: CrowdSec keeps alerts for 7 days, so last week exists only here. */
127
127
  weeks: { date: string; bans: number }[];
128
+ /** The newest NAS MEDIA loss (rule 100012) and return (100013); the Arcane container watch expects media restarts around them. */
129
+ nas: { lostAt: string | null; restoredAt: string | null };
128
130
  }
129
131
 
130
132
  export function emptyState(): SecurityState {
@@ -140,6 +142,7 @@ export function emptyState(): SecurityState {
140
142
  poll: { failures: 0, warned: false },
141
143
  vulns: { reported: {}, processedUntil: null, since: null },
142
144
  weeks: [],
145
+ nas: { lostAt: null, restoredAt: null },
143
146
  };
144
147
  }
145
148
 
@@ -165,6 +168,23 @@ export function writeSecurityState(state: SecurityState): void {
165
168
 
166
169
  const DAY = 86_400_000;
167
170
 
171
+ /** Keeps the newest NAS loss and return the checks have seen. */
172
+ export function noteNas(state: SecurityState, e: WazuhEvent): void {
173
+ const key = e.rule === "100012" ? "lostAt" : e.rule === "100013" ? "restoredAt" : null;
174
+ if (key && (state.nas[key] === null || Date.parse(e.timestamp) > Date.parse(state.nas[key] as string))) state.nas[key] = e.timestamp;
175
+ }
176
+
177
+ /** The media watchdog restarts the media containers once access returns, within a few minutes. */
178
+ const NAS_SETTLE_MS = 30 * 60_000;
179
+
180
+ /** Whether the NAS share is lost now, or came back within the last half hour. */
181
+ export function nasOutage(nas: SecurityState["nas"], now: number): boolean {
182
+ const lost = nas.lostAt ? Date.parse(nas.lostAt) : null;
183
+ const restored = nas.restoredAt ? Date.parse(nas.restoredAt) : null;
184
+ if (lost !== null && (restored === null || restored < lost)) return true;
185
+ return restored !== null && now - restored < NAS_SETTLE_MS;
186
+ }
187
+
168
188
  /** Drops what no check, report, or button can need any more. */
169
189
  export function pruneState(state: SecurityState, now: number): void {
170
190
  const keep = (iso: string, days: number) => Date.parse(iso) >= now - days * DAY;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@astrofoundry/pi-astro",
3
- "version": "0.37.2",
3
+ "version": "0.37.3",
4
4
  "description": "Personal pi customizations (extensions, subagents, skills, prompts, themes) for the pi coding agent.",
5
5
  "keywords": [
6
6
  "pi-package"
@@ -89,6 +89,8 @@ Public routes (Supabase, Grimoire, Mealie URLs) are `astro.edge`'s `route-probe`
89
89
 
90
90
  `["change-origin", "<path>", "<sha256|->", "<utc>"]` answers whether a deployment-file change Wazuh saw on the Arcane VM (`/opt/docker/<project>/compose.yaml`, `docker-compose.yml`, or `pre-deploy.sh`; rule 100020) is ours. `known` when the new content (its SHA-256 from the alert's `syscheck.sha256_after`), or `-` for a deletion, equals a commit of `<project>/<file>` at origin, any author, CI included; or when our `stack-update`, `manager-upgrade`, `manager-rollback`, or `host db-restore` on that stack ran then (the specialists' audit log). Otherwise `unknown`, with the newest commit touching the file. Arcane syncs the files verbatim, so a GitOps deploy always matches. The Discord bridge calls it for every such alert; an `unknown` change pages the owner.
91
91
 
92
+ `["container-watch"]` lists every container with its state, health, exit code, and uptime; containers started or created in the last 30 minutes also carry the start time, the Compose project, and the `operation` (our stack-update, manager-upgrade, manager-rollback, or host db-restore on that stack) whose run covers the start. The Discord bridge polls it every five minutes to detect restarts and redeploys; when it asks you to investigate a container that keeps restarting, read only and propose the fix.
93
+
92
94
  `["project-status"]` (or with a project name) lists projects with their GitOps sync: commit, state, hook state, without error text. `["deployment", "check"|"wait", "<project>", "--commit", "<sha>", "--container", "NAME=sha256:ID", ...]` compares the sync and each container's project, image ID, state, and health; `wait` polls every 10 s up to `--timeout` (default 600 s). Read `status` in the answer.
93
95
 
94
96
  ## Output
@@ -6,6 +6,7 @@ import { buildHostRemote, compareAudit, HOST_COMMANDS, type AuditFile } from "..
6
6
  import { type CommitVersion, coveringOperation, judgeChange, parseChangeQuery, readTail } from "../lib/change-origin.ts";
7
7
  import { failedChecks, MANAGER_COMPOSE, MANAGER_IMAGE, normalizeVersion, olderThan, pinCompose, projectSync, syncReady, utcStamp, type VerifyCheck } from "../lib/arcane-manager.ts";
8
8
  import { readConfig } from "../lib/config.ts";
9
+ import { isRecent, listContainer, type WatchedContainer, withDetail } from "../lib/container-watch.ts";
9
10
  import { containerFacts, deploymentChecks, listRows, parseDeploymentArgs, syncFacts, type DeploymentExpectation } from "../lib/deployment.ts";
10
11
  import { ServiceError, UsageError } from "../lib/errors.ts";
11
12
  import { withLock } from "../lib/lock.ts";
@@ -116,6 +117,9 @@ ${Object.entries(HOST_COMMANDS)
116
117
  docker-compose.yml, pre-deploy.sh) is ours: known when the new content, or the deletion,
117
118
  is a commit of <project>/<file> in the GitOps repository, or when our stack-update,
118
119
  manager-upgrade, manager-rollback, or host db-restore on that stack ran then (audit log)
120
+ container-watch every container's state, health, exit code, and uptime; for those started or created in
121
+ the last 30 minutes also the start time, the Compose project, and our stack-update,
122
+ manager-upgrade, manager-rollback, or host db-restore whose run covers the start (JSON)
119
123
  release-notes <stack>[/<service>]
120
124
  GitHub release notes between the running and the latest version of one service's image,
121
125
  found through the image's org.opencontainers.image.source label
@@ -303,6 +307,29 @@ async function changeOrigin(config: ArcaneConfig, args: string[]): Promise<numbe
303
307
  return 0;
304
308
  }
305
309
 
310
+ /** Container state for the Discord bridge's restart and redeploy checks: one list, plus details only for recent starts. */
311
+ async function containerWatch(config: ArcaneConfig, args: string[]): Promise<number> {
312
+ if (args.length > 0) throw new UsageError("container-watch takes no arguments");
313
+ let audit: string;
314
+ try {
315
+ audit = readTail(auditLogPath(), AUDIT_TAIL_BYTES);
316
+ } catch {
317
+ audit = "";
318
+ }
319
+ const now = Date.now();
320
+ const containers = await withCli(config, async (cli) => {
321
+ const out: WatchedContainer[] = [];
322
+ for (const row of listRows(await cliJson(cli, ["containers", "list", "--all"]))) {
323
+ const c = listContainer(row);
324
+ if (!c) continue;
325
+ out.push(isRecent(c, now) ? withDetail(c, await cliJson(cli, ["containers", "get", c.id]), audit) : c);
326
+ }
327
+ return out;
328
+ });
329
+ printJson({ checkedAt: new Date(now).toISOString(), containers });
330
+ return 0;
331
+ }
332
+
306
333
  // ---------------------------------------------------------------- Arcane VM
307
334
 
308
335
  const NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;
@@ -742,6 +769,7 @@ export async function command(args: string[]): Promise<number> {
742
769
  if (args[0] === "update-report") return updateReport(config, args.slice(1));
743
770
  if (args[0] === "release-notes") return releaseNotes(config, args.slice(1));
744
771
  if (args[0] === "change-origin") return changeOrigin(config, args.slice(1));
772
+ if (args[0] === "container-watch") return containerWatch(config, args.slice(1));
745
773
  if (args[0] === "host") return hostCommand(config, args.slice(1));
746
774
  if (args[0] === "project-status") return projectStatus(config, args.slice(1));
747
775
  if (args[0] === "deployment") return deployment(config, args.slice(1));
@@ -0,0 +1,54 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { durationSeconds, exitCodeOf, healthOf, isRecent, listContainer, uptimeSeconds, withDetail } from "./container-watch.ts";
3
+
4
+ const NOW = Date.parse("2026-10-22T08:00:00Z");
5
+
6
+ describe("container-watch parsing", () => {
7
+ it("reads Docker's human durations as a lower bound", () => {
8
+ expect(durationSeconds("Less than a second")).toBe(0);
9
+ expect(durationSeconds("1 second")).toBe(1);
10
+ expect(durationSeconds("About a minute")).toBe(60);
11
+ expect(durationSeconds("12 minutes")).toBe(720);
12
+ expect(durationSeconds("About an hour")).toBe(3600);
13
+ expect(durationSeconds("3 weeks")).toBe(1_814_400);
14
+ expect(durationSeconds("soon")).toBeNull();
15
+ });
16
+
17
+ it("reads uptime, health, and exit code from the list status", () => {
18
+ expect(uptimeSeconds("Up 3 minutes (healthy)")).toBe(180);
19
+ expect(uptimeSeconds("Up About a minute")).toBe(60);
20
+ expect(uptimeSeconds("Up 2 hours (Paused)")).toBe(7200);
21
+ expect(uptimeSeconds("Exited (0) 2 days ago")).toBeNull();
22
+ expect(healthOf("Up 3 weeks (healthy)")).toBe("healthy");
23
+ expect(healthOf("Up 14 hours (unhealthy)")).toBe("unhealthy");
24
+ expect(healthOf("Up 5 seconds (health: starting)")).toBe("starting");
25
+ expect(healthOf("Up 6 days")).toBe("none");
26
+ expect(exitCodeOf("Exited (137) 1 minute ago")).toBe(137);
27
+ expect(exitCodeOf("Up 6 days")).toBeNull();
28
+ });
29
+
30
+ it("turns a list row into a container and skips rows without a name or id", () => {
31
+ const row = { id: "abc", names: ["/syncthing"], imageId: "sha256:1", state: "running", status: "Up 14 hours (unhealthy)", created: 1792305689 };
32
+ expect(listContainer(row)).toEqual({ name: "syncthing", id: "abc", imageId: "sha256:1", state: "running", health: "unhealthy", exitCode: null, uptimeSeconds: 50_400, created: new Date(1792305689 * 1000).toISOString() });
33
+ expect(listContainer({ id: "abc", names: [] })).toBeNull();
34
+ expect(listContainer({ names: ["x"] })).toBeNull();
35
+ });
36
+
37
+ it("asks for details only about recent starts and creations", () => {
38
+ const base = { name: "x", id: "1", imageId: null, health: "none" as const, exitCode: null, created: "2026-10-01T00:00:00Z" };
39
+ expect(isRecent({ ...base, state: "running", uptimeSeconds: 120 }, NOW)).toBe(true);
40
+ expect(isRecent({ ...base, state: "running", uptimeSeconds: 7200 }, NOW)).toBe(false);
41
+ expect(isRecent({ ...base, state: "restarting", uptimeSeconds: null }, NOW)).toBe(true);
42
+ expect(isRecent({ ...base, state: "exited", uptimeSeconds: null, created: "2026-10-22T07:50:00Z" }, NOW)).toBe(true);
43
+ });
44
+
45
+ it("adds the start time, the project, and our operation that covers the start", () => {
46
+ const c = { name: "mealie", id: "1", imageId: "sha256:2", state: "running", health: "healthy" as const, exitCode: null, uptimeSeconds: 120, created: "2026-10-22T07:57:00Z" };
47
+ const detail = { data: { imageId: "sha256:2", state: { status: "running", running: true, startedAt: "2026-10-22T07:58:00Z", health: { status: "healthy" } }, composeInfo: { projectName: "mealie" } } };
48
+ const audit = `${JSON.stringify({ ts: "2026-10-22T07:59:00Z", service: "arcane", caller: "discord-1", argv: ["stack-update", "mealie"], code: 0, durationMs: 180_000 })}\n`;
49
+ expect(withDetail(c, detail, audit)).toEqual({ ...c, startedAt: "2026-10-22T07:58:00Z", project: "mealie", operation: { command: "stack-update mealie", caller: "discord-1", from: "2026-10-22T07:56:00.000Z", to: "2026-10-22T07:59:00Z" } });
50
+ expect(withDetail(c, detail, "")).toEqual({ ...c, startedAt: "2026-10-22T07:58:00Z", project: "mealie" });
51
+ const never = { data: { state: { status: "created", startedAt: "0001-01-01T00:00:00Z" } } };
52
+ expect(withDetail({ ...c, state: "created" }, never, audit)).toEqual({ ...c, state: "created" });
53
+ });
54
+ });
@@ -0,0 +1,94 @@
1
+ import { coveringOperation, type Operation } from "./change-origin.ts";
2
+ import { containerFacts } from "./deployment.ts";
3
+
4
+ export type Health = "healthy" | "unhealthy" | "starting" | "none";
5
+
6
+ /** One container as `container-watch` reports it; start time, project, and operation only for recent ones. */
7
+ export interface WatchedContainer {
8
+ name: string;
9
+ id: string;
10
+ imageId: string | null;
11
+ state: string;
12
+ health: Health;
13
+ exitCode: number | null;
14
+ uptimeSeconds: number | null;
15
+ created: string | null;
16
+ startedAt?: string;
17
+ project?: string;
18
+ operation?: Operation;
19
+ }
20
+
21
+ /** Started or created within this window: the bridge polls every five minutes, so half an hour leaves room for a missed poll. */
22
+ export const RECENT_S = 1800;
23
+
24
+ const UNITS: Record<string, number> = { second: 1, minute: 60, hour: 3600, day: 86_400, week: 604_800, month: 2_592_000, year: 31_536_000 };
25
+
26
+ /** Docker's human duration (`Less than a second`, `About a minute`, `3 minutes`, `2 weeks`) as a lower bound in seconds. */
27
+ export function durationSeconds(text: string): number | null {
28
+ if (text === "Less than a second") return 0;
29
+ const about = /^About an? (minute|hour)$/.exec(text);
30
+ if (about) return UNITS[about[1]];
31
+ const m = /^(\d+) (second|minute|hour|day|week|month|year)s?$/.exec(text);
32
+ return m ? Number(m[1]) * UNITS[m[2]] : null;
33
+ }
34
+
35
+ /** Seconds since the start from a list status such as `Up 3 minutes (healthy)`; null when the container is not up. */
36
+ export function uptimeSeconds(status: string): number | null {
37
+ const m = /^Up (.+?)(?: \(.*\))?$/.exec(status);
38
+ return m ? durationSeconds(m[1]) : null;
39
+ }
40
+
41
+ export function healthOf(status: string): Health {
42
+ const m = /\((healthy|unhealthy|health: starting)\)/.exec(status);
43
+ if (!m) return "none";
44
+ return m[1] === "health: starting" ? "starting" : (m[1] as Health);
45
+ }
46
+
47
+ export function exitCodeOf(status: string): number | null {
48
+ const m = /^Exited \((-?\d+)\)/.exec(status);
49
+ return m ? Number(m[1]) : null;
50
+ }
51
+
52
+ function str(v: unknown): string | null {
53
+ return typeof v === "string" && v.length > 0 ? v : null;
54
+ }
55
+
56
+ /** One row of `containers list --all`. */
57
+ export function listContainer(row: Record<string, unknown>): WatchedContainer | null {
58
+ const names = Array.isArray(row.names) ? row.names : [];
59
+ const name = typeof names[0] === "string" ? names[0].replace(/^\//, "") : null;
60
+ const id = str(row.id);
61
+ if (!name || !id) return null;
62
+ const status = typeof row.status === "string" ? row.status : "";
63
+ return {
64
+ name,
65
+ id,
66
+ imageId: str(row.imageId),
67
+ state: str(row.state) ?? "unknown",
68
+ health: healthOf(status),
69
+ exitCode: exitCodeOf(status),
70
+ uptimeSeconds: uptimeSeconds(status),
71
+ created: typeof row.created === "number" ? new Date(row.created * 1000).toISOString() : null,
72
+ };
73
+ }
74
+
75
+ /** Whether the bridge needs the exact start time and project: started or created recently, or restarting now. */
76
+ export function isRecent(c: WatchedContainer, now: number): boolean {
77
+ if (c.state === "restarting") return true;
78
+ if (c.uptimeSeconds !== null && c.uptimeSeconds < RECENT_S) return true;
79
+ return c.created !== null && now - Date.parse(c.created) < RECENT_S * 1000;
80
+ }
81
+
82
+ /** Adds the start time and project from `containers get`, and our operation on that stack whose run covers the start. */
83
+ export function withDetail(c: WatchedContainer, detail: unknown, auditText: string): WatchedContainer {
84
+ const facts = containerFacts(detail, c.name);
85
+ const top = ((detail as { data?: unknown } | null)?.data ?? detail) as { state?: { startedAt?: unknown } } | null;
86
+ const startedAt = str(top?.state?.startedAt);
87
+ const out: WatchedContainer = { ...c, ...(startedAt && !startedAt.startsWith("0001-") ? { startedAt } : {}), ...(facts.project ? { project: facts.project } : {}) };
88
+ const at = Date.parse(out.startedAt ?? c.created ?? "");
89
+ if (out.project && !Number.isNaN(at)) {
90
+ const operation = coveringOperation(auditText, out.project, at);
91
+ if (operation) out.operation = operation;
92
+ }
93
+ return out;
94
+ }