@astrofoundry/pi-astro 0.37.2 → 0.37.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,133 @@
1
+ import {
2
+ escalationText,
3
+ observe,
4
+ parseSnapshot,
5
+ readWatchState,
6
+ restartInvestigationTask,
7
+ unhealthyText,
8
+ WATCH_POLL_MS,
9
+ WATCH_RECOVERED_TEXT,
10
+ watchFailingText,
11
+ writeWatchState,
12
+ } from "./arcane-watch.ts";
13
+ import { type DiscordConfig, type ReportSchedule, reportSchedule } from "./config.ts";
14
+ import type { CreateMessage, DiscordMessage } from "./rest.ts";
15
+ import type { InvestigationRequest } from "./security-monitor.ts";
16
+
17
+ export interface WatchRest {
18
+ createMessage(channelId: string, message: CreateMessage): Promise<DiscordMessage>;
19
+ startThread(channelId: string, messageId: string, name: string): Promise<{ id: string }>;
20
+ }
21
+
22
+ export interface WatchDeps {
23
+ rest: WatchRest;
24
+ runEntry: (service: string, caller: string, args: string[], timeoutMs: number) => Promise<string>;
25
+ config: () => DiscordConfig;
26
+ log: (line: string) => void;
27
+ /** Runs astro.arcane on the task in the thread; resolves true when it answered. */
28
+ investigate: (request: InvestigationRequest) => Promise<boolean>;
29
+ /** Whether the NAS share is lost or just came back (the security monitor's record of rules 100012 and 100013). */
30
+ nasOutage: (now: number) => boolean;
31
+ stopping: () => boolean;
32
+ now: () => number;
33
+ }
34
+
35
+ const MINUTE = 60_000;
36
+ /** Six failed polls (30 minutes) tag the owners once. */
37
+ const FAILS_TO_WARN = 6;
38
+ const CALLER = "arcane-watch";
39
+
40
+ function reason(err: unknown): string {
41
+ return (err instanceof Error ? err.message : String(err)).split("\n")[0].slice(0, 300);
42
+ }
43
+
44
+ /**
45
+ * Restarts and redeploys on the Arcane VM, every five minutes through the
46
+ * arcane wrapper's `container-watch`, without a model. An `arcane` report entry
47
+ * turns it on in that report's channel.
48
+ */
49
+ export class ArcaneWatch {
50
+ private readonly deps: WatchDeps;
51
+ private running = false;
52
+ private last = 0;
53
+
54
+ constructor(deps: WatchDeps) {
55
+ this.deps = deps;
56
+ }
57
+
58
+ tick(): void {
59
+ const s = reportSchedule(this.deps.config(), "arcane");
60
+ if (!s || this.deps.stopping() || this.running) return;
61
+ const now = this.deps.now();
62
+ if (now - this.last < WATCH_POLL_MS - 5000) return;
63
+ this.running = true;
64
+ this.last = now;
65
+ void this.poll(s)
66
+ .catch((err) => this.deps.log(`arcane watch: check failed: ${reason(err)}`))
67
+ .finally(() => (this.running = false));
68
+ }
69
+
70
+ private async post(channelId: string, message: CreateMessage): Promise<DiscordMessage | null> {
71
+ try {
72
+ return await this.deps.rest.createMessage(channelId, message);
73
+ } catch (err) {
74
+ this.deps.log(`arcane watch: message failed: ${reason(err)}`);
75
+ return null;
76
+ }
77
+ }
78
+
79
+ async poll(s: ReportSchedule): Promise<void> {
80
+ const owners = this.deps.config().owners;
81
+ const state = readWatchState();
82
+ let snapshot;
83
+ try {
84
+ snapshot = parseSnapshot(JSON.parse(await this.deps.runEntry("arcane", CALLER, ["container-watch"], 3 * MINUTE)));
85
+ if (!snapshot) throw new Error("container-watch returned an unexpected answer");
86
+ } catch (err) {
87
+ const why = reason(err);
88
+ this.deps.log(`arcane watch: container-watch failed: ${why}`);
89
+ state.poll.failures++;
90
+ if (state.poll.failures >= FAILS_TO_WARN && !state.poll.warned) {
91
+ state.poll.warned = (await this.post(s.channelId, { content: watchFailingText((state.poll.failures * WATCH_POLL_MS) / MINUTE, why, owners), mentionUsers: owners })) !== null;
92
+ }
93
+ writeWatchState(state);
94
+ return;
95
+ }
96
+ if (state.poll.warned) await this.post(s.channelId, { content: WATCH_RECOVERED_TEXT });
97
+ state.poll = { failures: 0, warned: false };
98
+ const now = this.deps.now();
99
+ const seen = observe(state, snapshot, now, this.deps.nasOutage(now));
100
+ for (const note of seen.notes) this.deps.log(`arcane watch: ${note}`);
101
+ for (const u of seen.unhealthy) {
102
+ const posted = await this.post(s.channelId, { content: unhealthyText(u, owners, s.timeZone, now), mentionUsers: owners });
103
+ if (posted) this.deps.log(`arcane watch: ${u.name} ${u.problem} after a redeploy`);
104
+ else state.pending.push({ name: u.name, id: state.containers[u.name]?.id ?? "", since: u.since, newImage: u.newImage, ...(u.project ? { project: u.project } : {}) });
105
+ }
106
+ const started: Promise<void>[] = [];
107
+ for (const e of seen.escalations) {
108
+ const posted = await this.post(s.channelId, { content: escalationText(e, owners, s.timeZone), mentionUsers: owners });
109
+ if (!posted) {
110
+ delete state.escalated[e.name];
111
+ continue;
112
+ }
113
+ this.deps.log(`arcane watch: ${e.name} restarted ${e.restarts.length} times in 24 h`);
114
+ started.push(this.investigate(s.channelId, posted.id, e.name, restartInvestigationTask(e, now)));
115
+ }
116
+ writeWatchState(state);
117
+ // Investigations take minutes; the next poll does not wait for them.
118
+ void Promise.all(started);
119
+ }
120
+
121
+ private async investigate(channelId: string, messageId: string, name: string, task: string): Promise<void> {
122
+ try {
123
+ const thread = await this.deps.rest.startThread(channelId, messageId, `Restarts of ${name}`.slice(0, 100));
124
+ const starter = await this.deps.rest.createMessage(thread.id, { content: "🔎 The arcane specialist reads only; a risky read still asks an owner here. Write in this thread to continue." });
125
+ const owner = this.deps.config().owners[0];
126
+ const ok = await this.deps.investigate({ threadId: thread.id, starterId: starter.id, userId: owner, task });
127
+ if (!ok) this.deps.log(`arcane watch: the investigation of ${name} did not answer`);
128
+ } catch (err) {
129
+ this.deps.log(`arcane watch: investigation of ${name} failed: ${reason(err)}`);
130
+ await this.post(channelId, { content: `❌ Could not start the investigation of ${name} (${reason(err)}); ask the arcane specialist in the channel.`, replyTo: messageId });
131
+ }
132
+ }
133
+ }
@@ -0,0 +1,242 @@
1
+ import { mkdtempSync, rmSync } from "node:fs";
2
+ import { tmpdir } from "node:os";
3
+ import { join } from "node:path";
4
+ import { afterEach, beforeEach, describe, expect, it } from "vitest";
5
+ import type { WatchedContainer } from "../../specialists/lib/container-watch.ts";
6
+ import { ArcaneWatch } from "./arcane-watch-monitor.ts";
7
+ import { emptyWatchState, escalationText, healthProblem, observe, parseSnapshot, readWatchState, restartInvestigationTask, unhealthyText, type WatchState } from "./arcane-watch.ts";
8
+ import type { DiscordConfig } from "./config.ts";
9
+ import type { CreateMessage } from "./rest.ts";
10
+ import type { InvestigationRequest } from "./security-monitor.ts";
11
+
12
+ const OWNER = "178096425509453824";
13
+ const CHANNEL = "1549873938140499999";
14
+ const TZ = "Europe/Madrid";
15
+ const T0 = Date.parse("2026-10-22T08:00:00Z");
16
+ const MIN = 60_000;
17
+
18
+ function ct(over: Partial<WatchedContainer> = {}): WatchedContainer {
19
+ return { name: "syncthing", id: "a", imageId: "sha256:1", state: "running", health: "healthy", exitCode: null, uptimeSeconds: 86_400, created: "2026-10-07T21:39:38Z", ...over };
20
+ }
21
+
22
+ function iso(ms: number): string {
23
+ return new Date(ms).toISOString();
24
+ }
25
+
26
+ /** A state that has already seen `list` at `at`. */
27
+ function seenAt(list: WatchedContainer[], at: number): WatchState {
28
+ const state = emptyWatchState();
29
+ observe(state, list, at, false);
30
+ return state;
31
+ }
32
+
33
+ describe("container watch rules", () => {
34
+ it("only records a baseline the first time", () => {
35
+ const state = emptyWatchState();
36
+ const out = observe(state, [ct({ startedAt: iso(T0 - MIN) })], T0, false);
37
+ expect(out).toEqual({ escalations: [], unhealthy: [], notes: [] });
38
+ expect(state.containers.syncthing).toEqual({ id: "a", imageId: "sha256:1", startedAt: iso(T0 - MIN), seenAt: iso(T0) });
39
+ });
40
+
41
+ it("escalates the third restart within 24 hours once a day", () => {
42
+ const state = seenAt([ct(), ct({ name: "mealie", id: "m", project: "mealie" })], T0);
43
+ for (let i = 1; i <= 2; i++) {
44
+ const now = T0 + i * 5 * MIN;
45
+ expect(observe(state, [ct({ startedAt: iso(now - MIN), project: "syncthing" })], now, false).escalations).toEqual([]);
46
+ }
47
+ const now = T0 + 15 * MIN;
48
+ const out = observe(state, [ct({ startedAt: iso(now - MIN), project: "syncthing" })], now, false);
49
+ expect(out.escalations).toEqual([{ name: "syncthing", project: "syncthing", restarts: [iso(T0 + 4 * MIN), iso(T0 + 9 * MIN), iso(T0 + 14 * MIN)] }]);
50
+ expect(state.containers.mealie).toBeUndefined();
51
+ const later = T0 + 20 * MIN;
52
+ expect(observe(state, [ct({ startedAt: iso(later - MIN) })], later, false).escalations).toEqual([]);
53
+ expect(state.restarts.syncthing).toHaveLength(4);
54
+ });
55
+
56
+ it("does not count a start it already saw, or one before the last observation", () => {
57
+ const state = seenAt([ct({ startedAt: iso(T0 - 2 * MIN) })], T0);
58
+ observe(state, [ct({ startedAt: iso(T0 - 2 * MIN) })], T0 + 5 * MIN, false);
59
+ observe(state, [ct()], T0 + 10 * MIN, false);
60
+ expect(state.restarts).toEqual({});
61
+ expect(state.containers.syncthing.startedAt).toBe(iso(T0 - 2 * MIN));
62
+ });
63
+
64
+ it("leaves out restarts our operations or a NAS outage explain", () => {
65
+ const state = seenAt([ct({ name: "slskd", id: "s", project: "media" }), ct({ name: "mealie", id: "m", project: "mealie" })], T0);
66
+ const now = T0 + 5 * MIN;
67
+ const operation = { command: "host db-restore mealie 20261022T0700Z", caller: "discord-1", from: iso(now - 3 * MIN), to: iso(now) };
68
+ const out = observe(state, [ct({ name: "slskd", id: "s", project: "media", startedAt: iso(now - MIN) }), ct({ name: "mealie", id: "m", project: "mealie", startedAt: iso(now - MIN), operation })], now, true);
69
+ expect(out.notes).toEqual(["slskd restarted during a NAS outage", "mealie restarted during host db-restore mealie 20261022T0700Z (discord-1)"]);
70
+ expect(state.restarts).toEqual({});
71
+ });
72
+
73
+ it("checks a redeployed container after five minutes and reports what is wrong", () => {
74
+ const state = seenAt([ct({ name: "mealie", id: "old", imageId: "sha256:1", project: "mealie" })], T0);
75
+ const started = T0 + MIN;
76
+ const fresh = ct({ name: "mealie", id: "new", imageId: "sha256:2", project: "mealie", startedAt: iso(started), health: "starting", uptimeSeconds: 60 });
77
+ expect(observe(state, [fresh], T0 + 2 * MIN, false).unhealthy).toEqual([]);
78
+ expect(state.pending).toEqual([{ name: "mealie", id: "new", since: iso(started), newImage: true, project: "mealie" }]);
79
+ expect(observe(state, [{ ...fresh, health: "starting" }], T0 + 7 * MIN, false).unhealthy).toEqual([]);
80
+ const out = observe(state, [{ ...fresh, health: "unhealthy" }], T0 + 12 * MIN, false);
81
+ expect(out.unhealthy).toEqual([{ name: "mealie", project: "mealie", since: iso(started), newImage: true, problem: "unhealthy" }]);
82
+ expect(state.pending).toEqual([]);
83
+ });
84
+
85
+ it("drops a redeploy that came up fine, was replaced, or that our operation reports", () => {
86
+ const state = seenAt([ct({ name: "a", id: "1" }), ct({ name: "b", id: "1b" }), ct({ name: "c", id: "1c", project: "c" })], T0);
87
+ const operation = { command: "stack-update c", caller: "discord-1", from: iso(T0), to: iso(T0 + MIN) };
88
+ const created = iso(T0);
89
+ observe(state, [ct({ name: "a", id: "2", created }), ct({ name: "b", id: "2b", created }), ct({ name: "c", id: "2c", project: "c", operation, state: "exited", exitCode: 1, created })], T0 + MIN, false);
90
+ expect(state.pending.map((p) => p.id)).toEqual(["2", "2b", "2c"]);
91
+ const out = observe(state, [ct({ name: "a", id: "2", created }), ct({ name: "b", id: "3b", created: iso(T0 + 6 * MIN) }), ct({ name: "c", id: "2c", project: "c", state: "exited", exitCode: 1, created })], T0 + 7 * MIN, false);
92
+ expect(out.unhealthy).toEqual([]);
93
+ expect(out.notes).toEqual(["c exited with code 1 after stack-update c (discord-1), which reports it"]);
94
+ expect(state.pending.map((p) => p.id)).toEqual(["3b"]);
95
+ });
96
+
97
+ it("names what is wrong with a container", () => {
98
+ expect(healthProblem(ct({ state: "exited", exitCode: 0 }), 0)).toBeNull();
99
+ expect(healthProblem(ct({ state: "exited", exitCode: 137 }), 0)).toBe("exited with code 137");
100
+ expect(healthProblem(ct({ state: "restarting" }), 0)).toBe("restarting");
101
+ expect(healthProblem(ct({ state: "created" }), 0)).toBe("never started");
102
+ expect(healthProblem(ct({ health: "starting" }), 16 * MIN)).toBe("health check still starting");
103
+ expect(healthProblem(ct({ health: "none" }), 16 * MIN)).toBeNull();
104
+ });
105
+
106
+ it("forgets restarts and escalations after a day", () => {
107
+ const state = emptyWatchState();
108
+ state.containers.x = { id: "1", imageId: null, startedAt: null, seenAt: iso(T0) };
109
+ state.restarts.x = [iso(T0 - 25 * 60 * MIN)];
110
+ state.escalated.x = iso(T0 - 25 * 60 * MIN);
111
+ observe(state, [ct({ name: "x", id: "1" })], T0, false);
112
+ expect(state.restarts).toEqual({});
113
+ expect(state.escalated).toEqual({});
114
+ });
115
+ });
116
+
117
+ describe("container watch texts", () => {
118
+ it("tags the owners and says what happened", () => {
119
+ const e = { name: "syncthing", project: "syncthing", restarts: [iso(T0), iso(T0 + 5 * MIN), iso(T0 + 10 * MIN)] };
120
+ expect(escalationText(e, [OWNER], TZ)).toBe(
121
+ `🔁 **Container restarting** · syncthing · 3 restarts in 24 h, last 22 Oct 10:10 CEST\nAuto-heal or Docker restarted it each time. The arcane specialist investigates in the thread and changes nothing.\n<@${OWNER}>`,
122
+ );
123
+ expect(unhealthyText({ name: "worker", project: "grimoire", since: iso(T0), newImage: true, problem: "unhealthy" }, [OWNER], TZ, T0 + 7 * MIN)).toBe(
124
+ `⚠️ **Container unhealthy after a redeploy** · worker (stack grimoire) · recreated 22 Oct 10:00 CEST with a new image, still unhealthy 7 min later\nNo stack-update or other specialist operation explains the redeploy. Ask the arcane specialist here to investigate.\n<@${OWNER}>`,
125
+ );
126
+ });
127
+
128
+ it("asks the specialist to read only, within the host's one-day window", () => {
129
+ const now = T0 + 24 * 60 * MIN;
130
+ const task = restartInvestigationTask({ name: "syncthing", restarts: [iso(T0 + MIN), iso(now - MIN)] }, now);
131
+ expect(task).toContain("Read only: change nothing");
132
+ expect(task).toContain(`host container-logs syncthing ${iso(T0 + MIN)} ${iso(now)}`);
133
+ });
134
+
135
+ it("accepts only a well-formed snapshot", () => {
136
+ expect(parseSnapshot({ containers: [ct()] })).toEqual([ct()]);
137
+ expect(parseSnapshot({ containers: [{ name: "x" }] })).toBeNull();
138
+ expect(parseSnapshot({})).toBeNull();
139
+ });
140
+ });
141
+
142
+ function config(): DiscordConfig {
143
+ return {
144
+ applicationId: "1",
145
+ guildId: "2",
146
+ channels: { [CHANNEL]: { trigger: "always", specialists: ["arcane"] } },
147
+ adminChannelId: CHANNEL,
148
+ owners: [OWNER],
149
+ access: {},
150
+ approvalTimeoutMinutes: 10,
151
+ reports: [{ report: "arcane", channelId: CHANNEL, time: "06:00", timeZone: TZ }],
152
+ };
153
+ }
154
+
155
+ describe("container watch monitor", () => {
156
+ let dir: string;
157
+ beforeEach(() => {
158
+ dir = mkdtempSync(join(tmpdir(), "astro-watch-"));
159
+ process.env.ASTRO_DISCORD_DIR = dir;
160
+ });
161
+ afterEach(() => {
162
+ delete process.env.ASTRO_DISCORD_DIR;
163
+ rmSync(dir, { recursive: true, force: true });
164
+ });
165
+
166
+ function harness() {
167
+ const messages: { channelId: string; message: CreateMessage }[] = [];
168
+ const threads: string[] = [];
169
+ const investigations: InvestigationRequest[] = [];
170
+ const logs: string[] = [];
171
+ const clock = { now: T0 };
172
+ const answer: { next: () => string } = { next: () => JSON.stringify({ checkedAt: iso(clock.now), containers: [ct()] }) };
173
+ let id = 500;
174
+ const watch = new ArcaneWatch({
175
+ rest: {
176
+ createMessage: async (channelId, message) => {
177
+ messages.push({ channelId, message });
178
+ return { id: String(id++), channel_id: channelId };
179
+ },
180
+ startThread: async (_channelId, messageId) => {
181
+ threads.push(messageId);
182
+ return { id: `t${messageId}` };
183
+ },
184
+ },
185
+ runEntry: async (service, caller, args) => {
186
+ expect([service, caller, ...args]).toEqual(["arcane", "arcane-watch", "container-watch"]);
187
+ return answer.next();
188
+ },
189
+ config,
190
+ log: (line) => logs.push(line),
191
+ investigate: async (request) => {
192
+ investigations.push(request);
193
+ return true;
194
+ },
195
+ nasOutage: () => false,
196
+ stopping: () => false,
197
+ now: () => clock.now,
198
+ });
199
+ return { watch, messages, threads, investigations, logs, clock, answer };
200
+ }
201
+
202
+ const SCHEDULE = config().reports[0];
203
+
204
+ it("posts the escalation with a mention, opens a thread, and investigates there", async () => {
205
+ const h = harness();
206
+ await h.watch.poll(SCHEDULE);
207
+ for (let i = 1; i <= 3; i++) {
208
+ h.clock.now = T0 + i * 5 * MIN;
209
+ const started = iso(h.clock.now - MIN);
210
+ h.answer.next = () => JSON.stringify({ containers: [ct({ startedAt: started, project: "syncthing" })] });
211
+ await h.watch.poll(SCHEDULE);
212
+ }
213
+ await new Promise((resolve) => setTimeout(resolve, 0));
214
+ expect(h.messages[0].message.content).toContain("🔁 **Container restarting** · syncthing · 3 restarts in 24 h");
215
+ expect(h.messages[0].message.mentionUsers).toEqual([OWNER]);
216
+ expect(h.threads).toEqual(["500"]);
217
+ expect(h.investigations).toHaveLength(1);
218
+ expect(h.investigations[0]).toMatchObject({ threadId: "t500", starterId: "501", userId: OWNER });
219
+ expect(h.investigations[0].task).toContain("restarted 3 times");
220
+ });
221
+
222
+ it("warns once after 30 minutes of failed checks and says when they work again", async () => {
223
+ const h = harness();
224
+ h.answer.next = () => "not json";
225
+ for (let i = 0; i < 7; i++) await h.watch.poll(SCHEDULE);
226
+ expect(h.messages).toHaveLength(1);
227
+ expect(h.messages[0].message.content).toContain("⚠️ **Container checks failing:** the bridge could not read Arcane's containers for 30 minutes");
228
+ h.answer.next = () => JSON.stringify({ containers: [ct()] });
229
+ await h.watch.poll(SCHEDULE);
230
+ expect(h.messages[1].message.content).toBe("✅ Container checks work again.");
231
+ expect(readWatchState().poll).toEqual({ failures: 0, warned: false });
232
+ });
233
+
234
+ it("polls only with an arcane report entry and every five minutes", () => {
235
+ const h = harness();
236
+ h.watch.tick();
237
+ h.watch.tick();
238
+ h.clock.now = T0 + 5 * MIN;
239
+ h.watch.tick();
240
+ expect(h.logs).toEqual([]);
241
+ });
242
+ });
@@ -0,0 +1,215 @@
1
+ import { readFileSync, renameSync, writeFileSync } from "node:fs";
2
+ import { join } from "node:path";
3
+ import type { Operation } from "../../specialists/lib/change-origin.ts";
4
+ import type { WatchedContainer } from "../../specialists/lib/container-watch.ts";
5
+ import { configDir } from "./config.ts";
6
+ import { escapeDiscord } from "./media.ts";
7
+ import { clockText } from "./security.ts";
8
+
9
+ const MINUTE = 60_000;
10
+ const DAY = 86_400_000;
11
+ export const WATCH_POLL_MS = 5 * MINUTE;
12
+ /** Three restarts of one container within 24 hours start an investigation, at most once a day per container. */
13
+ export const RESTARTS_TO_ESCALATE = 3;
14
+ /** A redeployed container gets five minutes to come up before its health counts. */
15
+ export const HEALTH_GRACE_MS = 5 * MINUTE;
16
+ /** A health check still starting after this long counts as failing. */
17
+ const STARTING_LIMIT_MS = 15 * MINUTE;
18
+ /** The only stack that mounts the NAS share (`nas_media` in the arcane repository's media/compose.yaml). */
19
+ export const NAS_STACKS: readonly string[] = ["media"];
20
+
21
+ export interface WatchState {
22
+ /** The last observation of each container by name. */
23
+ containers: Record<string, { id: string; imageId: string | null; startedAt: string | null; seenAt: string }>;
24
+ /** Restart times per container within the last 24 hours, ours and NAS-related ones left out. */
25
+ restarts: Record<string, string[]>;
26
+ /** When each container was last escalated. */
27
+ escalated: Record<string, string>;
28
+ /** Redeployed containers whose health is checked once the grace time has passed. */
29
+ pending: PendingHealth[];
30
+ poll: { failures: number; warned: boolean };
31
+ }
32
+
33
+ export interface PendingHealth {
34
+ name: string;
35
+ id: string;
36
+ since: string;
37
+ project?: string;
38
+ newImage: boolean;
39
+ operation?: string;
40
+ }
41
+
42
+ export interface Escalation {
43
+ name: string;
44
+ project?: string;
45
+ restarts: string[];
46
+ }
47
+
48
+ export interface Unhealthy {
49
+ name: string;
50
+ project?: string;
51
+ since: string;
52
+ newImage: boolean;
53
+ problem: string;
54
+ }
55
+
56
+ export interface Observation {
57
+ escalations: Escalation[];
58
+ unhealthy: Unhealthy[];
59
+ /** Log lines for what was seen but is expected: our operations and NAS outages. */
60
+ notes: string[];
61
+ }
62
+
63
+ export function emptyWatchState(): WatchState {
64
+ return { containers: {}, restarts: {}, escalated: {}, pending: [], poll: { failures: 0, warned: false } };
65
+ }
66
+
67
+ function statePath(): string {
68
+ return join(configDir(), "arcane-watch-state.json");
69
+ }
70
+
71
+ export function readWatchState(): WatchState {
72
+ try {
73
+ return { ...emptyWatchState(), ...(JSON.parse(readFileSync(statePath(), "utf-8")) as Partial<WatchState>) };
74
+ } catch {
75
+ return emptyWatchState();
76
+ }
77
+ }
78
+
79
+ export function writeWatchState(state: WatchState): void {
80
+ const path = statePath();
81
+ writeFileSync(`${path}.tmp`, `${JSON.stringify(state)}\n`, { mode: 0o600 });
82
+ renameSync(`${path}.tmp`, path);
83
+ }
84
+
85
+ function isContainer(value: unknown): value is WatchedContainer {
86
+ const c = value as Partial<WatchedContainer> | null;
87
+ return !!c && typeof c.name === "string" && typeof c.id === "string" && typeof c.state === "string" && typeof c.health === "string";
88
+ }
89
+
90
+ /** The `container-watch` answer of the arcane wrapper. */
91
+ export function parseSnapshot(value: unknown): WatchedContainer[] | null {
92
+ const r = value as { containers?: unknown } | null;
93
+ return r && Array.isArray(r.containers) && r.containers.every(isContainer) ? r.containers : null;
94
+ }
95
+
96
+ function operationText(o: Operation): string {
97
+ return `${o.command} (${o.caller})`;
98
+ }
99
+
100
+ /** What is wrong with a container that should be up, or null. */
101
+ export function healthProblem(c: WatchedContainer, sinceMs: number): string | null {
102
+ if (c.state === "exited") return c.exitCode === 0 ? null : `exited with code ${c.exitCode ?? "?"}`;
103
+ if (c.state === "restarting" || c.state === "dead" || c.state === "created") return c.state === "created" ? "never started" : c.state;
104
+ if (c.health === "unhealthy") return "unhealthy";
105
+ if (c.health === "starting" && sinceMs >= STARTING_LIMIT_MS) return "health check still starting";
106
+ return null;
107
+ }
108
+
109
+ /**
110
+ * Compares a fresh snapshot with the last one: a new id is a redeploy, whose
111
+ * health is checked after the grace time; a newer start time with the same id
112
+ * is a restart. The first snapshot only records the baseline.
113
+ */
114
+ export function observe(state: WatchState, snapshot: readonly WatchedContainer[], now: number, nasOutage: boolean): Observation {
115
+ const out: Observation = { escalations: [], unhealthy: [], notes: [] };
116
+ const baseline = Object.keys(state.containers).length === 0;
117
+ const at = new Date(now).toISOString();
118
+ const nasExpected = (c: { project?: string }) => nasOutage && c.project !== undefined && NAS_STACKS.includes(c.project);
119
+ const current = new Map(snapshot.map((c) => [c.name, c]));
120
+ for (const c of snapshot) {
121
+ const prev = state.containers[c.name];
122
+ if (prev && !baseline) {
123
+ if (prev.id !== c.id) {
124
+ if (!state.pending.some((p) => p.id === c.id)) {
125
+ state.pending.push({ name: c.name, id: c.id, since: c.startedAt ?? c.created ?? at, newImage: prev.imageId !== c.imageId, ...(c.project ? { project: c.project } : {}), ...(c.operation ? { operation: operationText(c.operation) } : {}) });
126
+ }
127
+ } else if (c.startedAt && c.startedAt !== prev.startedAt && Date.parse(c.startedAt) > Date.parse(prev.seenAt)) {
128
+ if (c.operation) out.notes.push(`${c.name} restarted during ${operationText(c.operation)}`);
129
+ else if (nasExpected(c)) out.notes.push(`${c.name} restarted during a NAS outage`);
130
+ else state.restarts[c.name] = [...(state.restarts[c.name] ?? []), c.startedAt];
131
+ }
132
+ }
133
+ const startedAt = c.startedAt ?? (prev?.id === c.id ? prev.startedAt : null);
134
+ state.containers[c.name] = { id: c.id, imageId: c.imageId, startedAt, seenAt: at };
135
+ }
136
+ for (const name of Object.keys(state.containers)) if (!current.has(name)) delete state.containers[name];
137
+ for (const [name, times] of Object.entries(state.restarts)) {
138
+ const recent = times.filter((t) => Date.parse(t) >= now - DAY);
139
+ if (recent.length === 0) delete state.restarts[name];
140
+ else state.restarts[name] = recent;
141
+ }
142
+ state.escalated = Object.fromEntries(Object.entries(state.escalated).filter(([, t]) => Date.parse(t) >= now - DAY));
143
+ for (const [name, times] of Object.entries(state.restarts)) {
144
+ if (times.length < RESTARTS_TO_ESCALATE || state.escalated[name]) continue;
145
+ state.escalated[name] = at;
146
+ const project = current.get(name)?.project;
147
+ out.escalations.push({ name, ...(project ? { project } : {}), restarts: [...times] });
148
+ }
149
+ const waiting: PendingHealth[] = [];
150
+ for (const p of state.pending) {
151
+ const c = current.get(p.name);
152
+ if (!c || c.id !== p.id) continue;
153
+ const age = now - Date.parse(p.since);
154
+ if (age < HEALTH_GRACE_MS) {
155
+ waiting.push(p);
156
+ continue;
157
+ }
158
+ const problem = healthProblem(c, age);
159
+ if (problem === null && c.health === "starting") {
160
+ waiting.push(p);
161
+ continue;
162
+ }
163
+ if (problem === null) continue;
164
+ if (p.operation) out.notes.push(`${p.name} ${problem} after ${p.operation}, which reports it`);
165
+ else if (nasExpected(p)) out.notes.push(`${p.name} ${problem} during a NAS outage`);
166
+ else out.unhealthy.push({ name: p.name, ...(p.project ? { project: p.project } : {}), since: p.since, newImage: p.newImage, problem });
167
+ }
168
+ state.pending = waiting;
169
+ return out;
170
+ }
171
+
172
+ function mention(owners: readonly string[]): string {
173
+ return owners.map((id) => `<@${id}>`).join(" ");
174
+ }
175
+
176
+ function where(name: string, project?: string): string {
177
+ return `${escapeDiscord(name)}${project && project !== name ? ` (stack ${escapeDiscord(project)})` : ""}`;
178
+ }
179
+
180
+ export function escalationText(e: Escalation, owners: readonly string[], timeZone: string): string {
181
+ const last = e.restarts[e.restarts.length - 1];
182
+ return [
183
+ `🔁 **Container restarting** · ${where(e.name, e.project)} · ${e.restarts.length} restarts in 24 h, last ${clockText(last, timeZone)}`,
184
+ "Auto-heal or Docker restarted it each time. The arcane specialist investigates in the thread and changes nothing.",
185
+ mention(owners),
186
+ ].join("\n");
187
+ }
188
+
189
+ export function unhealthyText(u: Unhealthy, owners: readonly string[], timeZone: string, now: number): string {
190
+ const minutes = Math.round((now - Date.parse(u.since)) / MINUTE);
191
+ return [
192
+ `⚠️ **Container ${u.problem} after a redeploy** · ${where(u.name, u.project)} · recreated ${clockText(u.since, timeZone)}${u.newImage ? " with a new image" : ""}, still ${u.problem} ${minutes} min later`,
193
+ "No stack-update or other specialist operation explains the redeploy. Ask the arcane specialist here to investigate.",
194
+ mention(owners),
195
+ ].join("\n");
196
+ }
197
+
198
+ export function watchFailingText(minutes: number, reason: string, owners: readonly string[]): string {
199
+ return `⚠️ **Container checks failing:** the bridge could not read Arcane's containers for ${minutes} minutes (${escapeDiscord(reason.slice(0, 200))}). Restarts and redeploys are not checked meanwhile; the bridge log on Cortex has details.\n${mention(owners)}`;
200
+ }
201
+
202
+ export const WATCH_RECOVERED_TEXT = "✅ Container checks work again.";
203
+
204
+ /** The read-only task astro.arcane gets in the escalation's thread. */
205
+ export function restartInvestigationTask(e: Escalation, now: number): string {
206
+ // The host's log and OOM windows span at most one day.
207
+ const since = new Date(Math.max(Date.parse(e.restarts[0]) - 10 * MINUTE, now - DAY + MINUTE)).toISOString();
208
+ const until = new Date(now).toISOString();
209
+ return [
210
+ `Investigate why the container ${e.name}${e.project ? ` (stack ${e.project})` : ""} restarted ${e.restarts.length} times in the last 24 hours, at ${e.restarts.join(", ")} (UTC).`,
211
+ "Read only: change nothing, restart nothing, and run no stack-update, manager-upgrade, git write, or other write command. A fix goes through GitOps and needs an owner's approval, so describe it instead.",
212
+ `Collect evidence with \`containers get ${e.name}\`, \`host container-logs ${e.name} ${since} ${until}\`, \`host container-oom ${e.name} ${since} ${until}\`, \`host kernel-oom ${since} ${until}\`, and \`host container-memory ${e.name}\`. Name each command whose output you rely on.`,
213
+ "Answer with the likely cause, the evidence for it, and the change you would propose.",
214
+ ].join("\n");
215
+ }
@@ -71,7 +71,9 @@ import {
71
71
  } from "./tools-report.ts";
72
72
  import { type Quote, type QuotedMessage, withQuotes } from "./quote.ts";
73
73
  import { DiscordRest, type MessageComponent } from "./rest.ts";
74
+ import { ArcaneWatch } from "./arcane-watch-monitor.ts";
74
75
  import { type InvestigationRequest, SecurityMonitor } from "./security-monitor.ts";
76
+ import { nasOutage, readSecurityState } from "./security.ts";
75
77
 
76
78
  const REACTION = { running: "⏳", waiting: "🔒", done: "✅", failed: "❌" } as const;
77
79
  /** Answers longer than this go as a file attachment with a short summary. */
@@ -222,6 +224,7 @@ class Bridge {
222
224
  private readonly ctx: ExtensionContext;
223
225
  private readonly token: string;
224
226
  private readonly security: SecurityMonitor;
227
+ private readonly arcaneWatch: ArcaneWatch;
225
228
 
226
229
  constructor(ctx: ExtensionContext, config: DiscordConfig, token: string) {
227
230
  this.ctx = ctx;
@@ -235,7 +238,17 @@ class Bridge {
235
238
  runEntry: (service, caller, args, timeoutMs) => this.runEntry(service, caller, args, timeoutMs),
236
239
  config: () => this.config,
237
240
  log,
238
- investigate: (request) => this.investigateSecurity(request),
241
+ investigate: (request) => this.investigateIn("security", request),
242
+ stopping: () => this.stopping,
243
+ now: () => Date.now(),
244
+ });
245
+ this.arcaneWatch = new ArcaneWatch({
246
+ rest: this.rest,
247
+ runEntry: (service, caller, args, timeoutMs) => this.runEntry(service, caller, args, timeoutMs),
248
+ config: () => this.config,
249
+ log,
250
+ investigate: (request) => this.investigateIn("arcane", request),
251
+ nasOutage: (now) => nasOutage(readSecurityState().nas, now),
239
252
  stopping: () => this.stopping,
240
253
  now: () => Date.now(),
241
254
  });
@@ -346,10 +359,12 @@ class Bridge {
346
359
  void this.maybeReports();
347
360
  void this.processRestarts();
348
361
  this.security.tick();
362
+ this.arcaneWatch.tick();
349
363
  }, 60_000);
350
364
  this.reportTimer.unref();
351
365
  void this.maybeReports();
352
366
  this.security.tick();
367
+ this.arcaneWatch.tick();
353
368
  log(`connected as ${me.username} (${me.id}); channels ${Object.keys(this.config.channels).join(", ")}; specialists ${this.specialists().names.join(", ")}`);
354
369
  }
355
370
 
@@ -1118,29 +1133,30 @@ class Bridge {
1118
1133
  }
1119
1134
 
1120
1135
  /**
1121
- * Investigate on a security alert: astro.security in the alert's thread,
1122
- * queued behind any other security task, as a fresh conversation that the
1136
+ * An investigation in a thread: astro.security on a security alert, or
1137
+ * astro.arcane on a container that keeps restarting. It waits behind any
1138
+ * other task of that specialist and starts a fresh conversation that the
1123
1139
  * owner can continue by writing in the thread.
1124
1140
  */
1125
- private async investigateSecurity(request: InvestigationRequest): Promise<boolean> {
1126
- const agent = this.specialists().agents.find((a) => a.name === "astro.security");
1141
+ private async investigateIn(specialist: "security" | "arcane", request: InvestigationRequest): Promise<boolean> {
1142
+ const agent = this.specialists().agents.find((a) => a.name === `astro.${specialist}`);
1127
1143
  if (!agent) {
1128
- await this.rest.createMessage(request.threadId, { content: "❌ astro.security is not configured on this host." }).catch(() => undefined);
1144
+ await this.rest.createMessage(request.threadId, { content: `❌ astro.${specialist} is not configured on this host.` }).catch(() => undefined);
1129
1145
  return false;
1130
1146
  }
1131
1147
  const now = Date.now();
1132
- const conversation = resolveConversation(this.conversations, { channelId: request.threadId, specialist: "security", fresh: true, now }, () => newConversationId("security", now));
1148
+ const conversation = resolveConversation(this.conversations, { channelId: request.threadId, specialist, fresh: true, now }, () => newConversationId(specialist, now));
1133
1149
  this.conversations = startConversation(this.conversations, conversation.key, conversation.id, now);
1134
1150
  saveState(this.conversations);
1135
- const task: ActiveTask = { specialist: "security", channelId: request.threadId, messageId: request.starterId, userId: request.userId, startedAt: now, conversation, images: [] };
1151
+ const task: ActiveTask = { specialist, channelId: request.threadId, messageId: request.starterId, userId: request.userId, startedAt: now, conversation, images: [] };
1136
1152
  let failed = true;
1137
- const previous = this.queues.get("security") ?? Promise.resolve();
1153
+ const previous = this.queues.get(specialist) ?? Promise.resolve();
1138
1154
  const next = previous
1139
1155
  .then(async () => {
1140
1156
  failed = await this.runTask(agent, request.task, task);
1141
1157
  })
1142
- .catch((err) => log(`security investigation failed: ${err instanceof Error ? err.message : String(err)}`));
1143
- this.queues.set("security", next);
1158
+ .catch((err) => log(`${specialist} investigation failed: ${err instanceof Error ? err.message : String(err)}`));
1159
+ this.queues.set(specialist, next);
1144
1160
  await next;
1145
1161
  return !failed;
1146
1162
  }
@@ -27,6 +27,7 @@ import {
27
27
  isAcked,
28
28
  MAX_POSTS_PER_CHECK,
29
29
  needsOrigin,
30
+ noteNas,
30
31
  noticeText,
31
32
  ORIGIN_RETRY_MS,
32
33
  type OriginVerdict,
@@ -277,6 +278,7 @@ export class SecurityMonitor {
277
278
  for (const e of fresh) {
278
279
  if (state.seen[e.id] !== undefined) continue;
279
280
  state.seen[e.id] = e.timestamp;
281
+ noteNas(state, e);
280
282
  if (needsOrigin(e)) settle(e, new Date(now).toISOString());
281
283
  else ready.push(e);
282
284
  }
@@ -39,6 +39,8 @@ import {
39
39
  type ImageVulnFacts,
40
40
  offeredImages,
41
41
  parseImageVulns,
42
+ nasOutage,
43
+ noteNas,
42
44
  repeatText,
43
45
  resticStatus,
44
46
  type VulnEvent,
@@ -363,6 +365,21 @@ describe("vulnerabilities", () => {
363
365
  });
364
366
  });
365
367
 
368
+ describe("NAS outage record", () => {
369
+ it("keeps the newest loss and return and calls the share out until half an hour after it returns", () => {
370
+ const state = emptyState();
371
+ noteNas(state, event({ rule: "100012", timestamp: "2026-10-17T22:47:00.000+0000" }));
372
+ noteNas(state, event({ rule: "100012", timestamp: "2026-10-17T20:00:00.000+0000" }));
373
+ noteNas(state, event({ rule: "5712", timestamp: "2026-10-18T09:00:00.000+0000" }));
374
+ expect(state.nas).toEqual({ lostAt: "2026-10-17T22:47:00.000+0000", restoredAt: null });
375
+ expect(nasOutage(state.nas, Date.parse("2026-10-18T03:00:00Z"))).toBe(true);
376
+ noteNas(state, event({ rule: "100013", timestamp: "2026-10-18T06:25:00.000+0000" }));
377
+ expect(nasOutage(state.nas, Date.parse("2026-10-18T06:50:00Z"))).toBe(true);
378
+ expect(nasOutage(state.nas, Date.parse("2026-10-18T07:00:00Z"))).toBe(false);
379
+ expect(nasOutage(emptyState().nas, Date.parse("2026-10-18T07:00:00Z"))).toBe(false);
380
+ });
381
+ });
382
+
366
383
  describe("Arcane image scans", () => {
367
384
  const item = (over: Record<string, unknown> = {}) => ({ vulnerabilityId: "CVE-2026-1", pkgName: "openssl", installedVersion: "3.0.1", fixedVersion: "3.0.2", severity: "CRITICAL", ignored: false, imageName: "ghcr.io/mealie-recipes/mealie:latest", ...over });
368
385
  const stacks = [
@@ -125,6 +125,8 @@ export interface SecurityState {
125
125
  vulns: { reported: Record<string, ReportedVuln>; processedUntil: string | null; since: string | null };
126
126
  /** Ban totals of earlier weekly reports: CrowdSec keeps alerts for 7 days, so last week exists only here. */
127
127
  weeks: { date: string; bans: number }[];
128
+ /** The newest NAS MEDIA loss (rule 100012) and return (100013); the Arcane container watch expects media restarts around them. */
129
+ nas: { lostAt: string | null; restoredAt: string | null };
128
130
  }
129
131
 
130
132
  export function emptyState(): SecurityState {
@@ -140,6 +142,7 @@ export function emptyState(): SecurityState {
140
142
  poll: { failures: 0, warned: false },
141
143
  vulns: { reported: {}, processedUntil: null, since: null },
142
144
  weeks: [],
145
+ nas: { lostAt: null, restoredAt: null },
143
146
  };
144
147
  }
145
148
 
@@ -165,6 +168,23 @@ export function writeSecurityState(state: SecurityState): void {
165
168
 
166
169
  const DAY = 86_400_000;
167
170
 
171
+ /** Keeps the newest NAS loss and return the checks have seen. */
172
+ export function noteNas(state: SecurityState, e: WazuhEvent): void {
173
+ const key = e.rule === "100012" ? "lostAt" : e.rule === "100013" ? "restoredAt" : null;
174
+ if (key && (state.nas[key] === null || Date.parse(e.timestamp) > Date.parse(state.nas[key] as string))) state.nas[key] = e.timestamp;
175
+ }
176
+
177
+ /** The media watchdog restarts the media containers once access returns, within a few minutes. */
178
+ const NAS_SETTLE_MS = 30 * 60_000;
179
+
180
+ /** Whether the NAS share is lost now, or came back within the last half hour. */
181
+ export function nasOutage(nas: SecurityState["nas"], now: number): boolean {
182
+ const lost = nas.lostAt ? Date.parse(nas.lostAt) : null;
183
+ const restored = nas.restoredAt ? Date.parse(nas.restoredAt) : null;
184
+ if (lost !== null && (restored === null || restored < lost)) return true;
185
+ return restored !== null && now - restored < NAS_SETTLE_MS;
186
+ }
187
+
168
188
  /** Drops what no check, report, or button can need any more. */
169
189
  export function pruneState(state: SecurityState, now: number): void {
170
190
  const keep = (iso: string, days: number) => Date.parse(iso) >= now - days * DAY;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@astrofoundry/pi-astro",
3
- "version": "0.37.2",
3
+ "version": "0.37.4",
4
4
  "description": "Personal pi customizations (extensions, subagents, skills, prompts, themes) for the pi coding agent.",
5
5
  "keywords": [
6
6
  "pi-package"
@@ -44,9 +44,9 @@ Messages that start with "Arcane Daily Report:" are the bridge's scheduled repor
44
44
 
45
45
  Apply updates only when asked, one stack at a time, with `["stack-update", "<stack>"]`. It applies every pending update of the stack in one call, without judgment:
46
46
 
47
- - Pinned tags (`edit tag`) and digests (`edit digest`) move in the stack's Compose file through the GitOps checkout: commit, push, then a wait of up to 20 minutes for the stack's sync to report success at that commit. Only `image:` lines that name exactly the old reference change.
48
- - `projects redeploy <stack> --yes` then pulls and restarts the stack (up to 35 minutes for large pulls).
49
- - Checks: every service has a container that runs (a one-shot job may have exited 0), none is unhealthy or still starting after 5 minutes, and a fresh check finds every updated image current.
47
+ - Pinned tags (`edit tag`) and digests (`edit digest`) move in the stack's Compose file through the GitOps checkout: commit, push, start the stack's GitOps sync at once (`gitops sync`, so a long sync interval does not matter), then a wait of up to 20 minutes for it to report success at that commit. Only `image:` lines that name exactly the old reference change.
48
+ - `projects redeploy <stack> --yes` then pulls and restarts the stack (up to 35 minutes for large pulls). Arcane can leave a running container on its old image although the pull moved its tag (Docker then lists it by image id); each such container is recreated with `containers redeploy <id>` and named in `recreated`.
49
+ - Checks: every service has a container that runs (a one-shot job may have exited 0), found through its Compose project, none still runs an older image than its tag, none is unhealthy or still starting after 5 minutes, and a fresh check finds every updated image current.
50
50
  - A failed check after a Compose change reverts the commit, pushes, waits for the sync, and redeploys the previous file. A pull alone cannot be undone; say so when it fails.
51
51
 
52
52
  The answer is JSON with `status`: `updated`, `rolled back`, `failed` (with `reason`), or `nothing to do`, plus `services` (`service`, `from`, `to`, `applied`: `pull` or the new reference, `result`: `current`, `behind`, `unchecked`) and `commit`/`revert` when the Compose file changed. Report it in plain words.
@@ -89,6 +89,8 @@ Public routes (Supabase, Grimoire, Mealie URLs) are `astro.edge`'s `route-probe`
89
89
 
90
90
  `["change-origin", "<path>", "<sha256|->", "<utc>"]` answers whether a deployment-file change Wazuh saw on the Arcane VM (`/opt/docker/<project>/compose.yaml`, `docker-compose.yml`, or `pre-deploy.sh`; rule 100020) is ours. `known` when the new content (its SHA-256 from the alert's `syscheck.sha256_after`), or `-` for a deletion, equals a commit of `<project>/<file>` at origin, any author, CI included; or when our `stack-update`, `manager-upgrade`, `manager-rollback`, or `host db-restore` on that stack ran then (the specialists' audit log). Otherwise `unknown`, with the newest commit touching the file. Arcane syncs the files verbatim, so a GitOps deploy always matches. The Discord bridge calls it for every such alert; an `unknown` change pages the owner.
91
91
 
92
+ `["container-watch"]` lists every container with its state, health, exit code, and uptime; containers started or created in the last 30 minutes also carry the start time, the Compose project, and the `operation` (our stack-update, manager-upgrade, manager-rollback, or host db-restore on that stack) whose run covers the start. The Discord bridge polls it every five minutes to detect restarts and redeploys; when it asks you to investigate a container that keeps restarting, read only and propose the fix.
93
+
92
94
  `["project-status"]` (or with a project name) lists projects with their GitOps sync: commit, state, hook state, without error text. `["deployment", "check"|"wait", "<project>", "--commit", "<sha>", "--container", "NAME=sha256:ID", ...]` compares the sync and each container's project, image ID, state, and health; `wait` polls every 10 s up to `--timeout` (default 600 s). Read `status` in the answer.
93
95
 
94
96
  ## Output
@@ -4,8 +4,9 @@ import { tmpdir } from "node:os";
4
4
  import { dirname, join } from "node:path";
5
5
  import { buildHostRemote, compareAudit, HOST_COMMANDS, type AuditFile } from "../lib/arcane-host.ts";
6
6
  import { type CommitVersion, coveringOperation, judgeChange, parseChangeQuery, readTail } from "../lib/change-origin.ts";
7
- import { failedChecks, MANAGER_COMPOSE, MANAGER_IMAGE, normalizeVersion, olderThan, pinCompose, projectSync, syncReady, utcStamp, type VerifyCheck } from "../lib/arcane-manager.ts";
7
+ import { failedChecks, MANAGER_COMPOSE, MANAGER_IMAGE, normalizeVersion, olderThan, pinCompose, projectSync, syncId, syncReady, utcStamp, type VerifyCheck } from "../lib/arcane-manager.ts";
8
8
  import { readConfig } from "../lib/config.ts";
9
+ import { isRecent, listContainer, type WatchedContainer, withDetail } from "../lib/container-watch.ts";
9
10
  import { containerFacts, deploymentChecks, listRows, parseDeploymentArgs, syncFacts, type DeploymentExpectation } from "../lib/deployment.ts";
10
11
  import { ServiceError, UsageError } from "../lib/errors.ts";
11
12
  import { withLock } from "../lib/lock.ts";
@@ -17,7 +18,7 @@ import { run, type RunResult } from "../lib/proc.ts";
17
18
  import { HttpRegistry, parseImageRef } from "../lib/registry.ts";
18
19
  import { releaseNotesFor } from "../lib/releases.ts";
19
20
  import { ensureCheckout, git, REPO_HELP, repoCommand, repoStatus, safeRepoPath, type RepoSpec } from "../lib/repo.ts";
20
- import { commitMessage, containersProblem, crossesMajor, HOLDS_FILE, nextImageRef, parseHolds, pendingRows, replaceImageRef, type ServiceResult, type StackContainer, type StackStatus } from "../lib/stack-update.ts";
21
+ import { commitMessage, containersProblem, crossesMajor, HOLDS_FILE, isStaleImage, nextImageRef, parseHolds, pendingRows, replaceImageRef, type ServiceResult, type StackContainer, type StackStatus } from "../lib/stack-update.ts";
21
22
  import { ARCANE_DENIED_KEYS, filterJsonText, stripKeys } from "../lib/sanitize.ts";
22
23
  import { readSecret, secretPath } from "../lib/secrets.ts";
23
24
  import { sshFixed } from "../lib/ssh.ts";
@@ -106,8 +107,9 @@ ${Object.entries(HOST_COMMANDS)
106
107
  the held stacks of ${HOLDS_FILE}. Checks only: never pulls images or redeploys.
107
108
  stack-update <stack> [--confirm]
108
109
  apply every pending update of one stack: pinned tags and digests move in the stack's
109
- Compose file (commit, push, wait for the sync), then the stack is redeployed with a pull
110
- and checked (containers running and healthy, images current). A failed check after a
110
+ Compose file (commit, push, start the sync and wait for it), then the stack is redeployed
111
+ with a pull, containers the redeploy left on an old image are recreated one by one, and
112
+ the stack is checked (containers running and healthy, images current). A failed check after a
111
113
  Compose change reverts it and redeploys; a pull alone cannot be undone. Refuses held
112
114
  stacks, stopped stacks, and arcane (manager-upgrade). An update across a major version
113
115
  needs --confirm, which asks an owner for approval.
@@ -116,6 +118,9 @@ ${Object.entries(HOST_COMMANDS)
116
118
  docker-compose.yml, pre-deploy.sh) is ours: known when the new content, or the deletion,
117
119
  is a commit of <project>/<file> in the GitOps repository, or when our stack-update,
118
120
  manager-upgrade, manager-rollback, or host db-restore on that stack ran then (audit log)
121
+ container-watch every container's state, health, exit code, and uptime; for those started or created in
122
+ the last 30 minutes also the start time, the Compose project, and our stack-update,
123
+ manager-upgrade, manager-rollback, or host db-restore whose run covers the start (JSON)
119
124
  release-notes <stack>[/<service>]
120
125
  GitHub release notes between the running and the latest version of one service's image,
121
126
  found through the image's org.opencontainers.image.source label
@@ -303,6 +308,29 @@ async function changeOrigin(config: ArcaneConfig, args: string[]): Promise<numbe
303
308
  return 0;
304
309
  }
305
310
 
311
+ /** Container state for the Discord bridge's restart and redeploy checks: one list, plus details only for recent starts. */
312
+ async function containerWatch(config: ArcaneConfig, args: string[]): Promise<number> {
313
+ if (args.length > 0) throw new UsageError("container-watch takes no arguments");
314
+ let audit: string;
315
+ try {
316
+ audit = readTail(auditLogPath(), AUDIT_TAIL_BYTES);
317
+ } catch {
318
+ audit = "";
319
+ }
320
+ const now = Date.now();
321
+ const containers = await withCli(config, async (cli) => {
322
+ const out: WatchedContainer[] = [];
323
+ for (const row of listRows(await cliJson(cli, ["containers", "list", "--all"]))) {
324
+ const c = listContainer(row);
325
+ if (!c) continue;
326
+ out.push(isRecent(c, now) ? withDetail(c, await cliJson(cli, ["containers", "get", c.id]), audit) : c);
327
+ }
328
+ return out;
329
+ });
330
+ printJson({ checkedAt: new Date(now).toISOString(), containers });
331
+ return 0;
332
+ }
333
+
306
334
  // ---------------------------------------------------------------- Arcane VM
307
335
 
308
336
  const NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;
@@ -561,6 +589,8 @@ async function managerRollback(args: string[]): Promise<number> {
561
589
  const STACK_SYNC_WAIT_MS = 20 * MINUTE;
562
590
  /** arcane-cli allows a project redeploy up to 30 minutes for large pulls. */
563
591
  const REDEPLOY_TIMEOUT_MS = 35 * MINUTE;
592
+ /** `gitops sync` answers once the files are copied, about a minute even with a pre-deploy hook. */
593
+ const SYNC_START_TIMEOUT_MS = 10 * MINUTE;
564
594
  /** Containers with a health check get this long to leave "starting". */
565
595
  const HEALTH_WAIT_MS = 5 * MINUTE;
566
596
 
@@ -593,22 +623,58 @@ async function stackState(cli: Cli, stack: string, startedAt: number): Promise<S
593
623
  return { rows, services: services.length, stopped: services.some((s) => s.stopped), composePath };
594
624
  }
595
625
 
596
- /** The stack's containers as `containers get` reports them, found by the stack's images and kept when Compose says they belong to it. */
597
- async function stackContainers(cli: Cli, stack: string, refs: readonly string[]): Promise<StackContainer[]> {
598
- const listed = listRows(await cliJson(cli, ["containers", "list", "--all"])).filter((c) => typeof c.image === "string" && refs.includes(c.image));
599
- const out: StackContainer[] = [];
626
+ /**
627
+ * The stack's containers as `containers get` reports them, kept when Compose
628
+ * says they belong to it. Candidates are the containers listed with one of the
629
+ * stack's image references, or with a bare image id: Docker shows that once a
630
+ * pull moved the container's tag to a newer image it does not run.
631
+ */
632
+ async function stackContainers(cli: Cli, stack: string, refs: readonly string[]): Promise<(StackContainer & { id: string })[]> {
633
+ const listed = listRows(await cliJson(cli, ["containers", "list", "--all"])).filter((c) => (typeof c.image === "string" && refs.includes(c.image)) || isStaleImage(c.image));
634
+ const out: (StackContainer & { id: string })[] = [];
600
635
  for (const c of listed) {
601
636
  if (typeof c.id !== "string") continue;
602
637
  const name = Array.isArray(c.names) && typeof c.names[0] === "string" ? c.names[0].replace(/^\//, "") : c.id.slice(0, 12);
603
638
  const facts = containerFacts(await cliJson(cli, ["containers", "get", c.id]), name);
604
- if (facts.project === stack) out.push({ name, running: facts.running, state: facts.state, exitCode: facts.exitCode, health: facts.health });
639
+ if (facts.project === stack) out.push({ id: c.id, name, running: facts.running, state: facts.state, exitCode: facts.exitCode, health: facts.health, stale: isStaleImage(c.image) });
605
640
  }
606
641
  return out;
607
642
  }
608
643
 
609
- /** Waits until the stack's sync reports success at `commit`; false when it does not within the limit. */
644
+ async function stackRefs(cli: Cli, stack: string): Promise<string[]> {
645
+ return stackServices(await cliJson(cli, ["projects", "list", "--all"]))
646
+ .filter((s) => s.stack === stack)
647
+ .map((s) => s.ref)
648
+ .filter((r): r is string => r !== null);
649
+ }
650
+
651
+ /**
652
+ * Arcane's project redeploy can leave a running container on its old image
653
+ * although the pull moved its tag (paperless-db and paperless-redis on
654
+ * 2026-10-22). Arcane's container redeploy pulls and recreates that one
655
+ * container through Compose; returns the names it recreated, or the error.
656
+ */
657
+ async function recreateStale(config: ArcaneConfig, stack: string): Promise<{ recreated: string[]; problem: string | null }> {
658
+ return withCli(config, async (cli) => {
659
+ const stale = (await stackContainers(cli, stack, await stackRefs(cli, stack))).filter((c) => c.running === true && c.stale);
660
+ const recreated: string[] = [];
661
+ for (const c of stale) {
662
+ const result = await cli(["containers", "redeploy", c.id], REDEPLOY_TIMEOUT_MS);
663
+ if (result.code !== 0) return { recreated, problem: `redeploy of ${c.name} failed: ${(result.stderr.trim().split("\n").pop() ?? "").slice(0, 200) || `exited ${result.code}`}` };
664
+ recreated.push(c.name);
665
+ }
666
+ return { recreated, problem: null };
667
+ });
668
+ }
669
+
670
+ /**
671
+ * Starts the stack's sync at once, so its interval (hourly for profiles) does
672
+ * not matter, then waits until it reports success at `commit`; false when it
673
+ * does not within the limit. A failed start leaves the scheduled sync to it.
674
+ */
610
675
  async function waitForSync(config: ArcaneConfig, stack: string, commit: string): Promise<{ ok: boolean; last: string }> {
611
676
  const deadline = Date.now() + STACK_SYNC_WAIT_MS;
677
+ await withCli(config, async (cli) => cli(["gitops", "sync", syncId(await cliJson(cli, ["projects", "list", "--all"]), await cliJson(cli, ["gitops", "list", "--all"]), stack)], SYNC_START_TIMEOUT_MS)).catch(() => undefined);
612
678
  for (;;) {
613
679
  const sync = await withCli(config, async (cli) => projectSync(await cliJson(cli, ["projects", "list", "--all"]), await cliJson(cli, ["gitops", "list", "--all"]), stack));
614
680
  const sameCommit = sync.commit !== null && (commit.startsWith(sync.commit) || sync.commit.startsWith(commit));
@@ -627,13 +693,7 @@ async function redeploy(config: ArcaneConfig, stack: string): Promise<string | n
627
693
  async function settle(config: ArcaneConfig, stack: string, services: number): Promise<string | null> {
628
694
  const deadline = Date.now() + HEALTH_WAIT_MS;
629
695
  for (;;) {
630
- const problem = await withCli(config, async (cli) => {
631
- const refs = stackServices(await cliJson(cli, ["projects", "list", "--all"]))
632
- .filter((s) => s.stack === stack)
633
- .map((s) => s.ref)
634
- .filter((r): r is string => r !== null);
635
- return containersProblem(await stackContainers(cli, stack, refs), services);
636
- });
696
+ const problem = await withCli(config, async (cli) => containersProblem(await stackContainers(cli, stack, await stackRefs(cli, stack)), services));
637
697
  if (problem === null || Date.now() > deadline) return problem;
638
698
  await new Promise((r) => setTimeout(r, 15_000));
639
699
  }
@@ -699,6 +759,12 @@ async function stackUpdate(config: ArcaneConfig, args: string[]): Promise<number
699
759
  }
700
760
 
701
761
  let problem = await redeploy(config, stack);
762
+ let recreated: string[] = [];
763
+ if (problem === null) {
764
+ const stale = await recreateStale(config, stack);
765
+ recreated = stale.recreated;
766
+ problem = stale.problem;
767
+ }
702
768
  if (problem === null) problem = await settle(config, stack, before.services);
703
769
  if (problem === null) {
704
770
  const after = await withCli(config, (cli) => stackState(cli, stack, Date.now()));
@@ -711,7 +777,7 @@ async function stackUpdate(config: ArcaneConfig, args: string[]): Promise<number
711
777
  if (behind.length > 0) problem = `still behind after the redeploy: ${behind.join(", ")}`;
712
778
  }
713
779
  if (problem === null) {
714
- answer("updated", { services: results, ...(commit ? { commit } : {}) });
780
+ answer("updated", { services: results, ...(commit ? { commit } : {}), ...(recreated.length > 0 ? { recreated } : {}) });
715
781
  return 0;
716
782
  }
717
783
  if (commit === null) {
@@ -742,6 +808,7 @@ export async function command(args: string[]): Promise<number> {
742
808
  if (args[0] === "update-report") return updateReport(config, args.slice(1));
743
809
  if (args[0] === "release-notes") return releaseNotes(config, args.slice(1));
744
810
  if (args[0] === "change-origin") return changeOrigin(config, args.slice(1));
811
+ if (args[0] === "container-watch") return containerWatch(config, args.slice(1));
745
812
  if (args[0] === "host") return hostCommand(config, args.slice(1));
746
813
  if (args[0] === "project-status") return projectStatus(config, args.slice(1));
747
814
  if (args[0] === "deployment") return deployment(config, args.slice(1));
@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
4
4
  import { join } from "node:path";
5
5
  import { afterEach, describe, expect, it } from "vitest";
6
6
  import { buildHostRemote, compareAudit, HOST_COMMANDS } from "./arcane-host.ts";
7
- import { failedChecks, normalizeVersion, olderThan, pinCompose, pinnedDigest, projectSync, syncReady, utcStamp } from "./arcane-manager.ts";
7
+ import { failedChecks, normalizeVersion, olderThan, pinCompose, pinnedDigest, projectSync, syncId, syncReady, utcStamp } from "./arcane-manager.ts";
8
8
  import { containerFacts, deploymentChecks, listRows, parseDeploymentArgs, syncFacts } from "./deployment.ts";
9
9
  import { UsageError } from "./errors.ts";
10
10
 
@@ -130,6 +130,9 @@ describe("manager upgrade helpers", () => {
130
130
  expect(syncReady({ ...sync, status: "success" }, "abc1234def", "x", "y")).toBe(false);
131
131
  expect(syncReady({ ...sync, status: "success", commit: "fff0000" }, "abc1234def", "x", "x")).toBe(false);
132
132
  expect(() => projectSync(projects, { data: [] }, "arcane")).toThrow(UsageError);
133
+ expect(syncId(projects, { data: [{ id: "s1", projectId: "p1" }] }, "arcane")).toBe("s1");
134
+ expect(() => syncId(projects, { data: [{ id: "s1", projectId: "p1" }, { id: "s2", projectId: "p1" }] }, "arcane")).toThrow(UsageError);
135
+ expect(() => syncId(projects, syncs, "other")).toThrow(UsageError);
133
136
  expect(failedChecks([{ check: "version", ok: false, observed: "2.16.1" }, { check: "digest", ok: true }])).toBe('version ("2.16.1")');
134
137
  });
135
138
  });
@@ -56,6 +56,14 @@ export function projectSync(projects: unknown, syncs: unknown, name: string): Sy
56
56
  return { status: text(s.lastSyncStatus), commit: text(s.lastSyncCommit), at: text(s.lastSyncAt) };
57
57
  }
58
58
 
59
+ /** The id of the project's one GitOps sync, for `gitops sync <id>`. */
60
+ export function syncId(projects: unknown, syncs: unknown, name: string): string {
61
+ const project = listRows(projects).filter((p) => p.name === name);
62
+ const mine = project.length === 1 ? listRows(syncs).filter((s) => s.projectId === project[0].id) : [];
63
+ if (mine.length !== 1 || typeof mine[0].id !== "string") throw new UsageError(`project ${name} needs exactly one GitOps sync`);
64
+ return mine[0].id;
65
+ }
66
+
59
67
  /**
60
68
  * Ready for the host step when the sync reports success at the pin commit and
61
69
  * the deployed Compose file is the committed one. A sync that copied the file
@@ -0,0 +1,54 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { durationSeconds, exitCodeOf, healthOf, isRecent, listContainer, uptimeSeconds, withDetail } from "./container-watch.ts";
3
+
4
+ const NOW = Date.parse("2026-10-22T08:00:00Z");
5
+
6
+ describe("container-watch parsing", () => {
7
+ it("reads Docker's human durations as a lower bound", () => {
8
+ expect(durationSeconds("Less than a second")).toBe(0);
9
+ expect(durationSeconds("1 second")).toBe(1);
10
+ expect(durationSeconds("About a minute")).toBe(60);
11
+ expect(durationSeconds("12 minutes")).toBe(720);
12
+ expect(durationSeconds("About an hour")).toBe(3600);
13
+ expect(durationSeconds("3 weeks")).toBe(1_814_400);
14
+ expect(durationSeconds("soon")).toBeNull();
15
+ });
16
+
17
+ it("reads uptime, health, and exit code from the list status", () => {
18
+ expect(uptimeSeconds("Up 3 minutes (healthy)")).toBe(180);
19
+ expect(uptimeSeconds("Up About a minute")).toBe(60);
20
+ expect(uptimeSeconds("Up 2 hours (Paused)")).toBe(7200);
21
+ expect(uptimeSeconds("Exited (0) 2 days ago")).toBeNull();
22
+ expect(healthOf("Up 3 weeks (healthy)")).toBe("healthy");
23
+ expect(healthOf("Up 14 hours (unhealthy)")).toBe("unhealthy");
24
+ expect(healthOf("Up 5 seconds (health: starting)")).toBe("starting");
25
+ expect(healthOf("Up 6 days")).toBe("none");
26
+ expect(exitCodeOf("Exited (137) 1 minute ago")).toBe(137);
27
+ expect(exitCodeOf("Up 6 days")).toBeNull();
28
+ });
29
+
30
+ it("turns a list row into a container and skips rows without a name or id", () => {
31
+ const row = { id: "abc", names: ["/syncthing"], imageId: "sha256:1", state: "running", status: "Up 14 hours (unhealthy)", created: 1792305689 };
32
+ expect(listContainer(row)).toEqual({ name: "syncthing", id: "abc", imageId: "sha256:1", state: "running", health: "unhealthy", exitCode: null, uptimeSeconds: 50_400, created: new Date(1792305689 * 1000).toISOString() });
33
+ expect(listContainer({ id: "abc", names: [] })).toBeNull();
34
+ expect(listContainer({ names: ["x"] })).toBeNull();
35
+ });
36
+
37
+ it("asks for details only about recent starts and creations", () => {
38
+ const base = { name: "x", id: "1", imageId: null, health: "none" as const, exitCode: null, created: "2026-10-01T00:00:00Z" };
39
+ expect(isRecent({ ...base, state: "running", uptimeSeconds: 120 }, NOW)).toBe(true);
40
+ expect(isRecent({ ...base, state: "running", uptimeSeconds: 7200 }, NOW)).toBe(false);
41
+ expect(isRecent({ ...base, state: "restarting", uptimeSeconds: null }, NOW)).toBe(true);
42
+ expect(isRecent({ ...base, state: "exited", uptimeSeconds: null, created: "2026-10-22T07:50:00Z" }, NOW)).toBe(true);
43
+ });
44
+
45
+ it("adds the start time, the project, and our operation that covers the start", () => {
46
+ const c = { name: "mealie", id: "1", imageId: "sha256:2", state: "running", health: "healthy" as const, exitCode: null, uptimeSeconds: 120, created: "2026-10-22T07:57:00Z" };
47
+ const detail = { data: { imageId: "sha256:2", state: { status: "running", running: true, startedAt: "2026-10-22T07:58:00Z", health: { status: "healthy" } }, composeInfo: { projectName: "mealie" } } };
48
+ const audit = `${JSON.stringify({ ts: "2026-10-22T07:59:00Z", service: "arcane", caller: "discord-1", argv: ["stack-update", "mealie"], code: 0, durationMs: 180_000 })}\n`;
49
+ expect(withDetail(c, detail, audit)).toEqual({ ...c, startedAt: "2026-10-22T07:58:00Z", project: "mealie", operation: { command: "stack-update mealie", caller: "discord-1", from: "2026-10-22T07:56:00.000Z", to: "2026-10-22T07:59:00Z" } });
50
+ expect(withDetail(c, detail, "")).toEqual({ ...c, startedAt: "2026-10-22T07:58:00Z", project: "mealie" });
51
+ const never = { data: { state: { status: "created", startedAt: "0001-01-01T00:00:00Z" } } };
52
+ expect(withDetail({ ...c, state: "created" }, never, audit)).toEqual({ ...c, state: "created" });
53
+ });
54
+ });
@@ -0,0 +1,94 @@
1
+ import { coveringOperation, type Operation } from "./change-origin.ts";
2
+ import { containerFacts } from "./deployment.ts";
3
+
4
+ export type Health = "healthy" | "unhealthy" | "starting" | "none";
5
+
6
+ /** One container as `container-watch` reports it; start time, project, and operation only for recent ones. */
7
+ export interface WatchedContainer {
8
+ name: string;
9
+ id: string;
10
+ imageId: string | null;
11
+ state: string;
12
+ health: Health;
13
+ exitCode: number | null;
14
+ uptimeSeconds: number | null;
15
+ created: string | null;
16
+ startedAt?: string;
17
+ project?: string;
18
+ operation?: Operation;
19
+ }
20
+
21
+ /** Started or created within this window: the bridge polls every five minutes, so half an hour leaves room for a missed poll. */
22
+ export const RECENT_S = 1800;
23
+
24
+ const UNITS: Record<string, number> = { second: 1, minute: 60, hour: 3600, day: 86_400, week: 604_800, month: 2_592_000, year: 31_536_000 };
25
+
26
+ /** Docker's human duration (`Less than a second`, `About a minute`, `3 minutes`, `2 weeks`) as a lower bound in seconds. */
27
+ export function durationSeconds(text: string): number | null {
28
+ if (text === "Less than a second") return 0;
29
+ const about = /^About an? (minute|hour)$/.exec(text);
30
+ if (about) return UNITS[about[1]];
31
+ const m = /^(\d+) (second|minute|hour|day|week|month|year)s?$/.exec(text);
32
+ return m ? Number(m[1]) * UNITS[m[2]] : null;
33
+ }
34
+
35
+ /** Seconds since the start from a list status such as `Up 3 minutes (healthy)`; null when the container is not up. */
36
+ export function uptimeSeconds(status: string): number | null {
37
+ const m = /^Up (.+?)(?: \(.*\))?$/.exec(status);
38
+ return m ? durationSeconds(m[1]) : null;
39
+ }
40
+
41
+ export function healthOf(status: string): Health {
42
+ const m = /\((healthy|unhealthy|health: starting)\)/.exec(status);
43
+ if (!m) return "none";
44
+ return m[1] === "health: starting" ? "starting" : (m[1] as Health);
45
+ }
46
+
47
+ export function exitCodeOf(status: string): number | null {
48
+ const m = /^Exited \((-?\d+)\)/.exec(status);
49
+ return m ? Number(m[1]) : null;
50
+ }
51
+
52
+ function str(v: unknown): string | null {
53
+ return typeof v === "string" && v.length > 0 ? v : null;
54
+ }
55
+
56
+ /** One row of `containers list --all`. */
57
+ export function listContainer(row: Record<string, unknown>): WatchedContainer | null {
58
+ const names = Array.isArray(row.names) ? row.names : [];
59
+ const name = typeof names[0] === "string" ? names[0].replace(/^\//, "") : null;
60
+ const id = str(row.id);
61
+ if (!name || !id) return null;
62
+ const status = typeof row.status === "string" ? row.status : "";
63
+ return {
64
+ name,
65
+ id,
66
+ imageId: str(row.imageId),
67
+ state: str(row.state) ?? "unknown",
68
+ health: healthOf(status),
69
+ exitCode: exitCodeOf(status),
70
+ uptimeSeconds: uptimeSeconds(status),
71
+ created: typeof row.created === "number" ? new Date(row.created * 1000).toISOString() : null,
72
+ };
73
+ }
74
+
75
+ /** Whether the bridge needs the exact start time and project: started or created recently, or restarting now. */
76
+ export function isRecent(c: WatchedContainer, now: number): boolean {
77
+ if (c.state === "restarting") return true;
78
+ if (c.uptimeSeconds !== null && c.uptimeSeconds < RECENT_S) return true;
79
+ return c.created !== null && now - Date.parse(c.created) < RECENT_S * 1000;
80
+ }
81
+
82
+ /** Adds the start time and project from `containers get`, and our operation on that stack whose run covers the start. */
83
+ export function withDetail(c: WatchedContainer, detail: unknown, auditText: string): WatchedContainer {
84
+ const facts = containerFacts(detail, c.name);
85
+ const top = ((detail as { data?: unknown } | null)?.data ?? detail) as { state?: { startedAt?: unknown } } | null;
86
+ const startedAt = str(top?.state?.startedAt);
87
+ const out: WatchedContainer = { ...c, ...(startedAt && !startedAt.startsWith("0001-") ? { startedAt } : {}), ...(facts.project ? { project: facts.project } : {}) };
88
+ const at = Date.parse(out.startedAt ?? c.created ?? "");
89
+ if (out.project && !Number.isNaN(at)) {
90
+ const operation = coveringOperation(auditText, out.project, at);
91
+ if (operation) out.operation = operation;
92
+ }
93
+ return out;
94
+ }
@@ -1,6 +1,6 @@
1
1
  import { describe, expect, it } from "vitest";
2
2
  import type { UpdateRow } from "./image-updates.ts";
3
- import { commitMessage, containersProblem, crossesMajor, nextImageRef, parseHolds, pendingRows, replaceImageRef, type StackContainer } from "./stack-update.ts";
3
+ import { commitMessage, containersProblem, crossesMajor, isStaleImage, nextImageRef, parseHolds, pendingRows, replaceImageRef, type StackContainer } from "./stack-update.ts";
4
4
 
5
5
  const A = `sha256:${"a".repeat(64)}`;
6
6
  const B = `sha256:${"b".repeat(64)}`;
@@ -80,7 +80,7 @@ describe("Compose rewrite", () => {
80
80
  });
81
81
 
82
82
  describe("stack health after a redeploy", () => {
83
- const up = (name: string, over: Partial<StackContainer> = {}): StackContainer => ({ name, running: true, state: "running", exitCode: null, health: "healthy", ...over });
83
+ const up = (name: string, over: Partial<StackContainer> = {}): StackContainer => ({ name, running: true, state: "running", exitCode: null, health: "healthy", stale: false, ...over });
84
84
 
85
85
  it("passes running containers and finished one-shot jobs", () => {
86
86
  expect(containersProblem([up("db"), up("app", { health: "none" }), up("init", { running: false, state: "exited", exitCode: 0 })], 3)).toBeNull();
@@ -91,6 +91,15 @@ describe("stack health after a redeploy", () => {
91
91
  expect(containersProblem([up("db"), up("init", { running: false, state: "exited", exitCode: 1 })], 2)).toBe("not running: init");
92
92
  expect(containersProblem([up("db", { health: "unhealthy" }), up("app", { health: "starting" })], 2)).toBe("health: db (unhealthy), app (starting)");
93
93
  });
94
+
95
+ it("names running containers left on an older image than their tag, not finished jobs", () => {
96
+ expect(containersProblem([up("db", { stale: true }), up("redis", { stale: true }), up("app")], 3)).toBe("still on an older image than its tag: db, redis");
97
+ expect(containersProblem([up("init", { running: false, state: "exited", exitCode: 0, stale: true })], 1)).toBeNull();
98
+ expect(isStaleImage(A)).toBe(true);
99
+ expect(isStaleImage("postgres:18")).toBe(false);
100
+ expect(isStaleImage(`postgres@${A}`)).toBe(false);
101
+ expect(isStaleImage(undefined)).toBe(false);
102
+ });
94
103
  });
95
104
 
96
105
  describe("commit message", () => {
@@ -95,6 +95,13 @@ export interface StackContainer {
95
95
  state: string | null;
96
96
  exitCode: number | null;
97
97
  health: string;
98
+ /** Docker lists the container by image id: its Compose tag now names a newer image that it does not run. */
99
+ stale: boolean;
100
+ }
101
+
102
+ /** A container's listed image is a bare id once its tag moved to another image. */
103
+ export function isStaleImage(image: unknown): boolean {
104
+ return typeof image === "string" && /^sha256:[0-9a-f]{64}$/.test(image);
98
105
  }
99
106
 
100
107
  /**
@@ -105,6 +112,8 @@ export function containersProblem(containers: readonly StackContainer[], expecte
105
112
  if (containers.length < expected) return `${containers.length} of ${expected} services have a container`;
106
113
  const down = containers.filter((c) => c.running !== true && !(c.state === "exited" && c.exitCode === 0)).map((c) => c.name);
107
114
  if (down.length > 0) return `not running: ${down.join(", ")}`;
115
+ const old = containers.filter((c) => c.running === true && c.stale).map((c) => c.name);
116
+ if (old.length > 0) return `still on an older image than its tag: ${old.join(", ")}`;
108
117
  const sick = containers.filter((c) => c.health === "unhealthy" || c.health === "starting").map((c) => `${c.name} (${c.health})`);
109
118
  return sick.length > 0 ? `health: ${sick.join(", ")}` : null;
110
119
  }