@astrofoundry/pi-astro 0.37.1 → 0.37.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/astro-discord/arcane-watch-monitor.ts +133 -0
- package/extensions/astro-discord/arcane-watch.test.ts +242 -0
- package/extensions/astro-discord/arcane-watch.ts +215 -0
- package/extensions/astro-discord/daily-report.ts +3 -1
- package/extensions/astro-discord/index.ts +27 -11
- package/extensions/astro-discord/security-monitor.test.ts +5 -1
- package/extensions/astro-discord/security-monitor.ts +21 -1
- package/extensions/astro-discord/security.test.ts +81 -2
- package/extensions/astro-discord/security.ts +126 -3
- package/package.json +1 -1
- package/skills/arcane/SKILL.md +2 -0
- package/skills/security/SKILL.md +1 -1
- package/specialists/arcane/run.ts +28 -0
- package/specialists/entry/remote/spc-security-obs-entry.sh +1 -1
- package/specialists/lib/container-watch.test.ts +54 -0
- package/specialists/lib/container-watch.ts +94 -0
- package/specialists/security/run.ts +1 -1
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import {
|
|
2
|
+
escalationText,
|
|
3
|
+
observe,
|
|
4
|
+
parseSnapshot,
|
|
5
|
+
readWatchState,
|
|
6
|
+
restartInvestigationTask,
|
|
7
|
+
unhealthyText,
|
|
8
|
+
WATCH_POLL_MS,
|
|
9
|
+
WATCH_RECOVERED_TEXT,
|
|
10
|
+
watchFailingText,
|
|
11
|
+
writeWatchState,
|
|
12
|
+
} from "./arcane-watch.ts";
|
|
13
|
+
import { type DiscordConfig, type ReportSchedule, reportSchedule } from "./config.ts";
|
|
14
|
+
import type { CreateMessage, DiscordMessage } from "./rest.ts";
|
|
15
|
+
import type { InvestigationRequest } from "./security-monitor.ts";
|
|
16
|
+
|
|
17
|
+
export interface WatchRest {
|
|
18
|
+
createMessage(channelId: string, message: CreateMessage): Promise<DiscordMessage>;
|
|
19
|
+
startThread(channelId: string, messageId: string, name: string): Promise<{ id: string }>;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface WatchDeps {
|
|
23
|
+
rest: WatchRest;
|
|
24
|
+
runEntry: (service: string, caller: string, args: string[], timeoutMs: number) => Promise<string>;
|
|
25
|
+
config: () => DiscordConfig;
|
|
26
|
+
log: (line: string) => void;
|
|
27
|
+
/** Runs astro.arcane on the task in the thread; resolves true when it answered. */
|
|
28
|
+
investigate: (request: InvestigationRequest) => Promise<boolean>;
|
|
29
|
+
/** Whether the NAS share is lost or just came back (the security monitor's record of rules 100012 and 100013). */
|
|
30
|
+
nasOutage: (now: number) => boolean;
|
|
31
|
+
stopping: () => boolean;
|
|
32
|
+
now: () => number;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const MINUTE = 60_000;
|
|
36
|
+
/** Six failed polls (30 minutes) tag the owners once. */
|
|
37
|
+
const FAILS_TO_WARN = 6;
|
|
38
|
+
const CALLER = "arcane-watch";
|
|
39
|
+
|
|
40
|
+
function reason(err: unknown): string {
|
|
41
|
+
return (err instanceof Error ? err.message : String(err)).split("\n")[0].slice(0, 300);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Restarts and redeploys on the Arcane VM, every five minutes through the
|
|
46
|
+
* arcane wrapper's `container-watch`, without a model. An `arcane` report entry
|
|
47
|
+
* turns it on in that report's channel.
|
|
48
|
+
*/
|
|
49
|
+
export class ArcaneWatch {
|
|
50
|
+
private readonly deps: WatchDeps;
|
|
51
|
+
private running = false;
|
|
52
|
+
private last = 0;
|
|
53
|
+
|
|
54
|
+
constructor(deps: WatchDeps) {
|
|
55
|
+
this.deps = deps;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
tick(): void {
|
|
59
|
+
const s = reportSchedule(this.deps.config(), "arcane");
|
|
60
|
+
if (!s || this.deps.stopping() || this.running) return;
|
|
61
|
+
const now = this.deps.now();
|
|
62
|
+
if (now - this.last < WATCH_POLL_MS - 5000) return;
|
|
63
|
+
this.running = true;
|
|
64
|
+
this.last = now;
|
|
65
|
+
void this.poll(s)
|
|
66
|
+
.catch((err) => this.deps.log(`arcane watch: check failed: ${reason(err)}`))
|
|
67
|
+
.finally(() => (this.running = false));
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
private async post(channelId: string, message: CreateMessage): Promise<DiscordMessage | null> {
|
|
71
|
+
try {
|
|
72
|
+
return await this.deps.rest.createMessage(channelId, message);
|
|
73
|
+
} catch (err) {
|
|
74
|
+
this.deps.log(`arcane watch: message failed: ${reason(err)}`);
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async poll(s: ReportSchedule): Promise<void> {
|
|
80
|
+
const owners = this.deps.config().owners;
|
|
81
|
+
const state = readWatchState();
|
|
82
|
+
let snapshot;
|
|
83
|
+
try {
|
|
84
|
+
snapshot = parseSnapshot(JSON.parse(await this.deps.runEntry("arcane", CALLER, ["container-watch"], 3 * MINUTE)));
|
|
85
|
+
if (!snapshot) throw new Error("container-watch returned an unexpected answer");
|
|
86
|
+
} catch (err) {
|
|
87
|
+
const why = reason(err);
|
|
88
|
+
this.deps.log(`arcane watch: container-watch failed: ${why}`);
|
|
89
|
+
state.poll.failures++;
|
|
90
|
+
if (state.poll.failures >= FAILS_TO_WARN && !state.poll.warned) {
|
|
91
|
+
state.poll.warned = (await this.post(s.channelId, { content: watchFailingText((state.poll.failures * WATCH_POLL_MS) / MINUTE, why, owners), mentionUsers: owners })) !== null;
|
|
92
|
+
}
|
|
93
|
+
writeWatchState(state);
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
if (state.poll.warned) await this.post(s.channelId, { content: WATCH_RECOVERED_TEXT });
|
|
97
|
+
state.poll = { failures: 0, warned: false };
|
|
98
|
+
const now = this.deps.now();
|
|
99
|
+
const seen = observe(state, snapshot, now, this.deps.nasOutage(now));
|
|
100
|
+
for (const note of seen.notes) this.deps.log(`arcane watch: ${note}`);
|
|
101
|
+
for (const u of seen.unhealthy) {
|
|
102
|
+
const posted = await this.post(s.channelId, { content: unhealthyText(u, owners, s.timeZone, now), mentionUsers: owners });
|
|
103
|
+
if (posted) this.deps.log(`arcane watch: ${u.name} ${u.problem} after a redeploy`);
|
|
104
|
+
else state.pending.push({ name: u.name, id: state.containers[u.name]?.id ?? "", since: u.since, newImage: u.newImage, ...(u.project ? { project: u.project } : {}) });
|
|
105
|
+
}
|
|
106
|
+
const started: Promise<void>[] = [];
|
|
107
|
+
for (const e of seen.escalations) {
|
|
108
|
+
const posted = await this.post(s.channelId, { content: escalationText(e, owners, s.timeZone), mentionUsers: owners });
|
|
109
|
+
if (!posted) {
|
|
110
|
+
delete state.escalated[e.name];
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
this.deps.log(`arcane watch: ${e.name} restarted ${e.restarts.length} times in 24 h`);
|
|
114
|
+
started.push(this.investigate(s.channelId, posted.id, e.name, restartInvestigationTask(e, now)));
|
|
115
|
+
}
|
|
116
|
+
writeWatchState(state);
|
|
117
|
+
// Investigations take minutes; the next poll does not wait for them.
|
|
118
|
+
void Promise.all(started);
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
private async investigate(channelId: string, messageId: string, name: string, task: string): Promise<void> {
|
|
122
|
+
try {
|
|
123
|
+
const thread = await this.deps.rest.startThread(channelId, messageId, `Restarts of ${name}`.slice(0, 100));
|
|
124
|
+
const starter = await this.deps.rest.createMessage(thread.id, { content: "🔎 The arcane specialist reads only; a risky read still asks an owner here. Write in this thread to continue." });
|
|
125
|
+
const owner = this.deps.config().owners[0];
|
|
126
|
+
const ok = await this.deps.investigate({ threadId: thread.id, starterId: starter.id, userId: owner, task });
|
|
127
|
+
if (!ok) this.deps.log(`arcane watch: the investigation of ${name} did not answer`);
|
|
128
|
+
} catch (err) {
|
|
129
|
+
this.deps.log(`arcane watch: investigation of ${name} failed: ${reason(err)}`);
|
|
130
|
+
await this.post(channelId, { content: `❌ Could not start the investigation of ${name} (${reason(err)}); ask the arcane specialist in the channel.`, replyTo: messageId });
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
@@ -0,0 +1,242 @@
|
|
|
1
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
5
|
+
import type { WatchedContainer } from "../../specialists/lib/container-watch.ts";
|
|
6
|
+
import { ArcaneWatch } from "./arcane-watch-monitor.ts";
|
|
7
|
+
import { emptyWatchState, escalationText, healthProblem, observe, parseSnapshot, readWatchState, restartInvestigationTask, unhealthyText, type WatchState } from "./arcane-watch.ts";
|
|
8
|
+
import type { DiscordConfig } from "./config.ts";
|
|
9
|
+
import type { CreateMessage } from "./rest.ts";
|
|
10
|
+
import type { InvestigationRequest } from "./security-monitor.ts";
|
|
11
|
+
|
|
12
|
+
const OWNER = "178096425509453824";
|
|
13
|
+
const CHANNEL = "1549873938140499999";
|
|
14
|
+
const TZ = "Europe/Madrid";
|
|
15
|
+
const T0 = Date.parse("2026-10-22T08:00:00Z");
|
|
16
|
+
const MIN = 60_000;
|
|
17
|
+
|
|
18
|
+
function ct(over: Partial<WatchedContainer> = {}): WatchedContainer {
|
|
19
|
+
return { name: "syncthing", id: "a", imageId: "sha256:1", state: "running", health: "healthy", exitCode: null, uptimeSeconds: 86_400, created: "2026-10-07T21:39:38Z", ...over };
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function iso(ms: number): string {
|
|
23
|
+
return new Date(ms).toISOString();
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** A state that has already seen `list` at `at`. */
|
|
27
|
+
function seenAt(list: WatchedContainer[], at: number): WatchState {
|
|
28
|
+
const state = emptyWatchState();
|
|
29
|
+
observe(state, list, at, false);
|
|
30
|
+
return state;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
describe("container watch rules", () => {
|
|
34
|
+
it("only records a baseline the first time", () => {
|
|
35
|
+
const state = emptyWatchState();
|
|
36
|
+
const out = observe(state, [ct({ startedAt: iso(T0 - MIN) })], T0, false);
|
|
37
|
+
expect(out).toEqual({ escalations: [], unhealthy: [], notes: [] });
|
|
38
|
+
expect(state.containers.syncthing).toEqual({ id: "a", imageId: "sha256:1", startedAt: iso(T0 - MIN), seenAt: iso(T0) });
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it("escalates the third restart within 24 hours once a day", () => {
|
|
42
|
+
const state = seenAt([ct(), ct({ name: "mealie", id: "m", project: "mealie" })], T0);
|
|
43
|
+
for (let i = 1; i <= 2; i++) {
|
|
44
|
+
const now = T0 + i * 5 * MIN;
|
|
45
|
+
expect(observe(state, [ct({ startedAt: iso(now - MIN), project: "syncthing" })], now, false).escalations).toEqual([]);
|
|
46
|
+
}
|
|
47
|
+
const now = T0 + 15 * MIN;
|
|
48
|
+
const out = observe(state, [ct({ startedAt: iso(now - MIN), project: "syncthing" })], now, false);
|
|
49
|
+
expect(out.escalations).toEqual([{ name: "syncthing", project: "syncthing", restarts: [iso(T0 + 4 * MIN), iso(T0 + 9 * MIN), iso(T0 + 14 * MIN)] }]);
|
|
50
|
+
expect(state.containers.mealie).toBeUndefined();
|
|
51
|
+
const later = T0 + 20 * MIN;
|
|
52
|
+
expect(observe(state, [ct({ startedAt: iso(later - MIN) })], later, false).escalations).toEqual([]);
|
|
53
|
+
expect(state.restarts.syncthing).toHaveLength(4);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it("does not count a start it already saw, or one before the last observation", () => {
|
|
57
|
+
const state = seenAt([ct({ startedAt: iso(T0 - 2 * MIN) })], T0);
|
|
58
|
+
observe(state, [ct({ startedAt: iso(T0 - 2 * MIN) })], T0 + 5 * MIN, false);
|
|
59
|
+
observe(state, [ct()], T0 + 10 * MIN, false);
|
|
60
|
+
expect(state.restarts).toEqual({});
|
|
61
|
+
expect(state.containers.syncthing.startedAt).toBe(iso(T0 - 2 * MIN));
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it("leaves out restarts our operations or a NAS outage explain", () => {
|
|
65
|
+
const state = seenAt([ct({ name: "slskd", id: "s", project: "media" }), ct({ name: "mealie", id: "m", project: "mealie" })], T0);
|
|
66
|
+
const now = T0 + 5 * MIN;
|
|
67
|
+
const operation = { command: "host db-restore mealie 20261022T0700Z", caller: "discord-1", from: iso(now - 3 * MIN), to: iso(now) };
|
|
68
|
+
const out = observe(state, [ct({ name: "slskd", id: "s", project: "media", startedAt: iso(now - MIN) }), ct({ name: "mealie", id: "m", project: "mealie", startedAt: iso(now - MIN), operation })], now, true);
|
|
69
|
+
expect(out.notes).toEqual(["slskd restarted during a NAS outage", "mealie restarted during host db-restore mealie 20261022T0700Z (discord-1)"]);
|
|
70
|
+
expect(state.restarts).toEqual({});
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
it("checks a redeployed container after five minutes and reports what is wrong", () => {
|
|
74
|
+
const state = seenAt([ct({ name: "mealie", id: "old", imageId: "sha256:1", project: "mealie" })], T0);
|
|
75
|
+
const started = T0 + MIN;
|
|
76
|
+
const fresh = ct({ name: "mealie", id: "new", imageId: "sha256:2", project: "mealie", startedAt: iso(started), health: "starting", uptimeSeconds: 60 });
|
|
77
|
+
expect(observe(state, [fresh], T0 + 2 * MIN, false).unhealthy).toEqual([]);
|
|
78
|
+
expect(state.pending).toEqual([{ name: "mealie", id: "new", since: iso(started), newImage: true, project: "mealie" }]);
|
|
79
|
+
expect(observe(state, [{ ...fresh, health: "starting" }], T0 + 7 * MIN, false).unhealthy).toEqual([]);
|
|
80
|
+
const out = observe(state, [{ ...fresh, health: "unhealthy" }], T0 + 12 * MIN, false);
|
|
81
|
+
expect(out.unhealthy).toEqual([{ name: "mealie", project: "mealie", since: iso(started), newImage: true, problem: "unhealthy" }]);
|
|
82
|
+
expect(state.pending).toEqual([]);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it("drops a redeploy that came up fine, was replaced, or that our operation reports", () => {
|
|
86
|
+
const state = seenAt([ct({ name: "a", id: "1" }), ct({ name: "b", id: "1b" }), ct({ name: "c", id: "1c", project: "c" })], T0);
|
|
87
|
+
const operation = { command: "stack-update c", caller: "discord-1", from: iso(T0), to: iso(T0 + MIN) };
|
|
88
|
+
const created = iso(T0);
|
|
89
|
+
observe(state, [ct({ name: "a", id: "2", created }), ct({ name: "b", id: "2b", created }), ct({ name: "c", id: "2c", project: "c", operation, state: "exited", exitCode: 1, created })], T0 + MIN, false);
|
|
90
|
+
expect(state.pending.map((p) => p.id)).toEqual(["2", "2b", "2c"]);
|
|
91
|
+
const out = observe(state, [ct({ name: "a", id: "2", created }), ct({ name: "b", id: "3b", created: iso(T0 + 6 * MIN) }), ct({ name: "c", id: "2c", project: "c", state: "exited", exitCode: 1, created })], T0 + 7 * MIN, false);
|
|
92
|
+
expect(out.unhealthy).toEqual([]);
|
|
93
|
+
expect(out.notes).toEqual(["c exited with code 1 after stack-update c (discord-1), which reports it"]);
|
|
94
|
+
expect(state.pending.map((p) => p.id)).toEqual(["3b"]);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it("names what is wrong with a container", () => {
|
|
98
|
+
expect(healthProblem(ct({ state: "exited", exitCode: 0 }), 0)).toBeNull();
|
|
99
|
+
expect(healthProblem(ct({ state: "exited", exitCode: 137 }), 0)).toBe("exited with code 137");
|
|
100
|
+
expect(healthProblem(ct({ state: "restarting" }), 0)).toBe("restarting");
|
|
101
|
+
expect(healthProblem(ct({ state: "created" }), 0)).toBe("never started");
|
|
102
|
+
expect(healthProblem(ct({ health: "starting" }), 16 * MIN)).toBe("health check still starting");
|
|
103
|
+
expect(healthProblem(ct({ health: "none" }), 16 * MIN)).toBeNull();
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it("forgets restarts and escalations after a day", () => {
|
|
107
|
+
const state = emptyWatchState();
|
|
108
|
+
state.containers.x = { id: "1", imageId: null, startedAt: null, seenAt: iso(T0) };
|
|
109
|
+
state.restarts.x = [iso(T0 - 25 * 60 * MIN)];
|
|
110
|
+
state.escalated.x = iso(T0 - 25 * 60 * MIN);
|
|
111
|
+
observe(state, [ct({ name: "x", id: "1" })], T0, false);
|
|
112
|
+
expect(state.restarts).toEqual({});
|
|
113
|
+
expect(state.escalated).toEqual({});
|
|
114
|
+
});
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
describe("container watch texts", () => {
|
|
118
|
+
it("tags the owners and says what happened", () => {
|
|
119
|
+
const e = { name: "syncthing", project: "syncthing", restarts: [iso(T0), iso(T0 + 5 * MIN), iso(T0 + 10 * MIN)] };
|
|
120
|
+
expect(escalationText(e, [OWNER], TZ)).toBe(
|
|
121
|
+
`🔁 **Container restarting** · syncthing · 3 restarts in 24 h, last 22 Oct 10:10 CEST\nAuto-heal or Docker restarted it each time. The arcane specialist investigates in the thread and changes nothing.\n<@${OWNER}>`,
|
|
122
|
+
);
|
|
123
|
+
expect(unhealthyText({ name: "worker", project: "grimoire", since: iso(T0), newImage: true, problem: "unhealthy" }, [OWNER], TZ, T0 + 7 * MIN)).toBe(
|
|
124
|
+
`⚠️ **Container unhealthy after a redeploy** · worker (stack grimoire) · recreated 22 Oct 10:00 CEST with a new image, still unhealthy 7 min later\nNo stack-update or other specialist operation explains the redeploy. Ask the arcane specialist here to investigate.\n<@${OWNER}>`,
|
|
125
|
+
);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it("asks the specialist to read only, within the host's one-day window", () => {
|
|
129
|
+
const now = T0 + 24 * 60 * MIN;
|
|
130
|
+
const task = restartInvestigationTask({ name: "syncthing", restarts: [iso(T0 + MIN), iso(now - MIN)] }, now);
|
|
131
|
+
expect(task).toContain("Read only: change nothing");
|
|
132
|
+
expect(task).toContain(`host container-logs syncthing ${iso(T0 + MIN)} ${iso(now)}`);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it("accepts only a well-formed snapshot", () => {
|
|
136
|
+
expect(parseSnapshot({ containers: [ct()] })).toEqual([ct()]);
|
|
137
|
+
expect(parseSnapshot({ containers: [{ name: "x" }] })).toBeNull();
|
|
138
|
+
expect(parseSnapshot({})).toBeNull();
|
|
139
|
+
});
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
function config(): DiscordConfig {
|
|
143
|
+
return {
|
|
144
|
+
applicationId: "1",
|
|
145
|
+
guildId: "2",
|
|
146
|
+
channels: { [CHANNEL]: { trigger: "always", specialists: ["arcane"] } },
|
|
147
|
+
adminChannelId: CHANNEL,
|
|
148
|
+
owners: [OWNER],
|
|
149
|
+
access: {},
|
|
150
|
+
approvalTimeoutMinutes: 10,
|
|
151
|
+
reports: [{ report: "arcane", channelId: CHANNEL, time: "06:00", timeZone: TZ }],
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
describe("container watch monitor", () => {
|
|
156
|
+
let dir: string;
|
|
157
|
+
beforeEach(() => {
|
|
158
|
+
dir = mkdtempSync(join(tmpdir(), "astro-watch-"));
|
|
159
|
+
process.env.ASTRO_DISCORD_DIR = dir;
|
|
160
|
+
});
|
|
161
|
+
afterEach(() => {
|
|
162
|
+
delete process.env.ASTRO_DISCORD_DIR;
|
|
163
|
+
rmSync(dir, { recursive: true, force: true });
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
function harness() {
|
|
167
|
+
const messages: { channelId: string; message: CreateMessage }[] = [];
|
|
168
|
+
const threads: string[] = [];
|
|
169
|
+
const investigations: InvestigationRequest[] = [];
|
|
170
|
+
const logs: string[] = [];
|
|
171
|
+
const clock = { now: T0 };
|
|
172
|
+
const answer: { next: () => string } = { next: () => JSON.stringify({ checkedAt: iso(clock.now), containers: [ct()] }) };
|
|
173
|
+
let id = 500;
|
|
174
|
+
const watch = new ArcaneWatch({
|
|
175
|
+
rest: {
|
|
176
|
+
createMessage: async (channelId, message) => {
|
|
177
|
+
messages.push({ channelId, message });
|
|
178
|
+
return { id: String(id++), channel_id: channelId };
|
|
179
|
+
},
|
|
180
|
+
startThread: async (_channelId, messageId) => {
|
|
181
|
+
threads.push(messageId);
|
|
182
|
+
return { id: `t${messageId}` };
|
|
183
|
+
},
|
|
184
|
+
},
|
|
185
|
+
runEntry: async (service, caller, args) => {
|
|
186
|
+
expect([service, caller, ...args]).toEqual(["arcane", "arcane-watch", "container-watch"]);
|
|
187
|
+
return answer.next();
|
|
188
|
+
},
|
|
189
|
+
config,
|
|
190
|
+
log: (line) => logs.push(line),
|
|
191
|
+
investigate: async (request) => {
|
|
192
|
+
investigations.push(request);
|
|
193
|
+
return true;
|
|
194
|
+
},
|
|
195
|
+
nasOutage: () => false,
|
|
196
|
+
stopping: () => false,
|
|
197
|
+
now: () => clock.now,
|
|
198
|
+
});
|
|
199
|
+
return { watch, messages, threads, investigations, logs, clock, answer };
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
const SCHEDULE = config().reports[0];
|
|
203
|
+
|
|
204
|
+
it("posts the escalation with a mention, opens a thread, and investigates there", async () => {
|
|
205
|
+
const h = harness();
|
|
206
|
+
await h.watch.poll(SCHEDULE);
|
|
207
|
+
for (let i = 1; i <= 3; i++) {
|
|
208
|
+
h.clock.now = T0 + i * 5 * MIN;
|
|
209
|
+
const started = iso(h.clock.now - MIN);
|
|
210
|
+
h.answer.next = () => JSON.stringify({ containers: [ct({ startedAt: started, project: "syncthing" })] });
|
|
211
|
+
await h.watch.poll(SCHEDULE);
|
|
212
|
+
}
|
|
213
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
214
|
+
expect(h.messages[0].message.content).toContain("🔁 **Container restarting** · syncthing · 3 restarts in 24 h");
|
|
215
|
+
expect(h.messages[0].message.mentionUsers).toEqual([OWNER]);
|
|
216
|
+
expect(h.threads).toEqual(["500"]);
|
|
217
|
+
expect(h.investigations).toHaveLength(1);
|
|
218
|
+
expect(h.investigations[0]).toMatchObject({ threadId: "t500", starterId: "501", userId: OWNER });
|
|
219
|
+
expect(h.investigations[0].task).toContain("restarted 3 times");
|
|
220
|
+
});
|
|
221
|
+
|
|
222
|
+
it("warns once after 30 minutes of failed checks and says when they work again", async () => {
|
|
223
|
+
const h = harness();
|
|
224
|
+
h.answer.next = () => "not json";
|
|
225
|
+
for (let i = 0; i < 7; i++) await h.watch.poll(SCHEDULE);
|
|
226
|
+
expect(h.messages).toHaveLength(1);
|
|
227
|
+
expect(h.messages[0].message.content).toContain("⚠️ **Container checks failing:** the bridge could not read Arcane's containers for 30 minutes");
|
|
228
|
+
h.answer.next = () => JSON.stringify({ containers: [ct()] });
|
|
229
|
+
await h.watch.poll(SCHEDULE);
|
|
230
|
+
expect(h.messages[1].message.content).toBe("✅ Container checks work again.");
|
|
231
|
+
expect(readWatchState().poll).toEqual({ failures: 0, warned: false });
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
it("polls only with an arcane report entry and every five minutes", () => {
|
|
235
|
+
const h = harness();
|
|
236
|
+
h.watch.tick();
|
|
237
|
+
h.watch.tick();
|
|
238
|
+
h.clock.now = T0 + 5 * MIN;
|
|
239
|
+
h.watch.tick();
|
|
240
|
+
expect(h.logs).toEqual([]);
|
|
241
|
+
});
|
|
242
|
+
});
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
import { readFileSync, renameSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import type { Operation } from "../../specialists/lib/change-origin.ts";
|
|
4
|
+
import type { WatchedContainer } from "../../specialists/lib/container-watch.ts";
|
|
5
|
+
import { configDir } from "./config.ts";
|
|
6
|
+
import { escapeDiscord } from "./media.ts";
|
|
7
|
+
import { clockText } from "./security.ts";
|
|
8
|
+
|
|
9
|
+
const MINUTE = 60_000;
|
|
10
|
+
const DAY = 86_400_000;
|
|
11
|
+
export const WATCH_POLL_MS = 5 * MINUTE;
|
|
12
|
+
/** Three restarts of one container within 24 hours start an investigation, at most once a day per container. */
|
|
13
|
+
export const RESTARTS_TO_ESCALATE = 3;
|
|
14
|
+
/** A redeployed container gets five minutes to come up before its health counts. */
|
|
15
|
+
export const HEALTH_GRACE_MS = 5 * MINUTE;
|
|
16
|
+
/** A health check still starting after this long counts as failing. */
|
|
17
|
+
const STARTING_LIMIT_MS = 15 * MINUTE;
|
|
18
|
+
/** The only stack that mounts the NAS share (`nas_media` in the arcane repository's media/compose.yaml). */
|
|
19
|
+
export const NAS_STACKS: readonly string[] = ["media"];
|
|
20
|
+
|
|
21
|
+
export interface WatchState {
|
|
22
|
+
/** The last observation of each container by name. */
|
|
23
|
+
containers: Record<string, { id: string; imageId: string | null; startedAt: string | null; seenAt: string }>;
|
|
24
|
+
/** Restart times per container within the last 24 hours, ours and NAS-related ones left out. */
|
|
25
|
+
restarts: Record<string, string[]>;
|
|
26
|
+
/** When each container was last escalated. */
|
|
27
|
+
escalated: Record<string, string>;
|
|
28
|
+
/** Redeployed containers whose health is checked once the grace time has passed. */
|
|
29
|
+
pending: PendingHealth[];
|
|
30
|
+
poll: { failures: number; warned: boolean };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface PendingHealth {
|
|
34
|
+
name: string;
|
|
35
|
+
id: string;
|
|
36
|
+
since: string;
|
|
37
|
+
project?: string;
|
|
38
|
+
newImage: boolean;
|
|
39
|
+
operation?: string;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export interface Escalation {
|
|
43
|
+
name: string;
|
|
44
|
+
project?: string;
|
|
45
|
+
restarts: string[];
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface Unhealthy {
|
|
49
|
+
name: string;
|
|
50
|
+
project?: string;
|
|
51
|
+
since: string;
|
|
52
|
+
newImage: boolean;
|
|
53
|
+
problem: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export interface Observation {
|
|
57
|
+
escalations: Escalation[];
|
|
58
|
+
unhealthy: Unhealthy[];
|
|
59
|
+
/** Log lines for what was seen but is expected: our operations and NAS outages. */
|
|
60
|
+
notes: string[];
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export function emptyWatchState(): WatchState {
|
|
64
|
+
return { containers: {}, restarts: {}, escalated: {}, pending: [], poll: { failures: 0, warned: false } };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function statePath(): string {
|
|
68
|
+
return join(configDir(), "arcane-watch-state.json");
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function readWatchState(): WatchState {
|
|
72
|
+
try {
|
|
73
|
+
return { ...emptyWatchState(), ...(JSON.parse(readFileSync(statePath(), "utf-8")) as Partial<WatchState>) };
|
|
74
|
+
} catch {
|
|
75
|
+
return emptyWatchState();
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function writeWatchState(state: WatchState): void {
|
|
80
|
+
const path = statePath();
|
|
81
|
+
writeFileSync(`${path}.tmp`, `${JSON.stringify(state)}\n`, { mode: 0o600 });
|
|
82
|
+
renameSync(`${path}.tmp`, path);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function isContainer(value: unknown): value is WatchedContainer {
|
|
86
|
+
const c = value as Partial<WatchedContainer> | null;
|
|
87
|
+
return !!c && typeof c.name === "string" && typeof c.id === "string" && typeof c.state === "string" && typeof c.health === "string";
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** The `container-watch` answer of the arcane wrapper. */
|
|
91
|
+
export function parseSnapshot(value: unknown): WatchedContainer[] | null {
|
|
92
|
+
const r = value as { containers?: unknown } | null;
|
|
93
|
+
return r && Array.isArray(r.containers) && r.containers.every(isContainer) ? r.containers : null;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function operationText(o: Operation): string {
|
|
97
|
+
return `${o.command} (${o.caller})`;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** What is wrong with a container that should be up, or null. */
|
|
101
|
+
export function healthProblem(c: WatchedContainer, sinceMs: number): string | null {
|
|
102
|
+
if (c.state === "exited") return c.exitCode === 0 ? null : `exited with code ${c.exitCode ?? "?"}`;
|
|
103
|
+
if (c.state === "restarting" || c.state === "dead" || c.state === "created") return c.state === "created" ? "never started" : c.state;
|
|
104
|
+
if (c.health === "unhealthy") return "unhealthy";
|
|
105
|
+
if (c.health === "starting" && sinceMs >= STARTING_LIMIT_MS) return "health check still starting";
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Compares a fresh snapshot with the last one: a new id is a redeploy, whose
|
|
111
|
+
* health is checked after the grace time; a newer start time with the same id
|
|
112
|
+
* is a restart. The first snapshot only records the baseline.
|
|
113
|
+
*/
|
|
114
|
+
export function observe(state: WatchState, snapshot: readonly WatchedContainer[], now: number, nasOutage: boolean): Observation {
|
|
115
|
+
const out: Observation = { escalations: [], unhealthy: [], notes: [] };
|
|
116
|
+
const baseline = Object.keys(state.containers).length === 0;
|
|
117
|
+
const at = new Date(now).toISOString();
|
|
118
|
+
const nasExpected = (c: { project?: string }) => nasOutage && c.project !== undefined && NAS_STACKS.includes(c.project);
|
|
119
|
+
const current = new Map(snapshot.map((c) => [c.name, c]));
|
|
120
|
+
for (const c of snapshot) {
|
|
121
|
+
const prev = state.containers[c.name];
|
|
122
|
+
if (prev && !baseline) {
|
|
123
|
+
if (prev.id !== c.id) {
|
|
124
|
+
if (!state.pending.some((p) => p.id === c.id)) {
|
|
125
|
+
state.pending.push({ name: c.name, id: c.id, since: c.startedAt ?? c.created ?? at, newImage: prev.imageId !== c.imageId, ...(c.project ? { project: c.project } : {}), ...(c.operation ? { operation: operationText(c.operation) } : {}) });
|
|
126
|
+
}
|
|
127
|
+
} else if (c.startedAt && c.startedAt !== prev.startedAt && Date.parse(c.startedAt) > Date.parse(prev.seenAt)) {
|
|
128
|
+
if (c.operation) out.notes.push(`${c.name} restarted during ${operationText(c.operation)}`);
|
|
129
|
+
else if (nasExpected(c)) out.notes.push(`${c.name} restarted during a NAS outage`);
|
|
130
|
+
else state.restarts[c.name] = [...(state.restarts[c.name] ?? []), c.startedAt];
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
const startedAt = c.startedAt ?? (prev?.id === c.id ? prev.startedAt : null);
|
|
134
|
+
state.containers[c.name] = { id: c.id, imageId: c.imageId, startedAt, seenAt: at };
|
|
135
|
+
}
|
|
136
|
+
for (const name of Object.keys(state.containers)) if (!current.has(name)) delete state.containers[name];
|
|
137
|
+
for (const [name, times] of Object.entries(state.restarts)) {
|
|
138
|
+
const recent = times.filter((t) => Date.parse(t) >= now - DAY);
|
|
139
|
+
if (recent.length === 0) delete state.restarts[name];
|
|
140
|
+
else state.restarts[name] = recent;
|
|
141
|
+
}
|
|
142
|
+
state.escalated = Object.fromEntries(Object.entries(state.escalated).filter(([, t]) => Date.parse(t) >= now - DAY));
|
|
143
|
+
for (const [name, times] of Object.entries(state.restarts)) {
|
|
144
|
+
if (times.length < RESTARTS_TO_ESCALATE || state.escalated[name]) continue;
|
|
145
|
+
state.escalated[name] = at;
|
|
146
|
+
const project = current.get(name)?.project;
|
|
147
|
+
out.escalations.push({ name, ...(project ? { project } : {}), restarts: [...times] });
|
|
148
|
+
}
|
|
149
|
+
const waiting: PendingHealth[] = [];
|
|
150
|
+
for (const p of state.pending) {
|
|
151
|
+
const c = current.get(p.name);
|
|
152
|
+
if (!c || c.id !== p.id) continue;
|
|
153
|
+
const age = now - Date.parse(p.since);
|
|
154
|
+
if (age < HEALTH_GRACE_MS) {
|
|
155
|
+
waiting.push(p);
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
const problem = healthProblem(c, age);
|
|
159
|
+
if (problem === null && c.health === "starting") {
|
|
160
|
+
waiting.push(p);
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
if (problem === null) continue;
|
|
164
|
+
if (p.operation) out.notes.push(`${p.name} ${problem} after ${p.operation}, which reports it`);
|
|
165
|
+
else if (nasExpected(p)) out.notes.push(`${p.name} ${problem} during a NAS outage`);
|
|
166
|
+
else out.unhealthy.push({ name: p.name, ...(p.project ? { project: p.project } : {}), since: p.since, newImage: p.newImage, problem });
|
|
167
|
+
}
|
|
168
|
+
state.pending = waiting;
|
|
169
|
+
return out;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function mention(owners: readonly string[]): string {
|
|
173
|
+
return owners.map((id) => `<@${id}>`).join(" ");
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function where(name: string, project?: string): string {
|
|
177
|
+
return `${escapeDiscord(name)}${project && project !== name ? ` (stack ${escapeDiscord(project)})` : ""}`;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
export function escalationText(e: Escalation, owners: readonly string[], timeZone: string): string {
|
|
181
|
+
const last = e.restarts[e.restarts.length - 1];
|
|
182
|
+
return [
|
|
183
|
+
`🔁 **Container restarting** · ${where(e.name, e.project)} · ${e.restarts.length} restarts in 24 h, last ${clockText(last, timeZone)}`,
|
|
184
|
+
"Auto-heal or Docker restarted it each time. The arcane specialist investigates in the thread and changes nothing.",
|
|
185
|
+
mention(owners),
|
|
186
|
+
].join("\n");
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
export function unhealthyText(u: Unhealthy, owners: readonly string[], timeZone: string, now: number): string {
|
|
190
|
+
const minutes = Math.round((now - Date.parse(u.since)) / MINUTE);
|
|
191
|
+
return [
|
|
192
|
+
`⚠️ **Container ${u.problem} after a redeploy** · ${where(u.name, u.project)} · recreated ${clockText(u.since, timeZone)}${u.newImage ? " with a new image" : ""}, still ${u.problem} ${minutes} min later`,
|
|
193
|
+
"No stack-update or other specialist operation explains the redeploy. Ask the arcane specialist here to investigate.",
|
|
194
|
+
mention(owners),
|
|
195
|
+
].join("\n");
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
export function watchFailingText(minutes: number, reason: string, owners: readonly string[]): string {
|
|
199
|
+
return `⚠️ **Container checks failing:** the bridge could not read Arcane's containers for ${minutes} minutes (${escapeDiscord(reason.slice(0, 200))}). Restarts and redeploys are not checked meanwhile; the bridge log on Cortex has details.\n${mention(owners)}`;
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
export const WATCH_RECOVERED_TEXT = "✅ Container checks work again.";
|
|
203
|
+
|
|
204
|
+
/** The read-only task astro.arcane gets in the escalation's thread. */
|
|
205
|
+
export function restartInvestigationTask(e: Escalation, now: number): string {
|
|
206
|
+
// The host's log and OOM windows span at most one day.
|
|
207
|
+
const since = new Date(Math.max(Date.parse(e.restarts[0]) - 10 * MINUTE, now - DAY + MINUTE)).toISOString();
|
|
208
|
+
const until = new Date(now).toISOString();
|
|
209
|
+
return [
|
|
210
|
+
`Investigate why the container ${e.name}${e.project ? ` (stack ${e.project})` : ""} restarted ${e.restarts.length} times in the last 24 hours, at ${e.restarts.join(", ")} (UTC).`,
|
|
211
|
+
"Read only: change nothing, restart nothing, and run no stack-update, manager-upgrade, git write, or other write command. A fix goes through GitOps and needs an owner's approval, so describe it instead.",
|
|
212
|
+
`Collect evidence with \`containers get ${e.name}\`, \`host container-logs ${e.name} ${since} ${until}\`, \`host container-oom ${e.name} ${since} ${until}\`, \`host kernel-oom ${since} ${until}\`, and \`host container-memory ${e.name}\`. Name each command whose output you rely on.`,
|
|
213
|
+
"Answer with the likely cause, the evidence for it, and the change you would propose.",
|
|
214
|
+
].join("\n");
|
|
215
|
+
}
|
|
@@ -68,6 +68,8 @@ export interface ServiceLine {
|
|
|
68
68
|
latest: string | null;
|
|
69
69
|
/** The tag the Compose file names (`latest`, `18`, `2.9.13`); what "new build of" refers to. */
|
|
70
70
|
tag: string;
|
|
71
|
+
/** The image reference in the Compose file; the Security Daily Report matches scanned images to stacks by it. */
|
|
72
|
+
ref?: string;
|
|
71
73
|
major: boolean;
|
|
72
74
|
note: string;
|
|
73
75
|
review?: Review;
|
|
@@ -146,7 +148,7 @@ export function buildStacks(report: unknown): ReportStacks | null {
|
|
|
146
148
|
.map((r): ServiceLine => {
|
|
147
149
|
const running = releaseTag(r.running);
|
|
148
150
|
const latest = releaseTag(r.latest);
|
|
149
|
-
return { service: r.service, status: r.status, action: r.action ?? "", running, latest, tag: tagOf(r.ref ?? ""), major: PENDING.has(r.status) && crossesMajor(running, latest), note: r.note ?? "" };
|
|
151
|
+
return { service: r.service, status: r.status, action: r.action ?? "", running, latest, tag: tagOf(r.ref ?? ""), ...(r.ref ? { ref: r.ref } : {}), major: PENDING.has(r.status) && crossesMajor(running, latest), note: r.note ?? "" };
|
|
150
152
|
});
|
|
151
153
|
const pending = lines.some((l) => PENDING.has(l.status));
|
|
152
154
|
const held = holds[stack];
|