@astrofoundry/pi-astro 0.40.64 → 0.40.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agents/media.md +3 -2
- package/extensions/astro-discord/arcane-watch-monitor.ts +2 -5
- package/extensions/astro-discord/arcane-watch.test.ts +4 -11
- package/extensions/astro-discord/arcane-watch.ts +0 -5
- package/extensions/astro-discord/daily-report.test.ts +8 -0
- package/extensions/astro-discord/daily-report.ts +32 -4
- package/extensions/astro-discord/daily-run.test.ts +56 -17
- package/extensions/astro-discord/daily-run.ts +106 -20
- package/extensions/astro-discord/index.ts +11 -8
- package/extensions/astro-discord/reports.ts +2 -0
- package/extensions/astro-discord/security-monitor.ts +2 -1
- package/extensions/astro-discord/security.test.ts +49 -21
- package/extensions/astro-discord/security.ts +79 -23
- package/extensions/specialist-gate/approvals.test.ts +2 -0
- package/extensions/specialist-gate/index.ts +2 -1
- package/package.json +1 -1
- package/skills/arcane/SKILL.md +4 -0
- package/skills/media/SKILL.md +9 -1
- package/specialists/AGENTS.md +2 -2
- package/specialists/README.md +2 -2
- package/specialists/arcane/run.ts +121 -25
- package/specialists/lib/image-updates.test.ts +12 -2
- package/specialists/lib/image-updates.ts +25 -10
- package/specialists/media/run.ts +90 -1
- package/specialists/wrappers.test.ts +19 -1
package/agents/media.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: media
|
|
3
|
-
description: Manages the movie and series library through the media tool (trendarr), covering weekly trending with IMDb ratings, recommendations from the library, title search, adds to Radarr and Sonarr,
|
|
3
|
+
description: Manages the movie and series library through the media tool (trendarr), covering weekly trending with IMDb ratings, recommendations from the library, title search, adds to Radarr and Sonarr, the SABnzbd queue pause state, and imports stuck in the Radarr and Sonarr queues. Use for what is trending, adding a movie or series by name, pausing or resuming downloads, and clearing a stuck import.
|
|
4
4
|
tools: read, media, subagent
|
|
5
5
|
skills: media
|
|
6
6
|
systemPromptMode: replace
|
|
@@ -20,5 +20,6 @@ Rules:
|
|
|
20
20
|
- Success needs an explicit `added` or `already_exists` in `results`; `already_exists` means it was already there, not newly added. Report every ID of a batch.
|
|
21
21
|
- Pause or resume downloads only when the task asks for it, then report the returned state. Adding never implies a queue change.
|
|
22
22
|
- Keep replies short: canonical title and year, and the outcome. When an add returned `sabnzbd_paused: true`, end with "Downloads are paused."; say nothing about the queue otherwise. Outside failures, name no tool or service (SABnzbd, Radarr, Sonarr, trendarr, TMDB, OMDb): say downloads, library, search.
|
|
23
|
-
-
|
|
23
|
+
- Remove a queue record only with `queue-remove` after `queue` shows it and its reason, following the skill's stuck-import rules; the tool asks an owner for approval.
|
|
24
|
+
- Library settings, quality profiles, Radarr and Sonarr configuration, and deletions other than a stuck queue record are operator tasks; say so and stop.
|
|
24
25
|
- Delegate through `subagent` only to another specialist (`astro.arcane` for the media containers on the Arcane VM) and only for that specialist's own area.
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
2
|
escalationText,
|
|
3
|
-
hostRestartText,
|
|
4
3
|
observe,
|
|
5
4
|
parseSnapshot,
|
|
6
5
|
readWatchState,
|
|
@@ -100,10 +99,8 @@ export class ArcaneWatch {
|
|
|
100
99
|
const now = this.deps.now();
|
|
101
100
|
const seen = observe(state, snapshot, now, this.deps.nasOutage(now));
|
|
102
101
|
for (const note of seen.notes) this.deps.log(`arcane watch: ${note}`);
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
await this.post(s.channelId, { content: hostRestartText(e, s.timeZone) });
|
|
106
|
-
}
|
|
102
|
+
// Docker or the VM restarted every container (the nightly upgrade): the log is enough, nobody has to act.
|
|
103
|
+
for (const e of seen.hostRestarts) this.deps.log(`arcane watch: host-wide restart, ${e.count} of ${e.total} containers restarted between ${e.from} and ${e.to}`);
|
|
107
104
|
for (const u of seen.unhealthy) {
|
|
108
105
|
const posted = await this.post(s.channelId, { content: unhealthyText(u, owners, s.timeZone, now), mentionUsers: owners });
|
|
109
106
|
if (posted) this.deps.log(`arcane watch: ${u.name} ${u.problem} after a redeploy`);
|
|
@@ -4,7 +4,7 @@ import { join } from "node:path";
|
|
|
4
4
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
5
5
|
import type { WatchedContainer } from "../../specialists/lib/container-watch.ts";
|
|
6
6
|
import { ArcaneWatch } from "./arcane-watch-monitor.ts";
|
|
7
|
-
import { emptyWatchState, escalationText, healthProblem,
|
|
7
|
+
import { emptyWatchState, escalationText, healthProblem, observe, parseSnapshot, readWatchState, restartInvestigationTask, unhealthyText, type WatchState } from "./arcane-watch.ts";
|
|
8
8
|
import type { DiscordConfig } from "./config.ts";
|
|
9
9
|
import type { CreateMessage } from "./rest.ts";
|
|
10
10
|
import type { InvestigationRequest } from "./security-monitor.ts";
|
|
@@ -152,12 +152,6 @@ describe("container watch texts", () => {
|
|
|
152
152
|
);
|
|
153
153
|
});
|
|
154
154
|
|
|
155
|
-
it("names a host-wide restart without tagging anyone", () => {
|
|
156
|
-
expect(hostRestartText({ count: 46, total: 52, from: iso(T0), to: iso(T0 + 8_000) }, TZ)).toBe(
|
|
157
|
-
"🔁 **Host-wide restart** · 46 of 52 containers restarted at 22 Oct 10:00 CEST, so Docker or the Arcane VM restarted. They do not count as container restarts.",
|
|
158
|
-
);
|
|
159
|
-
});
|
|
160
|
-
|
|
161
155
|
it("asks the specialist to read only, within the host's one-day window", () => {
|
|
162
156
|
const now = T0 + 24 * 60 * MIN;
|
|
163
157
|
const task = restartInvestigationTask({ name: "syncthing", restarts: [iso(T0 + MIN), iso(now - MIN)] }, now);
|
|
@@ -257,7 +251,7 @@ describe("container watch monitor", () => {
|
|
|
257
251
|
expect(h.archived).toEqual(["t500"]);
|
|
258
252
|
});
|
|
259
253
|
|
|
260
|
-
it("
|
|
254
|
+
it("logs a host-wide restart, posts nothing, and investigates nothing", async () => {
|
|
261
255
|
const h = harness();
|
|
262
256
|
const three = ["a", "b", "c"].map((name) => ct({ name, id: name }));
|
|
263
257
|
h.answer.next = () => JSON.stringify({ containers: three });
|
|
@@ -266,9 +260,8 @@ describe("container watch monitor", () => {
|
|
|
266
260
|
const started = iso(T0 + 4 * MIN);
|
|
267
261
|
h.answer.next = () => JSON.stringify({ containers: three.map((c) => ({ ...c, startedAt: started })) });
|
|
268
262
|
await h.watch.poll(SCHEDULE);
|
|
269
|
-
expect(h.messages).
|
|
270
|
-
expect(h.
|
|
271
|
-
expect(h.messages[0].message.mentionUsers).toBeUndefined();
|
|
263
|
+
expect(h.messages).toEqual([]);
|
|
264
|
+
expect(h.logs.some((l) => l.includes("host-wide restart, 3 of 3 containers"))).toBe(true);
|
|
272
265
|
expect(h.investigations).toEqual([]);
|
|
273
266
|
expect(readWatchState().restarts).toEqual({});
|
|
274
267
|
});
|
|
@@ -218,11 +218,6 @@ export function escalationText(e: Escalation, owners: readonly string[], timeZon
|
|
|
218
218
|
].join("\n");
|
|
219
219
|
}
|
|
220
220
|
|
|
221
|
-
/** Untagged: Docker or the VM restarting is planned maintenance far more often than a fault. */
|
|
222
|
-
export function hostRestartText(e: HostRestart, timeZone: string): string {
|
|
223
|
-
return `🔁 **Host-wide restart** · ${e.count} of ${e.total} containers restarted at ${clockText(e.from, timeZone)}, so Docker or the Arcane VM restarted. They do not count as container restarts.`;
|
|
224
|
-
}
|
|
225
|
-
|
|
226
221
|
export function unhealthyText(u: Unhealthy, owners: readonly string[], timeZone: string, now: number): string {
|
|
227
222
|
const minutes = Math.round((now - Date.parse(u.since)) / MINUTE);
|
|
228
223
|
return [
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
releaseNotesFromJson,
|
|
21
21
|
REPORT_PREFIX,
|
|
22
22
|
serviceText,
|
|
23
|
+
serviceUsage,
|
|
23
24
|
type ServiceLine,
|
|
24
25
|
stackComponent,
|
|
25
26
|
type StackEntry,
|
|
@@ -172,6 +173,13 @@ describe("Arcane Daily Report layout", () => {
|
|
|
172
173
|
if (!empty) throw new Error("no report");
|
|
173
174
|
expect(headText({ ...empty, timeZone: "Europe/Madrid" }).content).toBe(`**${REPORT_PREFIX}** no updates (44 images, 2 pinned private excluded, checked 06:00 CEST).\n-# ⚠️ update-holds.json is not JSON; Update still refuses held stacks`);
|
|
174
175
|
expect(buildStacks({ nonsense: true })).toBeNull();
|
|
176
|
+
const remote = buildStacks({ ...updateReport, holds: {}, rows: [{ ...rows[2], stack: "media-mira", environment: "mira" }], environmentErrors: [{ environment: "lab", error: "agent offline" }, { broken: true }], agentsBehind: [{ environment: "mira", agent: "v2.17.3", manager: "v2.18.0" }] });
|
|
177
|
+
if (!remote) throw new Error("no report");
|
|
178
|
+
expect(remote.stacks[0].lines[0].environment).toBe("mira");
|
|
179
|
+
expect(serviceUsage("media-mira", remote.stacks[0].lines[0])).toContain("on the Docker host of the Arcane environment mira,");
|
|
180
|
+
expect(remote.environmentErrors).toEqual([{ environment: "lab", error: "agent offline" }]);
|
|
181
|
+
expect(headText({ ...remote, timeZone: "Europe/Madrid" }).content).toContain("\n-# ⚠️ environment lab was not checked: agent offline");
|
|
182
|
+
expect(headText({ ...remote, timeZone: "Europe/Madrid" }).content).toContain("\n-# ⚠️ the Arcane agent of mira runs v2.17.3, the manager v2.18.0");
|
|
175
183
|
expect(failureMessage().startsWith(REPORT_PREFIX)).toBe(true);
|
|
176
184
|
});
|
|
177
185
|
|
|
@@ -23,6 +23,20 @@ interface ReportRow {
|
|
|
23
23
|
action: string;
|
|
24
24
|
note: string;
|
|
25
25
|
stopped?: boolean;
|
|
26
|
+
environment?: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** A remote Arcane environment the update check could not reach. */
|
|
30
|
+
export interface EnvironmentError {
|
|
31
|
+
environment: string;
|
|
32
|
+
error: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** A remote environment whose agent runs another version than the manager. */
|
|
36
|
+
export interface AgentBehind {
|
|
37
|
+
environment: string;
|
|
38
|
+
agent: string;
|
|
39
|
+
manager: string;
|
|
26
40
|
}
|
|
27
41
|
|
|
28
42
|
interface UpdateReportJson {
|
|
@@ -32,6 +46,8 @@ interface UpdateReportJson {
|
|
|
32
46
|
rows: ReportRow[];
|
|
33
47
|
holds?: Record<string, string>;
|
|
34
48
|
holdsError?: string;
|
|
49
|
+
environmentErrors?: EnvironmentError[];
|
|
50
|
+
agentsBehind?: AgentBehind[];
|
|
35
51
|
}
|
|
36
52
|
|
|
37
53
|
function isRow(value: unknown): value is ReportRow {
|
|
@@ -85,6 +101,8 @@ export interface ServiceLine {
|
|
|
85
101
|
own?: boolean;
|
|
86
102
|
/** Why this service is held (`<stack>/<service>` in `update-holds.json`); stack-update skips it. */
|
|
87
103
|
held?: string;
|
|
104
|
+
/** The Arcane remote environment that runs the stack; absent for the Arcane VM. */
|
|
105
|
+
environment?: string;
|
|
88
106
|
}
|
|
89
107
|
|
|
90
108
|
export type StackState = "offered" | "running" | "updated" | "rolled back" | "failed" | "current" | "planning";
|
|
@@ -122,6 +140,8 @@ export interface ArcaneReport {
|
|
|
122
140
|
images: number;
|
|
123
141
|
privateCount: number;
|
|
124
142
|
holdsError?: string;
|
|
143
|
+
environmentErrors?: EnvironmentError[];
|
|
144
|
+
agentsBehind?: AgentBehind[];
|
|
125
145
|
stacks: StackEntry[];
|
|
126
146
|
messages: PostedPage[];
|
|
127
147
|
/** Owners tagged on the first message because an update needs them. */
|
|
@@ -148,6 +168,8 @@ export interface ReportStacks {
|
|
|
148
168
|
privateCount: number;
|
|
149
169
|
checkedAt: string;
|
|
150
170
|
holdsError?: string;
|
|
171
|
+
environmentErrors?: EnvironmentError[];
|
|
172
|
+
agentsBehind?: AgentBehind[];
|
|
151
173
|
}
|
|
152
174
|
|
|
153
175
|
/**
|
|
@@ -159,6 +181,8 @@ export interface ReportStacks {
|
|
|
159
181
|
export function buildStacks(report: unknown): ReportStacks | null {
|
|
160
182
|
if (!isReport(report)) return null;
|
|
161
183
|
const holds = report.holds ?? {};
|
|
184
|
+
const environmentErrors = (Array.isArray(report.environmentErrors) ? report.environmentErrors : []).filter((e): e is EnvironmentError => typeof e?.environment === "string" && typeof e?.error === "string");
|
|
185
|
+
const agentsBehind = (Array.isArray(report.agentsBehind) ? report.agentsBehind : []).filter((a): a is AgentBehind => typeof a?.environment === "string" && typeof a?.agent === "string" && typeof a?.manager === "string");
|
|
162
186
|
const shown = report.rows.filter((r) => !r.stopped && (PENDING.has(r.status) || r.status === "unknown" || /newer major \S+ available/.test(r.note ?? "")));
|
|
163
187
|
const names = [...new Set(shown.map((r) => r.stack))].sort((a, b) => a.localeCompare(b));
|
|
164
188
|
const stacks = names.map((stack): StackEntry => {
|
|
@@ -169,7 +193,7 @@ export function buildStacks(report: unknown): ReportStacks | null {
|
|
|
169
193
|
const latest = releaseTag(r.latest);
|
|
170
194
|
const own = r.ref ? isOwnImage(r.ref) : false;
|
|
171
195
|
const held = holds[`${stack}/${r.service}`];
|
|
172
|
-
return { service: r.service, status: r.status, action: r.action ?? "", running, latest, tag: tagOf(r.ref ?? ""), ...(r.ref ? { ref: r.ref } : {}), major: PENDING.has(r.status) && crossesMajor(running, latest), note: r.note ?? "", ...(own ? { own } : {}), ...(held ? { held } : {}) };
|
|
196
|
+
return { service: r.service, status: r.status, action: r.action ?? "", running, latest, tag: tagOf(r.ref ?? ""), ...(r.ref ? { ref: r.ref } : {}), major: PENDING.has(r.status) && crossesMajor(running, latest), note: r.note ?? "", ...(own ? { own } : {}), ...(held ? { held } : {}), ...(r.environment ? { environment: r.environment } : {}) };
|
|
173
197
|
});
|
|
174
198
|
const pending = lines.some(isPending);
|
|
175
199
|
const held = holds[stack];
|
|
@@ -179,7 +203,7 @@ export function buildStacks(report: unknown): ReportStacks | null {
|
|
|
179
203
|
}
|
|
180
204
|
return { stack, lines, actionable: pending && !held, ...(held ? { held } : {}), state: "offered" };
|
|
181
205
|
});
|
|
182
|
-
return { stacks, images: report.images, privateCount: report.privateImages.length, checkedAt: report.checkedAt, ...(report.holdsError ? { holdsError: report.holdsError } : {}) };
|
|
206
|
+
return { stacks, images: report.images, privateCount: report.privateImages.length, checkedAt: report.checkedAt, ...(report.holdsError ? { holdsError: report.holdsError } : {}), ...(environmentErrors.length > 0 ? { environmentErrors } : {}), ...(agentsBehind.length > 0 ? { agentsBehind } : {}) };
|
|
183
207
|
}
|
|
184
208
|
|
|
185
209
|
/** Services whose update crosses a major version; each gets a release-notes review before the report posts. */
|
|
@@ -302,7 +326,10 @@ export function headText(report: ReportView): MessageComponent {
|
|
|
302
326
|
report.stacks.length === 0
|
|
303
327
|
? `**${REPORT_PREFIX}** no updates (${scope}).`
|
|
304
328
|
: `**${REPORT_PREFIX}** ${withUpdates} stack${withUpdates === 1 ? "" : "s"} with updates (${scope}).`;
|
|
305
|
-
const warning =
|
|
329
|
+
const warning =
|
|
330
|
+
(report.holdsError ? `\n-# ⚠️ ${escapeDiscord(report.holdsError)}; Update still refuses held stacks` : "") +
|
|
331
|
+
(report.environmentErrors ?? []).map((e) => `\n-# ⚠️ environment ${escapeDiscord(e.environment)} was not checked: ${escapeDiscord(e.error.slice(0, 200))}`).join("") +
|
|
332
|
+
(report.agentsBehind ?? []).map((a) => `\n-# ⚠️ the Arcane agent of ${escapeDiscord(a.environment)} runs ${escapeDiscord(a.agent)}, the manager ${escapeDiscord(a.manager)}`).join("");
|
|
306
333
|
const attention = report.stacks.filter((s) => s.plan === "update" || s.plan === "careful");
|
|
307
334
|
const automatic = report.stacks.filter((s) => s.plan === "auto").length;
|
|
308
335
|
const plan = automatic > 0 ? `\n-# ${automatic} stack${automatic === 1 ? " updates" : "s update"} automatically and ${automatic === 1 ? "is" : "are"} checked again 5 minutes later.` : "";
|
|
@@ -407,7 +434,8 @@ export function releaseNotesFromJson(value: unknown): StackNotes {
|
|
|
407
434
|
|
|
408
435
|
/** What the reviewer is told about a service of a stack, so it can tell what a release would break. */
|
|
409
436
|
export function serviceUsage(stack: string, line: ServiceLine): string {
|
|
410
|
-
|
|
437
|
+
const host = line.environment ? `on the Docker host of the Arcane environment ${line.environment}` : "on the Arcane VM";
|
|
438
|
+
return `${line.service} is a service of the Docker Compose stack ${stack} ${host}, deployed through GitOps. The update replaces its image (tag ${line.tag}) from ${line.running ?? "an older build"} to ${line.latest ?? "a newer build"} and changes nothing else in the Compose file: no environment variables, volumes, ports, or companion services. Judge whether the release needs a configuration, data, or companion change before or after the swap.`;
|
|
411
439
|
}
|
|
412
440
|
|
|
413
441
|
// ---------------------------------------------------------------- posted report state
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { describe, expect, it } from "vitest";
|
|
2
2
|
import type { ArcaneReport } from "./daily-report.ts";
|
|
3
3
|
import type { ToolsReport } from "./tools-report.ts";
|
|
4
|
-
import { agoSeconds, arcaneSummary, guestOwner, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, pomeriumButtonId, wazuhRelease, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemSpecialist, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
4
|
+
import { type AreaStatus, agoSeconds, arcaneSummary, guestOwner, backupStatus, dnsStatus, macosStatus, edgeStatus, identityStatus, pomeriumButtonId, wazuhRelease, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemSpecialist, problemsToPost, problemText, proxmoxStatus, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
5
5
|
|
|
6
6
|
const EDGE = `nginx active
|
|
7
7
|
wg-quick@wg0 active
|
|
@@ -119,6 +119,9 @@ describe("morning status lines", () => {
|
|
|
119
119
|
expect(newer.line).toBe("pomerium, nginx, lego-renew.timer active; Pomerium v0.33.1, v0.33.6 available");
|
|
120
120
|
expect(newer.problem).toContain(`Pomerium v0.33.6 is out (running v0.33.1; notes: <${notes}>)`);
|
|
121
121
|
expect(newer.buttons).toEqual([{ label: "Update Pomerium to v0.33.6", customId: pomeriumButtonId("v0.33.6") }]);
|
|
122
|
+
expect(newer.once).toBe("pomerium:v0.33.6");
|
|
123
|
+
const failedUnit = DMZ.replace("Active: active (running) since Thu 2026-09-24 17:21:24 CEST; 4 weeks 0 days ago\n● nginx", "Active: failed (Result: signal)\n● nginx");
|
|
124
|
+
expect(identityStatus(failedUnit, { installed: "v0.33.1", newest: "v0.33.6", update: true, notes }).once).toBeUndefined();
|
|
122
125
|
expect(identityStatus(DMZ, null).line).toBe("pomerium, nginx, lego-renew.timer active; Pomerium version unknown");
|
|
123
126
|
});
|
|
124
127
|
|
|
@@ -128,16 +131,25 @@ describe("morning status lines", () => {
|
|
|
128
131
|
expect(dnsStatus({ technitium1: null, technitium2: { status: "error" }, technitium3: { status: "ok" } })).toMatchObject({ problem: "DNS: technitium1 and technitium2 do not answer" });
|
|
129
132
|
});
|
|
130
133
|
|
|
131
|
-
it("flags
|
|
132
|
-
const nexus = "26.
|
|
133
|
-
const cortex = "26.
|
|
134
|
-
expect(macosStatus([{ name: "
|
|
135
|
-
|
|
134
|
+
it("flags any newer macOS of the installed major for its owner and notes a major without flagging it", () => {
|
|
135
|
+
const nexus = "26.7.2\nTitle: Command Line Tools for Xcode 26.6, Version: 26.6, Size: 920431KiB, Recommended: YES,\nTitle: Safari, Version: 27.2, Size: 250130KiB, Recommended: YES,\nTitle: macOS 27.2.1, Version: 27.2.1, Size: 17362949KiB, Recommended: YES, Action: restart,";
|
|
136
|
+
const cortex = "26.7.2\nTitle: macOS Tahoe 26.7.3, Version: 26.7.3, Size: 2506663KiB, Recommended: YES, Action: restart,\nTitle: macOS 27.2.1, Version: 27.2.1, Size: 17362949KiB, Recommended: YES, Action: restart,";
|
|
137
|
+
expect(macosStatus([{ name: "Nexus", output: nexus }])).toEqual({ area: "macOS", line: "Nexus 26.7.2, macOS 27.2.1 offered (major, by hand)" });
|
|
138
|
+
const waiting = "Cortex runs macOS 26.7.2; macOS Tahoe 26.7.3 waits: install it with softwareupdate as a volume owner, then check that Cortex comes back";
|
|
139
|
+
expect(macosStatus([{ name: "Cortex", output: cortex }, { name: "Nexus", output: nexus.replace("Title: Safari", "Title: macOS Tahoe 26.7.3, Version: 26.7.3, Size: 1KiB,\nTitle: Safari") }])).toEqual({
|
|
136
140
|
area: "macOS",
|
|
137
|
-
line: "
|
|
138
|
-
problem:
|
|
141
|
+
line: "Cortex 26.7.2, 26.7.3 waiting, macOS 27.2.1 offered (major, by hand); Nexus 26.7.2, 26.7.3 waiting, macOS 27.2.1 offered (major, by hand)",
|
|
142
|
+
problem: `macOS: ${waiting}; ${waiting.replaceAll("Cortex", "Nexus")}`,
|
|
143
|
+
items: [
|
|
144
|
+
{ owner: "Cortex", text: waiting },
|
|
145
|
+
{ owner: "Inference", text: waiting.replaceAll("Cortex", "Nexus") },
|
|
146
|
+
],
|
|
139
147
|
route: "admin",
|
|
148
|
+
once: "macos:Cortex:26.7.3,Nexus:26.7.3",
|
|
140
149
|
});
|
|
150
|
+
const unknown = macosStatus([{ name: "Cortex", output: cortex }, { name: "Nexus", output: null }]);
|
|
151
|
+
expect(unknown.items?.at(-1)).toEqual({ owner: "Inference", text: "no macOS update list from Nexus" });
|
|
152
|
+
expect(unknown.once).toBeUndefined();
|
|
141
153
|
expect(macosStatus([{ name: "Cortex", output: "26.7.2" }]).line).toBe("Cortex 26.7.2");
|
|
142
154
|
});
|
|
143
155
|
|
|
@@ -155,8 +167,26 @@ describe("morning status lines", () => {
|
|
|
155
167
|
expect(dnsStatus(all, { major: true, from: "15.6.2", to: "16.0.0" }).problem).toMatch(/^DNS: Technitium 16.0.0 is a major release \(running 15.6.2\)/);
|
|
156
168
|
expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=200\nrerank \tstate = running \tpid = 81670 health=200")).toEqual({ area: "Inference", line: "Nexus embed, rerank running and healthy" });
|
|
157
169
|
expect(inferenceStatus("embed \tstate = running \tpid = 80042 health=503")).toMatchObject({ problem: "Inference: embed not healthy on Nexus" });
|
|
158
|
-
|
|
159
|
-
expect(mediaStatus({ sabnzbd_paused:
|
|
170
|
+
const empty = { records: [], warnings: [] };
|
|
171
|
+
expect(mediaStatus({ sabnzbd_paused: true, warnings: [] }, empty, NOW)).toEqual({ area: "Media", line: "downloads paused" });
|
|
172
|
+
expect(mediaStatus({ sabnzbd_paused: false, warnings: [{ text: "login failed" }] }, empty, NOW)).toMatchObject({ line: "downloads running, 1 warning", problem: "Media: the downloader shows 1 warning", route: "media" });
|
|
173
|
+
expect(mediaStatus({ sabnzbd_paused: false, warnings: [] }, null, NOW)).toMatchObject({ line: "downloads running, import queue unknown", problem: "Media: the import queue check failed" });
|
|
174
|
+
expect(mediaStatus({ sabnzbd_paused: false, warnings: [] }, { records: [], warnings: ["sonarr: sonarr answered 401 to GET /api/v3/queue"] }, NOW)).toMatchObject({ problem: "Media: the import queue of sonarr could not be read", route: "media" });
|
|
175
|
+
});
|
|
176
|
+
|
|
177
|
+
it("names imports that wait in the queue for more than a day, and tags them once", () => {
|
|
178
|
+
const record = (id: number, added: string) => ({ app: "radarr", id, title: "Primetime.2026.REPACK.2160p", state: "importPending", status: "warning", messages: ["Not a Custom Format upgrade for existing movie file(s)."], added, size: 13406787332 });
|
|
179
|
+
const queue = { records: [record(7, "2026-10-22T16:03:52Z"), record(8, "2026-10-23T20:00:00Z")], warnings: [] };
|
|
180
|
+
const item = "Primetime.2026.REPACK.2160p (radarr) waits in the queue since 2026-10-22 (importPending: Not a Custom Format upgrade for existing movie file(s).); reply here to remove it (queue-remove radarr 7)";
|
|
181
|
+
expect(mediaStatus({ sabnzbd_paused: false, warnings: [] }, queue, NOW)).toEqual({
|
|
182
|
+
area: "Media",
|
|
183
|
+
line: "downloads running, 1 import stuck",
|
|
184
|
+
problem: `Media: ${item}`,
|
|
185
|
+
items: [{ owner: "Media", text: item }],
|
|
186
|
+
route: "media",
|
|
187
|
+
once: "queue:radarr:7",
|
|
188
|
+
});
|
|
189
|
+
expect(mediaStatus({ sabnzbd_paused: false, warnings: ["x"] }, queue, NOW).once).toBeUndefined();
|
|
160
190
|
});
|
|
161
191
|
|
|
162
192
|
it("checks last night's Proxmox archives and the restic check", () => {
|
|
@@ -171,7 +201,7 @@ describe("morning status lines", () => {
|
|
|
171
201
|
});
|
|
172
202
|
|
|
173
203
|
describe("morning summary", () => {
|
|
174
|
-
it("gives each
|
|
204
|
+
it("gives each item its own line under its owner and names the owners with nothing open last, without a mention", () => {
|
|
175
205
|
const text = summaryText({
|
|
176
206
|
date: "Fri 24 Oct",
|
|
177
207
|
guildId: "1",
|
|
@@ -191,16 +221,23 @@ describe("morning summary", () => {
|
|
|
191
221
|
expect(text).toBe(
|
|
192
222
|
[
|
|
193
223
|
"**Daily** · Fri 24 Oct · 🟠 3 items need you",
|
|
194
|
-
"
|
|
195
|
-
"
|
|
196
|
-
"Arcane · stirling-pdf waits for you (major version) ([report](https://discord.com/channels/1/22/32)); arcane: docker-ce did not install overnight (<#11>)",
|
|
224
|
+
"Arcane · stirling-pdf waits for you (major version) ([report](https://discord.com/channels/1/22/32))",
|
|
225
|
+
"Arcane · arcane: docker-ce did not install overnight (<#11>)",
|
|
197
226
|
"Backup · no Proxmox backup last night for identity (<#11>)",
|
|
198
|
-
"
|
|
199
|
-
"DNS · All clear boss!",
|
|
227
|
+
"All clear: Security, Cortex, Proxmox, DNS",
|
|
200
228
|
].join("\n"),
|
|
201
229
|
);
|
|
202
230
|
expect(text).not.toContain("<@");
|
|
203
|
-
expect(summaryText({ date: "Fri 24 Oct", guildId: "1", routes: { admin: "11", media: "12" }, reports: [], areas: [{ area: "Media", line: "downloads running" }] })).toBe("**Daily** · Fri 24 Oct · 🟢 All clear boss!\
|
|
231
|
+
expect(summaryText({ date: "Fri 24 Oct", guildId: "1", routes: { admin: "11", media: "12" }, reports: [], areas: [{ area: "Media", line: "downloads running" }] })).toBe("**Daily** · Fri 24 Oct · 🟢 All clear boss!\nAll clear: Media");
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
it("tags the owners for a waiting release on its first morning only", () => {
|
|
235
|
+
const pomerium: AreaStatus = { area: "Identity", line: "x", problem: "Identity: Pomerium v0.34.0 is out", once: "pomerium:v0.34.0", route: "admin" };
|
|
236
|
+
const backup: AreaStatus = { area: "Backup", line: "x", problem: "Backup: a backup status check failed", route: "admin" };
|
|
237
|
+
const quiet: AreaStatus = { area: "DNS", line: "x" };
|
|
238
|
+
expect(problemsToPost([pomerium, backup, quiet], [])).toEqual({ post: [pomerium, backup], notified: ["pomerium:v0.34.0"] });
|
|
239
|
+
expect(problemsToPost([pomerium, backup, quiet], ["pomerium:v0.34.0", "macos:Cortex:26.7.3"])).toEqual({ post: [backup], notified: ["pomerium:v0.34.0"] });
|
|
240
|
+
expect(problemsToPost([{ ...pomerium, once: "pomerium:v0.34.1" }], ["pomerium:v0.34.0"]).post).toHaveLength(1);
|
|
204
241
|
});
|
|
205
242
|
|
|
206
243
|
it("tags the owners for an area's problem in its channel", () => {
|
|
@@ -257,5 +294,7 @@ describe("report summaries", () => {
|
|
|
257
294
|
channelId: "6",
|
|
258
295
|
messageId: "8",
|
|
259
296
|
});
|
|
297
|
+
const unchecked = { ...report, stacks: [], environmentErrors: [{ environment: "mira", error: "environment offline" }], agentsBehind: [{ environment: "lab", agent: "v2.17.3", manager: "v2.18.0" }] } as unknown as ArcaneReport;
|
|
298
|
+
expect(arcaneSummary(unchecked, "2026-10-24", "6").needs).toEqual(["Arcane environment mira was not checked for updates", "the Arcane agent of lab runs v2.17.3, the manager v2.18.0: move its tag on the host"]);
|
|
260
299
|
});
|
|
261
300
|
});
|
|
@@ -27,6 +27,12 @@ export interface AreaStatus {
|
|
|
27
27
|
items?: SummaryItem[];
|
|
28
28
|
route?: Route;
|
|
29
29
|
buttons?: AreaButton[];
|
|
30
|
+
/**
|
|
31
|
+
* Set when the problem only waits for the owners, such as a release with its
|
|
32
|
+
* button: its tagged message posts on the first morning with this key, and
|
|
33
|
+
* the summary keeps listing it every morning.
|
|
34
|
+
*/
|
|
35
|
+
once?: string;
|
|
30
36
|
}
|
|
31
37
|
|
|
32
38
|
const HOUR = 3600;
|
|
@@ -255,6 +261,7 @@ export function identityStatus(status: string | null, pomerium?: unknown): AreaS
|
|
|
255
261
|
line: [unitLine, releaseLine].filter(Boolean).join("; "),
|
|
256
262
|
...(problems.length > 0 ? { problem: `Identity: ${problems.join("; ")}`, route: "admin" as const } : {}),
|
|
257
263
|
...(newer ? { buttons: [{ label: `Update Pomerium to ${newer.newest}`, customId: pomeriumButtonId(newer.newest) }] } : {}),
|
|
264
|
+
...(newer && problems.length === 1 ? { once: `pomerium:${newer.newest}` } : {}),
|
|
258
265
|
};
|
|
259
266
|
}
|
|
260
267
|
|
|
@@ -310,13 +317,58 @@ export function inferenceStatus(status: string | null): AreaStatus {
|
|
|
310
317
|
return { area: "Inference", line, ...(bad.length > 0 || services.length === 0 ? { problem: `Inference: ${bad.length > 0 ? `${bad.join(", ")} not healthy on Nexus` : "Nexus lists no service"}`, route: "admin" as const } : {}) };
|
|
311
318
|
}
|
|
312
319
|
|
|
313
|
-
/**
|
|
314
|
-
export
|
|
320
|
+
/** One download that Radarr or Sonarr cannot import, from `media queue`. */
|
|
321
|
+
export interface StuckImport {
|
|
322
|
+
app: string;
|
|
323
|
+
id: number;
|
|
324
|
+
title: string;
|
|
325
|
+
state: string;
|
|
326
|
+
messages: string[];
|
|
327
|
+
added: string;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/** An import that waits this long has stopped waiting for anything but the owners. */
|
|
331
|
+
const STUCK_IMPORT_S = 24 * HOUR;
|
|
332
|
+
|
|
333
|
+
function stuckImports(queue: unknown): StuckImport[] | null {
|
|
334
|
+
const records = (queue as { records?: unknown } | null)?.records;
|
|
335
|
+
if (!Array.isArray(records)) return null;
|
|
336
|
+
return (records as Record<string, unknown>[])
|
|
337
|
+
.filter((r) => typeof r.app === "string" && typeof r.id === "number" && typeof r.title === "string" && typeof r.added === "string")
|
|
338
|
+
.map((r) => ({ app: r.app as string, id: r.id as number, title: r.title as string, state: typeof r.state === "string" ? r.state : "?", messages: Array.isArray(r.messages) ? r.messages.filter((m): m is string => typeof m === "string") : [], added: r.added as string }));
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/** The apps whose queue `media queue` could not read: its warnings start with the app name. */
|
|
342
|
+
function queueWarnings(queue: unknown): string[] {
|
|
343
|
+
const warnings = (queue as { warnings?: unknown } | null)?.warnings;
|
|
344
|
+
return Array.isArray(warnings) ? warnings.filter((w): w is string => typeof w === "string").map((w) => w.split(":")[0]) : [];
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/** `media sabnzbd status` and `media queue`: paused or downloading, SABnzbd's own warnings, and imports stuck for a day. */
|
|
348
|
+
export function mediaStatus(sab: unknown, queue: unknown, now: number): AreaStatus {
|
|
315
349
|
const r = sab as { sabnzbd_paused?: unknown; warnings?: unknown } | null;
|
|
316
350
|
if (!r || typeof r.sabnzbd_paused !== "boolean") return failed("Media", "media", "sabnzbd status");
|
|
317
351
|
const warnings = Array.isArray(r.warnings) ? r.warnings.length : 0;
|
|
318
|
-
const
|
|
319
|
-
|
|
352
|
+
const imports = stuckImports(queue);
|
|
353
|
+
const unread = imports === null ? [] : queueWarnings(queue);
|
|
354
|
+
const stuck = imports?.filter((i) => now / 1000 - Date.parse(i.added) / 1000 > STUCK_IMPORT_S) ?? [];
|
|
355
|
+
const line = [`downloads ${r.sabnzbd_paused ? "paused" : "running"}${warnings > 0 ? `, ${warnings} warning${warnings === 1 ? "" : "s"}` : ""}`, imports === null ? "import queue unknown" : stuck.length > 0 ? `${stuck.length} import${stuck.length === 1 ? "" : "s"} stuck` : null].filter(Boolean).join(", ");
|
|
356
|
+
const items: SummaryItem[] = [
|
|
357
|
+
...(warnings > 0 ? [{ owner: "Media", text: `the downloader shows ${warnings} warning${warnings === 1 ? "" : "s"}` }] : []),
|
|
358
|
+
...(imports === null ? [{ owner: "Media", text: "the import queue check failed" }] : []),
|
|
359
|
+
...unread.map((w) => ({ owner: "Media", text: `the import queue of ${w} could not be read` })),
|
|
360
|
+
...stuck.map((i) => ({
|
|
361
|
+
owner: "Media",
|
|
362
|
+
text: `${i.title} (${i.app}) waits in the queue since ${i.added.slice(0, 10)} (${i.state}${i.messages[0] ? `: ${i.messages[0].slice(0, 160)}` : ""}); reply here to remove it (queue-remove ${i.app} ${i.id})`,
|
|
363
|
+
})),
|
|
364
|
+
];
|
|
365
|
+
const onlyStuck = items.length > 0 && items.length === stuck.length;
|
|
366
|
+
return {
|
|
367
|
+
area: "Media",
|
|
368
|
+
line,
|
|
369
|
+
...(items.length > 0 ? { problem: `Media: ${items.map((i) => i.text).join("; ")}`, items, route: "media" as const } : {}),
|
|
370
|
+
...(onlyStuck ? { once: `queue:${stuck.map((i) => `${i.app}:${i.id}`).sort().join(",")}` } : {}),
|
|
371
|
+
};
|
|
320
372
|
}
|
|
321
373
|
|
|
322
374
|
/** Every guest is backed up nightly from 02:00; an archive older than this means last night's run missed it. */
|
|
@@ -339,30 +391,48 @@ export interface MacUpdates {
|
|
|
339
391
|
output: string | null;
|
|
340
392
|
}
|
|
341
393
|
|
|
342
|
-
/**
|
|
343
|
-
const
|
|
394
|
+
/** The summary line each Mac's items go to: Cortex is this host, Nexus is the inference specialist's. */
|
|
395
|
+
const MAC_OWNER: Readonly<Record<string, string>> = { Cortex: "Cortex", Nexus: "Inference" };
|
|
344
396
|
|
|
345
|
-
/**
|
|
397
|
+
/**
|
|
398
|
+
* Each Mac's macOS version and offers. Apple silicon installs a macOS update
|
|
399
|
+
* only with a volume owner's password, so any newer update of the installed
|
|
400
|
+
* major waits for the owners; a new major is noted for a manual upgrade.
|
|
401
|
+
*/
|
|
346
402
|
export function macosStatus(macs: readonly MacUpdates[]): AreaStatus {
|
|
347
403
|
const parts: string[] = [];
|
|
348
|
-
const
|
|
404
|
+
const items: SummaryItem[] = [];
|
|
405
|
+
const waiting: string[] = [];
|
|
349
406
|
for (const m of macs) {
|
|
407
|
+
const owner = MAC_OWNER[m.name] ?? "Cortex";
|
|
350
408
|
if (m.output === null) {
|
|
351
409
|
parts.push(`${m.name} unknown`);
|
|
352
|
-
|
|
410
|
+
items.push({ owner, text: `no macOS update list from ${m.name}` });
|
|
353
411
|
continue;
|
|
354
412
|
}
|
|
355
413
|
const [installed = "", ...rest] = m.output.split("\n").map((l) => l.trim());
|
|
356
|
-
const offers = rest
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
const
|
|
361
|
-
const
|
|
362
|
-
|
|
363
|
-
|
|
414
|
+
const offers = rest
|
|
415
|
+
.map((l) => /Title: (macOS[^,]*), Version: ([\d.]+)/.exec(l))
|
|
416
|
+
.filter((x): x is RegExpExecArray => x !== null)
|
|
417
|
+
.map((x) => ({ title: x[1], version: x[2] }));
|
|
418
|
+
const major = Number(installed.split(".")[0]);
|
|
419
|
+
const compare = (a: string, b: string) => a.localeCompare(b, undefined, { numeric: true });
|
|
420
|
+
const newer = offers.filter((o) => Number(o.version.split(".")[0]) === major && compare(o.version, installed) > 0).sort((a, b) => compare(a.version, b.version));
|
|
421
|
+
const majors = offers.filter((o) => Number(o.version.split(".")[0]) > major);
|
|
422
|
+
const latest = newer.at(-1);
|
|
423
|
+
parts.push(`${m.name} ${installed}${latest ? `, ${latest.version} waiting` : ""}${majors.length > 0 ? `, macOS ${majors[0].version} offered (major, by hand)` : ""}`);
|
|
424
|
+
if (latest) {
|
|
425
|
+
items.push({ owner, text: `${m.name} runs macOS ${installed}; ${latest.title} waits: install it with softwareupdate as a volume owner, then check that ${m.name} comes back` });
|
|
426
|
+
waiting.push(`${m.name}:${latest.version}`);
|
|
427
|
+
}
|
|
364
428
|
}
|
|
365
|
-
|
|
429
|
+
const onlyWaiting = items.length > 0 && items.length === waiting.length;
|
|
430
|
+
return {
|
|
431
|
+
area: "macOS",
|
|
432
|
+
line: parts.join("; "),
|
|
433
|
+
...(items.length > 0 ? { problem: `macOS: ${items.map((i) => i.text).join("; ")}`, items, route: "admin" as const } : {}),
|
|
434
|
+
...(onlyWaiting ? { once: `macos:${waiting.join(",")}` } : {}),
|
|
435
|
+
};
|
|
366
436
|
}
|
|
367
437
|
|
|
368
438
|
/** One report the run posted in its own channel, as the summary shows it. */
|
|
@@ -431,6 +501,8 @@ export function arcaneSummary(report: ArcaneReport | null, date: string, channel
|
|
|
431
501
|
...report.stacks.filter((s) => s.plan === "auto" && (s.state === "rolled back" || s.state === "failed")).map((s) => `${escapeDiscord(s.stack)} automatic update ${s.state}`),
|
|
432
502
|
...report.stacks.flatMap((s) => s.lines.map((l) => byHand(s.stack, l)).filter((t): t is string => t !== null)),
|
|
433
503
|
...(report.holdsError ? [`update-holds.json is unreadable: ${escapeDiscord(report.holdsError)}`] : []),
|
|
504
|
+
...(report.environmentErrors ?? []).map((e) => `Arcane environment ${escapeDiscord(e.environment)} was not checked for updates`),
|
|
505
|
+
...(report.agentsBehind ?? []).map((a) => `the Arcane agent of ${escapeDiscord(a.environment)} runs ${escapeDiscord(a.agent)}, the manager ${escapeDiscord(a.manager)}: move its tag on the host`),
|
|
434
506
|
];
|
|
435
507
|
return { area: "Arcane", line: parts.join(", ") || "no updates", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
|
|
436
508
|
}
|
|
@@ -460,7 +532,7 @@ function ownerOf(area: string): string {
|
|
|
460
532
|
return AREA_OWNER[area] ?? area;
|
|
461
533
|
}
|
|
462
534
|
|
|
463
|
-
/** The #daily message: one line per specialist
|
|
535
|
+
/** The #daily message: one line per item, under the specialist or Cortex that owns it, then the owners with nothing open; no mention. */
|
|
464
536
|
export function summaryText(s: Summary): string {
|
|
465
537
|
const lines = new Map<string, string[]>();
|
|
466
538
|
const add = (owner: string, items: readonly string[]) => lines.set(owner, [...(lines.get(owner) ?? []), ...items]);
|
|
@@ -477,7 +549,21 @@ export function summaryText(s: Summary): string {
|
|
|
477
549
|
const head = `**Daily** · ${s.date} · ${count === 0 ? `🟢 ${ALL_CLEAR}` : `🟠 ${count} item${count === 1 ? " needs" : "s need"} you`}`;
|
|
478
550
|
const rank = (owner: string) => (OWNER_ORDER.includes(owner) ? OWNER_ORDER.indexOf(owner) : OWNER_ORDER.length);
|
|
479
551
|
const owners = [...lines.keys()].sort((x, y) => rank(x) - rank(y));
|
|
480
|
-
|
|
552
|
+
const clear = owners.filter((o) => (lines.get(o) ?? []).length === 0);
|
|
553
|
+
return [head, ...owners.flatMap((o) => (lines.get(o) ?? []).map((item) => `${o} · ${item}`)), ...(clear.length > 0 ? [`All clear: ${clear.join(", ")}`] : [])].join("\n");
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
/**
|
|
557
|
+
* The problems whose tagged message posts this morning: all of them, except a
|
|
558
|
+
* problem with a `once` key an earlier morning already posted. `notified` is
|
|
559
|
+
* every `once` key still open, the record for the next morning.
|
|
560
|
+
*/
|
|
561
|
+
export function problemsToPost(areas: readonly AreaStatus[], notifiedBefore: readonly string[]): { post: AreaStatus[]; notified: string[] } {
|
|
562
|
+
const problems = areas.filter((a) => a.problem);
|
|
563
|
+
return {
|
|
564
|
+
post: problems.filter((a) => a.once === undefined || !notifiedBefore.includes(a.once)),
|
|
565
|
+
notified: problems.map((a) => a.once).filter((k): k is string => k !== undefined),
|
|
566
|
+
};
|
|
481
567
|
}
|
|
482
568
|
|
|
483
569
|
/** Areas read by a specialist of another name, or by none: Cortex's macOS is this host's own. */
|
|
@@ -79,7 +79,7 @@ import { ArcaneWatch } from "./arcane-watch-monitor.ts";
|
|
|
79
79
|
import { parseSnapshot } from "./arcane-watch.ts";
|
|
80
80
|
import { type Applied, autoFailureText, foremanTask, stackProblems, stackUpdateAnswer, SUPERVISE_DELAY_MS, SUPERVISE_LIMIT_MS } from "./supervise.ts";
|
|
81
81
|
import { stackPlan, toolPlan } from "./update-policy.ts";
|
|
82
|
-
import { type AreaStatus, arcaneSummary, backupStatus, type DnsUpgrade, dnsStatus, macosStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemSpecialist, problemText, type ProxmoxHost, proxmoxStatus, type ReportLine, type Route, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
82
|
+
import { type AreaStatus, arcaneSummary, backupStatus, type DnsUpgrade, dnsStatus, macosStatus, edgeStatus, identityStatus, inferenceStatus, mediaStatus, networkStatus, pendingPackageLines, problemSpecialist, problemsToPost, problemText, type ProxmoxHost, proxmoxStatus, type ReportLine, type Route, summaryText, technitiumPlan, toolsSummary, updatesStatus } from "./daily-run.ts";
|
|
83
83
|
import { type InvestigationRequest, SecurityMonitor } from "./security-monitor.ts";
|
|
84
84
|
import { nasOutage, readSecurityState } from "./security.ts";
|
|
85
85
|
|
|
@@ -630,17 +630,21 @@ class Bridge {
|
|
|
630
630
|
if (arcane) reports.push(arcaneSummary(readPostedArcane(), date, arcane.channelId));
|
|
631
631
|
const routes = this.routes();
|
|
632
632
|
const owners = this.config.owners;
|
|
633
|
-
for
|
|
633
|
+
// A problem that only waits for the owners tags them on its first morning; the summary lists it every day.
|
|
634
|
+
const prior = readReportState("daily")?.notified;
|
|
635
|
+
const { post, notified } = problemsToPost(areas, Array.isArray(prior) ? prior : []);
|
|
636
|
+
for (const a of areas.filter((x) => x.problem && !post.includes(x))) log(`daily ${date}: ${a.area} problem already posted (${a.once ?? ""})`);
|
|
637
|
+
for (const a of post) {
|
|
634
638
|
const components = a.buttons ? [{ type: 1, components: a.buttons.map((b) => ({ type: 2, style: 4, label: b.label, custom_id: b.customId })) }] : undefined;
|
|
635
639
|
await this.rest.createMessage(routes[a.route ?? "admin"], { content: problemText(a, owners), mentionUsers: owners, ...(components ? { components } : {}) }).catch((err) => log(`daily ${date}: ${a.area} problem message failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
636
640
|
}
|
|
637
641
|
const label = new Intl.DateTimeFormat("en-GB", { timeZone: schedule.timeZone, weekday: "short", day: "numeric", month: "short" }).format(new Date());
|
|
638
642
|
const text = summaryText({ date: label, guildId: this.config.guildId, reports, areas, routes });
|
|
639
643
|
// Recorded only now: a run a crash cut short runs again and skips the reports it already posted.
|
|
640
|
-
writeReportState("daily", { date, status: "sending" });
|
|
644
|
+
writeReportState("daily", { date, status: "sending", notified });
|
|
641
645
|
let first = "";
|
|
642
646
|
for (const chunk of chunkMessage(text)) first ||= (await this.rest.createMessage(schedule.channelId, { content: chunk })).id;
|
|
643
|
-
writeReportState("daily", { date, status: "sent", messageId: first });
|
|
647
|
+
writeReportState("daily", { date, status: "sent", messageId: first, notified });
|
|
644
648
|
log(`daily ${date}: posted`);
|
|
645
649
|
}
|
|
646
650
|
|
|
@@ -727,7 +731,7 @@ class Bridge {
|
|
|
727
731
|
);
|
|
728
732
|
}
|
|
729
733
|
if (present.has("inference")) areas.push((async () => inferenceStatus(await text("inference", ["nexus-status"])))());
|
|
730
|
-
if (present.has(MEDIA)) areas.push((async () => mediaStatus(await json(MEDIA, ["sabnzbd", "status"])))());
|
|
734
|
+
if (present.has(MEDIA)) areas.push((async () => mediaStatus(await json(MEDIA, ["sabnzbd", "status"]), await json(MEDIA, ["queue"]), Date.now()))());
|
|
731
735
|
areas.push(
|
|
732
736
|
(async () => {
|
|
733
737
|
// Cortex is this host; Nexus answers through the inference specialist.
|
|
@@ -1401,9 +1405,8 @@ class Bridge {
|
|
|
1401
1405
|
/** Restarts one service; the bridge's own restart runs detached, because launchd stops this process with it. */
|
|
1402
1406
|
private async restartService(p: PendingRestart, byUser: string | null): Promise<void> {
|
|
1403
1407
|
const who = byUser ? ` by <@${byUser}>` : "";
|
|
1404
|
-
|
|
1405
|
-
if (p.prompt) await this.rest.editMessage(p.prompt.channelId, p.prompt.messageId,
|
|
1406
|
-
else await this.rest.createMessage(this.restartChannel(), { content: text.slice(0, 1900) }).catch(() => undefined);
|
|
1408
|
+
// Only the owners' Restart now prompt is edited; a routine restart goes to the log, not to Discord.
|
|
1409
|
+
if (p.prompt) await this.rest.editMessage(p.prompt.channelId, p.prompt.messageId, `♻️ Restarting ${SERVICE_LABEL[p.service]}${who} for ${p.reasons.join(", ")}.`, []).catch(() => undefined);
|
|
1407
1410
|
const uid = process.getuid?.();
|
|
1408
1411
|
if (uid === undefined) {
|
|
1409
1412
|
log(`restart ${p.service}: this platform has no user id for launchctl; restart it by hand`);
|
|
@@ -8,6 +8,8 @@ export interface ReportState {
|
|
|
8
8
|
/** `skipped`: the Security Daily Report on a Monday, when the weekly report takes its place. */
|
|
9
9
|
status: "sending" | "sent" | "failed" | "skipped";
|
|
10
10
|
messageId?: string;
|
|
11
|
+
/** The morning run: `once` keys of the problems it has tagged the owners for, so a waiting release tags them once. */
|
|
12
|
+
notified?: string[];
|
|
11
13
|
}
|
|
12
14
|
|
|
13
15
|
/** Local date and minutes past midnight in a time zone. */
|