@astrofoundry/pi-astro 0.40.75 → 0.40.77

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -33,12 +33,24 @@ export interface AreaStatus {
33
33
  * the summary keeps listing it every morning.
34
34
  */
35
35
  once?: string;
36
+ /** The specialist that answers replies to the problem's message, when not the area's own; null for none. */
37
+ specialist?: string | null;
38
+ /** The problem's tagged message, which its summary line links to. */
39
+ messageId?: string;
36
40
  }
37
41
 
38
42
  const HOUR = 3600;
39
43
 
40
44
  function failed(area: string, route: Route, why: string): AreaStatus {
41
- return { area, line: `the check failed (${why.slice(0, 120)})`, problem: `${area}: the status check failed`, route };
45
+ return { area, line: `the check failed (${why.slice(0, 120)})`, problem: `${area}: \`${why}\` failed`, route };
46
+ }
47
+
48
+ /** Escapes Discord markdown outside inline code, so commands in a problem stay copyable code. */
49
+ export function escapeOutsideCode(text: string): string {
50
+ return text
51
+ .split(/(`[^`\n]+`)/)
52
+ .map((part, i) => (i % 2 === 1 ? part : escapeDiscord(part)))
53
+ .join("");
42
54
  }
43
55
 
44
56
  /** Seconds from `21s ago`, `3m ago`, `2h ago`, `1d ago`; null when the text is not one of those. */
@@ -67,7 +79,7 @@ export function edgeStatus(status: string | null, updates: string | null): AreaS
67
79
  upgradable === null ? "updates unknown" : `${upgradable} upgradable package${upgradable === 1 ? "" : "s"}`,
68
80
  ];
69
81
  const tunnel = age === null || age > STALE_HANDSHAKE_S;
70
- const problem = down.length > 0 ? `Edge: ${down.join(", ")} not active on the front door` : tunnel ? `Edge: the WireGuard handshake is ${handshake || "missing"}` : undefined;
82
+ const problem = down.length > 0 ? `Edge: ${down.join(", ")} down on the front door` : tunnel ? `Edge: WireGuard handshake ${handshake || "missing"}` : undefined;
71
83
  return { area: "Edge", line: `front door: ${parts.join(", ")}`, ...(problem ? { problem, route: "admin" as const } : {}) };
72
84
  }
73
85
 
@@ -112,9 +124,9 @@ function proxmoxHostFacts(h: ProxmoxHost): ProxmoxFacts {
112
124
  const rebootNeeded = reboot !== undefined && reboot !== "no";
113
125
  const line = [up ? `${h.name} up ${up}` : `${h.name} up`, mem ? `RAM ${mem[2]} of ${mem[1]} used` : null, root ? `root disk ${root}%` : null, updates, rebootNeeded ? "reboot needed" : null].filter(Boolean).join(", ");
114
126
  const problems = [
115
- h.upgrade === null ? `the host upgrade on ${h.name} failed` : null,
116
- held ? `${h.name} waits for you: ${held}` : null,
117
- rebootNeeded ? `${h.name} needs a reboot (${reboot}); every guest on it restarts for a few minutes` : null,
127
+ h.upgrade === null ? `${h.name}: host upgrade failed` : null,
128
+ held ? `${h.name}: ${held}, held for you` : null,
129
+ rebootNeeded ? `${h.name}: reboot needed (${reboot}); its guests stop for a few minutes` : null,
118
130
  ].filter((p): p is string => p !== null);
119
131
  return { line, problems, ...(rebootNeeded ? { button: { label: `Reboot ${h.name}`, customId: rebootButtonId(h.id) } } : {}) };
120
132
  }
@@ -135,8 +147,8 @@ export function proxmoxStatus(hosts: readonly ProxmoxHost[]): AreaStatus {
135
147
  /** Packages the nightly run leaves alone on purpose: a Wazuh agent must never be newer than the manager. */
136
148
  const HELD_PACKAGES = new Set(["wazuh-agent"]);
137
149
 
138
- /** The `[upgradable from` lines of `apt list --upgradable`, also when a VM's guest agent wraps them in JSON (`out-data`). */
139
- function upgradableLines(output: string): string[] {
150
+ /** The lines of an `updates` read, also when a VM's guest agent wraps them in JSON (`out-data`). */
151
+ function aptLines(output: string): string[] {
140
152
  let text = output;
141
153
  try {
142
154
  const wrapped = (JSON.parse(output) as { "out-data"?: unknown } | null)?.["out-data"];
@@ -144,7 +156,19 @@ function upgradableLines(output: string): string[] {
144
156
  } catch {
145
157
  /* plain apt output */
146
158
  }
147
- return text.split("\n").filter((l) => l.includes("[upgradable from"));
159
+ return text.split("\n");
160
+ }
161
+
162
+ /** The `[upgradable from` lines of `apt list --upgradable`. */
163
+ function upgradableLines(output: string): string[] {
164
+ return aptLines(output).filter((l) => l.includes("[upgradable from"));
165
+ }
166
+
167
+ /** What a full upgrade would add and remove: the `new:` and `remove:` lines of the `updates` dry run. Packages wait when this is not empty. */
168
+ export function keptBack(output: string): { adds: string[]; removes: string[] } {
169
+ const lines = aptLines(output).map((l) => l.trim());
170
+ const pick = (prefix: string) => lines.filter((l) => l.startsWith(prefix)).map((l) => l.slice(prefix.length).trim());
171
+ return { adds: pick("new: "), removes: pick("remove: ") };
148
172
  }
149
173
 
150
174
  /** The upgradable packages of one host, held packages left out. */
@@ -177,15 +201,18 @@ export function guestOwner(name: string): string {
177
201
  return "Proxmox";
178
202
  }
179
203
 
180
- /** Each guest's and Synapse's `updates` after the 02:25 run: anything still waiting did not install and needs a look. */
181
- export function updatesStatus(hosts: Readonly<Record<string, string | null>>): AreaStatus {
204
+ /**
205
+ * Each guest's and Synapse's `updates` after the 02:25 run: anything still waiting did not install.
206
+ * One problem per owner, so a reply reaches the specialist that runs the host; a kept-back upgrade names its fix.
207
+ */
208
+ export function updatesStatus(hosts: Readonly<Record<string, string | null>>): AreaStatus[] {
182
209
  const names = Object.keys(hosts);
183
210
  const wazuh = wazuhRelease(hosts);
184
211
  const unknown = names.filter((n) => hosts[n] === null);
185
212
  const left = names.flatMap((n) => {
186
213
  const out = hosts[n];
187
214
  const packages = out === null ? [] : pendingPackageLines(out).map((l) => l.split("/")[0]);
188
- return packages.length > 0 ? [{ name: n, packages }] : [];
215
+ return out !== null && packages.length > 0 ? [{ name: n, packages, ...keptBack(out) }] : [];
189
216
  });
190
217
  const checked = names.length - unknown.length;
191
218
  const parts = [
@@ -193,15 +220,24 @@ export function updatesStatus(hosts: Readonly<Record<string, string | null>>): A
193
220
  unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null,
194
221
  wazuh ? `Wazuh ${wazuh.to} is out (agents held at ${wazuh.from})` : null,
195
222
  ].filter(Boolean);
196
- const packages = (h: { packages: string[] }) => `${h.packages.slice(0, 4).join(", ")}${h.packages.length > 4 ? ", ..." : ""}`;
197
- const wazuhText = wazuh ? `Wazuh ${wazuh.to} is out: Claude Code on the Mac upgrades the manager (\`observability/compose.yaml\`, then \`deploy.sh\`), then the agents (\`scripts/wazuh-agents/deploy.sh --upgrade\`); an agent must never be newer than the manager` : null;
198
- const problems = [left.length > 0 ? `${left.map((h) => `${h.name} (${packages(h)})`).join("; ")} did not install overnight` : null, unknown.length > 0 ? `no update list from ${unknown.join(", ")}` : null, wazuhText].filter(Boolean);
199
- const items: SummaryItem[] = [
200
- ...left.map((h) => ({ owner: guestOwner(h.name), text: `${h.name}: ${packages(h)} did not install overnight` })),
201
- ...unknown.map((n) => ({ owner: guestOwner(n), text: `no update list from ${n}` })),
202
- ...(wazuhText ? [{ owner: "Security", text: wazuhText }] : []),
203
- ];
204
- return { area: "Updates", line: parts.join("; "), ...(problems.length > 0 ? { problem: `Updates: ${problems.join("; ")}`, items, route: "admin" as const } : {}) };
223
+ const line = parts.join("; ");
224
+ const list = (items: readonly string[]) => `${items.slice(0, 4).join(", ")}${items.length > 4 ? ", ..." : ""}`;
225
+ const hostText = (h: (typeof left)[number]) => {
226
+ if (h.adds.length === 0 && h.removes.length === 0) return `${h.name}: ${list(h.packages)} did not install`;
227
+ const changes = [h.adds.length > 0 ? `adds ${list(h.adds)}` : null, h.removes.length > 0 ? `removes ${list(h.removes)}` : null].filter(Boolean).join(", ");
228
+ return `${h.name}: ${list(h.packages)} kept back. Fix by hand: \`apt-get full-upgrade\` (${changes})`;
229
+ };
230
+ const items: SummaryItem[] = [...left.map((h) => ({ owner: guestOwner(h.name), text: hostText(h) })), ...unknown.map((n) => ({ owner: guestOwner(n), text: `${n}: no update list` }))];
231
+ const owners = [...new Set(items.map((i) => i.owner))];
232
+ const areas: AreaStatus[] = owners.map((owner) => {
233
+ const mine = items.filter((i) => i.owner === owner);
234
+ return { area: "Updates", line, problem: `Updates: ${mine.map((i) => i.text).join("; ")}`, items: mine, route: "admin", specialist: owner.toLowerCase() };
235
+ });
236
+ if (wazuh) {
237
+ const text = `Wazuh ${wazuh.to} is out (agents on ${wazuh.from}). Claude Code on the Mac upgrades the manager first, then the agents`;
238
+ areas.push({ area: "Updates", line, problem: `Updates: ${text}`, items: [{ owner: "Security", text }], route: "admin", specialist: null });
239
+ }
240
+ return areas.length > 0 ? areas : [{ area: "Updates", line }];
205
241
  }
206
242
 
207
243
  /** `network pulsar radius-status` and `tailscale-status`: FreeRADIUS running, and every `tailscale*` subnet router (37 and Mira) online. */
@@ -218,9 +254,10 @@ export function networkStatus(radius: string | null, tailscale: string | null):
218
254
  routers === null ? "Tailscale unknown" : down.length === 0 ? `Tailscale router${routers.length === 1 ? "" : "s"} ${routers.map((r) => r.name).join(" and ")} online` : `Tailscale offline: ${down.join(", ")}`,
219
255
  ];
220
256
  const problems = [
221
- radiusUp === false ? "FreeRADIUS is not running" : null,
222
- down.length > 0 ? `the Tailscale subnet router ${down.join(" and ")} ${down.length === 1 ? "is" : "are"} offline` : null,
223
- radiusUp === null || routers === null ? "a network status check failed" : null,
257
+ radiusUp === false ? "FreeRADIUS down" : null,
258
+ down.length > 0 ? `Tailscale router ${down.join(" and ")} offline` : null,
259
+ radiusUp === null ? "`pulsar radius-status` failed" : null,
260
+ routers === null ? "`pulsar tailscale-status` failed" : null,
224
261
  ].filter(Boolean);
225
262
  return { area: "Network", line: parts.join(", "), ...(problems.length > 0 ? { problem: `Network: ${problems.join("; ")}`, route: "admin" as const } : {}) };
226
263
  }
@@ -253,8 +290,8 @@ export function identityStatus(status: string | null, pomerium?: unknown): AreaS
253
290
  const unitLine = units.length === 0 ? "no unit status read" : bad.length === 0 ? `${units.map((u) => u.unit.replace(/\.service$/, "")).join(", ")} active` : `not active: ${bad.join(", ")}`;
254
291
  const releaseLine = pomerium === undefined ? null : release === null ? "Pomerium version unknown" : newer ? `Pomerium ${newer.installed}, ${newer.newest} available` : `Pomerium ${release.installed}`;
255
292
  const problems = [
256
- bad.length > 0 || units.length === 0 ? `${bad.length > 0 ? bad.join(", ") : "no unit status read"} on the DMZ VM` : null,
257
- newer ? `Pomerium ${newer.newest} is out (running ${newer.installed}; notes: <${newer.notes}>). The button installs the official release after checking its sha256, restarts Pomerium for a few seconds, and puts ${newer.installed} back if the login route fails` : null,
293
+ bad.length > 0 || units.length === 0 ? `DMZ: ${bad.length > 0 ? bad.join(", ") : "no unit status read"}` : null,
294
+ newer ? `Pomerium ${newer.installed} → ${newer.newest} (notes: <${newer.notes}>). The button checks the sha256, restarts Pomerium for seconds, and reverts if login fails` : null,
258
295
  ].filter(Boolean);
259
296
  return {
260
297
  area: "Identity",
@@ -296,9 +333,9 @@ export function dnsStatus(answers: Readonly<Record<string, unknown>>, upgrade?:
296
333
  const names = down.join(" and ");
297
334
  const upgradeLine = upgrade === undefined ? null : "major" in upgrade ? `Technitium ${upgrade.to} is a major release and waits for you` : upgrade.ok ? `${upgrade.host} upgraded ${upgrade.from} -> ${upgrade.to} (one node per morning, primary last)` : `the upgrade of ${upgrade.host} to ${upgrade.to} failed`;
298
335
  const problems = [
299
- down.length > 0 ? `${names} ${down.length === 1 ? "does" : "do"} not answer` : null,
300
- upgrade && "major" in upgrade ? `Technitium ${upgrade.to} is a major release (running ${upgrade.from}); read its change log, then upgrade one node at a time with \`dns technitium host <node> upgrade\`` : null,
301
- upgrade && "ok" in upgrade && !upgrade.ok ? `the upgrade of ${upgrade.host} to ${upgrade.to} failed (${upgrade.why.slice(0, 200)})` : null,
336
+ down.length > 0 ? `${names} not answering` : null,
337
+ upgrade && "major" in upgrade ? `Technitium ${upgrade.from} → ${upgrade.to} is a major release: read its change log, then reply "upgrade <node>", one node at a time, technitium1 last` : null,
338
+ upgrade && "ok" in upgrade && !upgrade.ok ? `${upgrade.host}: upgrade to ${upgrade.to} failed (${upgrade.why.slice(0, 200)})` : null,
302
339
  ].filter(Boolean);
303
340
  const line = [down.length === 0 ? `${nodes.join(", ")} answer` : `no answer from ${names}`, upgradeLine].filter(Boolean).join("; ");
304
341
  return { area: "DNS", line, ...(problems.length > 0 ? { problem: `DNS: ${problems.join("; ")}`, route: "admin" as const } : {}) };
@@ -314,7 +351,7 @@ export function inferenceStatus(status: string | null): AreaStatus {
314
351
  .map((m) => ({ name: m[1], running: m[2] === "running", healthy: m[3] === "200" }));
315
352
  const bad = services.filter((s) => !s.running || !s.healthy).map((s) => s.name);
316
353
  const line = services.length === 0 ? "no Nexus service listed" : bad.length === 0 ? `Nexus ${services.map((s) => s.name).join(", ")} running and healthy` : `Nexus not healthy: ${bad.join(", ")}`;
317
- return { area: "Inference", line, ...(bad.length > 0 || services.length === 0 ? { problem: `Inference: ${bad.length > 0 ? `${bad.join(", ")} not healthy on Nexus` : "Nexus lists no service"}`, route: "admin" as const } : {}) };
354
+ return { area: "Inference", line, ...(bad.length > 0 || services.length === 0 ? { problem: `Inference: ${bad.length > 0 ? `${bad.join(", ")} unhealthy on Nexus` : "Nexus lists no service"}`, route: "admin" as const } : {}) };
318
355
  }
319
356
 
320
357
  /**
@@ -368,12 +405,12 @@ export function mediaStatus(sab: unknown, queue: unknown, now: number): AreaStat
368
405
  const stuck = imports?.filter((i) => now / 1000 - Date.parse(i.added) / 1000 > STUCK_IMPORT_S) ?? [];
369
406
  const line = [`downloads ${r.sabnzbd_paused ? "paused" : "running"}${warnings > 0 ? `, ${warnings} warning${warnings === 1 ? "" : "s"}` : ""}`, imports === null ? "import queue unknown" : stuck.length > 0 ? `${stuck.length} import${stuck.length === 1 ? "" : "s"} stuck` : null].filter(Boolean).join(", ");
370
407
  const items: SummaryItem[] = [
371
- ...(warnings > 0 ? [{ owner: "Media", text: `the downloader shows ${warnings} warning${warnings === 1 ? "" : "s"}` }] : []),
372
- ...(imports === null ? [{ owner: "Media", text: "the import queue check failed" }] : []),
373
- ...unread.map((w) => ({ owner: "Media", text: `the import queue of ${w} could not be read` })),
408
+ ...(warnings > 0 ? [{ owner: "Media", text: `SABnzbd: ${warnings} warning${warnings === 1 ? "" : "s"}` }] : []),
409
+ ...(imports === null ? [{ owner: "Media", text: "`queue` failed" }] : []),
410
+ ...unread.map((w) => ({ owner: "Media", text: `${w} queue unreadable` })),
374
411
  ...stuck.map((i) => ({
375
412
  owner: "Media",
376
- text: `${i.title} (${i.app}) waits in the queue since ${i.added.slice(0, 10)} (${i.state}${i.messages[0] ? `: ${i.messages[0].slice(0, 160)}` : ""}); reply here to remove it (queue-remove ${i.app} ${i.id})`,
413
+ text: `${i.title} (${i.app}) stuck since ${i.added.slice(0, 10)}: ${i.messages[0] ? i.messages[0].slice(0, 160).replace(/\.+$/, "") : i.state}. Reply "remove" to delete it (\`queue-remove ${i.app} ${i.id}\`)`,
377
414
  })),
378
415
  ];
379
416
  const onlyStuck = items.length > 0 && items.length === stuck.length;
@@ -395,7 +432,12 @@ export function backupStatus(archives: unknown, resticText: string | null, now:
395
432
  const missed = guests?.filter((g) => now / 1000 - g.last > MISSED_BACKUP_S) ?? [];
396
433
  const proxmox = guests === null ? "Proxmox archives unknown" : missed.length === 0 ? `${guests.length === 1 ? "the only guest" : `all ${guests.length} guests`} backed up last night` : `no backup last night for ${missed.map((g) => escapeDiscord(g.name)).join(", ")}`;
397
434
  const check = restic ? `restic check ${escapeDiscord(restic.result)} ${restic.at}` : "restic check unknown";
398
- const problems = [missed.length > 0 ? `no Proxmox backup last night for ${missed.map((g) => g.name).join(", ")}` : null, restic && restic.result !== "success" ? `the restic check ended with ${restic.result}` : null, guests === null || restic === null ? "a backup status check failed" : null].filter(Boolean);
435
+ const problems = [
436
+ missed.length > 0 ? `no backup last night: ${missed.map((g) => g.name).join(", ")}` : null,
437
+ restic && restic.result !== "success" ? `restic check: ${restic.result}` : null,
438
+ guests === null ? "`vzdump-archives` failed" : null,
439
+ restic === null ? "`restic-check-status` failed" : null,
440
+ ].filter(Boolean);
399
441
  return { area: "Backup", line: `${proxmox}, ${check}`, ...(problems.length > 0 ? { problem: `Backup: ${problems.join("; ")}`, route: "admin" as const } : {}) };
400
442
  }
401
443
 
@@ -408,6 +450,9 @@ export interface MacUpdates {
408
450
  /** The summary line each Mac's items go to: Cortex is this host, Nexus is the inference specialist's. */
409
451
  const MAC_OWNER: Readonly<Record<string, string>> = { Cortex: "Cortex", Nexus: "Inference" };
410
452
 
453
+ /** The volume owner whose password authorizes a macOS install on each Mac: user `nexus` has no secure token. */
454
+ const MAC_VOLUME_OWNER: Readonly<Record<string, string>> = { Cortex: "cortex", Nexus: "astronaute" };
455
+
411
456
  /**
412
457
  * Each Mac's macOS version and offers. Apple silicon installs a macOS update
413
458
  * only with a volume owner's password, so any newer update of the installed
@@ -421,14 +466,22 @@ export function macosStatus(macs: readonly MacUpdates[]): AreaStatus {
421
466
  const owner = MAC_OWNER[m.name] ?? "Cortex";
422
467
  if (m.output === null) {
423
468
  parts.push(`${m.name} unknown`);
424
- items.push({ owner, text: `no macOS update list from ${m.name}` });
469
+ items.push({ owner, text: `${m.name}: no macOS update list` });
425
470
  continue;
426
471
  }
427
472
  const [installed = "", ...rest] = m.output.split("\n").map((l) => l.trim());
428
- const offers = rest
429
- .map((l) => /Title: (macOS[^,]*), Version: ([\d.]+)/.exec(l))
430
- .filter((x): x is RegExpExecArray => x !== null)
431
- .map((x) => ({ title: x[1], version: x[2] }));
473
+ // Software Update prints each `* Label:` line right before its `Title:` line; installing needs the label.
474
+ const offers: { title: string; version: string; label: string }[] = [];
475
+ let label = "";
476
+ for (const l of rest) {
477
+ const named = /^\* Label: (.+)$/.exec(l);
478
+ if (named) label = named[1].trim();
479
+ const x = /Title: (macOS[^,]*), Version: ([\d.]+)/.exec(l);
480
+ if (x) {
481
+ offers.push({ title: x[1], version: x[2], label: label || x[1] });
482
+ label = "";
483
+ }
484
+ }
432
485
  const major = Number(installed.split(".")[0]);
433
486
  const compare = (a: string, b: string) => a.localeCompare(b, undefined, { numeric: true });
434
487
  const newer = offers.filter((o) => Number(o.version.split(".")[0]) === major && compare(o.version, installed) > 0).sort((a, b) => compare(a.version, b.version));
@@ -436,7 +489,7 @@ export function macosStatus(macs: readonly MacUpdates[]): AreaStatus {
436
489
  const latest = newer.at(-1);
437
490
  parts.push(`${m.name} ${installed}${latest ? `, ${latest.version} waiting` : ""}${majors.length > 0 ? `, macOS ${majors[0].version} offered (major, by hand)` : ""}`);
438
491
  if (latest) {
439
- items.push({ owner, text: `${m.name} runs macOS ${installed}; ${latest.title} waits: install it with softwareupdate as a volume owner, then check that ${m.name} comes back` });
492
+ items.push({ owner, text: `${m.name} macOS ${installed} → ${latest.version}. On ${m.name}: \`sudo softwareupdate -i "${latest.label}" -R --user ${MAC_VOLUME_OWNER[m.name] ?? "<volume owner>"}\` (restarts it)` });
440
493
  waiting.push(`${m.name}:${latest.version}`);
441
494
  }
442
495
  }
@@ -465,7 +518,7 @@ function names(items: readonly string[]): string {
465
518
 
466
519
  /** The Cortex Tools Report after its automatic updates ran: what changed, what failed, and what waits for a button. */
467
520
  export function toolsSummary(report: ToolsReport | null, date: string, channelId: string): ReportLine {
468
- if (!report || report.date !== date) return { area: "Cortex tools", line: "no report today; the bridge log has the reason", needs: ["the Cortex Tools Report did not run today; the bridge log has the reason"], channelId };
521
+ if (!report || report.date !== date) return { area: "Cortex tools", line: "no report today; reason in the bridge log", needs: ["Cortex Tools Report did not run today; reason in the bridge log"], channelId };
469
522
  const updated = report.lines.filter((l) => l.state === "updated");
470
523
  const failed = report.lines.filter((l) => l.state === "failed");
471
524
  const waiting = report.lines.filter((l) => (l.plan === "update" || l.plan === "button") && l.state === "offered");
@@ -477,26 +530,26 @@ export function toolsSummary(report: ToolsReport | null, date: string, channelId
477
530
  manual.length > 0 ? `${manual.length} by hand` : null,
478
531
  ].filter(Boolean);
479
532
  const needs = [
480
- ...report.lines.filter((l) => l.plan === "update" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} waits for its button (${l.major ? "major version" : "release notes"})`),
533
+ ...report.lines.filter((l) => l.plan === "update" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} waits for its button (${l.major ? "major" : "release notes"})`),
481
534
  ...report.lines.filter((l) => l.plan === "button" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} waits for its button`),
482
535
  ...failed.filter((l) => l.plan === "auto").map((l) => `${escapeDiscord(l.label)} automatic update failed`),
483
536
  ...manual.map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} by hand: ${escapeDiscord(l.manual ?? "")}`),
484
537
  ];
485
- return { area: "Cortex tools", line: parts.join(", ") || "everything up to date", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
538
+ return { area: "Cortex tools", line: parts.join(", ") || "all up to date", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
486
539
  }
487
540
 
488
541
  /** A service line only the owner can act on: a failed check, or a newer major that needs a tag change by hand; a held service is the owner's own decision. */
489
542
  function byHand(stack: string, l: ArcaneReport["stacks"][number]["lines"][number]): string | null {
490
543
  if (l.held !== undefined) return null;
491
544
  const where = `${escapeDiscord(stack)} ${escapeDiscord(l.service)}`;
492
- if (l.status === "unknown") return `${where}: the update check failed`;
545
+ if (l.status === "unknown") return `${where}: update check failed`;
493
546
  const newer = /newer major (\S+) available/.exec(l.note)?.[1];
494
547
  return newer ? `${where} ${escapeDiscord(l.running ?? l.tag)}: major ${escapeDiscord(newer)} needs a tag change by hand` : null;
495
548
  }
496
549
 
497
550
  /** The Arcane Daily Report after its automatic updates and checks: updated, rolled back, waiting for the owner, held, by hand. Holds are the owner's own decisions, so they are not items. */
498
551
  export function arcaneSummary(report: ArcaneReport | null, date: string, channelId: string): ReportLine {
499
- if (!report || report.date !== date) return { area: "Arcane", line: "no report today; the bridge log has the reason", needs: ["the Arcane Daily Report did not run today; the bridge log has the reason"], channelId };
552
+ if (!report || report.date !== date) return { area: "Arcane", line: "no report today; reason in the bridge log", needs: ["Arcane Daily Report did not run today; reason in the bridge log"], channelId };
500
553
  const by = (pick: (s: ArcaneReport["stacks"][number]) => boolean) => report.stacks.filter(pick).map((s) => s.stack);
501
554
  const updated = by((s) => s.state === "updated");
502
555
  const rolled = by((s) => s.state === "rolled back");
@@ -511,12 +564,12 @@ export function arcaneSummary(report: ArcaneReport | null, date: string, channel
511
564
  held.length > 0 ? `${names(held)} held` : null,
512
565
  ].filter(Boolean);
513
566
  const needs = [
514
- ...waiting.map((s) => `${escapeDiscord(s.stack)} waits for you (${s.plan === "careful" ? "Upgrade carefully" : "major version"})`),
567
+ ...waiting.map((s) => `${escapeDiscord(s.stack)} waits for you (${s.plan === "careful" ? "Upgrade carefully" : "major"})`),
515
568
  ...report.stacks.filter((s) => s.plan === "auto" && (s.state === "rolled back" || s.state === "failed")).map((s) => `${escapeDiscord(s.stack)} automatic update ${s.state}`),
516
569
  ...report.stacks.flatMap((s) => s.lines.map((l) => byHand(s.stack, l)).filter((t): t is string => t !== null)),
517
- ...(report.holdsError ? [`update-holds.json is unreadable: ${escapeDiscord(report.holdsError)}`] : []),
518
- ...(report.environmentErrors ?? []).map((e) => `Arcane environment ${escapeDiscord(e.environment)} was not checked for updates`),
519
- ...(report.agentsBehind ?? []).map((a) => `the Arcane agent of ${escapeDiscord(a.environment)} runs ${escapeDiscord(a.agent)}, the manager ${escapeDiscord(a.manager)}: move its tag on the host`),
570
+ ...(report.holdsError ? [`update-holds.json unreadable: ${escapeDiscord(report.holdsError)}`] : []),
571
+ ...(report.environmentErrors ?? []).map((e) => `Arcane environment ${escapeDiscord(e.environment)} unchecked for updates`),
572
+ ...(report.agentsBehind ?? []).map((a) => `Arcane agent of ${escapeDiscord(a.environment)} runs ${escapeDiscord(a.agent)}, manager ${escapeDiscord(a.manager)}: move its tag on the host`),
520
573
  ];
521
574
  return { area: "Arcane", line: parts.join(", ") || "no updates", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
522
575
  }
@@ -555,9 +608,10 @@ export function summaryText(s: Summary): string {
555
608
  add(ownerOf(r.area), r.needs.map((n) => `${n}${where}`));
556
609
  }
557
610
  for (const a of s.areas) {
558
- const where = ` (<#${s.routes[a.route ?? "admin"]}>)`;
611
+ const channel = s.routes[a.route ?? "admin"];
612
+ const where = a.messageId ? ` ([message](${link(s.guildId, channel, a.messageId)}))` : ` (<#${channel}>)`;
559
613
  add(ownerOf(a.area), []);
560
- for (const item of a.items ?? (a.problem ? [{ owner: ownerOf(a.area), text: a.problem.replace(`${a.area}: `, "") }] : [])) add(item.owner, [`${escapeDiscord(item.text)}${where}`]);
614
+ for (const item of a.items ?? (a.problem ? [{ owner: ownerOf(a.area), text: a.problem.replace(`${a.area}: `, "") }] : [])) add(item.owner, [`${escapeOutsideCode(item.text)}${where}`]);
561
615
  }
562
616
  const count = [...lines.values()].reduce((n, items) => n + items.length, 0);
563
617
  const head = `**Daily** · ${s.date} · ${count === 0 ? `🟢 ${ALL_CLEAR}` : `🟠 ${count} item${count === 1 ? " needs" : "s need"} you`}`;
@@ -583,14 +637,17 @@ export function problemsToPost(areas: readonly AreaStatus[], notifiedBefore: rea
583
637
  /** Areas read by a specialist of another name, or by none: Cortex's macOS is this host's own. */
584
638
  const AREA_SPECIALIST: Readonly<Record<string, string | null>> = { Updates: "proxmox", macOS: null };
585
639
 
586
- /** The specialist a morning-run problem message names, so a reply to it goes there without a prefix. */
640
+ /** The specialist a morning-run problem message names, so a reply to it goes there without a prefix; messages before 0.40.76 use the longer line. */
587
641
  export function problemSpecialist(content: string): string | null {
588
- return /^Found by the morning run; the (\S+) specialist can look into it here\.$/m.exec(content)?.[1] ?? null;
642
+ const m = /^(?:Replies go to the (\S+) specialist\.|Found by the morning run; the (\S+) specialist can look into it here\.)$/m.exec(content);
643
+ return m?.[1] ?? m?.[2] ?? null;
589
644
  }
590
645
 
591
- /** The tagged message for one area's problem in its routed channel. */
646
+ /** The tagged message for one area's problem in its routed channel: the problem, one item per line, and who answers replies. */
592
647
  export function problemText(a: AreaStatus, owners: readonly string[]): string {
593
- const specialist = Object.hasOwn(AREA_SPECIALIST, a.area) ? AREA_SPECIALIST[a.area] : a.area.toLowerCase();
594
- const who = specialist === null ? "ask about it here" : `the ${escapeDiscord(specialist)} specialist can look into it here`;
595
- return `⚠️ **${escapeDiscord(a.area)}** · ${escapeDiscord(a.problem ?? a.line)}\nFound by the morning run; ${who}.\n${owners.map((id) => `<@${id}>`).join(" ")}`;
648
+ const specialist = a.specialist !== undefined ? a.specialist : Object.hasOwn(AREA_SPECIALIST, a.area) ? AREA_SPECIALIST[a.area] : a.area.toLowerCase();
649
+ const texts = a.items && a.items.length > 0 ? a.items.map((i) => i.text) : [(a.problem ?? a.line).replace(`${a.area}: `, "")];
650
+ const head = `⚠️ **${escapeDiscord(a.area)}**`;
651
+ const body = texts.length === 1 ? `${head} · ${escapeOutsideCode(texts[0])}` : [head, ...texts.map((t) => `- ${escapeOutsideCode(t)}`)].join("\n");
652
+ return [body, specialist === null ? null : `Replies go to the ${escapeDiscord(specialist)} specialist.`, owners.map((id) => `<@${id}>`).join(" ")].filter((l) => l !== null).join("\n");
596
653
  }
@@ -631,20 +631,30 @@ class Bridge {
631
631
  const routes = this.routes();
632
632
  const owners = this.config.owners;
633
633
  // A problem that only waits for the owners tags them on its first morning; the summary lists it every day.
634
- const prior = readReportState("daily")?.notified;
634
+ const priorState = readReportState("daily");
635
+ const prior = priorState?.notified;
636
+ const priorLinks = priorState?.links ?? {};
635
637
  const { post, notified } = problemsToPost(areas, Array.isArray(prior) ? prior : []);
636
638
  for (const a of areas.filter((x) => x.problem && !post.includes(x))) log(`daily ${date}: ${a.area} problem already posted (${a.once ?? ""})`);
639
+ // Each summary line links to its problem's message; a problem posted on an earlier morning keeps that message.
640
+ const posted = new Map<AreaStatus, string>();
637
641
  for (const a of post) {
638
642
  const components = a.buttons ? [{ type: 1, components: a.buttons.map((b) => ({ type: 2, style: 4, label: b.label, custom_id: b.customId })) }] : undefined;
639
- await this.rest.createMessage(routes[a.route ?? "admin"], { content: problemText(a, owners), mentionUsers: owners, ...(components ? { components } : {}) }).catch((err) => log(`daily ${date}: ${a.area} problem message failed: ${err instanceof Error ? err.message : String(err)}`));
643
+ const message = await this.rest.createMessage(routes[a.route ?? "admin"], { content: problemText(a, owners), mentionUsers: owners, ...(components ? { components } : {}) }).catch((err) => log(`daily ${date}: ${a.area} problem message failed: ${err instanceof Error ? err.message : String(err)}`));
644
+ if (message) posted.set(a, message.id);
640
645
  }
646
+ const linked = areas.map((a) => {
647
+ const messageId = posted.get(a) ?? (a.once !== undefined ? priorLinks[a.once] : undefined);
648
+ return messageId ? { ...a, messageId } : a;
649
+ });
650
+ const links = Object.fromEntries(linked.flatMap((a) => (a.once !== undefined && a.messageId ? [[a.once, a.messageId]] : [])));
641
651
  const label = new Intl.DateTimeFormat("en-GB", { timeZone: schedule.timeZone, weekday: "short", day: "numeric", month: "short" }).format(new Date());
642
- const text = summaryText({ date: label, guildId: this.config.guildId, reports, areas, routes });
652
+ const text = summaryText({ date: label, guildId: this.config.guildId, reports, areas: linked, routes });
643
653
  // Recorded only now: a run a crash cut short runs again and skips the reports it already posted.
644
- writeReportState("daily", { date, status: "sending", notified });
654
+ writeReportState("daily", { date, status: "sending", notified, links });
645
655
  let first = "";
646
656
  for (const chunk of chunkMessage(text)) first ||= (await this.rest.createMessage(schedule.channelId, { content: chunk })).id;
647
- writeReportState("daily", { date, status: "sent", messageId: first, notified });
657
+ writeReportState("daily", { date, status: "sent", messageId: first, notified, links });
648
658
  log(`daily ${date}: posted`);
649
659
  }
650
660
 
@@ -667,7 +677,7 @@ class Bridge {
667
677
  return null;
668
678
  }
669
679
  };
670
- const areas: Promise<AreaStatus>[] = [];
680
+ const areas: Promise<AreaStatus | AreaStatus[]>[] = [];
671
681
  if (present.has("backup")) areas.push((async () => backupStatus(await json("backup", ["vzdump-archives"]), await text("backup", ["restic-check-status"]), Date.now()))());
672
682
  if (present.has("edge")) areas.push((async () => edgeStatus(await text("edge", ["vps", "status"]), await text("edge", ["vps", "updates"])))());
673
683
  if (present.has("proxmox")) {
@@ -746,7 +756,7 @@ class Bridge {
746
756
  ]);
747
757
  })(),
748
758
  );
749
- return Promise.all(areas);
759
+ return (await Promise.all(areas)).flat();
750
760
  }
751
761
 
752
762
  private async arcaneReport(schedule: ReportSchedule, date: string): Promise<void> {
@@ -834,7 +844,7 @@ class Bridge {
834
844
  log(`arcane auto: container-watch failed: ${err instanceof Error ? err.message.split("\n")[0] : String(err)}`);
835
845
  }
836
846
  if (!snapshot) {
837
- await this.arcaneAttention(report, open[0].stack, `⚠️ The automatic updates of ${open.map((a) => a.stack).join(", ")} could not be checked again: container-watch failed. Check them with the arcane specialist.`);
847
+ await this.arcaneAttention(report, open[0].stack, `⚠️ Recheck of ${open.map((a) => a.stack).join(", ")} failed (container-watch). Ask the arcane specialist.`);
838
848
  return;
839
849
  }
840
850
  const now = Date.now();
@@ -864,9 +874,9 @@ class Bridge {
864
874
  rollback = `The rollback failed: ${(err instanceof Error ? err.message : String(err)).split("\n")[0].slice(0, 200)}.`;
865
875
  }
866
876
  } else if (a.stack === "arcane") {
867
- rollback = `Not rolled back: \`manager-rollback ${a.stamp ?? "<stamp>"}\` restores the previous manager, and the pin commit in git is reverted by hand.`;
877
+ rollback = `Not rolled back. Fix: \`manager-rollback ${a.stamp ?? "<stamp>"}\`, then revert the pin commit in git.`;
868
878
  } else {
869
- rollback = "Not rolled back: the update only pulled newer images of floating tags, which cannot be undone.";
879
+ rollback = "Not rolled back: only floating tags were pulled, which cannot be undone.";
870
880
  }
871
881
  log(`arcane auto: ${a.stack} failed its check (${problems.join(", ")}); ${rollback}`);
872
882
  await this.markArcane(report, a.stack, state, `${state} after the automatic update: ${problems.join(", ")}`);
@@ -875,7 +885,7 @@ class Bridge {
875
885
  try {
876
886
  const posted = await this.rest.createMessage(report.channelId, { content: autoFailureText(a, problems, rollback, owners), mentionUsers: owners, ...(page ? { replyTo: page.id } : {}) });
877
887
  const thread = await this.rest.startThread(report.channelId, posted.id, `Update of ${a.stack}`.slice(0, 100));
878
- const starter = await this.rest.createMessage(thread.id, { content: "🔎 The foreman reads only and asks the specialists for evidence. Write here to continue with the arcane specialist." });
888
+ const starter = await this.rest.createMessage(thread.id, { content: "🔎 The foreman investigates read-only. Write here to continue with the arcane specialist." });
879
889
  void this.investigateIn("foreman", { threadId: thread.id, starterId: starter.id, userId: owners[0], task: foremanTask(a, problems, rollback, Date.now()) }).then((ok) => (ok ? this.archiveThread(thread.id) : undefined));
880
890
  } catch (err) {
881
891
  log(`arcane auto: reporting ${a.stack} failed: ${err instanceof Error ? err.message : String(err)}`);
@@ -982,7 +992,7 @@ class Bridge {
982
992
  });
983
993
  const verdict = isFailed(result) ? null : parseVerdict(finalOutput(result.messages));
984
994
  log(`release review: ${subject}: ${verdict?.verdict ?? `no answer (${isFailed(result) ? "failed" : "unparsed"}: ${resultOutput(result).replace(/\s+/g, " ").slice(0, 300)})`}`);
985
- return verdict ?? { verdict: null, reason: "the review gave no usable answer; read the notes before updating" };
995
+ return verdict ?? { verdict: null, reason: "no usable review; read the notes before updating" };
986
996
  } catch (err) {
987
997
  return { verdict: null, reason: `the review failed: ${err instanceof Error ? err.message.slice(0, 120) : String(err)}` };
988
998
  }
@@ -996,7 +1006,7 @@ class Bridge {
996
1006
  if (interrupted.length === 0 && report.brewAll !== "running") return;
997
1007
  for (const l of interrupted) {
998
1008
  l.state = "failed";
999
- l.detail = "interrupted by a bridge restart; check the version before pressing Retry";
1009
+ l.detail = "interrupted by a bridge restart; check the version before Retry";
1000
1010
  }
1001
1011
  if (report.brewAll === "running") {
1002
1012
  report.brewAll = "failed";
@@ -1013,7 +1023,7 @@ class Bridge {
1013
1023
  if (!report || stuck.length === 0) return;
1014
1024
  for (const s of stuck) {
1015
1025
  s.state = "failed";
1016
- s.detail = "interrupted by a bridge restart; check the stack and the GitOps repository with the arcane specialist before pressing Retry";
1026
+ s.detail = "interrupted by a bridge restart; check the stack and the GitOps repository with the arcane specialist before Retry";
1017
1027
  }
1018
1028
  writePostedArcane(report);
1019
1029
  for (const m of report.messages.filter((x) => stuck.some((s) => x.stacks.includes(s.stack)))) await this.rest.editComponents(report.channelId, m.id, arcaneComponents(report, m)).catch(() => undefined);
@@ -10,6 +10,8 @@ export interface ReportState {
10
10
  messageId?: string;
11
11
  /** The morning run: `once` keys of the problems it has tagged the owners for, so a waiting release tags them once. */
12
12
  notified?: string[];
13
+ /** The morning run: the message of each `once` problem still open, so later summaries link to it. */
14
+ links?: Record<string, string>;
13
15
  }
14
16
 
15
17
  /** Local date and minutes past midnight in a time zone. */
@@ -139,7 +139,7 @@ describe("security monitor", () => {
139
139
  h.answers.set("security wazuh-events", events([wazuh({ id: "a1" }), wazuh({ id: "a2", timestamp: "2026-10-21T10:00:01.000+0000" })]));
140
140
  await h.monitor.poll(SCHEDULE);
141
141
  expect(h.rest.components).toHaveLength(1);
142
- expect(h.rest.components[0].components[0].content).toContain("first time this week · × 2 in this check");
142
+ expect(h.rest.components[0].components[0].content).toContain("first this week · × 2 this check");
143
143
  const messageId = readSecurityState().alerts[0].messageId;
144
144
  await h.monitor.onButton({ interactionId: "i", token: "t", userId: OWNER, messageId, customId: "security:ack" });
145
145
  expect(h.rest.deferred).toBe(1);
@@ -188,7 +188,7 @@ describe("security monitor", () => {
188
188
  h.clock.now += 7 * 60_000;
189
189
  await h.monitor.poll(SCHEDULE);
190
190
  expect(h.rest.components).toHaveLength(1);
191
- expect(h.rest.components[0].components[0].content).toContain("origin unchecked: the origin check kept failing: arcane: token request failed");
191
+ expect(h.rest.components[0].components[0].content).toContain("origin unchecked: origin check kept failing: arcane: token request failed");
192
192
  expect(readSecurityState().pending).toHaveLength(0);
193
193
  });
194
194
 
@@ -221,7 +221,7 @@ describe("security monitor", () => {
221
221
  expect(warnings[0].message.mentionUsers).toEqual([OWNER]);
222
222
  h.answers.set("security wazuh-events", events([]));
223
223
  await h.monitor.poll(SCHEDULE);
224
- expect(h.rest.messages.at(-1)?.message.content).toBe("✅ **Security checks** read Wazuh alerts again; anything from the gap is checked now.");
224
+ expect(h.rest.messages.at(-1)?.message.content).toBe("✅ **Security checks** read Wazuh alerts again; the gap is checked now.");
225
225
  });
226
226
 
227
227
  it("posts health changes: a new problem with a tag, recovery without", async () => {
@@ -237,7 +237,7 @@ describe("security monitor", () => {
237
237
  expect(h.rest.messages).toHaveLength(1);
238
238
  h.answers.set("security attention", attention([]));
239
239
  await h.monitor.checkHealth(SCHEDULE);
240
- expect(h.rest.messages[1].message).toEqual({ content: "✅ **Security health recovered:** every check is clear again." });
240
+ expect(h.rest.messages[1].message).toEqual({ content: "✅ **Security health recovered:** all checks clear." });
241
241
  });
242
242
 
243
243
  it("lets only owners press the buttons, and only on a tracked alert", async () => {
@@ -245,7 +245,7 @@ describe("security monitor", () => {
245
245
  await h.monitor.onButton({ interactionId: "i", token: "t", userId: "999999999999999999", messageId: "m", customId: "security:ack" });
246
246
  await h.monitor.onButton({ interactionId: "i", token: "t", userId: OWNER, messageId: "m", customId: "security:nope" });
247
247
  await h.monitor.onButton({ interactionId: "i", token: "t", userId: OWNER, messageId: "unknown", customId: "security:investigate" });
248
- expect(h.rest.replies).toEqual(["Only owners investigate or acknowledge security alerts.", "This button is no longer active.", "This alert is no longer tracked (older than 14 days); ask the security specialist in the channel."]);
248
+ expect(h.rest.replies).toEqual(["Owners only.", "Button no longer active.", "Alert no longer tracked (over 14 days old); ask the security specialist in the channel."]);
249
249
  expect(h.rest.deferred).toBe(0);
250
250
  });
251
251
 
@@ -305,14 +305,14 @@ describe("security monitor", () => {
305
305
  const summary = await h.monitor.dailyReport(SCHEDULE, "2026-10-21");
306
306
  expect(h.rest.messages).toEqual([]);
307
307
  const text = JSON.stringify(h.rest.components[0].components);
308
- expect(text).toContain("🐳 **Arcane images** · 1 critical CVE whose package has a fix, in 1 image\"");
309
- expect(text).not.toContain("waiting on the publisher");
308
+ expect(text).toContain("🐳 **Arcane images** · 1 fixable critical CVE in 1 image\"");
309
+ expect(text).not.toContain("waiting on publisher");
310
310
  expect(h.calls).toContainEqual(["arcane", "images", "list", "--all"]);
311
311
  expect(h.calls).toContainEqual(["arcane", "images", "vulnerabilities", "list", "--severity", "critical", "--all"]);
312
312
  expect(summary).toEqual({ headline: "quiet day (24 h to 06:05 CEST)", needs: [], messageId: "100" });
313
313
  expect(text).toContain("## 🛡️ Security Daily Report\\n🟢 quiet day · 24 h to 06:05 CEST");
314
314
  expect(text).toContain("🚫 **CrowdSec** · 2 bans, all front door SNI scanning");
315
- expect(text).toContain("📦 **Host packages** · 1 new CVE on 1 host, fixed versions exist\\n- CVE-1 (openssl, High 7.5) on dmz");
315
+ expect(text).toContain("📦 **Host packages** · 1 new CVE on 1 host, all fixable\\n- CVE-1 (openssl, High 7.5) on dmz");
316
316
  expect(h.rest.components[0].mentionUsers).toBeUndefined();
317
317
  expect(readReportState("security")).toMatchObject({ date: "2026-10-21", status: "sent" });
318
318
  expect(readSecurityState().vulns.reported["dmz|CVE-1"]).toMatchObject({ fix: true });