@astrofoundry/pi-astro 0.40.62 → 0.40.64

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,6 +10,7 @@ import {
10
10
  carefulTask,
11
11
  failureMessage,
12
12
  headText,
13
+ isPending,
13
14
  layoutMessages,
14
15
  majorLines,
15
16
  managerSummary,
@@ -18,6 +19,8 @@ import {
18
19
  readPostedArcane,
19
20
  releaseNotesFromJson,
20
21
  REPORT_PREFIX,
22
+ serviceText,
23
+ type ServiceLine,
21
24
  stackComponent,
22
25
  type StackEntry,
23
26
  stackOutcome,
@@ -94,6 +97,22 @@ function entry(r: ArcaneReport, stack: string): StackEntry {
94
97
  }
95
98
 
96
99
  describe("Arcane Daily Report layout", () => {
100
+ it("keeps a held service out of the stack's update and shows why it is held", () => {
101
+ const proxy = { stack: "bigcapital", service: "proxy", ref: "envoyproxy/envoy:v1.39.1", running: "v1.39.1", latest: "v1.39.6", status: "release", action: "edit tag", note: "" };
102
+ const holds = { "bigcapital/gotenberg": "Bigcapital needs Gotenberg 7", "bigcapital/proxy": "wait for the next bundle" };
103
+ const built = buildStacks({ ...updateReport, rows: [rows[5], proxy], holds });
104
+ const stack = built?.stacks[0];
105
+ expect(stack?.held).toBeUndefined();
106
+ expect(stack?.actionable).toBe(false);
107
+ expect(stack?.lines.map((l) => [l.service, l.held, isPending(l)])).toEqual([
108
+ ["gotenberg", "Bigcapital needs Gotenberg 7", false],
109
+ ["proxy", "wait for the next bundle", false],
110
+ ]);
111
+ expect(serviceText(stack?.lines[0] as ServiceLine)).toBe("• gotenberg: 7.10.2, newer major 8 available (a tag change by hand; Update does not apply it)\n-# ⏸ held: Bigcapital needs Gotenberg 7");
112
+ const partly = buildStacks({ ...updateReport, rows: [rows[5], proxy], holds: { "bigcapital/gotenberg": "x" } });
113
+ expect(partly?.stacks[0].actionable).toBe(true);
114
+ });
115
+
97
116
  it("marks the images our own CI builds and leaves out stopped stacks", () => {
98
117
  const own = { stack: "bigcapital", service: "server", ref: "ghcr.io/astrofoundry/bigcapital-server:astrofoundry", running: "?", latest: "new build", status: "update", action: "pull", note: "versions unavailable: private image" };
99
118
  const stopped = { ...own, stack: "grimoire-graphile-bootstrap", service: "bootstrap", ref: "ghcr.io/astronaute77/arcane-graphile-worker:latest", stopped: true };
@@ -63,9 +63,9 @@ export function crossesMajor(running: string | null, latest: string | null): boo
63
63
 
64
64
  const PENDING = new Set(["update", "release", "rebuild"]);
65
65
 
66
- /** A line whose stack-update would change something. */
67
- export function isPending(l: Pick<ServiceLine, "status">): boolean {
68
- return PENDING.has(l.status);
66
+ /** A line whose stack-update would change something; a held service keeps its version. */
67
+ export function isPending(l: Pick<ServiceLine, "status" | "held">): boolean {
68
+ return PENDING.has(l.status) && l.held === undefined;
69
69
  }
70
70
 
71
71
  export interface ServiceLine {
@@ -83,6 +83,8 @@ export interface ServiceLine {
83
83
  review?: Review;
84
84
  /** The image is one our own CI builds, so its update goes through Upgrade carefully. */
85
85
  own?: boolean;
86
+ /** Why this service is held (`<stack>/<service>` in `update-holds.json`); stack-update skips it. */
87
+ held?: string;
86
88
  }
87
89
 
88
90
  export type StackState = "offered" | "running" | "updated" | "rolled back" | "failed" | "current" | "planning";
@@ -166,12 +168,13 @@ export function buildStacks(report: unknown): ReportStacks | null {
166
168
  const running = releaseTag(r.running);
167
169
  const latest = releaseTag(r.latest);
168
170
  const own = r.ref ? isOwnImage(r.ref) : false;
169
- return { service: r.service, status: r.status, action: r.action ?? "", running, latest, tag: tagOf(r.ref ?? ""), ...(r.ref ? { ref: r.ref } : {}), major: PENDING.has(r.status) && crossesMajor(running, latest), note: r.note ?? "", ...(own ? { own } : {}) };
171
+ const held = holds[`${stack}/${r.service}`];
172
+ return { service: r.service, status: r.status, action: r.action ?? "", running, latest, tag: tagOf(r.ref ?? ""), ...(r.ref ? { ref: r.ref } : {}), major: PENDING.has(r.status) && crossesMajor(running, latest), note: r.note ?? "", ...(own ? { own } : {}), ...(held ? { held } : {}) };
170
173
  });
171
- const pending = lines.some((l) => PENDING.has(l.status));
174
+ const pending = lines.some(isPending);
172
175
  const held = holds[stack];
173
176
  if (stack === "arcane") {
174
- const offer = lines.map((l) => (PENDING.has(l.status) ? managerVersion(l) : null)).find((v) => v !== null) ?? null;
177
+ const offer = lines.map((l) => (isPending(l) ? managerVersion(l) : null)).find((v) => v !== null) ?? null;
175
178
  return { stack, lines, actionable: offer !== null && !held, ...(offer ? { manager: offer } : {}), ...(held ? { held } : {}), state: "offered" };
176
179
  }
177
180
  return { stack, lines, actionable: pending && !held, ...(held ? { held } : {}), state: "offered" };
@@ -212,7 +215,8 @@ export function serviceText(l: ServiceLine): string {
212
215
  const newer = PENDING.has(l.status) ? /newer major (\S+) available/.exec(l.note)?.[1] : undefined;
213
216
  if (newer) text += ` · newer major ${escapeDiscord(newer)} available`;
214
217
  const review = l.review ? `\n-# ${l.review.verdict ? `${VERDICT_MARK[l.review.verdict]}: ` : "notes: "}${escapeDiscord(l.review.reason)}` : "";
215
- return `• ${text}${review}`;
218
+ const held = l.held ? `\n-# ⏸ held: ${escapeDiscord(l.held)}` : "";
219
+ return `• ${text}${review}${held}`;
216
220
  }
217
221
 
218
222
  function stackText(s: StackEntry): string {
@@ -227,11 +227,11 @@ describe("report summaries", () => {
227
227
  const tool = (over: Partial<ToolsReport["lines"][number]>): ToolsReport["lines"][number] => ({ key: "brew:jq", label: "jq", installed: "1.8.1", latest: "1.8.2", major: false, action: { kind: "brew", name: "jq" }, state: "offered", ...over }) as ToolsReport["lines"][number];
228
228
 
229
229
  it("says what the tools report updated, what failed, and what waits for a button", () => {
230
- const report = { date: "2026-10-24", lines: [tool({ state: "updated", plan: "auto" }), tool({ key: "brew:herdr", label: "herdr", latest: "0.10.0", state: "failed", plan: "auto" }), tool({ key: "pi", label: "Pi", latest: "1.4.0", plan: "button" }), tool({ key: "spc:node", label: "Node", latest: "26.1.0", major: true, plan: "update" })], messages: [{ id: "9", keys: [], head: true, all: false }] } as unknown as ToolsReport;
230
+ const report = { date: "2026-10-24", lines: [tool({ state: "updated", plan: "auto" }), tool({ key: "brew:herdr", label: "herdr", latest: "0.10.0", state: "failed", plan: "auto" }), tool({ key: "pi", label: "Pi", latest: "1.4.0", plan: "button" }), tool({ key: "spc:node", label: "Node", latest: "26.1.0", major: true, plan: "update" }), tool({ key: "cask:temurin@17", label: "temurin@17", latest: "17.0.18", plan: "auto", manual: "its installer asks for your password; on Cortex: brew upgrade --cask temurin@17" })], messages: [{ id: "9", keys: [], head: true, all: false }] } as unknown as ToolsReport;
231
231
  expect(toolsSummary(report, "2026-10-24", "5")).toEqual({
232
232
  area: "Cortex tools",
233
- line: "1 updated (jq 1.8.2), 1 failed (herdr), Pi 1.4.0, Node 26.1.0 wait for a button",
234
- needs: ["Node 26.1.0 waits for its button (major version)", "Pi 1.4.0 waits for its button", "herdr automatic update failed"],
233
+ line: "1 updated (jq 1.8.2), 1 failed (herdr), Pi 1.4.0, Node 26.1.0 wait for a button, 1 by hand",
234
+ needs: ["Node 26.1.0 waits for its button (major version)", "Pi 1.4.0 waits for its button", "herdr automatic update failed", "temurin@17 17.0.18 by hand: its installer asks for your password; on Cortex: brew upgrade --cask temurin@17"],
235
235
  channelId: "5",
236
236
  messageId: "9",
237
237
  });
@@ -240,11 +240,20 @@ describe("report summaries", () => {
240
240
 
241
241
  it("says what the Arcane report updated, rolled back, and leaves to the owner", () => {
242
242
  const stack = (name: string, over: Partial<ArcaneReport["stacks"][number]>) => ({ stack: name, lines: [], actionable: true, state: "offered", ...over }) as ArcaneReport["stacks"][number];
243
- const report = { date: "2026-10-24", stacks: [stack("media", { plan: "auto", state: "updated" }), stack("mealie", { plan: "auto", state: "rolled back" }), stack("stirling-pdf", { plan: "careful" }), stack("supabase", { actionable: false, held: "review first" })], messages: [{ id: "8", stacks: [], head: true }] } as unknown as ArcaneReport;
243
+ const line = (over: Partial<ArcaneReport["stacks"][number]["lines"][number]>) => ({ service: "gotenberg", status: "current", action: "", running: "8.40.0", latest: "8.40.0", tag: "8.40.0", major: false, note: "newer major 9 available", ...over }) as ArcaneReport["stacks"][number]["lines"][number];
244
+ const stacks = [
245
+ stack("media", { plan: "auto", state: "updated" }),
246
+ stack("mealie", { plan: "auto", state: "rolled back" }),
247
+ stack("stirling-pdf", { plan: "careful" }),
248
+ stack("supabase", { actionable: false, held: "review first" }),
249
+ stack("paperless", { actionable: false, lines: [line({}), line({ service: "broker", status: "unknown", note: "registry timeout" })] }),
250
+ stack("bigcapital", { actionable: false, lines: [line({ held: "Bigcapital needs Gotenberg 8" })] }),
251
+ ];
252
+ const report = { date: "2026-10-24", stacks, messages: [{ id: "8", stacks: [], head: true }] } as unknown as ArcaneReport;
244
253
  expect(arcaneSummary(report, "2026-10-24", "6")).toEqual({
245
254
  area: "Arcane",
246
255
  line: "1 stack updated (media), mealie rolled back, stirling-pdf waits for you, supabase held",
247
- needs: ["stirling-pdf waits for you (Upgrade carefully)", "mealie automatic update rolled back"],
256
+ needs: ["stirling-pdf waits for you (Upgrade carefully)", "mealie automatic update rolled back", "paperless gotenberg 8.40.0: major 9 needs a tag change by hand", "paperless broker: the update check failed"],
248
257
  channelId: "6",
249
258
  messageId: "8",
250
259
  });
@@ -369,7 +369,7 @@ export function macosStatus(macs: readonly MacUpdates[]): AreaStatus {
369
369
  export interface ReportLine {
370
370
  area: string;
371
371
  line: string;
372
- /** Items the report tagged the owners for. */
372
+ /** Items that need the owners: what the report tagged them for, waits for them, or must be done by hand. */
373
373
  needs: string[];
374
374
  channelId?: string;
375
375
  messageId?: string;
@@ -396,11 +396,21 @@ export function toolsSummary(report: ToolsReport | null, date: string, channelId
396
396
  ...report.lines.filter((l) => l.plan === "update" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} waits for its button (${l.major ? "major version" : "release notes"})`),
397
397
  ...report.lines.filter((l) => l.plan === "button" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} waits for its button`),
398
398
  ...failed.filter((l) => l.plan === "auto").map((l) => `${escapeDiscord(l.label)} automatic update failed`),
399
+ ...manual.map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} by hand: ${escapeDiscord(l.manual ?? "")}`),
399
400
  ];
400
401
  return { area: "Cortex tools", line: parts.join(", ") || "everything up to date", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
401
402
  }
402
403
 
403
- /** The Arcane Daily Report after its automatic updates and checks: updated, rolled back, waiting for the owner, held. */
404
+ /** A service line only the owner can act on: a failed check, or a newer major that needs a tag change by hand; a held service is the owner's own decision. */
405
+ function byHand(stack: string, l: ArcaneReport["stacks"][number]["lines"][number]): string | null {
406
+ if (l.held !== undefined) return null;
407
+ const where = `${escapeDiscord(stack)} ${escapeDiscord(l.service)}`;
408
+ if (l.status === "unknown") return `${where}: the update check failed`;
409
+ const newer = /newer major (\S+) available/.exec(l.note)?.[1];
410
+ return newer ? `${where} ${escapeDiscord(l.running ?? l.tag)}: major ${escapeDiscord(newer)} needs a tag change by hand` : null;
411
+ }
412
+
413
+ /** The Arcane Daily Report after its automatic updates and checks: updated, rolled back, waiting for the owner, held, by hand. Holds are the owner's own decisions, so they are not items. */
404
414
  export function arcaneSummary(report: ArcaneReport | null, date: string, channelId: string): ReportLine {
405
415
  if (!report || report.date !== date) return { area: "Arcane", line: "no report today; the bridge log has the reason", needs: ["the Arcane Daily Report did not run today; the bridge log has the reason"], channelId };
406
416
  const by = (pick: (s: ArcaneReport["stacks"][number]) => boolean) => report.stacks.filter(pick).map((s) => s.stack);
@@ -419,6 +429,8 @@ export function arcaneSummary(report: ArcaneReport | null, date: string, channel
419
429
  const needs = [
420
430
  ...waiting.map((s) => `${escapeDiscord(s.stack)} waits for you (${s.plan === "careful" ? "Upgrade carefully" : "major version"})`),
421
431
  ...report.stacks.filter((s) => s.plan === "auto" && (s.state === "rolled back" || s.state === "failed")).map((s) => `${escapeDiscord(s.stack)} automatic update ${s.state}`),
432
+ ...report.stacks.flatMap((s) => s.lines.map((l) => byHand(s.stack, l)).filter((t): t is string => t !== null)),
433
+ ...(report.holdsError ? [`update-holds.json is unreadable: ${escapeDiscord(report.holdsError)}`] : []),
422
434
  ];
423
435
  return { area: "Arcane", line: parts.join(", ") || "no updates", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
424
436
  }
@@ -613,7 +613,7 @@ class Bridge {
613
613
  const name: ReportName = monday ? "security-weekly" : "security";
614
614
  const posted = done(name) ? undefined : await (monday ? this.security.weeklyReport(securitySchedule, date) : this.security.dailyReport(securitySchedule, date)).catch((err) => log(`daily ${date}: security report failed: ${err instanceof Error ? err.message : String(err)}`));
615
615
  const area = monday ? "Security (weekly)" : "Security";
616
- reports.push(posted ? { area, line: posted.headline, needs: posted.tagged ? [posted.headline] : [], channelId: securitySchedule.channelId, messageId: posted.messageId } : { area, line: done(name) ? "already posted today" : "the report failed; the bridge log has the reason", needs: done(name) ? [] : ["the security report failed; the bridge log has the reason"], channelId: securitySchedule.channelId });
616
+ reports.push(posted ? { area, line: posted.headline, needs: posted.needs, channelId: securitySchedule.channelId, messageId: posted.messageId } : { area, line: done(name) ? "already posted today" : "the report failed; the bridge log has the reason", needs: done(name) ? [] : ["the security report failed; the bridge log has the reason"], channelId: securitySchedule.channelId });
617
617
  }
618
618
  const tools = reportSchedule(this.config, "tools");
619
619
  if (tools && !this.stopping) {
@@ -309,7 +309,7 @@ describe("security monitor", () => {
309
309
  expect(text).not.toContain("waiting on the publisher");
310
310
  expect(h.calls).toContainEqual(["arcane", "images", "list", "--all"]);
311
311
  expect(h.calls).toContainEqual(["arcane", "images", "vulnerabilities", "list", "--severity", "critical", "--all"]);
312
- expect(summary).toEqual({ headline: "quiet day (24 h to 06:05 CEST)", tagged: false, messageId: "100" });
312
+ expect(summary).toEqual({ headline: "quiet day (24 h to 06:05 CEST)", needs: [], messageId: "100" });
313
313
  expect(text).toContain("## 🛡️ Security Daily Report\\n🟢 quiet day · 24 h to 06:05 CEST");
314
314
  expect(text).toContain("🚫 **CrowdSec** · 2 bans, all front door SNI scanning");
315
315
  expect(text).toContain("📦 **Host packages** · 1 new CVE on 1 host, fixed versions exist\\n- CVE-1 (openssl, High 7.5) on dmz");
@@ -94,7 +94,8 @@ export interface SecurityClick {
94
94
  /** What a posted security report tells the morning summary. */
95
95
  export interface PostedSummary {
96
96
  headline: string;
97
- tagged: boolean;
97
+ /** What the report lists under Needs you. */
98
+ needs: string[];
98
99
  messageId: string;
99
100
  }
100
101
 
@@ -460,7 +461,7 @@ export class SecurityMonitor {
460
461
  return (await this.deps.rest.createComponentsMessage(channelId, report.components, report.mention ? this.owners() : undefined)).id;
461
462
  }
462
463
 
463
- /** Posts the Security Daily Report; returns its headline and whether it tagged the owners, for the morning summary. */
464
+ /** Posts the Security Daily Report; returns its headline and what needs the owners, for the morning summary. */
464
465
  async dailyReport(s: ReportSchedule, date: string): Promise<PostedSummary | undefined> {
465
466
  const now = this.deps.now();
466
467
  if (reportSchedule(this.deps.config(), "security-weekly") && zonedWeekday(s.timeZone, new Date(now)) === MONDAY) {
@@ -478,13 +479,13 @@ export class SecurityMonitor {
478
479
  const state = readSecurityState();
479
480
  const wazuh = wazuhDigest(events, state.origins, (e) => acked(state, e));
480
481
  if (this.deps.stopping()) return;
481
- const data = { until: clockText(new Date(now).toISOString(), s.timeZone).replace(/^\d+ \w+ /, ""), crowdsec, wazuh, found: vulns?.found ?? null, closed: vulns?.closed ?? null, attention, ...images };
482
+ const data = { until: clockText(new Date(now).toISOString(), s.timeZone).replace(/^\d+ \w+ /, ""), timeZone: s.timeZone, crowdsec, wazuh, found: vulns?.found ?? null, closed: vulns?.closed ?? null, attention, ...images, alerts: state.alerts };
482
483
  writeReportState("security", { date, status: "sending" });
483
484
  const report = dailyCard(data, this.owners());
484
485
  const id = await this.postReport(s.channelId, report);
485
486
  writeReportState("security", { date, status: "sent", messageId: id });
486
487
  this.deps.log(`security report ${date}: posted`);
487
- return { headline: report.headline, tagged: report.mention, messageId: id };
488
+ return { headline: report.headline, needs: report.needs, messageId: id };
488
489
  }
489
490
 
490
491
  async weeklyReport(s: ReportSchedule, date: string): Promise<PostedSummary | undefined> {
@@ -543,7 +544,7 @@ export class SecurityMonitor {
543
544
  const id = await this.postReport(s.channelId, report);
544
545
  writeReportState("security-weekly", { date, status: "sent", messageId: id });
545
546
  this.deps.log(`security weekly report ${date}: posted`);
546
- return { headline: report.headline, tagged: report.mention, messageId: id };
547
+ return { headline: report.headline, needs: report.needs, messageId: id };
547
548
  }
548
549
 
549
550
  // ------------------------------------------------------------ buttons
@@ -17,6 +17,7 @@ import {
17
17
  crowdsecDailyLine,
18
18
  crowdsecWeeklyLine,
19
19
  dailyCard,
20
+ dailyNeeds,
20
21
  dailyNeedsOwner,
21
22
  emptyState,
22
23
  healthChange,
@@ -481,9 +482,10 @@ describe("daily report", () => {
481
482
  const attention = parseAttention({ checked_at: "x", bouncer: { count: 1, oldest_pull_seconds: 6 }, wazuh_agents: { total: 8, active: 8 }, health_issues: [] });
482
483
 
483
484
  it("reads quiet day in a green card and tags nobody when nothing is open", () => {
484
- const data = { until: "06:00 CEST", crowdsec: summary(), wazuh: wazuhDigest([], {}, () => false), found: [], closed: [], attention, images: { scanned: 57, total: 57, critical: [] }, offers: new Map() };
485
+ const data = { until: "06:00 CEST", timeZone: TZ, crowdsec: summary(), wazuh: wazuhDigest([], {}, () => false), found: [], closed: [], attention, images: { scanned: 57, total: 57, critical: [] }, offers: new Map(), alerts: [] };
485
486
  const report = dailyCard(data, [OWNER]);
486
487
  expect(report.headline).toBe("quiet day (24 h to 06:00 CEST)");
488
+ expect(report.needs).toEqual([]);
487
489
  expect(report.mention).toBe(false);
488
490
  expect(report.components).toEqual([
489
491
  {
@@ -507,14 +509,40 @@ describe("daily report", () => {
507
509
 
508
510
  it("names urgent alerts and failed checks in an orange card and tags the owner while something is open", () => {
509
511
  const wazuh = wazuhDigest([event({ rule: "5712", level: 12, description: "brute force" })], {}, () => false);
510
- const data = { until: "06:00 CEST", crowdsec: null, wazuh, found: [], closed: [], attention, images: { scanned: 57, total: 57, critical: [] }, offers: new Map() };
512
+ const data = { until: "06:00 CEST", timeZone: TZ, crowdsec: null, wazuh, found: [], closed: [], attention, images: { scanned: 57, total: 57, critical: [] }, offers: new Map(), alerts: [{ key: alertKey(event({ rule: "5712" })), channelId: "c9" }] };
511
513
  const report = dailyCard(data, [OWNER]);
512
514
  expect(report.headline).toBe("1 urgent alert, 1 check failed (24 h to 06:00 CEST)");
515
+ expect(report.needs).toEqual(["urgent alert not acknowledged: brute force on arcane, 20 Oct 14:50 CEST; Acknowledge it in <#c9>", "1 check of this report failed; the bridge log on Cortex has the reason"]);
513
516
  expect(report.mention).toBe(true);
514
517
  expect(report.components[0].accent_color).toBe(0xe67e22);
515
- expect(texts(report.components)[0]).toBe("## 🛡️ Security Daily Report\n🟠 1 urgent alert, 1 check failed · 24 h to 06:00 CEST");
518
+ expect(texts(report.components).slice(0, 2)).toEqual([
519
+ "## 🛡️ Security Daily Report\n🟠 1 urgent alert, 1 check failed · 24 h to 06:00 CEST",
520
+ "### Needs you\n1. urgent alert not acknowledged: brute force on arcane, 20 Oct 14:50 CEST; Acknowledge it in <#c9>\n2. 1 check of this report failed; the bridge log on Cortex has the reason",
521
+ ]);
516
522
  expect(report.components[1]).toEqual({ type: 10, content: `<@${OWNER}>` });
517
523
  });
524
+
525
+ it("leaves out a NAS loss that a later restore ended and names our images to rebuild", () => {
526
+ const lost = event({ id: "n1", rule: "100012", level: 12, timestamp: "2026-10-20T01:00:00.000+0000" });
527
+ const back = event({ id: "n2", rule: "100013", level: 10, timestamp: "2026-10-20T01:05:00.000+0000" });
528
+ const oom = event({ id: "o1", agent: "space-x", rule: "5108", level: 12, description: "System running out of memory. Availability of the system is in risk.", timestamp: "2026-10-20T06:24:00.000+0000" });
529
+ const wazuh = wazuhDigest([lost, back, oom], {}, () => false);
530
+ expect(wazuh.unacknowledged.map((e) => e.id)).toEqual(["n1", "o1"]);
531
+ expect(wazuh.open.map((e) => e.id)).toEqual(["o1"]);
532
+ const builds = new Map([["ghcr.io/astrofoundry/app:1", { created: 1, newer: false }], ["ghcr.io/astrofoundry/web:1", { created: 1, newer: true }]]);
533
+ const critical = [
534
+ { id: "CVE-1", pkg: "a", image: "ghcr.io/astrofoundry/app:1" },
535
+ { id: "CVE-2", pkg: "b", image: "ghcr.io/astrofoundry/app:1" },
536
+ { id: "CVE-3", pkg: "c", image: "ghcr.io/astrofoundry/web:1" },
537
+ { id: "CVE-4", pkg: "d", image: "docker.io/library/redis:7" },
538
+ ];
539
+ const data = { until: "06:00 CEST", timeZone: TZ, crowdsec: summary(), wazuh, found: [], closed: [], attention, images: { scanned: 57, total: 57, critical, builds }, offers: new Map(), alerts: [{ key: alertKey(oom), channelId: "c9" }] };
540
+ expect(dailyNeeds(data)).toEqual([
541
+ "urgent alert not acknowledged: System running out of memory. Availability of the system is in risk on space-x, 20 Oct 08:24 CEST; Acknowledge it in <#c9>",
542
+ "rebuild app:1 (2): our images carry critical CVEs that have a fixed version",
543
+ ]);
544
+ expect(dailyCard(data, [OWNER]).components[0].accent_color).toBe(0xe67e22);
545
+ });
518
546
  });
519
547
 
520
548
  describe("weekly report", () => {
@@ -410,6 +410,8 @@ export interface WazuhDigest {
410
410
  urgent: number;
411
411
  /** Urgent events that were not acknowledged when they came. */
412
412
  unacknowledged: WazuhEvent[];
413
+ /** Unacknowledged events still open: a NAS loss that a later restore on the same agent ended is not. */
414
+ open: WazuhEvent[];
413
415
  /** Deployment-file changes no commit or operation of ours explains. */
414
416
  unexplained: WazuhEvent[];
415
417
  }
@@ -421,8 +423,8 @@ function originCount(n: number, of: number): string {
421
423
 
422
424
  /** The Wazuh line of a report: alerts at level 10 or higher, grouped by issue and agent, urgent ones first. */
423
425
  export function wazuhDigest(events: readonly WazuhEvent[] | null, origins: SecurityState["origins"], acked: (e: WazuhEvent) => boolean, maxGroups = 4): WazuhDigest {
424
- if (!events) return { line: "Wazuh · the check failed", failed: true, count: 0, urgent: 0, unacknowledged: [], unexplained: [] };
425
- if (events.length === 0) return { line: "Wazuh · no alerts at level 10+", failed: false, count: 0, urgent: 0, unacknowledged: [], unexplained: [] };
426
+ if (!events) return { line: "Wazuh · the check failed", failed: true, count: 0, urgent: 0, unacknowledged: [], open: [], unexplained: [] };
427
+ if (events.length === 0) return { line: "Wazuh · no alerts at level 10+", failed: false, count: 0, urgent: 0, unacknowledged: [], open: [], unexplained: [] };
426
428
  const groups = new Map<string, { label: string; agent: string; events: WazuhEvent[]; urgent: number; known: number }>();
427
429
  let urgent = 0;
428
430
  const unacknowledged: WazuhEvent[] = [];
@@ -451,7 +453,10 @@ export function wazuhDigest(events: readonly WazuhEvent[] | null, origins: Secur
451
453
  const quiet = urgent - unacknowledged.length;
452
454
  const urgentText = urgent === 0 ? "none urgent" : `${urgent} urgent${quiet > 0 ? ` (${quiet} acknowledged)` : ""}`;
453
455
  const line = `Wazuh · ${events.length} alert${events.length === 1 ? "" : "s"} at level 10+, ${urgentText} · ${shown.join(" · ")}${more}`;
454
- return { line, failed: false, count: events.length, urgent, unacknowledged, unexplained };
456
+ const restored = new Map<string, number>();
457
+ for (const e of events) if (e.rule === "100013") restored.set(e.agent, Math.max(restored.get(e.agent) ?? 0, Date.parse(e.timestamp)));
458
+ const open = unacknowledged.filter((e) => !(e.rule === "100012" && (restored.get(e.agent) ?? 0) > Date.parse(e.timestamp)));
459
+ return { line, failed: false, count: events.length, urgent, unacknowledged, open, unexplained };
455
460
  }
456
461
 
457
462
  // ---------------------------------------------------------------- vulnerabilities
@@ -717,6 +722,19 @@ export function imageVulnLine(facts: ImageVulnFacts | null, offers: ReadonlyMap<
717
722
  return parts.join(" · ");
718
723
  }
719
724
 
725
+ /** Our own images with critical CVEs that have a fix and no newer build in the registry: only our rebuild closes them. */
726
+ export function rebuildItem(facts: ImageVulnFacts | null): string | null {
727
+ if (!facts) return null;
728
+ const counts = new Map<string, number>();
729
+ for (const v of facts.critical) {
730
+ const build = facts.builds?.get(v.image) ?? facts.builds?.get(v.image.split("@")[0]);
731
+ if (isOwnImage(v.image) && !build?.newer) counts.set(v.image, (counts.get(v.image) ?? 0) + 1);
732
+ }
733
+ if (counts.size === 0) return null;
734
+ const images = [...counts.entries()].sort((x, y) => y[1] - x[1] || x[0].localeCompare(y[0])).map(([image, n]) => `${escapeDiscord(shortImageName(image))} (${n})`);
735
+ return `rebuild ${images.join(", ")}: our images carry critical CVEs that have a fixed version`;
736
+ }
737
+
720
738
  // ---------------------------------------------------------------- reports
721
739
 
722
740
  const GREEN = 0x57f287;
@@ -733,10 +751,17 @@ export function section(icon: string, line: string): string {
733
751
  export interface ReportCard {
734
752
  /** The state without the report's name, for the morning summary: `quiet day (24 h to 06:00 CEST)`. */
735
753
  headline: string;
754
+ /** What the owners must act on, listed under Needs you and in the morning summary. */
755
+ needs: string[];
736
756
  mention: boolean;
737
757
  components: MessageComponent[];
738
758
  }
739
759
 
760
+ /** The numbered Needs you list right under a report's title, or nothing. */
761
+ function needsBlock(items: readonly string[]): string[] {
762
+ return items.length === 0 ? [] : [`### Needs you\n${items.map((t, i) => `${i + 1}. ${t}`).join("\n")}`];
763
+ }
764
+
740
765
  /** Text Displays in groups divided by separators. */
741
766
  function card(groups: readonly (readonly string[])[], attention: boolean, mention: string | null): MessageComponent[] {
742
767
  const components = groups.filter((g) => g.length > 0).flatMap((g, i) => [...(i > 0 ? [SEPARATOR] : []), ...g.map((content): MessageComponent => ({ type: 10, content }))]);
@@ -746,6 +771,7 @@ function card(groups: readonly (readonly string[])[], attention: boolean, mentio
746
771
  export interface DailyData {
747
772
  /** Local time the report covers up to, already formatted (`06:00 CEST`). */
748
773
  until: string;
774
+ timeZone: string;
749
775
  crowdsec: AlertSummary | null;
750
776
  wazuh: WazuhDigest;
751
777
  found: VulnEvent[] | null;
@@ -755,6 +781,8 @@ export interface DailyData {
755
781
  offers: ReadonlyMap<string, ImageOffer>;
756
782
  /** The Arcane Daily Report's channel, linked from offered updates. */
757
783
  arcaneChannelId?: string;
784
+ /** Alerts posted to Discord, to tell an unacknowledged alert from one that was never posted. */
785
+ alerts: readonly Pick<PostedAlert, "key" | "channelId">[];
758
786
  }
759
787
 
760
788
  function headline(urgent: number, unexplained: number, health: number, failed: number): string {
@@ -776,9 +804,23 @@ export function dailyNeedsOwner(d: DailyData): boolean {
776
804
  return (d.attention?.issues.length ?? 0) > 0 || failures(d) > 0;
777
805
  }
778
806
 
807
+ /** What the owners must act on today: health issues, unexplained changes, open urgent alerts, our images to rebuild, failed checks. */
808
+ export function dailyNeeds(d: DailyData): string[] {
809
+ const items: string[] = [];
810
+ for (const i of d.attention?.issues ?? []) items.push(escapeDiscord(i.message.replace(/\.$/, "")));
811
+ if (d.wazuh.unexplained.length > 0) items.push(`${d.wazuh.unexplained.length} unexplained deployment change${d.wazuh.unexplained.length === 1 ? "" : "s"}; Investigate on the alert`);
812
+ items.push(...unacknowledgedItems(d.wazuh, d.alerts, d.timeZone));
813
+ const rebuild = rebuildItem(d.images);
814
+ if (rebuild) items.push(rebuild);
815
+ const failed = failures(d);
816
+ if (failed > 0) items.push(`${failed} check${failed === 1 ? "" : "s"} of this report failed; the bridge log on Cortex has the reason`);
817
+ return items;
818
+ }
819
+
779
820
  export function dailyCard(d: DailyData, owners: readonly string[]): ReportCard {
780
821
  const state = headline(d.wazuh.urgent, d.wazuh.unexplained.length, d.attention?.issues.length ?? 0, failures(d));
781
822
  const mention = dailyNeedsOwner(d);
823
+ const needs = dailyNeeds(d);
782
824
  const sections = [
783
825
  section("🚫", crowdsecDailyLine(d.crowdsec)),
784
826
  section("👁️", d.wazuh.line),
@@ -789,8 +831,9 @@ export function dailyCard(d: DailyData, owners: readonly string[]): ReportCard {
789
831
  ];
790
832
  return {
791
833
  headline: `${state} (24 h to ${d.until})`,
834
+ needs,
792
835
  mention,
793
- components: card([[`## 🛡️ ${DAILY_PREFIX}\n${state === "quiet day" ? "🟢" : "🟠"} ${state} · 24 h to ${d.until}`], sections], state !== "quiet day", mention ? mentionLine(owners) : null),
836
+ components: card([[`## 🛡️ ${DAILY_PREFIX}\n${state === "quiet day" ? "🟢" : "🟠"} ${state} · 24 h to ${d.until}`], needsBlock(needs), sections], state !== "quiet day" || needs.length > 0, mention ? mentionLine(owners) : null),
794
837
  };
795
838
  }
796
839
 
@@ -899,15 +942,15 @@ function crlText(crl: WeeklyData["crl"], timeZone: string): string {
899
942
  return `RADIUS CRL: renew within ${crl.days} days (warned ${dayText(crl.warned, timeZone)})`;
900
943
  }
901
944
 
902
- /** One item per alert key the owners did not acknowledge: what changed, when, and where to acknowledge it, if anywhere. */
903
- function unacknowledgedItems(d: WeeklyData): string[] {
945
+ /** One item per open alert key the owners did not acknowledge: what changed, when, and where to acknowledge it, if anywhere. */
946
+ function unacknowledgedItems(wazuh: WazuhDigest, alerts: readonly Pick<PostedAlert, "key" | "channelId">[], timeZone: string): string[] {
904
947
  const groups = new Map<string, WazuhEvent[]>();
905
- for (const e of d.wazuh.unacknowledged) if (e.rule !== DEPLOYMENT_RULE) groups.set(alertKey(e), [...(groups.get(alertKey(e)) ?? []), e]);
948
+ for (const e of wazuh.open) if (e.rule !== DEPLOYMENT_RULE) groups.set(alertKey(e), [...(groups.get(alertKey(e)) ?? []), e]);
906
949
  return [...groups.entries()].map(([key, events]) => {
907
950
  const last = events.reduce((a, b) => (Date.parse(b.timestamp) > Date.parse(a.timestamp) ? b : a));
908
951
  const what = last.path ? `\`${last.path.replace(/`/g, "'")}\` ${escapeDiscord(last.event ?? "changed")}` : escapeDiscord(issueLabel(last));
909
- const when = events.length > 1 ? ` × ${events.length}, last ${clockText(last.timestamp, d.timeZone)}` : `, ${clockText(last.timestamp, d.timeZone)}`;
910
- const posted = d.alerts.find((a) => a.key === key);
952
+ const when = events.length > 1 ? ` × ${events.length}, last ${clockText(last.timestamp, timeZone)}` : `, ${clockText(last.timestamp, timeZone)}`;
953
+ const posted = alerts.find((a) => a.key === key);
911
954
  const fact = `${what} on ${escapeDiscord(last.agent)}${when}`;
912
955
  return posted ? `urgent alert not acknowledged: ${fact}; Acknowledge it in <#${posted.channelId}>` : `urgent alert never posted, so it has no Acknowledge button: ${fact}; check that it was expected`;
913
956
  });
@@ -917,12 +960,14 @@ export function needsYou(d: WeeklyData): string[] {
917
960
  const items: string[] = [];
918
961
  for (const i of d.attention?.issues ?? []) items.push(escapeDiscord(i.message.replace(/\.$/, "")));
919
962
  if (d.wazuh.unexplained.length > 0) items.push(`${d.wazuh.unexplained.length} unexplained deployment change${d.wazuh.unexplained.length === 1 ? "" : "s"} this week; Investigate on the alert`);
920
- items.push(...unacknowledgedItems(d));
963
+ items.push(...unacknowledgedItems(d.wazuh, d.alerts, d.timeZone));
921
964
  if (d.patching && d.patching.rebootPending.length > 0) items.push(`reboot ${d.patching.rebootPending.map(escapeDiscord).join(", ")} to load installed updates`);
922
965
  const stuck = d.openWithFix.filter((v) => d.now - Date.parse(v.since) > 3 * DAY);
923
966
  if (stuck.length > 0) items.push(`${stuck.length} CVE${stuck.length === 1 ? "" : "s"} with a fixed version still open after 3 days (${[...new Set(stuck.map((v) => v.agent))].map(escapeDiscord).join(", ")}); unattended-upgrades may be stuck`);
924
967
  const offered = offeredImages(d.images, d.offers);
925
968
  if (offered.length > 0) items.push(`update ${offered.map(escapeDiscord).join(", ")} in the Arcane Daily Report: the newer image replaces one with a critical CVE`);
969
+ const rebuild = rebuildItem(d.images);
970
+ if (rebuild) items.push(rebuild);
926
971
  if (d.crl !== null && d.crl !== "unknown") items.push(`renew the RADIUS CRL (wifi-device.sh renew-crl), ${d.crl.days} days left`);
927
972
  if (d.backups) {
928
973
  const stale = d.backups.guests.filter((g) => d.now / 1000 - g.last > STALE_BACKUP_S);
@@ -959,11 +1004,11 @@ export function weeklyCard(d: WeeklyData, owners: readonly string[]): ReportCard
959
1004
  section("🩺", `${healthLine(d.attention)} · ${crlText(d.crl, d.timeZone)}`),
960
1005
  section("💾", backupLine(d.backups, d.now, d.timeZone)),
961
1006
  ];
962
- const needsBlock = items.length === 0 ? [] : [`### Needs you\n${items.map((t, i) => `${i + 1}. ${t}`).join("\n")}`];
963
1007
  return {
964
1008
  headline: `${d.range} · ${state} · ${needs}`,
1009
+ needs: items,
965
1010
  mention: items.length > 0,
966
- components: card([[`## 🛡️ ${WEEKLY_PREFIX} · ${d.range}\n${state} · ${needs}`], needsBlock, sections, [`-# ${lastDayLine(d.last24h)}`]], incidents > 0 || items.length > 0, items.length > 0 ? mentionLine(owners) : null),
1011
+ components: card([[`## 🛡️ ${WEEKLY_PREFIX} · ${d.range}\n${state} · ${needs}`], needsBlock(items), sections, [`-# ${lastDayLine(d.last24h)}`]], incidents > 0 || items.length > 0, items.length > 0 ? mentionLine(owners) : null),
967
1012
  };
968
1013
  }
969
1014
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@astrofoundry/pi-astro",
3
- "version": "0.40.62",
3
+ "version": "0.40.64",
4
4
  "description": "Personal pi customizations (extensions, subagents, skills, prompts, themes) for the pi coding agent.",
5
5
  "keywords": [
6
6
  "pi-package"
@@ -53,7 +53,7 @@ Apply updates only when asked, one stack at a time, with `["stack-update", "<sta
53
53
 
54
54
  The answer is JSON with `status`: `updated`, `rolled back`, `failed` (with `reason`), or `nothing to do`, plus `services` (`service`, `from`, `to`, `applied`: `pull` or the new reference, `result`: `current`, `behind`, `unchecked`) and `commit`/`revert` when the Compose file changed. Report it in plain words.
55
55
 
56
- It refuses `arcane` (next section), stopped stacks (a redeploy would start them), a checkout with local or unpushed changes, and held stacks. Held stacks are listed in `update-holds.json` at the root of the GitOps repository, `{ "<stack>": "<reason>" }`; `update-report` returns them in `holds`, and the daily report shows the reason without a button. Change that file only when the caller asks, through the `git` commands.
56
+ It refuses `arcane` (next section), stopped stacks (a redeploy would start them), a checkout with local or unpushed changes, and held stacks. Held stacks are listed in `update-holds.json` at the root of the GitOps repository, `{ "<stack>": "<reason>" }`; a key `<stack>/<service>` holds one service, which `stack-update` skips while the rest of the stack updates. `update-report` returns them in `holds`, and the daily report shows the reason without a button. Change that file only when the caller asks, through the `git` commands.
57
57
 
58
58
  An update across a major version is refused without `--confirm`: summarise its release notes first (next section), get the caller's go, then run `["stack-update", "<stack>", "--confirm"]`, which asks an owner for approval.
59
59
 
@@ -58,7 +58,7 @@ Wrappers ship as TypeScript and run under Node 24 type stripping: erasable synta
58
58
  - `entry/specialist-install.sh` runs as root, so it runs only `/usr/bin`, `/bin`, `/usr/sbin`, `/sbin` and the Node it installed, never Homebrew or anything else the main user can write; it checks that `/usr/local/libexec/specialist-node` (and each `specialist-<tool>` folder) and its parents belong to root and are writable by nobody else. Arguments are checked before the root test (tests exercise that part). Downloads go to a root-only temporary directory and are verified before use: Node against `SHASUMS256.txt` and the LTS flag in `index.json`, npm tarballs against the registry's `sha512` integrity, arcane-cli against the release's checksums file. Inside `/Users/specialist` it works only as `specialist` (`sudo -u specialist -H`), from a staging directory root hands over, so a link the specialist plants cannot redirect a root write. Each component switches by rename after a check (every installed wrapper's `--help`, `--version` of the new binary) and restores the previous copy when the check fails. Older versions are refused; `rollback` swaps current and previous. `python`, `gcloud`, `bw`, and `jq` follow the same rules: version folders under `/usr/local/libexec/specialist-<tool>/` behind the links `current` and `previous`, sealed `root:wheel` and not writable by others, smoke-tested as `specialist` before `current` moves; `python` refuses a switch that breaks the current gcloud. Consumers use the `current` paths: `provision.sh` for `bw` and `jq`, `edge.json` for `gcloud` and `gcloudPython`. Nothing a wrapper runs with secrets may come from Homebrew.
59
59
  - `main()` waits for stdout and stderr to flush before `process.exit`: on a pipe, exiting right after a write drops everything past the 64 KiB buffer, which cut large answers such as `projects list --all` until 2026-09-25 (`lib/flush.test.ts`).
60
60
  - `arcane update-report [--all]` is `lib/image-updates.ts` over `lib/registry.ts`: a fresh `images updates check --all`, then `projects list --all` for the stack and service of each image reference, then the registries' tag lists and manifest digests (anonymous bearer tokens from each registry's challenge; digests by HEAD, which Docker Hub does not count as a pull; lscr.io is read from ghcr.io). A floating or major-series tag is resolved to the release tag whose digest matches the running and the latest build, searching the newest tags of the three commonest release shapes; a fixed tag is compared with the newest tag of the same shape; a digest pin with the newest release. Checks older than one minute before the run count as stale and show as unknown. Private images (anonymous access refused with 401 or 403) are excluded from rows, counts, and table, listed in `privateImages`, and counted in the summary line. The table is fixed-width text for a code block, because Discord renders no Markdown tables. The answer adds `holds` from `update-holds.json` at origin's branch head (`git show`, no working-tree read), or `holdsError` when that file is malformed.
61
- - `arcane stack-update <stack> [--confirm]` (`lib/stack-update.ts` for the pure parts) is deterministic: refuses `arcane`, held stacks (the checkout's `update-holds.json` after `pull --ff-only`; a malformed file is an error), a dirty or ahead checkout, stopped stacks, and an update across a major version without `--confirm` (which the gate turns into an owner approval). Pending rows come from a fresh per-image check resolved like `update-report`. `edit tag`/`edit digest` rows rewrite `image:` lines that name exactly the old reference (`replaceImageRef`, quoted or not, trailing comment kept) in the sync's `composePath`, then commit, push, and wait for `gitops list` to report success at that commit (20 min, polled every 30 s). Then `projects redeploy <stack> --yes` (35 min), container checks through `containers list --all` and `containers get` (`composeInfo.projectName`, health; 5 min, polled every 15 s; a job that exited with code 0 counts as done whatever health its image's inherited check left, as bigcapital's migration did on 2026-10-28), and a fresh check that each service is current. A failure after a commit reverts, pushes, waits for the sync, redeploys, and checks again (`rolled back`); a pull-only failure stays `failed`. Every outcome prints JSON with `status` and exits 0. Worst case about 125 minutes, so the gate's `timeoutMs` for arcane must exceed it (8,100,000). It holds the `arcane-gitops` lock, so a second checkout call waits up to 2 minutes and then fails as busy; the bridge also runs one Arcane update at a time.
61
+ - `arcane stack-update <stack> [--confirm]` (`lib/stack-update.ts` for the pure parts) is deterministic: refuses `arcane`, held stacks (the checkout's `update-holds.json` after `pull --ff-only`; a malformed file is an error; a `<stack>/<service>` key skips that service's rows instead), a dirty or ahead checkout, stopped stacks, and an update across a major version without `--confirm` (which the gate turns into an owner approval). Pending rows come from a fresh per-image check resolved like `update-report`. `edit tag`/`edit digest` rows rewrite `image:` lines that name exactly the old reference (`replaceImageRef`, quoted or not, trailing comment kept) in the sync's `composePath`, then commit, push, and wait for `gitops list` to report success at that commit (20 min, polled every 30 s). Then `projects redeploy <stack> --yes` (35 min), container checks through `containers list --all` and `containers get` (`composeInfo.projectName`, health; 5 min, polled every 15 s; a job that exited with code 0 counts as done whatever health its image's inherited check left, as bigcapital's migration did on 2026-10-28), and a fresh check that each service is current. A failure after a commit reverts, pushes, waits for the sync, redeploys, and checks again (`rolled back`); a pull-only failure stays `failed`. Every outcome prints JSON with `status` and exits 0. Worst case about 125 minutes, so the gate's `timeoutMs` for arcane must exceed it (8,100,000). It holds the `arcane-gitops` lock, so a second checkout call waits up to 2 minutes and then fails as busy; the bridge also runs one Arcane update at a time.
62
62
  - `arcane host <command>` runs over the forced-command key of user `spc-arcane` on VM 100. `HOST_COMMANDS` in `lib/arcane-host.ts` must list exactly the operations of `entry/remote/spc-arcane-op.py` (a test compares them); the tracked copies live in the arcane repository's `host/` (`spc-arcane-host-entry`, `spc-arcane-op`, `spc-arcane-sudoers`, `spc-arcane-install.sh`) with the probes in `host/spc-arcane-probes/`. The entry only splits words (letters, digits, space, `._:/+-`) and hands them to `sudo -n /usr/local/sbin/spc-arcane-op`, the one command its sudoers line allows; the op checks the name and every argument and runs the reviewed arcane-ops probes, which print only allowlisted fields. Manager steps (`manager-*`) are internal to `manager-upgrade`/`manager-rollback` and refused through `host`. Nothing on the host deletes data: a manager rollback restores the image and Compose file, not the data volume (R3 stays manual), `db-restore` only fills an empty database, and dumps are never pruned by the tool. `supabase-backup` writes the pre-change backup of `supabase/AGENTS.md` (both databases with `pg_dump -Fc`, each listed and fully decoded by `pg_restore`, `pg_dumpall --globals-only`, and the `supabase-db-config` volume, which must hold `pgsodium_root.key`) into `/var/backups/supabase-upgrades/<UTC>`, the folder of the earlier manual backups. `host-audit` compares the host's hashes with `host/<name>` of the `arcane-gitops` checkout after a pull. The `bigcapital-*` commands run the base-image upgrade of `bigcapital/README.md`: an online backup, a rehearsal on an internal Docker network (restore into the current MariaDB, upgrade with the build's, migrations, server health, Envoy validation; everything it created removed afterwards), the owner-approved switch (stop, quiescent dump, Redis `SAVE`, volume copies compared entry by entry, the stack left stopped for the GitOps commit, the old volumes untouched), and a verify against the switch's row counts and ledger totals.
63
63
  - `arcane manager-upgrade <version>` (`lib/arcane-manager.ts`) follows arcane/AGENTS.md "Manager upgrades": the official image's revision label must equal the release tag's commit (GitHub API), `manager-prepare` records `compose.before.yaml` and every container before the pin commit, the pin goes through the `arcane-gitops` checkout (`git` is exported from `lib/repo.ts` for this), the wait polls `gitops list` and the deployed Compose hash every 30 s for up to 30 minutes, then pull, cold backup, `up -d --no-deps --pull never`, and checks. Any failure before the restart reverts the pin (`stopped`); a failed check after it reverts the pin and runs `manager-rollback` (`rolled back`). The answer always exits 0 with `status`, so the bridge and the agent both see it.
64
64
  - `research` is a pure forced-command wrapper: every call is one JSON request on stdin to `request` on VM 110 (`10.1.100.10`, user `spc-research`, key `SSH_KEY_SANDBOX`), whose entry hands it to `spc-research-op` as `researcher` (tracked in the homelab repository's `06-space-x-proxmox/research/system/`). The op attaches one playwright-cli session per Discord thread (`t<id>`) to the shared Chrome over CDP on `127.0.0.1:9222` and opens its own tab; cookie, storage, `run-code`, and browser lifecycle commands are refused. `script` runs the agent's code in the thread folder as `researcher` (5 minutes), on a VM that reaches only the internet (Mira's Sandbox zone plus the VM's own nftables). Defaults stay on Cortex in `work/research/defaults.json`, out of the sandbox's reach. `captcha-help` is in `RISKY`, so its owner approval is the moment the owner solved the challenge in the live view (`research-browser.37pla.net`); the bridge renders that ticket with the channel's `liveView` link.
@@ -104,7 +104,7 @@ ${Object.entries(HOST_COMMANDS)
104
104
  .join("\n")}
105
105
  update-report [--all] fresh image update check for every stack, with running and latest release versions
106
106
  from the registries; JSON with a ready table (--all also lists up-to-date services) and
107
- the held stacks of ${HOLDS_FILE}. Images the registry throttled (toomanyrequests) are
107
+ the held stacks and services of ${HOLDS_FILE}. Images the registry throttled (toomanyrequests) are
108
108
  checked again after 5 and 20 seconds. Checks only: never pulls images or redeploys.
109
109
  stack-update <stack> [--confirm]
110
110
  apply every pending update of one stack: pinned tags and digests move in the stack's
@@ -112,7 +112,7 @@ ${Object.entries(HOST_COMMANDS)
112
112
  with a pull, containers the redeploy left on an old image are recreated one by one, and
113
113
  the stack is checked (containers running and healthy, images current). A failed check after a
114
114
  Compose change reverts it and redeploys; a pull alone cannot be undone. Refuses held
115
- stacks, stopped stacks, and arcane (manager-upgrade). An update across a major version
115
+ stacks, stopped stacks, and arcane (manager-upgrade); skips held services. An update across a major version
116
116
  needs --confirm, which asks an owner for approval.
117
117
  stack-rollback <stack> <commit>
118
118
  undo one earlier stack-update commit of the stack's Compose file: revert, push, start the
@@ -753,7 +753,8 @@ async function stackUpdate(config: ArcaneConfig, args: string[]): Promise<number
753
753
 
754
754
  const before = await withCli(config, (cli) => stackState(cli, stack, Date.now()));
755
755
  if (before.stopped) throw new UsageError(`${stack} is stopped; a redeploy would start it, so start it on purpose first`);
756
- const pending = pendingRows(before.rows);
756
+ // A held service keeps its version; the rest of the stack still updates.
757
+ const pending = pendingRows(before.rows).filter((r) => !holds[`${stack}/${r.service}`]);
757
758
  const answer = (status: StackStatus, extra: Record<string, unknown>) => printJson({ status, stack, ...extra });
758
759
  if (pending.length === 0) {
759
760
  answer("nothing to do", { unknown: before.rows.filter((r) => r.status === "unknown").map((r) => ({ service: r.service, note: r.note })) });
@@ -18,9 +18,23 @@ case "$name" in
18
18
  case "$unit" in pomerium|nginx|lego-renew) ;; *) echo "bad unit" >&2; exit 2 ;; esac
19
19
  since_ok "$since" || { echo "bad since" >&2; exit 2; }
20
20
  journalctl -u "$unit" --since "$since" --no-pager -n 200 ;;
21
- config-template) cat /etc/pomerium/config.yaml.template ;;
21
+ config-template) sudo cat /etc/pomerium/config.yaml.template ;;
22
22
  render) sudo /usr/local/sbin/render-pomerium ;;
23
23
  pomerium-restart) sudo systemctl restart pomerium && systemctl is-active pomerium ;;
24
+ pomerium-version) /usr/local/bin/pomerium --version ;;
25
+ pomerium-upgrade)
26
+ # stdin: the release tarball as base64; pomerium-upgrade checks it against the sha256 argument.
27
+ version="${rest%% *}"; sum="${rest#* }"
28
+ printf '%s' "$version" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$' || { echo "bad version" >&2; exit 2; }
29
+ printf '%s' "$sum" | grep -Eq '^[0-9a-f]{64}$' || { echo "bad sha256" >&2; exit 2; }
30
+ stage="$(mktemp -d -p /var/tmp)"
31
+ base64 -d > "$stage/pomerium.tar.gz"
32
+ if sudo /usr/local/sbin/pomerium-upgrade "$version" "$sum" "$stage/pomerium.tar.gz"; then
33
+ rm -rf "$stage"
34
+ else
35
+ rm -rf "$stage"
36
+ exit 1
37
+ fi ;;
24
38
  nginx-reload) sudo nginx -t && sudo systemctl reload nginx ;;
25
39
  lego-renew) sudo systemctl start lego-renew.service && systemctl status lego-renew.service --no-pager --lines=10 ;;
26
40
  deploy-config)
@@ -13,13 +13,16 @@ describe("update holds", () => {
13
13
  it("reads stack names with a reason", () => {
14
14
  expect(parseHolds('{"supabase": " coordinated release: review first "}')).toEqual({ supabase: "coordinated release: review first" });
15
15
  expect(parseHolds("{}")).toEqual({});
16
+ expect(parseHolds('{"paperless/gotenberg": "Paperless needs Gotenberg 8"}')).toEqual({ "paperless/gotenberg": "Paperless needs Gotenberg 8" });
16
17
  });
17
18
 
18
19
  it("refuses a malformed file instead of holding nothing", () => {
19
20
  expect(() => parseHolds("not json")).toThrow(/not JSON/);
20
- expect(() => parseHolds('["supabase"]')).toThrow(/object of stack name/);
21
+ expect(() => parseHolds('["supabase"]')).toThrow(/object of stack/);
21
22
  expect(() => parseHolds('{"supabase": ""}')).toThrow(/non-empty reason/);
22
- expect(() => parseHolds('{"../x": "why"}')).toThrow(/stack name/);
23
+ expect(() => parseHolds('{"../x": "why"}')).toThrow(/stack or stack\/service name/);
24
+ expect(() => parseHolds('{"a/b/c": "why"}')).toThrow(/stack or stack\/service name/);
25
+ expect(() => parseHolds('{"a/../b": "why"}')).toThrow(/stack or stack\/service name/);
23
26
  });
24
27
  });
25
28
 
@@ -2,11 +2,11 @@ import { UsageError } from "./errors.ts";
2
2
  import { parseTag, type UpdateRow } from "./image-updates.ts";
3
3
  import { parseImageRef } from "./registry.ts";
4
4
 
5
- /** Stacks whose updates wait for a person, tracked in the root of the GitOps repository: `{ "<stack>": "<reason>" }`. */
5
+ /** Stacks and services whose updates wait for a person, tracked in the root of the GitOps repository: `{ "<stack>": "<reason>", "<stack>/<service>": "<reason>" }`. */
6
6
  export const HOLDS_FILE = "update-holds.json";
7
- const STACK = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;
7
+ const HOLD_KEY = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}(\/[A-Za-z0-9][A-Za-z0-9_.-]{0,127})?$/;
8
8
 
9
- /** The held stacks with their reasons; a malformed file is an error, never "nothing held". */
9
+ /** The held stacks and services with their reasons; a malformed file is an error, never "nothing held". */
10
10
  export function parseHolds(text: string): Record<string, string> {
11
11
  let value: unknown;
12
12
  try {
@@ -14,10 +14,10 @@ export function parseHolds(text: string): Record<string, string> {
14
14
  } catch {
15
15
  throw new UsageError(`${HOLDS_FILE} is not JSON`);
16
16
  }
17
- if (typeof value !== "object" || value === null || Array.isArray(value)) throw new UsageError(`${HOLDS_FILE} must be an object of stack name to reason`);
17
+ if (typeof value !== "object" || value === null || Array.isArray(value)) throw new UsageError(`${HOLDS_FILE} must be an object of stack (or stack/service) name to reason`);
18
18
  const holds: Record<string, string> = {};
19
19
  for (const [stack, reason] of Object.entries(value)) {
20
- if (!STACK.test(stack) || typeof reason !== "string" || reason.trim().length === 0) throw new UsageError(`${HOLDS_FILE}: "${stack.slice(0, 40)}" needs a stack name and a non-empty reason`);
20
+ if (!HOLD_KEY.test(stack) || typeof reason !== "string" || reason.trim().length === 0) throw new UsageError(`${HOLDS_FILE}: "${stack.slice(0, 40)}" needs a stack or stack/service name and a non-empty reason`);
21
21
  holds[stack] = reason.trim().slice(0, 200);
22
22
  }
23
23
  return holds;