@astrofoundry/pi-astro 0.40.62 → 0.40.64
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/astro-discord/daily-report.test.ts +19 -0
- package/extensions/astro-discord/daily-report.ts +11 -7
- package/extensions/astro-discord/daily-run.test.ts +14 -5
- package/extensions/astro-discord/daily-run.ts +14 -2
- package/extensions/astro-discord/index.ts +1 -1
- package/extensions/astro-discord/security-monitor.test.ts +1 -1
- package/extensions/astro-discord/security-monitor.ts +6 -5
- package/extensions/astro-discord/security.test.ts +31 -3
- package/extensions/astro-discord/security.ts +57 -12
- package/package.json +1 -1
- package/skills/arcane/SKILL.md +1 -1
- package/specialists/AGENTS.md +1 -1
- package/specialists/arcane/run.ts +4 -3
- package/specialists/entry/remote/spc-identity-dmz-entry.sh +15 -1
- package/specialists/lib/stack-update.test.ts +5 -2
- package/specialists/lib/stack-update.ts +5 -5
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
carefulTask,
|
|
11
11
|
failureMessage,
|
|
12
12
|
headText,
|
|
13
|
+
isPending,
|
|
13
14
|
layoutMessages,
|
|
14
15
|
majorLines,
|
|
15
16
|
managerSummary,
|
|
@@ -18,6 +19,8 @@ import {
|
|
|
18
19
|
readPostedArcane,
|
|
19
20
|
releaseNotesFromJson,
|
|
20
21
|
REPORT_PREFIX,
|
|
22
|
+
serviceText,
|
|
23
|
+
type ServiceLine,
|
|
21
24
|
stackComponent,
|
|
22
25
|
type StackEntry,
|
|
23
26
|
stackOutcome,
|
|
@@ -94,6 +97,22 @@ function entry(r: ArcaneReport, stack: string): StackEntry {
|
|
|
94
97
|
}
|
|
95
98
|
|
|
96
99
|
describe("Arcane Daily Report layout", () => {
|
|
100
|
+
it("keeps a held service out of the stack's update and shows why it is held", () => {
|
|
101
|
+
const proxy = { stack: "bigcapital", service: "proxy", ref: "envoyproxy/envoy:v1.39.1", running: "v1.39.1", latest: "v1.39.6", status: "release", action: "edit tag", note: "" };
|
|
102
|
+
const holds = { "bigcapital/gotenberg": "Bigcapital needs Gotenberg 7", "bigcapital/proxy": "wait for the next bundle" };
|
|
103
|
+
const built = buildStacks({ ...updateReport, rows: [rows[5], proxy], holds });
|
|
104
|
+
const stack = built?.stacks[0];
|
|
105
|
+
expect(stack?.held).toBeUndefined();
|
|
106
|
+
expect(stack?.actionable).toBe(false);
|
|
107
|
+
expect(stack?.lines.map((l) => [l.service, l.held, isPending(l)])).toEqual([
|
|
108
|
+
["gotenberg", "Bigcapital needs Gotenberg 7", false],
|
|
109
|
+
["proxy", "wait for the next bundle", false],
|
|
110
|
+
]);
|
|
111
|
+
expect(serviceText(stack?.lines[0] as ServiceLine)).toBe("• gotenberg: 7.10.2, newer major 8 available (a tag change by hand; Update does not apply it)\n-# ⏸ held: Bigcapital needs Gotenberg 7");
|
|
112
|
+
const partly = buildStacks({ ...updateReport, rows: [rows[5], proxy], holds: { "bigcapital/gotenberg": "x" } });
|
|
113
|
+
expect(partly?.stacks[0].actionable).toBe(true);
|
|
114
|
+
});
|
|
115
|
+
|
|
97
116
|
it("marks the images our own CI builds and leaves out stopped stacks", () => {
|
|
98
117
|
const own = { stack: "bigcapital", service: "server", ref: "ghcr.io/astrofoundry/bigcapital-server:astrofoundry", running: "?", latest: "new build", status: "update", action: "pull", note: "versions unavailable: private image" };
|
|
99
118
|
const stopped = { ...own, stack: "grimoire-graphile-bootstrap", service: "bootstrap", ref: "ghcr.io/astronaute77/arcane-graphile-worker:latest", stopped: true };
|
|
@@ -63,9 +63,9 @@ export function crossesMajor(running: string | null, latest: string | null): boo
|
|
|
63
63
|
|
|
64
64
|
const PENDING = new Set(["update", "release", "rebuild"]);
|
|
65
65
|
|
|
66
|
-
/** A line whose stack-update would change something. */
|
|
67
|
-
export function isPending(l: Pick<ServiceLine, "status">): boolean {
|
|
68
|
-
return PENDING.has(l.status);
|
|
66
|
+
/** A line whose stack-update would change something; a held service keeps its version. */
|
|
67
|
+
export function isPending(l: Pick<ServiceLine, "status" | "held">): boolean {
|
|
68
|
+
return PENDING.has(l.status) && l.held === undefined;
|
|
69
69
|
}
|
|
70
70
|
|
|
71
71
|
export interface ServiceLine {
|
|
@@ -83,6 +83,8 @@ export interface ServiceLine {
|
|
|
83
83
|
review?: Review;
|
|
84
84
|
/** The image is one our own CI builds, so its update goes through Upgrade carefully. */
|
|
85
85
|
own?: boolean;
|
|
86
|
+
/** Why this service is held (`<stack>/<service>` in `update-holds.json`); stack-update skips it. */
|
|
87
|
+
held?: string;
|
|
86
88
|
}
|
|
87
89
|
|
|
88
90
|
export type StackState = "offered" | "running" | "updated" | "rolled back" | "failed" | "current" | "planning";
|
|
@@ -166,12 +168,13 @@ export function buildStacks(report: unknown): ReportStacks | null {
|
|
|
166
168
|
const running = releaseTag(r.running);
|
|
167
169
|
const latest = releaseTag(r.latest);
|
|
168
170
|
const own = r.ref ? isOwnImage(r.ref) : false;
|
|
169
|
-
|
|
171
|
+
const held = holds[`${stack}/${r.service}`];
|
|
172
|
+
return { service: r.service, status: r.status, action: r.action ?? "", running, latest, tag: tagOf(r.ref ?? ""), ...(r.ref ? { ref: r.ref } : {}), major: PENDING.has(r.status) && crossesMajor(running, latest), note: r.note ?? "", ...(own ? { own } : {}), ...(held ? { held } : {}) };
|
|
170
173
|
});
|
|
171
|
-
const pending = lines.some(
|
|
174
|
+
const pending = lines.some(isPending);
|
|
172
175
|
const held = holds[stack];
|
|
173
176
|
if (stack === "arcane") {
|
|
174
|
-
const offer = lines.map((l) => (
|
|
177
|
+
const offer = lines.map((l) => (isPending(l) ? managerVersion(l) : null)).find((v) => v !== null) ?? null;
|
|
175
178
|
return { stack, lines, actionable: offer !== null && !held, ...(offer ? { manager: offer } : {}), ...(held ? { held } : {}), state: "offered" };
|
|
176
179
|
}
|
|
177
180
|
return { stack, lines, actionable: pending && !held, ...(held ? { held } : {}), state: "offered" };
|
|
@@ -212,7 +215,8 @@ export function serviceText(l: ServiceLine): string {
|
|
|
212
215
|
const newer = PENDING.has(l.status) ? /newer major (\S+) available/.exec(l.note)?.[1] : undefined;
|
|
213
216
|
if (newer) text += ` · newer major ${escapeDiscord(newer)} available`;
|
|
214
217
|
const review = l.review ? `\n-# ${l.review.verdict ? `${VERDICT_MARK[l.review.verdict]}: ` : "notes: "}${escapeDiscord(l.review.reason)}` : "";
|
|
215
|
-
|
|
218
|
+
const held = l.held ? `\n-# ⏸ held: ${escapeDiscord(l.held)}` : "";
|
|
219
|
+
return `• ${text}${review}${held}`;
|
|
216
220
|
}
|
|
217
221
|
|
|
218
222
|
function stackText(s: StackEntry): string {
|
|
@@ -227,11 +227,11 @@ describe("report summaries", () => {
|
|
|
227
227
|
const tool = (over: Partial<ToolsReport["lines"][number]>): ToolsReport["lines"][number] => ({ key: "brew:jq", label: "jq", installed: "1.8.1", latest: "1.8.2", major: false, action: { kind: "brew", name: "jq" }, state: "offered", ...over }) as ToolsReport["lines"][number];
|
|
228
228
|
|
|
229
229
|
it("says what the tools report updated, what failed, and what waits for a button", () => {
|
|
230
|
-
const report = { date: "2026-10-24", lines: [tool({ state: "updated", plan: "auto" }), tool({ key: "brew:herdr", label: "herdr", latest: "0.10.0", state: "failed", plan: "auto" }), tool({ key: "pi", label: "Pi", latest: "1.4.0", plan: "button" }), tool({ key: "spc:node", label: "Node", latest: "26.1.0", major: true, plan: "update" })], messages: [{ id: "9", keys: [], head: true, all: false }] } as unknown as ToolsReport;
|
|
230
|
+
const report = { date: "2026-10-24", lines: [tool({ state: "updated", plan: "auto" }), tool({ key: "brew:herdr", label: "herdr", latest: "0.10.0", state: "failed", plan: "auto" }), tool({ key: "pi", label: "Pi", latest: "1.4.0", plan: "button" }), tool({ key: "spc:node", label: "Node", latest: "26.1.0", major: true, plan: "update" }), tool({ key: "cask:temurin@17", label: "temurin@17", latest: "17.0.18", plan: "auto", manual: "its installer asks for your password; on Cortex: brew upgrade --cask temurin@17" })], messages: [{ id: "9", keys: [], head: true, all: false }] } as unknown as ToolsReport;
|
|
231
231
|
expect(toolsSummary(report, "2026-10-24", "5")).toEqual({
|
|
232
232
|
area: "Cortex tools",
|
|
233
|
-
line: "1 updated (jq 1.8.2), 1 failed (herdr), Pi 1.4.0, Node 26.1.0 wait for a button",
|
|
234
|
-
needs: ["Node 26.1.0 waits for its button (major version)", "Pi 1.4.0 waits for its button", "herdr automatic update failed"],
|
|
233
|
+
line: "1 updated (jq 1.8.2), 1 failed (herdr), Pi 1.4.0, Node 26.1.0 wait for a button, 1 by hand",
|
|
234
|
+
needs: ["Node 26.1.0 waits for its button (major version)", "Pi 1.4.0 waits for its button", "herdr automatic update failed", "temurin@17 17.0.18 by hand: its installer asks for your password; on Cortex: brew upgrade --cask temurin@17"],
|
|
235
235
|
channelId: "5",
|
|
236
236
|
messageId: "9",
|
|
237
237
|
});
|
|
@@ -240,11 +240,20 @@ describe("report summaries", () => {
|
|
|
240
240
|
|
|
241
241
|
it("says what the Arcane report updated, rolled back, and leaves to the owner", () => {
|
|
242
242
|
const stack = (name: string, over: Partial<ArcaneReport["stacks"][number]>) => ({ stack: name, lines: [], actionable: true, state: "offered", ...over }) as ArcaneReport["stacks"][number];
|
|
243
|
-
const
|
|
243
|
+
const line = (over: Partial<ArcaneReport["stacks"][number]["lines"][number]>) => ({ service: "gotenberg", status: "current", action: "", running: "8.40.0", latest: "8.40.0", tag: "8.40.0", major: false, note: "newer major 9 available", ...over }) as ArcaneReport["stacks"][number]["lines"][number];
|
|
244
|
+
const stacks = [
|
|
245
|
+
stack("media", { plan: "auto", state: "updated" }),
|
|
246
|
+
stack("mealie", { plan: "auto", state: "rolled back" }),
|
|
247
|
+
stack("stirling-pdf", { plan: "careful" }),
|
|
248
|
+
stack("supabase", { actionable: false, held: "review first" }),
|
|
249
|
+
stack("paperless", { actionable: false, lines: [line({}), line({ service: "broker", status: "unknown", note: "registry timeout" })] }),
|
|
250
|
+
stack("bigcapital", { actionable: false, lines: [line({ held: "Bigcapital needs Gotenberg 8" })] }),
|
|
251
|
+
];
|
|
252
|
+
const report = { date: "2026-10-24", stacks, messages: [{ id: "8", stacks: [], head: true }] } as unknown as ArcaneReport;
|
|
244
253
|
expect(arcaneSummary(report, "2026-10-24", "6")).toEqual({
|
|
245
254
|
area: "Arcane",
|
|
246
255
|
line: "1 stack updated (media), mealie rolled back, stirling-pdf waits for you, supabase held",
|
|
247
|
-
needs: ["stirling-pdf waits for you (Upgrade carefully)", "mealie automatic update rolled back"],
|
|
256
|
+
needs: ["stirling-pdf waits for you (Upgrade carefully)", "mealie automatic update rolled back", "paperless gotenberg 8.40.0: major 9 needs a tag change by hand", "paperless broker: the update check failed"],
|
|
248
257
|
channelId: "6",
|
|
249
258
|
messageId: "8",
|
|
250
259
|
});
|
|
@@ -369,7 +369,7 @@ export function macosStatus(macs: readonly MacUpdates[]): AreaStatus {
|
|
|
369
369
|
export interface ReportLine {
|
|
370
370
|
area: string;
|
|
371
371
|
line: string;
|
|
372
|
-
/** Items the report tagged
|
|
372
|
+
/** Items that need the owners: what the report tagged them for, waits for them, or must be done by hand. */
|
|
373
373
|
needs: string[];
|
|
374
374
|
channelId?: string;
|
|
375
375
|
messageId?: string;
|
|
@@ -396,11 +396,21 @@ export function toolsSummary(report: ToolsReport | null, date: string, channelId
|
|
|
396
396
|
...report.lines.filter((l) => l.plan === "update" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} waits for its button (${l.major ? "major version" : "release notes"})`),
|
|
397
397
|
...report.lines.filter((l) => l.plan === "button" && l.state === "offered").map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} waits for its button`),
|
|
398
398
|
...failed.filter((l) => l.plan === "auto").map((l) => `${escapeDiscord(l.label)} automatic update failed`),
|
|
399
|
+
...manual.map((l) => `${escapeDiscord(l.label)} ${escapeDiscord(l.latest)} by hand: ${escapeDiscord(l.manual ?? "")}`),
|
|
399
400
|
];
|
|
400
401
|
return { area: "Cortex tools", line: parts.join(", ") || "everything up to date", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
|
|
401
402
|
}
|
|
402
403
|
|
|
403
|
-
/**
|
|
404
|
+
/** A service line only the owner can act on: a failed check, or a newer major that needs a tag change by hand; a held service is the owner's own decision. */
|
|
405
|
+
function byHand(stack: string, l: ArcaneReport["stacks"][number]["lines"][number]): string | null {
|
|
406
|
+
if (l.held !== undefined) return null;
|
|
407
|
+
const where = `${escapeDiscord(stack)} ${escapeDiscord(l.service)}`;
|
|
408
|
+
if (l.status === "unknown") return `${where}: the update check failed`;
|
|
409
|
+
const newer = /newer major (\S+) available/.exec(l.note)?.[1];
|
|
410
|
+
return newer ? `${where} ${escapeDiscord(l.running ?? l.tag)}: major ${escapeDiscord(newer)} needs a tag change by hand` : null;
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
/** The Arcane Daily Report after its automatic updates and checks: updated, rolled back, waiting for the owner, held, by hand. Holds are the owner's own decisions, so they are not items. */
|
|
404
414
|
export function arcaneSummary(report: ArcaneReport | null, date: string, channelId: string): ReportLine {
|
|
405
415
|
if (!report || report.date !== date) return { area: "Arcane", line: "no report today; the bridge log has the reason", needs: ["the Arcane Daily Report did not run today; the bridge log has the reason"], channelId };
|
|
406
416
|
const by = (pick: (s: ArcaneReport["stacks"][number]) => boolean) => report.stacks.filter(pick).map((s) => s.stack);
|
|
@@ -419,6 +429,8 @@ export function arcaneSummary(report: ArcaneReport | null, date: string, channel
|
|
|
419
429
|
const needs = [
|
|
420
430
|
...waiting.map((s) => `${escapeDiscord(s.stack)} waits for you (${s.plan === "careful" ? "Upgrade carefully" : "major version"})`),
|
|
421
431
|
...report.stacks.filter((s) => s.plan === "auto" && (s.state === "rolled back" || s.state === "failed")).map((s) => `${escapeDiscord(s.stack)} automatic update ${s.state}`),
|
|
432
|
+
...report.stacks.flatMap((s) => s.lines.map((l) => byHand(s.stack, l)).filter((t): t is string => t !== null)),
|
|
433
|
+
...(report.holdsError ? [`update-holds.json is unreadable: ${escapeDiscord(report.holdsError)}`] : []),
|
|
422
434
|
];
|
|
423
435
|
return { area: "Arcane", line: parts.join(", ") || "no updates", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
|
|
424
436
|
}
|
|
@@ -613,7 +613,7 @@ class Bridge {
|
|
|
613
613
|
const name: ReportName = monday ? "security-weekly" : "security";
|
|
614
614
|
const posted = done(name) ? undefined : await (monday ? this.security.weeklyReport(securitySchedule, date) : this.security.dailyReport(securitySchedule, date)).catch((err) => log(`daily ${date}: security report failed: ${err instanceof Error ? err.message : String(err)}`));
|
|
615
615
|
const area = monday ? "Security (weekly)" : "Security";
|
|
616
|
-
reports.push(posted ? { area, line: posted.headline, needs: posted.
|
|
616
|
+
reports.push(posted ? { area, line: posted.headline, needs: posted.needs, channelId: securitySchedule.channelId, messageId: posted.messageId } : { area, line: done(name) ? "already posted today" : "the report failed; the bridge log has the reason", needs: done(name) ? [] : ["the security report failed; the bridge log has the reason"], channelId: securitySchedule.channelId });
|
|
617
617
|
}
|
|
618
618
|
const tools = reportSchedule(this.config, "tools");
|
|
619
619
|
if (tools && !this.stopping) {
|
|
@@ -309,7 +309,7 @@ describe("security monitor", () => {
|
|
|
309
309
|
expect(text).not.toContain("waiting on the publisher");
|
|
310
310
|
expect(h.calls).toContainEqual(["arcane", "images", "list", "--all"]);
|
|
311
311
|
expect(h.calls).toContainEqual(["arcane", "images", "vulnerabilities", "list", "--severity", "critical", "--all"]);
|
|
312
|
-
expect(summary).toEqual({ headline: "quiet day (24 h to 06:05 CEST)",
|
|
312
|
+
expect(summary).toEqual({ headline: "quiet day (24 h to 06:05 CEST)", needs: [], messageId: "100" });
|
|
313
313
|
expect(text).toContain("## 🛡️ Security Daily Report\\n🟢 quiet day · 24 h to 06:05 CEST");
|
|
314
314
|
expect(text).toContain("🚫 **CrowdSec** · 2 bans, all front door SNI scanning");
|
|
315
315
|
expect(text).toContain("📦 **Host packages** · 1 new CVE on 1 host, fixed versions exist\\n- CVE-1 (openssl, High 7.5) on dmz");
|
|
@@ -94,7 +94,8 @@ export interface SecurityClick {
|
|
|
94
94
|
/** What a posted security report tells the morning summary. */
|
|
95
95
|
export interface PostedSummary {
|
|
96
96
|
headline: string;
|
|
97
|
-
|
|
97
|
+
/** What the report lists under Needs you. */
|
|
98
|
+
needs: string[];
|
|
98
99
|
messageId: string;
|
|
99
100
|
}
|
|
100
101
|
|
|
@@ -460,7 +461,7 @@ export class SecurityMonitor {
|
|
|
460
461
|
return (await this.deps.rest.createComponentsMessage(channelId, report.components, report.mention ? this.owners() : undefined)).id;
|
|
461
462
|
}
|
|
462
463
|
|
|
463
|
-
/** Posts the Security Daily Report; returns its headline and
|
|
464
|
+
/** Posts the Security Daily Report; returns its headline and what needs the owners, for the morning summary. */
|
|
464
465
|
async dailyReport(s: ReportSchedule, date: string): Promise<PostedSummary | undefined> {
|
|
465
466
|
const now = this.deps.now();
|
|
466
467
|
if (reportSchedule(this.deps.config(), "security-weekly") && zonedWeekday(s.timeZone, new Date(now)) === MONDAY) {
|
|
@@ -478,13 +479,13 @@ export class SecurityMonitor {
|
|
|
478
479
|
const state = readSecurityState();
|
|
479
480
|
const wazuh = wazuhDigest(events, state.origins, (e) => acked(state, e));
|
|
480
481
|
if (this.deps.stopping()) return;
|
|
481
|
-
const data = { until: clockText(new Date(now).toISOString(), s.timeZone).replace(/^\d+ \w+ /, ""), crowdsec, wazuh, found: vulns?.found ?? null, closed: vulns?.closed ?? null, attention, ...images };
|
|
482
|
+
const data = { until: clockText(new Date(now).toISOString(), s.timeZone).replace(/^\d+ \w+ /, ""), timeZone: s.timeZone, crowdsec, wazuh, found: vulns?.found ?? null, closed: vulns?.closed ?? null, attention, ...images, alerts: state.alerts };
|
|
482
483
|
writeReportState("security", { date, status: "sending" });
|
|
483
484
|
const report = dailyCard(data, this.owners());
|
|
484
485
|
const id = await this.postReport(s.channelId, report);
|
|
485
486
|
writeReportState("security", { date, status: "sent", messageId: id });
|
|
486
487
|
this.deps.log(`security report ${date}: posted`);
|
|
487
|
-
return { headline: report.headline,
|
|
488
|
+
return { headline: report.headline, needs: report.needs, messageId: id };
|
|
488
489
|
}
|
|
489
490
|
|
|
490
491
|
async weeklyReport(s: ReportSchedule, date: string): Promise<PostedSummary | undefined> {
|
|
@@ -543,7 +544,7 @@ export class SecurityMonitor {
|
|
|
543
544
|
const id = await this.postReport(s.channelId, report);
|
|
544
545
|
writeReportState("security-weekly", { date, status: "sent", messageId: id });
|
|
545
546
|
this.deps.log(`security weekly report ${date}: posted`);
|
|
546
|
-
return { headline: report.headline,
|
|
547
|
+
return { headline: report.headline, needs: report.needs, messageId: id };
|
|
547
548
|
}
|
|
548
549
|
|
|
549
550
|
// ------------------------------------------------------------ buttons
|
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
crowdsecDailyLine,
|
|
18
18
|
crowdsecWeeklyLine,
|
|
19
19
|
dailyCard,
|
|
20
|
+
dailyNeeds,
|
|
20
21
|
dailyNeedsOwner,
|
|
21
22
|
emptyState,
|
|
22
23
|
healthChange,
|
|
@@ -481,9 +482,10 @@ describe("daily report", () => {
|
|
|
481
482
|
const attention = parseAttention({ checked_at: "x", bouncer: { count: 1, oldest_pull_seconds: 6 }, wazuh_agents: { total: 8, active: 8 }, health_issues: [] });
|
|
482
483
|
|
|
483
484
|
it("reads quiet day in a green card and tags nobody when nothing is open", () => {
|
|
484
|
-
const data = { until: "06:00 CEST", crowdsec: summary(), wazuh: wazuhDigest([], {}, () => false), found: [], closed: [], attention, images: { scanned: 57, total: 57, critical: [] }, offers: new Map() };
|
|
485
|
+
const data = { until: "06:00 CEST", timeZone: TZ, crowdsec: summary(), wazuh: wazuhDigest([], {}, () => false), found: [], closed: [], attention, images: { scanned: 57, total: 57, critical: [] }, offers: new Map(), alerts: [] };
|
|
485
486
|
const report = dailyCard(data, [OWNER]);
|
|
486
487
|
expect(report.headline).toBe("quiet day (24 h to 06:00 CEST)");
|
|
488
|
+
expect(report.needs).toEqual([]);
|
|
487
489
|
expect(report.mention).toBe(false);
|
|
488
490
|
expect(report.components).toEqual([
|
|
489
491
|
{
|
|
@@ -507,14 +509,40 @@ describe("daily report", () => {
|
|
|
507
509
|
|
|
508
510
|
it("names urgent alerts and failed checks in an orange card and tags the owner while something is open", () => {
|
|
509
511
|
const wazuh = wazuhDigest([event({ rule: "5712", level: 12, description: "brute force" })], {}, () => false);
|
|
510
|
-
const data = { until: "06:00 CEST", crowdsec: null, wazuh, found: [], closed: [], attention, images: { scanned: 57, total: 57, critical: [] }, offers: new Map() };
|
|
512
|
+
const data = { until: "06:00 CEST", timeZone: TZ, crowdsec: null, wazuh, found: [], closed: [], attention, images: { scanned: 57, total: 57, critical: [] }, offers: new Map(), alerts: [{ key: alertKey(event({ rule: "5712" })), channelId: "c9" }] };
|
|
511
513
|
const report = dailyCard(data, [OWNER]);
|
|
512
514
|
expect(report.headline).toBe("1 urgent alert, 1 check failed (24 h to 06:00 CEST)");
|
|
515
|
+
expect(report.needs).toEqual(["urgent alert not acknowledged: brute force on arcane, 20 Oct 14:50 CEST; Acknowledge it in <#c9>", "1 check of this report failed; the bridge log on Cortex has the reason"]);
|
|
513
516
|
expect(report.mention).toBe(true);
|
|
514
517
|
expect(report.components[0].accent_color).toBe(0xe67e22);
|
|
515
|
-
expect(texts(report.components)
|
|
518
|
+
expect(texts(report.components).slice(0, 2)).toEqual([
|
|
519
|
+
"## 🛡️ Security Daily Report\n🟠 1 urgent alert, 1 check failed · 24 h to 06:00 CEST",
|
|
520
|
+
"### Needs you\n1. urgent alert not acknowledged: brute force on arcane, 20 Oct 14:50 CEST; Acknowledge it in <#c9>\n2. 1 check of this report failed; the bridge log on Cortex has the reason",
|
|
521
|
+
]);
|
|
516
522
|
expect(report.components[1]).toEqual({ type: 10, content: `<@${OWNER}>` });
|
|
517
523
|
});
|
|
524
|
+
|
|
525
|
+
it("leaves out a NAS loss that a later restore ended and names our images to rebuild", () => {
|
|
526
|
+
const lost = event({ id: "n1", rule: "100012", level: 12, timestamp: "2026-10-20T01:00:00.000+0000" });
|
|
527
|
+
const back = event({ id: "n2", rule: "100013", level: 10, timestamp: "2026-10-20T01:05:00.000+0000" });
|
|
528
|
+
const oom = event({ id: "o1", agent: "space-x", rule: "5108", level: 12, description: "System running out of memory. Availability of the system is in risk.", timestamp: "2026-10-20T06:24:00.000+0000" });
|
|
529
|
+
const wazuh = wazuhDigest([lost, back, oom], {}, () => false);
|
|
530
|
+
expect(wazuh.unacknowledged.map((e) => e.id)).toEqual(["n1", "o1"]);
|
|
531
|
+
expect(wazuh.open.map((e) => e.id)).toEqual(["o1"]);
|
|
532
|
+
const builds = new Map([["ghcr.io/astrofoundry/app:1", { created: 1, newer: false }], ["ghcr.io/astrofoundry/web:1", { created: 1, newer: true }]]);
|
|
533
|
+
const critical = [
|
|
534
|
+
{ id: "CVE-1", pkg: "a", image: "ghcr.io/astrofoundry/app:1" },
|
|
535
|
+
{ id: "CVE-2", pkg: "b", image: "ghcr.io/astrofoundry/app:1" },
|
|
536
|
+
{ id: "CVE-3", pkg: "c", image: "ghcr.io/astrofoundry/web:1" },
|
|
537
|
+
{ id: "CVE-4", pkg: "d", image: "docker.io/library/redis:7" },
|
|
538
|
+
];
|
|
539
|
+
const data = { until: "06:00 CEST", timeZone: TZ, crowdsec: summary(), wazuh, found: [], closed: [], attention, images: { scanned: 57, total: 57, critical, builds }, offers: new Map(), alerts: [{ key: alertKey(oom), channelId: "c9" }] };
|
|
540
|
+
expect(dailyNeeds(data)).toEqual([
|
|
541
|
+
"urgent alert not acknowledged: System running out of memory. Availability of the system is in risk on space-x, 20 Oct 08:24 CEST; Acknowledge it in <#c9>",
|
|
542
|
+
"rebuild app:1 (2): our images carry critical CVEs that have a fixed version",
|
|
543
|
+
]);
|
|
544
|
+
expect(dailyCard(data, [OWNER]).components[0].accent_color).toBe(0xe67e22);
|
|
545
|
+
});
|
|
518
546
|
});
|
|
519
547
|
|
|
520
548
|
describe("weekly report", () => {
|
|
@@ -410,6 +410,8 @@ export interface WazuhDigest {
|
|
|
410
410
|
urgent: number;
|
|
411
411
|
/** Urgent events that were not acknowledged when they came. */
|
|
412
412
|
unacknowledged: WazuhEvent[];
|
|
413
|
+
/** Unacknowledged events still open: a NAS loss that a later restore on the same agent ended is not. */
|
|
414
|
+
open: WazuhEvent[];
|
|
413
415
|
/** Deployment-file changes no commit or operation of ours explains. */
|
|
414
416
|
unexplained: WazuhEvent[];
|
|
415
417
|
}
|
|
@@ -421,8 +423,8 @@ function originCount(n: number, of: number): string {
|
|
|
421
423
|
|
|
422
424
|
/** The Wazuh line of a report: alerts at level 10 or higher, grouped by issue and agent, urgent ones first. */
|
|
423
425
|
export function wazuhDigest(events: readonly WazuhEvent[] | null, origins: SecurityState["origins"], acked: (e: WazuhEvent) => boolean, maxGroups = 4): WazuhDigest {
|
|
424
|
-
if (!events) return { line: "Wazuh · the check failed", failed: true, count: 0, urgent: 0, unacknowledged: [], unexplained: [] };
|
|
425
|
-
if (events.length === 0) return { line: "Wazuh · no alerts at level 10+", failed: false, count: 0, urgent: 0, unacknowledged: [], unexplained: [] };
|
|
426
|
+
if (!events) return { line: "Wazuh · the check failed", failed: true, count: 0, urgent: 0, unacknowledged: [], open: [], unexplained: [] };
|
|
427
|
+
if (events.length === 0) return { line: "Wazuh · no alerts at level 10+", failed: false, count: 0, urgent: 0, unacknowledged: [], open: [], unexplained: [] };
|
|
426
428
|
const groups = new Map<string, { label: string; agent: string; events: WazuhEvent[]; urgent: number; known: number }>();
|
|
427
429
|
let urgent = 0;
|
|
428
430
|
const unacknowledged: WazuhEvent[] = [];
|
|
@@ -451,7 +453,10 @@ export function wazuhDigest(events: readonly WazuhEvent[] | null, origins: Secur
|
|
|
451
453
|
const quiet = urgent - unacknowledged.length;
|
|
452
454
|
const urgentText = urgent === 0 ? "none urgent" : `${urgent} urgent${quiet > 0 ? ` (${quiet} acknowledged)` : ""}`;
|
|
453
455
|
const line = `Wazuh · ${events.length} alert${events.length === 1 ? "" : "s"} at level 10+, ${urgentText} · ${shown.join(" · ")}${more}`;
|
|
454
|
-
|
|
456
|
+
const restored = new Map<string, number>();
|
|
457
|
+
for (const e of events) if (e.rule === "100013") restored.set(e.agent, Math.max(restored.get(e.agent) ?? 0, Date.parse(e.timestamp)));
|
|
458
|
+
const open = unacknowledged.filter((e) => !(e.rule === "100012" && (restored.get(e.agent) ?? 0) > Date.parse(e.timestamp)));
|
|
459
|
+
return { line, failed: false, count: events.length, urgent, unacknowledged, open, unexplained };
|
|
455
460
|
}
|
|
456
461
|
|
|
457
462
|
// ---------------------------------------------------------------- vulnerabilities
|
|
@@ -717,6 +722,19 @@ export function imageVulnLine(facts: ImageVulnFacts | null, offers: ReadonlyMap<
|
|
|
717
722
|
return parts.join(" · ");
|
|
718
723
|
}
|
|
719
724
|
|
|
725
|
+
/** Our own images with critical CVEs that have a fix and no newer build in the registry: only our rebuild closes them. */
|
|
726
|
+
export function rebuildItem(facts: ImageVulnFacts | null): string | null {
|
|
727
|
+
if (!facts) return null;
|
|
728
|
+
const counts = new Map<string, number>();
|
|
729
|
+
for (const v of facts.critical) {
|
|
730
|
+
const build = facts.builds?.get(v.image) ?? facts.builds?.get(v.image.split("@")[0]);
|
|
731
|
+
if (isOwnImage(v.image) && !build?.newer) counts.set(v.image, (counts.get(v.image) ?? 0) + 1);
|
|
732
|
+
}
|
|
733
|
+
if (counts.size === 0) return null;
|
|
734
|
+
const images = [...counts.entries()].sort((x, y) => y[1] - x[1] || x[0].localeCompare(y[0])).map(([image, n]) => `${escapeDiscord(shortImageName(image))} (${n})`);
|
|
735
|
+
return `rebuild ${images.join(", ")}: our images carry critical CVEs that have a fixed version`;
|
|
736
|
+
}
|
|
737
|
+
|
|
720
738
|
// ---------------------------------------------------------------- reports
|
|
721
739
|
|
|
722
740
|
const GREEN = 0x57f287;
|
|
@@ -733,10 +751,17 @@ export function section(icon: string, line: string): string {
|
|
|
733
751
|
export interface ReportCard {
|
|
734
752
|
/** The state without the report's name, for the morning summary: `quiet day (24 h to 06:00 CEST)`. */
|
|
735
753
|
headline: string;
|
|
754
|
+
/** What the owners must act on, listed under Needs you and in the morning summary. */
|
|
755
|
+
needs: string[];
|
|
736
756
|
mention: boolean;
|
|
737
757
|
components: MessageComponent[];
|
|
738
758
|
}
|
|
739
759
|
|
|
760
|
+
/** The numbered Needs you list right under a report's title, or nothing. */
|
|
761
|
+
function needsBlock(items: readonly string[]): string[] {
|
|
762
|
+
return items.length === 0 ? [] : [`### Needs you\n${items.map((t, i) => `${i + 1}. ${t}`).join("\n")}`];
|
|
763
|
+
}
|
|
764
|
+
|
|
740
765
|
/** Text Displays in groups divided by separators. */
|
|
741
766
|
function card(groups: readonly (readonly string[])[], attention: boolean, mention: string | null): MessageComponent[] {
|
|
742
767
|
const components = groups.filter((g) => g.length > 0).flatMap((g, i) => [...(i > 0 ? [SEPARATOR] : []), ...g.map((content): MessageComponent => ({ type: 10, content }))]);
|
|
@@ -746,6 +771,7 @@ function card(groups: readonly (readonly string[])[], attention: boolean, mentio
|
|
|
746
771
|
export interface DailyData {
|
|
747
772
|
/** Local time the report covers up to, already formatted (`06:00 CEST`). */
|
|
748
773
|
until: string;
|
|
774
|
+
timeZone: string;
|
|
749
775
|
crowdsec: AlertSummary | null;
|
|
750
776
|
wazuh: WazuhDigest;
|
|
751
777
|
found: VulnEvent[] | null;
|
|
@@ -755,6 +781,8 @@ export interface DailyData {
|
|
|
755
781
|
offers: ReadonlyMap<string, ImageOffer>;
|
|
756
782
|
/** The Arcane Daily Report's channel, linked from offered updates. */
|
|
757
783
|
arcaneChannelId?: string;
|
|
784
|
+
/** Alerts posted to Discord, to tell an unacknowledged alert from one that was never posted. */
|
|
785
|
+
alerts: readonly Pick<PostedAlert, "key" | "channelId">[];
|
|
758
786
|
}
|
|
759
787
|
|
|
760
788
|
function headline(urgent: number, unexplained: number, health: number, failed: number): string {
|
|
@@ -776,9 +804,23 @@ export function dailyNeedsOwner(d: DailyData): boolean {
|
|
|
776
804
|
return (d.attention?.issues.length ?? 0) > 0 || failures(d) > 0;
|
|
777
805
|
}
|
|
778
806
|
|
|
807
|
+
/** What the owners must act on today: health issues, unexplained changes, open urgent alerts, our images to rebuild, failed checks. */
|
|
808
|
+
export function dailyNeeds(d: DailyData): string[] {
|
|
809
|
+
const items: string[] = [];
|
|
810
|
+
for (const i of d.attention?.issues ?? []) items.push(escapeDiscord(i.message.replace(/\.$/, "")));
|
|
811
|
+
if (d.wazuh.unexplained.length > 0) items.push(`${d.wazuh.unexplained.length} unexplained deployment change${d.wazuh.unexplained.length === 1 ? "" : "s"}; Investigate on the alert`);
|
|
812
|
+
items.push(...unacknowledgedItems(d.wazuh, d.alerts, d.timeZone));
|
|
813
|
+
const rebuild = rebuildItem(d.images);
|
|
814
|
+
if (rebuild) items.push(rebuild);
|
|
815
|
+
const failed = failures(d);
|
|
816
|
+
if (failed > 0) items.push(`${failed} check${failed === 1 ? "" : "s"} of this report failed; the bridge log on Cortex has the reason`);
|
|
817
|
+
return items;
|
|
818
|
+
}
|
|
819
|
+
|
|
779
820
|
export function dailyCard(d: DailyData, owners: readonly string[]): ReportCard {
|
|
780
821
|
const state = headline(d.wazuh.urgent, d.wazuh.unexplained.length, d.attention?.issues.length ?? 0, failures(d));
|
|
781
822
|
const mention = dailyNeedsOwner(d);
|
|
823
|
+
const needs = dailyNeeds(d);
|
|
782
824
|
const sections = [
|
|
783
825
|
section("🚫", crowdsecDailyLine(d.crowdsec)),
|
|
784
826
|
section("👁️", d.wazuh.line),
|
|
@@ -789,8 +831,9 @@ export function dailyCard(d: DailyData, owners: readonly string[]): ReportCard {
|
|
|
789
831
|
];
|
|
790
832
|
return {
|
|
791
833
|
headline: `${state} (24 h to ${d.until})`,
|
|
834
|
+
needs,
|
|
792
835
|
mention,
|
|
793
|
-
components: card([[`## 🛡️ ${DAILY_PREFIX}\n${state === "quiet day" ? "🟢" : "🟠"} ${state} · 24 h to ${d.until}`], sections], state !== "quiet day", mention ? mentionLine(owners) : null),
|
|
836
|
+
components: card([[`## 🛡️ ${DAILY_PREFIX}\n${state === "quiet day" ? "🟢" : "🟠"} ${state} · 24 h to ${d.until}`], needsBlock(needs), sections], state !== "quiet day" || needs.length > 0, mention ? mentionLine(owners) : null),
|
|
794
837
|
};
|
|
795
838
|
}
|
|
796
839
|
|
|
@@ -899,15 +942,15 @@ function crlText(crl: WeeklyData["crl"], timeZone: string): string {
|
|
|
899
942
|
return `RADIUS CRL: renew within ${crl.days} days (warned ${dayText(crl.warned, timeZone)})`;
|
|
900
943
|
}
|
|
901
944
|
|
|
902
|
-
/** One item per alert key the owners did not acknowledge: what changed, when, and where to acknowledge it, if anywhere. */
|
|
903
|
-
function unacknowledgedItems(
|
|
945
|
+
/** One item per open alert key the owners did not acknowledge: what changed, when, and where to acknowledge it, if anywhere. */
|
|
946
|
+
function unacknowledgedItems(wazuh: WazuhDigest, alerts: readonly Pick<PostedAlert, "key" | "channelId">[], timeZone: string): string[] {
|
|
904
947
|
const groups = new Map<string, WazuhEvent[]>();
|
|
905
|
-
for (const e of
|
|
948
|
+
for (const e of wazuh.open) if (e.rule !== DEPLOYMENT_RULE) groups.set(alertKey(e), [...(groups.get(alertKey(e)) ?? []), e]);
|
|
906
949
|
return [...groups.entries()].map(([key, events]) => {
|
|
907
950
|
const last = events.reduce((a, b) => (Date.parse(b.timestamp) > Date.parse(a.timestamp) ? b : a));
|
|
908
951
|
const what = last.path ? `\`${last.path.replace(/`/g, "'")}\` ${escapeDiscord(last.event ?? "changed")}` : escapeDiscord(issueLabel(last));
|
|
909
|
-
const when = events.length > 1 ? ` × ${events.length}, last ${clockText(last.timestamp,
|
|
910
|
-
const posted =
|
|
952
|
+
const when = events.length > 1 ? ` × ${events.length}, last ${clockText(last.timestamp, timeZone)}` : `, ${clockText(last.timestamp, timeZone)}`;
|
|
953
|
+
const posted = alerts.find((a) => a.key === key);
|
|
911
954
|
const fact = `${what} on ${escapeDiscord(last.agent)}${when}`;
|
|
912
955
|
return posted ? `urgent alert not acknowledged: ${fact}; Acknowledge it in <#${posted.channelId}>` : `urgent alert never posted, so it has no Acknowledge button: ${fact}; check that it was expected`;
|
|
913
956
|
});
|
|
@@ -917,12 +960,14 @@ export function needsYou(d: WeeklyData): string[] {
|
|
|
917
960
|
const items: string[] = [];
|
|
918
961
|
for (const i of d.attention?.issues ?? []) items.push(escapeDiscord(i.message.replace(/\.$/, "")));
|
|
919
962
|
if (d.wazuh.unexplained.length > 0) items.push(`${d.wazuh.unexplained.length} unexplained deployment change${d.wazuh.unexplained.length === 1 ? "" : "s"} this week; Investigate on the alert`);
|
|
920
|
-
items.push(...unacknowledgedItems(d));
|
|
963
|
+
items.push(...unacknowledgedItems(d.wazuh, d.alerts, d.timeZone));
|
|
921
964
|
if (d.patching && d.patching.rebootPending.length > 0) items.push(`reboot ${d.patching.rebootPending.map(escapeDiscord).join(", ")} to load installed updates`);
|
|
922
965
|
const stuck = d.openWithFix.filter((v) => d.now - Date.parse(v.since) > 3 * DAY);
|
|
923
966
|
if (stuck.length > 0) items.push(`${stuck.length} CVE${stuck.length === 1 ? "" : "s"} with a fixed version still open after 3 days (${[...new Set(stuck.map((v) => v.agent))].map(escapeDiscord).join(", ")}); unattended-upgrades may be stuck`);
|
|
924
967
|
const offered = offeredImages(d.images, d.offers);
|
|
925
968
|
if (offered.length > 0) items.push(`update ${offered.map(escapeDiscord).join(", ")} in the Arcane Daily Report: the newer image replaces one with a critical CVE`);
|
|
969
|
+
const rebuild = rebuildItem(d.images);
|
|
970
|
+
if (rebuild) items.push(rebuild);
|
|
926
971
|
if (d.crl !== null && d.crl !== "unknown") items.push(`renew the RADIUS CRL (wifi-device.sh renew-crl), ${d.crl.days} days left`);
|
|
927
972
|
if (d.backups) {
|
|
928
973
|
const stale = d.backups.guests.filter((g) => d.now / 1000 - g.last > STALE_BACKUP_S);
|
|
@@ -959,11 +1004,11 @@ export function weeklyCard(d: WeeklyData, owners: readonly string[]): ReportCard
|
|
|
959
1004
|
section("🩺", `${healthLine(d.attention)} · ${crlText(d.crl, d.timeZone)}`),
|
|
960
1005
|
section("💾", backupLine(d.backups, d.now, d.timeZone)),
|
|
961
1006
|
];
|
|
962
|
-
const needsBlock = items.length === 0 ? [] : [`### Needs you\n${items.map((t, i) => `${i + 1}. ${t}`).join("\n")}`];
|
|
963
1007
|
return {
|
|
964
1008
|
headline: `${d.range} · ${state} · ${needs}`,
|
|
1009
|
+
needs: items,
|
|
965
1010
|
mention: items.length > 0,
|
|
966
|
-
components: card([[`## 🛡️ ${WEEKLY_PREFIX} · ${d.range}\n${state} · ${needs}`], needsBlock, sections, [`-# ${lastDayLine(d.last24h)}`]], incidents > 0 || items.length > 0, items.length > 0 ? mentionLine(owners) : null),
|
|
1011
|
+
components: card([[`## 🛡️ ${WEEKLY_PREFIX} · ${d.range}\n${state} · ${needs}`], needsBlock(items), sections, [`-# ${lastDayLine(d.last24h)}`]], incidents > 0 || items.length > 0, items.length > 0 ? mentionLine(owners) : null),
|
|
967
1012
|
};
|
|
968
1013
|
}
|
|
969
1014
|
|
package/package.json
CHANGED
package/skills/arcane/SKILL.md
CHANGED
|
@@ -53,7 +53,7 @@ Apply updates only when asked, one stack at a time, with `["stack-update", "<sta
|
|
|
53
53
|
|
|
54
54
|
The answer is JSON with `status`: `updated`, `rolled back`, `failed` (with `reason`), or `nothing to do`, plus `services` (`service`, `from`, `to`, `applied`: `pull` or the new reference, `result`: `current`, `behind`, `unchecked`) and `commit`/`revert` when the Compose file changed. Report it in plain words.
|
|
55
55
|
|
|
56
|
-
It refuses `arcane` (next section), stopped stacks (a redeploy would start them), a checkout with local or unpushed changes, and held stacks. Held stacks are listed in `update-holds.json` at the root of the GitOps repository, `{ "<stack>": "<reason>" }`; `update-report` returns them in `holds`, and the daily report shows the reason without a button. Change that file only when the caller asks, through the `git` commands.
|
|
56
|
+
It refuses `arcane` (next section), stopped stacks (a redeploy would start them), a checkout with local or unpushed changes, and held stacks. Held stacks are listed in `update-holds.json` at the root of the GitOps repository, `{ "<stack>": "<reason>" }`; a key `<stack>/<service>` holds one service, which `stack-update` skips while the rest of the stack updates. `update-report` returns them in `holds`, and the daily report shows the reason without a button. Change that file only when the caller asks, through the `git` commands.
|
|
57
57
|
|
|
58
58
|
An update across a major version is refused without `--confirm`: summarise its release notes first (next section), get the caller's go, then run `["stack-update", "<stack>", "--confirm"]`, which asks an owner for approval.
|
|
59
59
|
|
package/specialists/AGENTS.md
CHANGED
|
@@ -58,7 +58,7 @@ Wrappers ship as TypeScript and run under Node 24 type stripping: erasable synta
|
|
|
58
58
|
- `entry/specialist-install.sh` runs as root, so it runs only `/usr/bin`, `/bin`, `/usr/sbin`, `/sbin` and the Node it installed, never Homebrew or anything else the main user can write; it checks that `/usr/local/libexec/specialist-node` (and each `specialist-<tool>` folder) and its parents belong to root and are writable by nobody else. Arguments are checked before the root test (tests exercise that part). Downloads go to a root-only temporary directory and are verified before use: Node against `SHASUMS256.txt` and the LTS flag in `index.json`, npm tarballs against the registry's `sha512` integrity, arcane-cli against the release's checksums file. Inside `/Users/specialist` it works only as `specialist` (`sudo -u specialist -H`), from a staging directory root hands over, so a link the specialist plants cannot redirect a root write. Each component switches by rename after a check (every installed wrapper's `--help`, `--version` of the new binary) and restores the previous copy when the check fails. Older versions are refused; `rollback` swaps current and previous. `python`, `gcloud`, `bw`, and `jq` follow the same rules: version folders under `/usr/local/libexec/specialist-<tool>/` behind the links `current` and `previous`, sealed `root:wheel` and not writable by others, smoke-tested as `specialist` before `current` moves; `python` refuses a switch that breaks the current gcloud. Consumers use the `current` paths: `provision.sh` for `bw` and `jq`, `edge.json` for `gcloud` and `gcloudPython`. Nothing a wrapper runs with secrets may come from Homebrew.
|
|
59
59
|
- `main()` waits for stdout and stderr to flush before `process.exit`: on a pipe, exiting right after a write drops everything past the 64 KiB buffer, which cut large answers such as `projects list --all` until 2026-09-25 (`lib/flush.test.ts`).
|
|
60
60
|
- `arcane update-report [--all]` is `lib/image-updates.ts` over `lib/registry.ts`: a fresh `images updates check --all`, then `projects list --all` for the stack and service of each image reference, then the registries' tag lists and manifest digests (anonymous bearer tokens from each registry's challenge; digests by HEAD, which Docker Hub does not count as a pull; lscr.io is read from ghcr.io). A floating or major-series tag is resolved to the release tag whose digest matches the running and the latest build, searching the newest tags of the three commonest release shapes; a fixed tag is compared with the newest tag of the same shape; a digest pin with the newest release. Checks older than one minute before the run count as stale and show as unknown. Private images (anonymous access refused with 401 or 403) are excluded from rows, counts, and table, listed in `privateImages`, and counted in the summary line. The table is fixed-width text for a code block, because Discord renders no Markdown tables. The answer adds `holds` from `update-holds.json` at origin's branch head (`git show`, no working-tree read), or `holdsError` when that file is malformed.
|
|
61
|
-
- `arcane stack-update <stack> [--confirm]` (`lib/stack-update.ts` for the pure parts) is deterministic: refuses `arcane`, held stacks (the checkout's `update-holds.json` after `pull --ff-only`; a malformed file is an error), a dirty or ahead checkout, stopped stacks, and an update across a major version without `--confirm` (which the gate turns into an owner approval). Pending rows come from a fresh per-image check resolved like `update-report`. `edit tag`/`edit digest` rows rewrite `image:` lines that name exactly the old reference (`replaceImageRef`, quoted or not, trailing comment kept) in the sync's `composePath`, then commit, push, and wait for `gitops list` to report success at that commit (20 min, polled every 30 s). Then `projects redeploy <stack> --yes` (35 min), container checks through `containers list --all` and `containers get` (`composeInfo.projectName`, health; 5 min, polled every 15 s; a job that exited with code 0 counts as done whatever health its image's inherited check left, as bigcapital's migration did on 2026-10-28), and a fresh check that each service is current. A failure after a commit reverts, pushes, waits for the sync, redeploys, and checks again (`rolled back`); a pull-only failure stays `failed`. Every outcome prints JSON with `status` and exits 0. Worst case about 125 minutes, so the gate's `timeoutMs` for arcane must exceed it (8,100,000). It holds the `arcane-gitops` lock, so a second checkout call waits up to 2 minutes and then fails as busy; the bridge also runs one Arcane update at a time.
|
|
61
|
+
- `arcane stack-update <stack> [--confirm]` (`lib/stack-update.ts` for the pure parts) is deterministic: refuses `arcane`, held stacks (the checkout's `update-holds.json` after `pull --ff-only`; a malformed file is an error; a `<stack>/<service>` key skips that service's rows instead), a dirty or ahead checkout, stopped stacks, and an update across a major version without `--confirm` (which the gate turns into an owner approval). Pending rows come from a fresh per-image check resolved like `update-report`. `edit tag`/`edit digest` rows rewrite `image:` lines that name exactly the old reference (`replaceImageRef`, quoted or not, trailing comment kept) in the sync's `composePath`, then commit, push, and wait for `gitops list` to report success at that commit (20 min, polled every 30 s). Then `projects redeploy <stack> --yes` (35 min), container checks through `containers list --all` and `containers get` (`composeInfo.projectName`, health; 5 min, polled every 15 s; a job that exited with code 0 counts as done whatever health its image's inherited check left, as bigcapital's migration did on 2026-10-28), and a fresh check that each service is current. A failure after a commit reverts, pushes, waits for the sync, redeploys, and checks again (`rolled back`); a pull-only failure stays `failed`. Every outcome prints JSON with `status` and exits 0. Worst case about 125 minutes, so the gate's `timeoutMs` for arcane must exceed it (8,100,000). It holds the `arcane-gitops` lock, so a second checkout call waits up to 2 minutes and then fails as busy; the bridge also runs one Arcane update at a time.
|
|
62
62
|
- `arcane host <command>` runs over the forced-command key of user `spc-arcane` on VM 100. `HOST_COMMANDS` in `lib/arcane-host.ts` must list exactly the operations of `entry/remote/spc-arcane-op.py` (a test compares them); the tracked copies live in the arcane repository's `host/` (`spc-arcane-host-entry`, `spc-arcane-op`, `spc-arcane-sudoers`, `spc-arcane-install.sh`) with the probes in `host/spc-arcane-probes/`. The entry only splits words (letters, digits, space, `._:/+-`) and hands them to `sudo -n /usr/local/sbin/spc-arcane-op`, the one command its sudoers line allows; the op checks the name and every argument and runs the reviewed arcane-ops probes, which print only allowlisted fields. Manager steps (`manager-*`) are internal to `manager-upgrade`/`manager-rollback` and refused through `host`. Nothing on the host deletes data: a manager rollback restores the image and Compose file, not the data volume (R3 stays manual), `db-restore` only fills an empty database, and dumps are never pruned by the tool. `supabase-backup` writes the pre-change backup of `supabase/AGENTS.md` (both databases with `pg_dump -Fc`, each listed and fully decoded by `pg_restore`, `pg_dumpall --globals-only`, and the `supabase-db-config` volume, which must hold `pgsodium_root.key`) into `/var/backups/supabase-upgrades/<UTC>`, the folder of the earlier manual backups. `host-audit` compares the host's hashes with `host/<name>` of the `arcane-gitops` checkout after a pull. The `bigcapital-*` commands run the base-image upgrade of `bigcapital/README.md`: an online backup, a rehearsal on an internal Docker network (restore into the current MariaDB, upgrade with the build's, migrations, server health, Envoy validation; everything it created removed afterwards), the owner-approved switch (stop, quiescent dump, Redis `SAVE`, volume copies compared entry by entry, the stack left stopped for the GitOps commit, the old volumes untouched), and a verify against the switch's row counts and ledger totals.
|
|
63
63
|
- `arcane manager-upgrade <version>` (`lib/arcane-manager.ts`) follows arcane/AGENTS.md "Manager upgrades": the official image's revision label must equal the release tag's commit (GitHub API), `manager-prepare` records `compose.before.yaml` and every container before the pin commit, the pin goes through the `arcane-gitops` checkout (`git` is exported from `lib/repo.ts` for this), the wait polls `gitops list` and the deployed Compose hash every 30 s for up to 30 minutes, then pull, cold backup, `up -d --no-deps --pull never`, and checks. Any failure before the restart reverts the pin (`stopped`); a failed check after it reverts the pin and runs `manager-rollback` (`rolled back`). The answer always exits 0 with `status`, so the bridge and the agent both see it.
|
|
64
64
|
- `research` is a pure forced-command wrapper: every call is one JSON request on stdin to `request` on VM 110 (`10.1.100.10`, user `spc-research`, key `SSH_KEY_SANDBOX`), whose entry hands it to `spc-research-op` as `researcher` (tracked in the homelab repository's `06-space-x-proxmox/research/system/`). The op attaches one playwright-cli session per Discord thread (`t<id>`) to the shared Chrome over CDP on `127.0.0.1:9222` and opens its own tab; cookie, storage, `run-code`, and browser lifecycle commands are refused. `script` runs the agent's code in the thread folder as `researcher` (5 minutes), on a VM that reaches only the internet (Mira's Sandbox zone plus the VM's own nftables). Defaults stay on Cortex in `work/research/defaults.json`, out of the sandbox's reach. `captcha-help` is in `RISKY`, so its owner approval is the moment the owner solved the challenge in the live view (`research-browser.37pla.net`); the bridge renders that ticket with the channel's `liveView` link.
|
|
@@ -104,7 +104,7 @@ ${Object.entries(HOST_COMMANDS)
|
|
|
104
104
|
.join("\n")}
|
|
105
105
|
update-report [--all] fresh image update check for every stack, with running and latest release versions
|
|
106
106
|
from the registries; JSON with a ready table (--all also lists up-to-date services) and
|
|
107
|
-
the held stacks of ${HOLDS_FILE}. Images the registry throttled (toomanyrequests) are
|
|
107
|
+
the held stacks and services of ${HOLDS_FILE}. Images the registry throttled (toomanyrequests) are
|
|
108
108
|
checked again after 5 and 20 seconds. Checks only: never pulls images or redeploys.
|
|
109
109
|
stack-update <stack> [--confirm]
|
|
110
110
|
apply every pending update of one stack: pinned tags and digests move in the stack's
|
|
@@ -112,7 +112,7 @@ ${Object.entries(HOST_COMMANDS)
|
|
|
112
112
|
with a pull, containers the redeploy left on an old image are recreated one by one, and
|
|
113
113
|
the stack is checked (containers running and healthy, images current). A failed check after a
|
|
114
114
|
Compose change reverts it and redeploys; a pull alone cannot be undone. Refuses held
|
|
115
|
-
stacks, stopped stacks, and arcane (manager-upgrade). An update across a major version
|
|
115
|
+
stacks, stopped stacks, and arcane (manager-upgrade); skips held services. An update across a major version
|
|
116
116
|
needs --confirm, which asks an owner for approval.
|
|
117
117
|
stack-rollback <stack> <commit>
|
|
118
118
|
undo one earlier stack-update commit of the stack's Compose file: revert, push, start the
|
|
@@ -753,7 +753,8 @@ async function stackUpdate(config: ArcaneConfig, args: string[]): Promise<number
|
|
|
753
753
|
|
|
754
754
|
const before = await withCli(config, (cli) => stackState(cli, stack, Date.now()));
|
|
755
755
|
if (before.stopped) throw new UsageError(`${stack} is stopped; a redeploy would start it, so start it on purpose first`);
|
|
756
|
-
|
|
756
|
+
// A held service keeps its version; the rest of the stack still updates.
|
|
757
|
+
const pending = pendingRows(before.rows).filter((r) => !holds[`${stack}/${r.service}`]);
|
|
757
758
|
const answer = (status: StackStatus, extra: Record<string, unknown>) => printJson({ status, stack, ...extra });
|
|
758
759
|
if (pending.length === 0) {
|
|
759
760
|
answer("nothing to do", { unknown: before.rows.filter((r) => r.status === "unknown").map((r) => ({ service: r.service, note: r.note })) });
|
|
@@ -18,9 +18,23 @@ case "$name" in
|
|
|
18
18
|
case "$unit" in pomerium|nginx|lego-renew) ;; *) echo "bad unit" >&2; exit 2 ;; esac
|
|
19
19
|
since_ok "$since" || { echo "bad since" >&2; exit 2; }
|
|
20
20
|
journalctl -u "$unit" --since "$since" --no-pager -n 200 ;;
|
|
21
|
-
config-template) cat /etc/pomerium/config.yaml.template ;;
|
|
21
|
+
config-template) sudo cat /etc/pomerium/config.yaml.template ;;
|
|
22
22
|
render) sudo /usr/local/sbin/render-pomerium ;;
|
|
23
23
|
pomerium-restart) sudo systemctl restart pomerium && systemctl is-active pomerium ;;
|
|
24
|
+
pomerium-version) /usr/local/bin/pomerium --version ;;
|
|
25
|
+
pomerium-upgrade)
|
|
26
|
+
# stdin: the release tarball as base64; pomerium-upgrade checks it against the sha256 argument.
|
|
27
|
+
version="${rest%% *}"; sum="${rest#* }"
|
|
28
|
+
printf '%s' "$version" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$' || { echo "bad version" >&2; exit 2; }
|
|
29
|
+
printf '%s' "$sum" | grep -Eq '^[0-9a-f]{64}$' || { echo "bad sha256" >&2; exit 2; }
|
|
30
|
+
stage="$(mktemp -d -p /var/tmp)"
|
|
31
|
+
base64 -d > "$stage/pomerium.tar.gz"
|
|
32
|
+
if sudo /usr/local/sbin/pomerium-upgrade "$version" "$sum" "$stage/pomerium.tar.gz"; then
|
|
33
|
+
rm -rf "$stage"
|
|
34
|
+
else
|
|
35
|
+
rm -rf "$stage"
|
|
36
|
+
exit 1
|
|
37
|
+
fi ;;
|
|
24
38
|
nginx-reload) sudo nginx -t && sudo systemctl reload nginx ;;
|
|
25
39
|
lego-renew) sudo systemctl start lego-renew.service && systemctl status lego-renew.service --no-pager --lines=10 ;;
|
|
26
40
|
deploy-config)
|
|
@@ -13,13 +13,16 @@ describe("update holds", () => {
|
|
|
13
13
|
it("reads stack names with a reason", () => {
|
|
14
14
|
expect(parseHolds('{"supabase": " coordinated release: review first "}')).toEqual({ supabase: "coordinated release: review first" });
|
|
15
15
|
expect(parseHolds("{}")).toEqual({});
|
|
16
|
+
expect(parseHolds('{"paperless/gotenberg": "Paperless needs Gotenberg 8"}')).toEqual({ "paperless/gotenberg": "Paperless needs Gotenberg 8" });
|
|
16
17
|
});
|
|
17
18
|
|
|
18
19
|
it("refuses a malformed file instead of holding nothing", () => {
|
|
19
20
|
expect(() => parseHolds("not json")).toThrow(/not JSON/);
|
|
20
|
-
expect(() => parseHolds('["supabase"]')).toThrow(/object of stack
|
|
21
|
+
expect(() => parseHolds('["supabase"]')).toThrow(/object of stack/);
|
|
21
22
|
expect(() => parseHolds('{"supabase": ""}')).toThrow(/non-empty reason/);
|
|
22
|
-
expect(() => parseHolds('{"../x": "why"}')).toThrow(/stack name/);
|
|
23
|
+
expect(() => parseHolds('{"../x": "why"}')).toThrow(/stack or stack\/service name/);
|
|
24
|
+
expect(() => parseHolds('{"a/b/c": "why"}')).toThrow(/stack or stack\/service name/);
|
|
25
|
+
expect(() => parseHolds('{"a/../b": "why"}')).toThrow(/stack or stack\/service name/);
|
|
23
26
|
});
|
|
24
27
|
});
|
|
25
28
|
|
|
@@ -2,11 +2,11 @@ import { UsageError } from "./errors.ts";
|
|
|
2
2
|
import { parseTag, type UpdateRow } from "./image-updates.ts";
|
|
3
3
|
import { parseImageRef } from "./registry.ts";
|
|
4
4
|
|
|
5
|
-
/** Stacks whose updates wait for a person, tracked in the root of the GitOps repository: `{ "<stack>": "<reason>" }`. */
|
|
5
|
+
/** Stacks and services whose updates wait for a person, tracked in the root of the GitOps repository: `{ "<stack>": "<reason>", "<stack>/<service>": "<reason>" }`. */
|
|
6
6
|
export const HOLDS_FILE = "update-holds.json";
|
|
7
|
-
const
|
|
7
|
+
const HOLD_KEY = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}(\/[A-Za-z0-9][A-Za-z0-9_.-]{0,127})?$/;
|
|
8
8
|
|
|
9
|
-
/** The held stacks with their reasons; a malformed file is an error, never "nothing held". */
|
|
9
|
+
/** The held stacks and services with their reasons; a malformed file is an error, never "nothing held". */
|
|
10
10
|
export function parseHolds(text: string): Record<string, string> {
|
|
11
11
|
let value: unknown;
|
|
12
12
|
try {
|
|
@@ -14,10 +14,10 @@ export function parseHolds(text: string): Record<string, string> {
|
|
|
14
14
|
} catch {
|
|
15
15
|
throw new UsageError(`${HOLDS_FILE} is not JSON`);
|
|
16
16
|
}
|
|
17
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new UsageError(`${HOLDS_FILE} must be an object of stack name to reason`);
|
|
17
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new UsageError(`${HOLDS_FILE} must be an object of stack (or stack/service) name to reason`);
|
|
18
18
|
const holds: Record<string, string> = {};
|
|
19
19
|
for (const [stack, reason] of Object.entries(value)) {
|
|
20
|
-
if (!
|
|
20
|
+
if (!HOLD_KEY.test(stack) || typeof reason !== "string" || reason.trim().length === 0) throw new UsageError(`${HOLDS_FILE}: "${stack.slice(0, 40)}" needs a stack or stack/service name and a non-empty reason`);
|
|
21
21
|
holds[stack] = reason.trim().slice(0, 200);
|
|
22
22
|
}
|
|
23
23
|
return holds;
|