@astrofoundry/pi-astro 0.40.63 → 0.40.65
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/astro-discord/daily-report.test.ts +27 -0
- package/extensions/astro-discord/daily-report.ts +42 -10
- package/extensions/astro-discord/daily-run.test.ts +4 -1
- package/extensions/astro-discord/daily-run.ts +5 -3
- package/package.json +1 -1
- package/skills/arcane/SKILL.md +3 -1
- package/specialists/AGENTS.md +1 -1
- package/specialists/arcane/run.ts +125 -28
- package/specialists/lib/image-updates.test.ts +12 -2
- package/specialists/lib/image-updates.ts +25 -10
- package/specialists/lib/stack-update.test.ts +5 -2
- package/specialists/lib/stack-update.ts +5 -5
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
carefulTask,
|
|
11
11
|
failureMessage,
|
|
12
12
|
headText,
|
|
13
|
+
isPending,
|
|
13
14
|
layoutMessages,
|
|
14
15
|
majorLines,
|
|
15
16
|
managerSummary,
|
|
@@ -18,6 +19,9 @@ import {
|
|
|
18
19
|
readPostedArcane,
|
|
19
20
|
releaseNotesFromJson,
|
|
20
21
|
REPORT_PREFIX,
|
|
22
|
+
serviceText,
|
|
23
|
+
serviceUsage,
|
|
24
|
+
type ServiceLine,
|
|
21
25
|
stackComponent,
|
|
22
26
|
type StackEntry,
|
|
23
27
|
stackOutcome,
|
|
@@ -94,6 +98,22 @@ function entry(r: ArcaneReport, stack: string): StackEntry {
|
|
|
94
98
|
}
|
|
95
99
|
|
|
96
100
|
describe("Arcane Daily Report layout", () => {
|
|
101
|
+
it("keeps a held service out of the stack's update and shows why it is held", () => {
|
|
102
|
+
const proxy = { stack: "bigcapital", service: "proxy", ref: "envoyproxy/envoy:v1.39.1", running: "v1.39.1", latest: "v1.39.6", status: "release", action: "edit tag", note: "" };
|
|
103
|
+
const holds = { "bigcapital/gotenberg": "Bigcapital needs Gotenberg 7", "bigcapital/proxy": "wait for the next bundle" };
|
|
104
|
+
const built = buildStacks({ ...updateReport, rows: [rows[5], proxy], holds });
|
|
105
|
+
const stack = built?.stacks[0];
|
|
106
|
+
expect(stack?.held).toBeUndefined();
|
|
107
|
+
expect(stack?.actionable).toBe(false);
|
|
108
|
+
expect(stack?.lines.map((l) => [l.service, l.held, isPending(l)])).toEqual([
|
|
109
|
+
["gotenberg", "Bigcapital needs Gotenberg 7", false],
|
|
110
|
+
["proxy", "wait for the next bundle", false],
|
|
111
|
+
]);
|
|
112
|
+
expect(serviceText(stack?.lines[0] as ServiceLine)).toBe("• gotenberg: 7.10.2, newer major 8 available (a tag change by hand; Update does not apply it)\n-# ⏸ held: Bigcapital needs Gotenberg 7");
|
|
113
|
+
const partly = buildStacks({ ...updateReport, rows: [rows[5], proxy], holds: { "bigcapital/gotenberg": "x" } });
|
|
114
|
+
expect(partly?.stacks[0].actionable).toBe(true);
|
|
115
|
+
});
|
|
116
|
+
|
|
97
117
|
it("marks the images our own CI builds and leaves out stopped stacks", () => {
|
|
98
118
|
const own = { stack: "bigcapital", service: "server", ref: "ghcr.io/astrofoundry/bigcapital-server:astrofoundry", running: "?", latest: "new build", status: "update", action: "pull", note: "versions unavailable: private image" };
|
|
99
119
|
const stopped = { ...own, stack: "grimoire-graphile-bootstrap", service: "bootstrap", ref: "ghcr.io/astronaute77/arcane-graphile-worker:latest", stopped: true };
|
|
@@ -153,6 +173,13 @@ describe("Arcane Daily Report layout", () => {
|
|
|
153
173
|
if (!empty) throw new Error("no report");
|
|
154
174
|
expect(headText({ ...empty, timeZone: "Europe/Madrid" }).content).toBe(`**${REPORT_PREFIX}** no updates (44 images, 2 pinned private excluded, checked 06:00 CEST).\n-# ⚠️ update-holds.json is not JSON; Update still refuses held stacks`);
|
|
155
175
|
expect(buildStacks({ nonsense: true })).toBeNull();
|
|
176
|
+
const remote = buildStacks({ ...updateReport, holds: {}, rows: [{ ...rows[2], stack: "media-mira", environment: "mira" }], environmentErrors: [{ environment: "lab", error: "agent offline" }, { broken: true }], agentsBehind: [{ environment: "mira", agent: "v2.17.3", manager: "v2.18.0" }] });
|
|
177
|
+
if (!remote) throw new Error("no report");
|
|
178
|
+
expect(remote.stacks[0].lines[0].environment).toBe("mira");
|
|
179
|
+
expect(serviceUsage("media-mira", remote.stacks[0].lines[0])).toContain("on the Docker host of the Arcane environment mira,");
|
|
180
|
+
expect(remote.environmentErrors).toEqual([{ environment: "lab", error: "agent offline" }]);
|
|
181
|
+
expect(headText({ ...remote, timeZone: "Europe/Madrid" }).content).toContain("\n-# ⚠️ environment lab was not checked: agent offline");
|
|
182
|
+
expect(headText({ ...remote, timeZone: "Europe/Madrid" }).content).toContain("\n-# ⚠️ the Arcane agent of mira runs v2.17.3, the manager v2.18.0");
|
|
156
183
|
expect(failureMessage().startsWith(REPORT_PREFIX)).toBe(true);
|
|
157
184
|
});
|
|
158
185
|
|
|
@@ -23,6 +23,20 @@ interface ReportRow {
|
|
|
23
23
|
action: string;
|
|
24
24
|
note: string;
|
|
25
25
|
stopped?: boolean;
|
|
26
|
+
environment?: string;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** A remote Arcane environment the update check could not reach. */
|
|
30
|
+
export interface EnvironmentError {
|
|
31
|
+
environment: string;
|
|
32
|
+
error: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** A remote environment whose agent runs another version than the manager. */
|
|
36
|
+
export interface AgentBehind {
|
|
37
|
+
environment: string;
|
|
38
|
+
agent: string;
|
|
39
|
+
manager: string;
|
|
26
40
|
}
|
|
27
41
|
|
|
28
42
|
interface UpdateReportJson {
|
|
@@ -32,6 +46,8 @@ interface UpdateReportJson {
|
|
|
32
46
|
rows: ReportRow[];
|
|
33
47
|
holds?: Record<string, string>;
|
|
34
48
|
holdsError?: string;
|
|
49
|
+
environmentErrors?: EnvironmentError[];
|
|
50
|
+
agentsBehind?: AgentBehind[];
|
|
35
51
|
}
|
|
36
52
|
|
|
37
53
|
function isRow(value: unknown): value is ReportRow {
|
|
@@ -63,9 +79,9 @@ export function crossesMajor(running: string | null, latest: string | null): boo
|
|
|
63
79
|
|
|
64
80
|
const PENDING = new Set(["update", "release", "rebuild"]);
|
|
65
81
|
|
|
66
|
-
/** A line whose stack-update would change something. */
|
|
67
|
-
export function isPending(l: Pick<ServiceLine, "status">): boolean {
|
|
68
|
-
return PENDING.has(l.status);
|
|
82
|
+
/** A line whose stack-update would change something; a held service keeps its version. */
|
|
83
|
+
export function isPending(l: Pick<ServiceLine, "status" | "held">): boolean {
|
|
84
|
+
return PENDING.has(l.status) && l.held === undefined;
|
|
69
85
|
}
|
|
70
86
|
|
|
71
87
|
export interface ServiceLine {
|
|
@@ -83,6 +99,10 @@ export interface ServiceLine {
|
|
|
83
99
|
review?: Review;
|
|
84
100
|
/** The image is one our own CI builds, so its update goes through Upgrade carefully. */
|
|
85
101
|
own?: boolean;
|
|
102
|
+
/** Why this service is held (`<stack>/<service>` in `update-holds.json`); stack-update skips it. */
|
|
103
|
+
held?: string;
|
|
104
|
+
/** The Arcane remote environment that runs the stack; absent for the Arcane VM. */
|
|
105
|
+
environment?: string;
|
|
86
106
|
}
|
|
87
107
|
|
|
88
108
|
export type StackState = "offered" | "running" | "updated" | "rolled back" | "failed" | "current" | "planning";
|
|
@@ -120,6 +140,8 @@ export interface ArcaneReport {
|
|
|
120
140
|
images: number;
|
|
121
141
|
privateCount: number;
|
|
122
142
|
holdsError?: string;
|
|
143
|
+
environmentErrors?: EnvironmentError[];
|
|
144
|
+
agentsBehind?: AgentBehind[];
|
|
123
145
|
stacks: StackEntry[];
|
|
124
146
|
messages: PostedPage[];
|
|
125
147
|
/** Owners tagged on the first message because an update needs them. */
|
|
@@ -146,6 +168,8 @@ export interface ReportStacks {
|
|
|
146
168
|
privateCount: number;
|
|
147
169
|
checkedAt: string;
|
|
148
170
|
holdsError?: string;
|
|
171
|
+
environmentErrors?: EnvironmentError[];
|
|
172
|
+
agentsBehind?: AgentBehind[];
|
|
149
173
|
}
|
|
150
174
|
|
|
151
175
|
/**
|
|
@@ -157,6 +181,8 @@ export interface ReportStacks {
|
|
|
157
181
|
export function buildStacks(report: unknown): ReportStacks | null {
|
|
158
182
|
if (!isReport(report)) return null;
|
|
159
183
|
const holds = report.holds ?? {};
|
|
184
|
+
const environmentErrors = (Array.isArray(report.environmentErrors) ? report.environmentErrors : []).filter((e): e is EnvironmentError => typeof e?.environment === "string" && typeof e?.error === "string");
|
|
185
|
+
const agentsBehind = (Array.isArray(report.agentsBehind) ? report.agentsBehind : []).filter((a): a is AgentBehind => typeof a?.environment === "string" && typeof a?.agent === "string" && typeof a?.manager === "string");
|
|
160
186
|
const shown = report.rows.filter((r) => !r.stopped && (PENDING.has(r.status) || r.status === "unknown" || /newer major \S+ available/.test(r.note ?? "")));
|
|
161
187
|
const names = [...new Set(shown.map((r) => r.stack))].sort((a, b) => a.localeCompare(b));
|
|
162
188
|
const stacks = names.map((stack): StackEntry => {
|
|
@@ -166,17 +192,18 @@ export function buildStacks(report: unknown): ReportStacks | null {
|
|
|
166
192
|
const running = releaseTag(r.running);
|
|
167
193
|
const latest = releaseTag(r.latest);
|
|
168
194
|
const own = r.ref ? isOwnImage(r.ref) : false;
|
|
169
|
-
|
|
195
|
+
const held = holds[`${stack}/${r.service}`];
|
|
196
|
+
return { service: r.service, status: r.status, action: r.action ?? "", running, latest, tag: tagOf(r.ref ?? ""), ...(r.ref ? { ref: r.ref } : {}), major: PENDING.has(r.status) && crossesMajor(running, latest), note: r.note ?? "", ...(own ? { own } : {}), ...(held ? { held } : {}), ...(r.environment ? { environment: r.environment } : {}) };
|
|
170
197
|
});
|
|
171
|
-
const pending = lines.some(
|
|
198
|
+
const pending = lines.some(isPending);
|
|
172
199
|
const held = holds[stack];
|
|
173
200
|
if (stack === "arcane") {
|
|
174
|
-
const offer = lines.map((l) => (
|
|
201
|
+
const offer = lines.map((l) => (isPending(l) ? managerVersion(l) : null)).find((v) => v !== null) ?? null;
|
|
175
202
|
return { stack, lines, actionable: offer !== null && !held, ...(offer ? { manager: offer } : {}), ...(held ? { held } : {}), state: "offered" };
|
|
176
203
|
}
|
|
177
204
|
return { stack, lines, actionable: pending && !held, ...(held ? { held } : {}), state: "offered" };
|
|
178
205
|
});
|
|
179
|
-
return { stacks, images: report.images, privateCount: report.privateImages.length, checkedAt: report.checkedAt, ...(report.holdsError ? { holdsError: report.holdsError } : {}) };
|
|
206
|
+
return { stacks, images: report.images, privateCount: report.privateImages.length, checkedAt: report.checkedAt, ...(report.holdsError ? { holdsError: report.holdsError } : {}), ...(environmentErrors.length > 0 ? { environmentErrors } : {}), ...(agentsBehind.length > 0 ? { agentsBehind } : {}) };
|
|
180
207
|
}
|
|
181
208
|
|
|
182
209
|
/** Services whose update crosses a major version; each gets a release-notes review before the report posts. */
|
|
@@ -212,7 +239,8 @@ export function serviceText(l: ServiceLine): string {
|
|
|
212
239
|
const newer = PENDING.has(l.status) ? /newer major (\S+) available/.exec(l.note)?.[1] : undefined;
|
|
213
240
|
if (newer) text += ` · newer major ${escapeDiscord(newer)} available`;
|
|
214
241
|
const review = l.review ? `\n-# ${l.review.verdict ? `${VERDICT_MARK[l.review.verdict]}: ` : "notes: "}${escapeDiscord(l.review.reason)}` : "";
|
|
215
|
-
|
|
242
|
+
const held = l.held ? `\n-# ⏸ held: ${escapeDiscord(l.held)}` : "";
|
|
243
|
+
return `• ${text}${review}${held}`;
|
|
216
244
|
}
|
|
217
245
|
|
|
218
246
|
function stackText(s: StackEntry): string {
|
|
@@ -298,7 +326,10 @@ export function headText(report: ReportView): MessageComponent {
|
|
|
298
326
|
report.stacks.length === 0
|
|
299
327
|
? `**${REPORT_PREFIX}** no updates (${scope}).`
|
|
300
328
|
: `**${REPORT_PREFIX}** ${withUpdates} stack${withUpdates === 1 ? "" : "s"} with updates (${scope}).`;
|
|
301
|
-
const warning =
|
|
329
|
+
const warning =
|
|
330
|
+
(report.holdsError ? `\n-# ⚠️ ${escapeDiscord(report.holdsError)}; Update still refuses held stacks` : "") +
|
|
331
|
+
(report.environmentErrors ?? []).map((e) => `\n-# ⚠️ environment ${escapeDiscord(e.environment)} was not checked: ${escapeDiscord(e.error.slice(0, 200))}`).join("") +
|
|
332
|
+
(report.agentsBehind ?? []).map((a) => `\n-# ⚠️ the Arcane agent of ${escapeDiscord(a.environment)} runs ${escapeDiscord(a.agent)}, the manager ${escapeDiscord(a.manager)}`).join("");
|
|
302
333
|
const attention = report.stacks.filter((s) => s.plan === "update" || s.plan === "careful");
|
|
303
334
|
const automatic = report.stacks.filter((s) => s.plan === "auto").length;
|
|
304
335
|
const plan = automatic > 0 ? `\n-# ${automatic} stack${automatic === 1 ? " updates" : "s update"} automatically and ${automatic === 1 ? "is" : "are"} checked again 5 minutes later.` : "";
|
|
@@ -403,7 +434,8 @@ export function releaseNotesFromJson(value: unknown): StackNotes {
|
|
|
403
434
|
|
|
404
435
|
/** What the reviewer is told about a service of a stack, so it can tell what a release would break. */
|
|
405
436
|
export function serviceUsage(stack: string, line: ServiceLine): string {
|
|
406
|
-
|
|
437
|
+
const host = line.environment ? `on the Docker host of the Arcane environment ${line.environment}` : "on the Arcane VM";
|
|
438
|
+
return `${line.service} is a service of the Docker Compose stack ${stack} ${host}, deployed through GitOps. The update replaces its image (tag ${line.tag}) from ${line.running ?? "an older build"} to ${line.latest ?? "a newer build"} and changes nothing else in the Compose file: no environment variables, volumes, ports, or companion services. Judge whether the release needs a configuration, data, or companion change before or after the swap.`;
|
|
407
439
|
}
|
|
408
440
|
|
|
409
441
|
// ---------------------------------------------------------------- posted report state
|
|
@@ -247,14 +247,17 @@ describe("report summaries", () => {
|
|
|
247
247
|
stack("stirling-pdf", { plan: "careful" }),
|
|
248
248
|
stack("supabase", { actionable: false, held: "review first" }),
|
|
249
249
|
stack("paperless", { actionable: false, lines: [line({}), line({ service: "broker", status: "unknown", note: "registry timeout" })] }),
|
|
250
|
+
stack("bigcapital", { actionable: false, lines: [line({ held: "Bigcapital needs Gotenberg 8" })] }),
|
|
250
251
|
];
|
|
251
252
|
const report = { date: "2026-10-24", stacks, messages: [{ id: "8", stacks: [], head: true }] } as unknown as ArcaneReport;
|
|
252
253
|
expect(arcaneSummary(report, "2026-10-24", "6")).toEqual({
|
|
253
254
|
area: "Arcane",
|
|
254
255
|
line: "1 stack updated (media), mealie rolled back, stirling-pdf waits for you, supabase held",
|
|
255
|
-
needs: ["stirling-pdf waits for you (Upgrade carefully)", "mealie automatic update rolled back", "
|
|
256
|
+
needs: ["stirling-pdf waits for you (Upgrade carefully)", "mealie automatic update rolled back", "paperless gotenberg 8.40.0: major 9 needs a tag change by hand", "paperless broker: the update check failed"],
|
|
256
257
|
channelId: "6",
|
|
257
258
|
messageId: "8",
|
|
258
259
|
});
|
|
260
|
+
const unchecked = { ...report, stacks: [], environmentErrors: [{ environment: "mira", error: "environment offline" }], agentsBehind: [{ environment: "lab", agent: "v2.17.3", manager: "v2.18.0" }] } as unknown as ArcaneReport;
|
|
261
|
+
expect(arcaneSummary(unchecked, "2026-10-24", "6").needs).toEqual(["Arcane environment mira was not checked for updates", "the Arcane agent of lab runs v2.17.3, the manager v2.18.0: move its tag on the host"]);
|
|
259
262
|
});
|
|
260
263
|
});
|
|
@@ -401,15 +401,16 @@ export function toolsSummary(report: ToolsReport | null, date: string, channelId
|
|
|
401
401
|
return { area: "Cortex tools", line: parts.join(", ") || "everything up to date", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
|
|
402
402
|
}
|
|
403
403
|
|
|
404
|
-
/** A service line only the owner can act on: a failed check, or a newer major that needs a tag change by hand. */
|
|
404
|
+
/** A service line only the owner can act on: a failed check, or a newer major that needs a tag change by hand; a held service is the owner's own decision. */
|
|
405
405
|
function byHand(stack: string, l: ArcaneReport["stacks"][number]["lines"][number]): string | null {
|
|
406
|
+
if (l.held !== undefined) return null;
|
|
406
407
|
const where = `${escapeDiscord(stack)} ${escapeDiscord(l.service)}`;
|
|
407
408
|
if (l.status === "unknown") return `${where}: the update check failed`;
|
|
408
409
|
const newer = /newer major (\S+) available/.exec(l.note)?.[1];
|
|
409
410
|
return newer ? `${where} ${escapeDiscord(l.running ?? l.tag)}: major ${escapeDiscord(newer)} needs a tag change by hand` : null;
|
|
410
411
|
}
|
|
411
412
|
|
|
412
|
-
/** The Arcane Daily Report after its automatic updates and checks: updated, rolled back, waiting for the owner, held, by hand. */
|
|
413
|
+
/** The Arcane Daily Report after its automatic updates and checks: updated, rolled back, waiting for the owner, held, by hand. Holds are the owner's own decisions, so they are not items. */
|
|
413
414
|
export function arcaneSummary(report: ArcaneReport | null, date: string, channelId: string): ReportLine {
|
|
414
415
|
if (!report || report.date !== date) return { area: "Arcane", line: "no report today; the bridge log has the reason", needs: ["the Arcane Daily Report did not run today; the bridge log has the reason"], channelId };
|
|
415
416
|
const by = (pick: (s: ArcaneReport["stacks"][number]) => boolean) => report.stacks.filter(pick).map((s) => s.stack);
|
|
@@ -428,9 +429,10 @@ export function arcaneSummary(report: ArcaneReport | null, date: string, channel
|
|
|
428
429
|
const needs = [
|
|
429
430
|
...waiting.map((s) => `${escapeDiscord(s.stack)} waits for you (${s.plan === "careful" ? "Upgrade carefully" : "major version"})`),
|
|
430
431
|
...report.stacks.filter((s) => s.plan === "auto" && (s.state === "rolled back" || s.state === "failed")).map((s) => `${escapeDiscord(s.stack)} automatic update ${s.state}`),
|
|
431
|
-
...report.stacks.filter((s) => s.held !== undefined).map((s) => `${escapeDiscord(s.stack)} update held: ${escapeDiscord(s.held ?? "")}`),
|
|
432
432
|
...report.stacks.flatMap((s) => s.lines.map((l) => byHand(s.stack, l)).filter((t): t is string => t !== null)),
|
|
433
433
|
...(report.holdsError ? [`update-holds.json is unreadable: ${escapeDiscord(report.holdsError)}`] : []),
|
|
434
|
+
...(report.environmentErrors ?? []).map((e) => `Arcane environment ${escapeDiscord(e.environment)} was not checked for updates`),
|
|
435
|
+
...(report.agentsBehind ?? []).map((a) => `the Arcane agent of ${escapeDiscord(a.environment)} runs ${escapeDiscord(a.agent)}, the manager ${escapeDiscord(a.manager)}: move its tag on the host`),
|
|
434
436
|
];
|
|
435
437
|
return { area: "Arcane", line: parts.join(", ") || "no updates", needs, channelId, ...(report.messages[0] ? { messageId: report.messages[0].id } : {}) };
|
|
436
438
|
}
|
package/package.json
CHANGED
package/skills/arcane/SKILL.md
CHANGED
|
@@ -7,6 +7,8 @@ description: How the Arcane specialist operates the Arcane container platform an
|
|
|
7
7
|
|
|
8
8
|
Arcane manages the Docker host on VM 100 (`10.0.40.30`). Every app is a Compose project synced from the GitOps repository `astronaute77/arcane`, branch `main`, into `/opt/docker`. The `arcane` tool wraps `arcane-cli` with a fresh short-lived token on every call and adds `git` subcommands for the repository checkout. `arcane-cli gitops ...` (server-side syncs) passes through untouched.
|
|
9
9
|
|
|
10
|
+
Since 2026-11-09 a second environment, `mira`, runs the `media-mira` project on media-mira (`10.1.40.30`, VM 111 on SPACE-X at Mira) through an Edge agent. `stack-update`, `stack-rollback`, `deployment`, and `release-notes` find a stack's environment themselves; `update-report`, `container-watch`, and `project-status` cover every environment and name the remote one in `environment`. A raw arcane-cli call on Mira needs `--env f1968df8-bce3-4a65-966f-72934727ae86`. The `host` commands reach VM 100 only.
|
|
11
|
+
|
|
10
12
|
## Calling the tool
|
|
11
13
|
|
|
12
14
|
`arcane` takes one parameter, `args`, an array of strings, one element per argument.
|
|
@@ -53,7 +55,7 @@ Apply updates only when asked, one stack at a time, with `["stack-update", "<sta
|
|
|
53
55
|
|
|
54
56
|
The answer is JSON with `status`: `updated`, `rolled back`, `failed` (with `reason`), or `nothing to do`, plus `services` (`service`, `from`, `to`, `applied`: `pull` or the new reference, `result`: `current`, `behind`, `unchecked`) and `commit`/`revert` when the Compose file changed. Report it in plain words.
|
|
55
57
|
|
|
56
|
-
It refuses `arcane` (next section), stopped stacks (a redeploy would start them), a checkout with local or unpushed changes, and held stacks. Held stacks are listed in `update-holds.json` at the root of the GitOps repository, `{ "<stack>": "<reason>" }`; `update-report` returns them in `holds`, and the daily report shows the reason without a button. Change that file only when the caller asks, through the `git` commands.
|
|
58
|
+
It refuses `arcane` (next section), stopped stacks (a redeploy would start them), a checkout with local or unpushed changes, and held stacks. Held stacks are listed in `update-holds.json` at the root of the GitOps repository, `{ "<stack>": "<reason>" }`; a key `<stack>/<service>` holds one service, which `stack-update` skips while the rest of the stack updates. `update-report` returns them in `holds`, and the daily report shows the reason without a button. Change that file only when the caller asks, through the `git` commands.
|
|
57
59
|
|
|
58
60
|
An update across a major version is refused without `--confirm`: summarise its release notes first (next section), get the caller's go, then run `["stack-update", "<stack>", "--confirm"]`, which asks an owner for approval.
|
|
59
61
|
|
package/specialists/AGENTS.md
CHANGED
|
@@ -58,7 +58,7 @@ Wrappers ship as TypeScript and run under Node 24 type stripping: erasable synta
|
|
|
58
58
|
- `entry/specialist-install.sh` runs as root, so it runs only `/usr/bin`, `/bin`, `/usr/sbin`, `/sbin` and the Node it installed, never Homebrew or anything else the main user can write; it checks that `/usr/local/libexec/specialist-node` (and each `specialist-<tool>` folder) and its parents belong to root and are writable by nobody else. Arguments are checked before the root test (tests exercise that part). Downloads go to a root-only temporary directory and are verified before use: Node against `SHASUMS256.txt` and the LTS flag in `index.json`, npm tarballs against the registry's `sha512` integrity, arcane-cli against the release's checksums file. Inside `/Users/specialist` it works only as `specialist` (`sudo -u specialist -H`), from a staging directory root hands over, so a link the specialist plants cannot redirect a root write. Each component switches by rename after a check (every installed wrapper's `--help`, `--version` of the new binary) and restores the previous copy when the check fails. Older versions are refused; `rollback` swaps current and previous. `python`, `gcloud`, `bw`, and `jq` follow the same rules: version folders under `/usr/local/libexec/specialist-<tool>/` behind the links `current` and `previous`, sealed `root:wheel` and not writable by others, smoke-tested as `specialist` before `current` moves; `python` refuses a switch that breaks the current gcloud. Consumers use the `current` paths: `provision.sh` for `bw` and `jq`, `edge.json` for `gcloud` and `gcloudPython`. Nothing a wrapper runs with secrets may come from Homebrew.
|
|
59
59
|
- `main()` waits for stdout and stderr to flush before `process.exit`: on a pipe, exiting right after a write drops everything past the 64 KiB buffer, which cut large answers such as `projects list --all` until 2026-09-25 (`lib/flush.test.ts`).
|
|
60
60
|
- `arcane update-report [--all]` is `lib/image-updates.ts` over `lib/registry.ts`: a fresh `images updates check --all`, then `projects list --all` for the stack and service of each image reference, then the registries' tag lists and manifest digests (anonymous bearer tokens from each registry's challenge; digests by HEAD, which Docker Hub does not count as a pull; lscr.io is read from ghcr.io). A floating or major-series tag is resolved to the release tag whose digest matches the running and the latest build, searching the newest tags of the three commonest release shapes; a fixed tag is compared with the newest tag of the same shape; a digest pin with the newest release. Checks older than one minute before the run count as stale and show as unknown. Private images (anonymous access refused with 401 or 403) are excluded from rows, counts, and table, listed in `privateImages`, and counted in the summary line. The table is fixed-width text for a code block, because Discord renders no Markdown tables. The answer adds `holds` from `update-holds.json` at origin's branch head (`git show`, no working-tree read), or `holdsError` when that file is malformed.
|
|
61
|
-
- `arcane stack-update <stack> [--confirm]` (`lib/stack-update.ts` for the pure parts) is deterministic: refuses `arcane`, held stacks (the checkout's `update-holds.json` after `pull --ff-only`; a malformed file is an error), a dirty or ahead checkout, stopped stacks, and an update across a major version without `--confirm` (which the gate turns into an owner approval). Pending rows come from a fresh per-image check resolved like `update-report`. `edit tag`/`edit digest` rows rewrite `image:` lines that name exactly the old reference (`replaceImageRef`, quoted or not, trailing comment kept) in the sync's `composePath`, then commit, push, and wait for `gitops list` to report success at that commit (20 min, polled every 30 s). Then `projects redeploy <stack> --yes` (35 min), container checks through `containers list --all` and `containers get` (`composeInfo.projectName`, health; 5 min, polled every 15 s; a job that exited with code 0 counts as done whatever health its image's inherited check left, as bigcapital's migration did on 2026-10-28), and a fresh check that each service is current. A failure after a commit reverts, pushes, waits for the sync, redeploys, and checks again (`rolled back`); a pull-only failure stays `failed`. Every outcome prints JSON with `status` and exits 0. Worst case about 125 minutes, so the gate's `timeoutMs` for arcane must exceed it (8,100,000). It holds the `arcane-gitops` lock, so a second checkout call waits up to 2 minutes and then fails as busy; the bridge also runs one Arcane update at a time.
|
|
61
|
+
- `arcane stack-update <stack> [--confirm]` (`lib/stack-update.ts` for the pure parts) is deterministic: refuses `arcane`, held stacks (the checkout's `update-holds.json` after `pull --ff-only`; a malformed file is an error; a `<stack>/<service>` key skips that service's rows instead), a dirty or ahead checkout, stopped stacks, and an update across a major version without `--confirm` (which the gate turns into an owner approval). Pending rows come from a fresh per-image check resolved like `update-report`. `edit tag`/`edit digest` rows rewrite `image:` lines that name exactly the old reference (`replaceImageRef`, quoted or not, trailing comment kept) in the sync's `composePath`, then commit, push, and wait for `gitops list` to report success at that commit (20 min, polled every 30 s). Then `projects redeploy <stack> --yes` (35 min), container checks through `containers list --all` and `containers get` (`composeInfo.projectName`, health; 5 min, polled every 15 s; a job that exited with code 0 counts as done whatever health its image's inherited check left, as bigcapital's migration did on 2026-10-28), and a fresh check that each service is current. A failure after a commit reverts, pushes, waits for the sync, redeploys, and checks again (`rolled back`); a pull-only failure stays `failed`. Every outcome prints JSON with `status` and exits 0. Worst case about 125 minutes, so the gate's `timeoutMs` for arcane must exceed it (8,100,000). It holds the `arcane-gitops` lock, so a second checkout call waits up to 2 minutes and then fails as busy; the bridge also runs one Arcane update at a time.
|
|
62
62
|
- `arcane host <command>` runs over the forced-command key of user `spc-arcane` on VM 100. `HOST_COMMANDS` in `lib/arcane-host.ts` must list exactly the operations of `entry/remote/spc-arcane-op.py` (a test compares them); the tracked copies live in the arcane repository's `host/` (`spc-arcane-host-entry`, `spc-arcane-op`, `spc-arcane-sudoers`, `spc-arcane-install.sh`) with the probes in `host/spc-arcane-probes/`. The entry only splits words (letters, digits, space, `._:/+-`) and hands them to `sudo -n /usr/local/sbin/spc-arcane-op`, the one command its sudoers line allows; the op checks the name and every argument and runs the reviewed arcane-ops probes, which print only allowlisted fields. Manager steps (`manager-*`) are internal to `manager-upgrade`/`manager-rollback` and refused through `host`. Nothing on the host deletes data: a manager rollback restores the image and Compose file, not the data volume (R3 stays manual), `db-restore` only fills an empty database, and dumps are never pruned by the tool. `supabase-backup` writes the pre-change backup of `supabase/AGENTS.md` (both databases with `pg_dump -Fc`, each listed and fully decoded by `pg_restore`, `pg_dumpall --globals-only`, and the `supabase-db-config` volume, which must hold `pgsodium_root.key`) into `/var/backups/supabase-upgrades/<UTC>`, the folder of the earlier manual backups. `host-audit` compares the host's hashes with `host/<name>` of the `arcane-gitops` checkout after a pull. The `bigcapital-*` commands run the base-image upgrade of `bigcapital/README.md`: an online backup, a rehearsal on an internal Docker network (restore into the current MariaDB, upgrade with the build's, migrations, server health, Envoy validation; everything it created removed afterwards), the owner-approved switch (stop, quiescent dump, Redis `SAVE`, volume copies compared entry by entry, the stack left stopped for the GitOps commit, the old volumes untouched), and a verify against the switch's row counts and ledger totals.
|
|
63
63
|
- `arcane manager-upgrade <version>` (`lib/arcane-manager.ts`) follows arcane/AGENTS.md "Manager upgrades": the official image's revision label must equal the release tag's commit (GitHub API), `manager-prepare` records `compose.before.yaml` and every container before the pin commit, the pin goes through the `arcane-gitops` checkout (`git` is exported from `lib/repo.ts` for this), the wait polls `gitops list` and the deployed Compose hash every 30 s for up to 30 minutes, then pull, cold backup, `up -d --no-deps --pull never`, and checks. Any failure before the restart reverts the pin (`stopped`); a failed check after it reverts the pin and runs `manager-rollback` (`rolled back`). The answer always exits 0 with `status`, so the bridge and the agent both see it.
|
|
64
64
|
- `research` is a pure forced-command wrapper: every call is one JSON request on stdin to `request` on VM 110 (`10.1.100.10`, user `spc-research`, key `SSH_KEY_SANDBOX`), whose entry hands it to `spc-research-op` as `researcher` (tracked in the homelab repository's `06-space-x-proxmox/research/system/`). The op attaches one playwright-cli session per Discord thread (`t<id>`) to the shared Chrome over CDP on `127.0.0.1:9222` and opens its own tab; cookie, storage, `run-code`, and browser lifecycle commands are refused. `script` runs the agent's code in the thread folder as `researcher` (5 minutes), on a VM that reaches only the internet (Mira's Sandbox zone plus the VM's own nftables). Defaults stay on Cortex in `work/research/defaults.json`, out of the sandbox's reach. `captcha-help` is in `RISKY`, so its owner approval is the moment the owner solved the challenge in the live view (`research-browser.37pla.net`); the bridge renders that ticket with the channel's `liveView` link.
|
|
@@ -11,7 +11,7 @@ import { containerFacts, deploymentChecks, listRows, parseDeploymentArgs, syncFa
|
|
|
11
11
|
import { ServiceError, UsageError } from "../lib/errors.ts";
|
|
12
12
|
import { withLock } from "../lib/lock.ts";
|
|
13
13
|
import { main } from "../lib/main.ts";
|
|
14
|
-
import { buildUpdateReport, checkEntries, parseTag, rateLimited, resolveRef, stackServices, type CheckEntry, type UpdateRow } from "../lib/image-updates.ts";
|
|
14
|
+
import { buildUpdateReport, checkEntries, parseTag, rateLimited, resolveRef, stackServices, type CheckEntry, type ReportSource, type UpdateRow } from "../lib/image-updates.ts";
|
|
15
15
|
import { printJson, printRaw } from "../lib/output.ts";
|
|
16
16
|
import { auditLogPath, specialistsHome } from "../lib/paths.ts";
|
|
17
17
|
import { run, type RunResult } from "../lib/proc.ts";
|
|
@@ -102,9 +102,9 @@ ${Object.entries(HOST_COMMANDS)
|
|
|
102
102
|
.filter(([, c]) => !c.internal)
|
|
103
103
|
.map(([, c]) => ` ${c.help}`)
|
|
104
104
|
.join("\n")}
|
|
105
|
-
update-report [--all] fresh image update check for every stack, with running and latest release versions
|
|
105
|
+
update-report [--all] fresh image update check for every stack of every environment, with running and latest release versions
|
|
106
106
|
from the registries; JSON with a ready table (--all also lists up-to-date services) and
|
|
107
|
-
the held stacks of ${HOLDS_FILE}. Images the registry throttled (toomanyrequests) are
|
|
107
|
+
the held stacks and services of ${HOLDS_FILE}. Images the registry throttled (toomanyrequests) are
|
|
108
108
|
checked again after 5 and 20 seconds. Checks only: never pulls images or redeploys.
|
|
109
109
|
stack-update <stack> [--confirm]
|
|
110
110
|
apply every pending update of one stack: pinned tags and digests move in the stack's
|
|
@@ -112,7 +112,7 @@ ${Object.entries(HOST_COMMANDS)
|
|
|
112
112
|
with a pull, containers the redeploy left on an old image are recreated one by one, and
|
|
113
113
|
the stack is checked (containers running and healthy, images current). A failed check after a
|
|
114
114
|
Compose change reverts it and redeploys; a pull alone cannot be undone. Refuses held
|
|
115
|
-
stacks, stopped stacks, and arcane (manager-upgrade). An update across a major version
|
|
115
|
+
stacks, stopped stacks, and arcane (manager-upgrade); skips held services. An update across a major version
|
|
116
116
|
needs --confirm, which asks an owner for approval.
|
|
117
117
|
stack-rollback <stack> <commit>
|
|
118
118
|
undo one earlier stack-update commit of the stack's Compose file: revert, push, start the
|
|
@@ -123,7 +123,7 @@ ${Object.entries(HOST_COMMANDS)
|
|
|
123
123
|
docker-compose.yml, pre-deploy.sh) is ours: known when the new content, or the deletion,
|
|
124
124
|
is a commit of <project>/<file> in the GitOps repository, or when our stack-update,
|
|
125
125
|
manager-upgrade, manager-rollback, or host db-restore on that stack ran then (audit log)
|
|
126
|
-
container-watch every container's state, health, exit code, and uptime; for those started or created in
|
|
126
|
+
container-watch every container's state in every environment, health, exit code, and uptime; for those started or created in
|
|
127
127
|
the last 30 minutes also the start time, the Compose project, and our stack-update,
|
|
128
128
|
manager-upgrade, manager-rollback, or host db-restore whose run covers the start (JSON)
|
|
129
129
|
release-notes <stack>[/<service>]
|
|
@@ -187,12 +187,16 @@ function writeCallConfig(config: ArcaneConfig, token: string): string {
|
|
|
187
187
|
|
|
188
188
|
type Cli = (args: string[], timeoutMs?: number) => Promise<{ stdout: string; stderr: string; code: number }>;
|
|
189
189
|
|
|
190
|
+
/** The Arcane environment the calls of this invocation target; unset is the manager's own Docker host. Stack commands set it to their stack's environment. */
|
|
191
|
+
let targetEnv: string | undefined;
|
|
192
|
+
|
|
190
193
|
/** One federated token and one private config copy for every arcane-cli call of this invocation. */
|
|
191
|
-
async function withCli<T>(config: ArcaneConfig, fn: (cli: Cli) => Promise<T
|
|
194
|
+
async function withCli<T>(config: ArcaneConfig, fn: (cli: Cli) => Promise<T>, env: string | undefined = targetEnv): Promise<T> {
|
|
192
195
|
const token = await federatedToken(config);
|
|
193
196
|
const callConfig = writeCallConfig(config, token);
|
|
197
|
+
const at = env === undefined ? [] : ["--env", env];
|
|
194
198
|
try {
|
|
195
|
-
return await fn((args, timeoutMs = TIMEOUT_MS) => run(config.arcaneCli, ["--config", callConfig, "--no-color", ...args], { env: { ARCANE_TOKEN: token }, timeoutMs }));
|
|
199
|
+
return await fn((args, timeoutMs = TIMEOUT_MS) => run(config.arcaneCli, ["--config", callConfig, "--no-color", ...at, ...args], { env: { ARCANE_TOKEN: token }, timeoutMs }));
|
|
196
200
|
} finally {
|
|
197
201
|
rmSync(dirname(callConfig), { recursive: true, force: true });
|
|
198
202
|
}
|
|
@@ -228,30 +232,102 @@ async function cliJson(cli: Cli, args: string[]): Promise<unknown> {
|
|
|
228
232
|
}
|
|
229
233
|
}
|
|
230
234
|
|
|
235
|
+
/** The manager's own Docker host, which arcane-cli reaches without --env. */
|
|
236
|
+
const LOCAL_ENV = "0";
|
|
237
|
+
|
|
238
|
+
interface ArcaneEnv {
|
|
239
|
+
id: string;
|
|
240
|
+
name: string;
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/** Every enabled environment, the manager's own first. */
|
|
244
|
+
async function arcaneEnvs(cli: Cli): Promise<ArcaneEnv[]> {
|
|
245
|
+
const envs = listRows(await cliJson(cli, ["environments", "list"]))
|
|
246
|
+
.filter((e) => e.enabled !== false && typeof e.id === "string" && typeof e.name === "string")
|
|
247
|
+
.map((e) => ({ id: e.id as string, name: e.name as string }));
|
|
248
|
+
return envs.sort((a, b) => Number(b.id === LOCAL_ENV) - Number(a.id === LOCAL_ENV) || a.name.localeCompare(b.name));
|
|
249
|
+
}
|
|
250
|
+
|
|
251
|
+
/** arcane-cli arguments that point one call at an environment. */
|
|
252
|
+
function envArgs(env: ArcaneEnv): string[] {
|
|
253
|
+
return env.id === LOCAL_ENV ? [] : ["--env", env.id];
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/** The remote environment that runs this stack, or undefined for the manager's own host; stack names are unique across environments. */
|
|
257
|
+
async function stackEnv(config: ArcaneConfig, stack: string): Promise<string | undefined> {
|
|
258
|
+
return withCli(
|
|
259
|
+
config,
|
|
260
|
+
async (cli) => {
|
|
261
|
+
const found: ArcaneEnv[] = [];
|
|
262
|
+
for (const env of await arcaneEnvs(cli)) {
|
|
263
|
+
const projects = listRows(await cliJson(cli, [...envArgs(env), "projects", "list", "--all"]));
|
|
264
|
+
if (projects.some((p) => p.name === stack)) found.push(env);
|
|
265
|
+
}
|
|
266
|
+
if (found.length > 1) throw new ServiceError(`${stack} is a project in several environments: ${found.map((e) => e.name).join(", ")}`);
|
|
267
|
+
return found[0] === undefined || found[0].id === LOCAL_ENV ? undefined : found[0].id;
|
|
268
|
+
},
|
|
269
|
+
undefined,
|
|
270
|
+
);
|
|
271
|
+
}
|
|
272
|
+
|
|
231
273
|
/** Pauses before each new check of the images the registry throttled. */
|
|
232
274
|
const THROTTLE_RETRY_WAITS_MS = [5000, 20000];
|
|
233
275
|
|
|
234
276
|
/** A fresh registry check first, then the stack list, so every result the report uses is from this run. */
|
|
277
|
+
async function freshSource(cli: Cli, env: ArcaneEnv): Promise<ReportSource> {
|
|
278
|
+
const at = envArgs(env);
|
|
279
|
+
const check = checkEntries(await cliJson(cli, [...at, "images", "updates", "check", "--all"]));
|
|
280
|
+
for (const wait of THROTTLE_RETRY_WAITS_MS) {
|
|
281
|
+
const limited = rateLimited(check);
|
|
282
|
+
if (limited.length === 0) break;
|
|
283
|
+
await new Promise((r) => setTimeout(r, wait));
|
|
284
|
+
for (const ref of limited) {
|
|
285
|
+
// A retry that fails again keeps the throttled entry for the next round or the report.
|
|
286
|
+
const again = await cliJson(cli, [...at, "images", "updates", "check", ref]).catch(() => null);
|
|
287
|
+
if (again && typeof again === "object" && "hasUpdate" in again) check[ref] = again as CheckEntry;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
const projects = await cliJson(cli, [...at, "projects", "list", "--all"]);
|
|
291
|
+
return { ...(env.id === LOCAL_ENV ? {} : { environment: env.name }), check, projects };
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/** Every environment's stacks in one report; a remote environment that fails is named in environmentErrors instead of failing the report. */
|
|
235
295
|
async function updateReport(config: ArcaneConfig, args: string[]): Promise<number> {
|
|
236
296
|
const extra = args.filter((a) => a !== "--all");
|
|
237
297
|
if (extra.length > 0) throw new UsageError(`update-report takes only --all, got: ${extra.join(" ")}`);
|
|
238
298
|
const startedAt = Date.now();
|
|
239
299
|
const holds = await holdsFromOrigin(config);
|
|
240
300
|
return withCli(config, async (cli) => {
|
|
241
|
-
const
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
301
|
+
const sources: ReportSource[] = [];
|
|
302
|
+
const environmentErrors: { environment: string; error: string }[] = [];
|
|
303
|
+
// Each remote agent should run the manager's version; manager-upgrade moves only the manager.
|
|
304
|
+
const agentsBehind: { environment: string; agent: string; manager: string }[] = [];
|
|
305
|
+
const versionOf = async (env: ArcaneEnv): Promise<string> => {
|
|
306
|
+
const answer = await cliJson(cli, ["environments", "version", env.id]);
|
|
307
|
+
const current = typeof answer === "object" && answer !== null ? (answer as { currentVersion?: unknown }).currentVersion : undefined;
|
|
308
|
+
return typeof current === "string" ? current : "?";
|
|
309
|
+
};
|
|
310
|
+
const envs = await arcaneEnvs(cli);
|
|
311
|
+
const manager = await versionOf(envs[0]);
|
|
312
|
+
for (const env of envs) {
|
|
313
|
+
try {
|
|
314
|
+
sources.push(await freshSource(cli, env));
|
|
315
|
+
if (env.id === LOCAL_ENV) continue;
|
|
316
|
+
const agent = await versionOf(env);
|
|
317
|
+
if (agent !== manager) agentsBehind.push({ environment: env.name, agent, manager });
|
|
318
|
+
} catch (e) {
|
|
319
|
+
if (env.id === LOCAL_ENV) throw e;
|
|
320
|
+
environmentErrors.push({ environment: env.name, error: e instanceof Error ? e.message : String(e) });
|
|
250
321
|
}
|
|
251
322
|
}
|
|
252
|
-
const
|
|
253
|
-
|
|
254
|
-
|
|
323
|
+
const report = await buildUpdateReport({ sources, registry: new HttpRegistry(), startedAt, all: args.includes("--all") });
|
|
324
|
+
printJson({
|
|
325
|
+
...report,
|
|
326
|
+
holds: holds.holds,
|
|
327
|
+
...(holds.error ? { holdsError: holds.error } : {}),
|
|
328
|
+
...(environmentErrors.length > 0 ? { environmentErrors } : {}),
|
|
329
|
+
...(agentsBehind.length > 0 ? { agentsBehind } : {}),
|
|
330
|
+
});
|
|
255
331
|
return 0;
|
|
256
332
|
});
|
|
257
333
|
}
|
|
@@ -274,6 +350,7 @@ async function releaseNotes(config: ArcaneConfig, args: string[]): Promise<numbe
|
|
|
274
350
|
if (args.length !== 1 || !/^[A-Za-z0-9_.-]+(\/[A-Za-z0-9_.-]+)?$/.test(args[0])) throw new UsageError("release-notes <stack>[/<service>]");
|
|
275
351
|
const [stack, service] = args[0].split("/");
|
|
276
352
|
const startedAt = Date.now();
|
|
353
|
+
targetEnv = await stackEnv(config, stack);
|
|
277
354
|
return withCli(config, async (cli) => {
|
|
278
355
|
const all = stackServices(await cliJson(cli, ["projects", "list", "--all"]));
|
|
279
356
|
const matches = all.filter((s) => s.stack === stack && (service === undefined || s.service === service));
|
|
@@ -342,16 +419,26 @@ async function containerWatch(config: ArcaneConfig, args: string[]): Promise<num
|
|
|
342
419
|
audit = "";
|
|
343
420
|
}
|
|
344
421
|
const now = Date.now();
|
|
422
|
+
const environmentErrors: { environment: string; error: string }[] = [];
|
|
345
423
|
const containers = await withCli(config, async (cli) => {
|
|
346
|
-
const out: WatchedContainer[] = [];
|
|
347
|
-
for (const
|
|
348
|
-
const
|
|
349
|
-
|
|
350
|
-
|
|
424
|
+
const out: (WatchedContainer & { environment?: string })[] = [];
|
|
425
|
+
for (const env of await arcaneEnvs(cli)) {
|
|
426
|
+
const at = envArgs(env);
|
|
427
|
+
const where = env.id === LOCAL_ENV ? {} : { environment: env.name };
|
|
428
|
+
try {
|
|
429
|
+
for (const row of listRows(await cliJson(cli, [...at, "containers", "list", "--all"]))) {
|
|
430
|
+
const c = listContainer(row);
|
|
431
|
+
if (!c) continue;
|
|
432
|
+
out.push({ ...(isRecent(c, now) ? withDetail(c, await cliJson(cli, [...at, "containers", "get", c.id]), audit) : c), ...where });
|
|
433
|
+
}
|
|
434
|
+
} catch (e) {
|
|
435
|
+
if (env.id === LOCAL_ENV) throw e;
|
|
436
|
+
environmentErrors.push({ environment: env.name, error: e instanceof Error ? e.message : String(e) });
|
|
437
|
+
}
|
|
351
438
|
}
|
|
352
439
|
return out;
|
|
353
440
|
});
|
|
354
|
-
printJson({ checkedAt: new Date(now).toISOString(), containers });
|
|
441
|
+
printJson({ checkedAt: new Date(now).toISOString(), containers, ...(environmentErrors.length > 0 ? { environmentErrors } : {}) });
|
|
355
442
|
return 0;
|
|
356
443
|
}
|
|
357
444
|
|
|
@@ -407,13 +494,19 @@ async function hostCommand(config: ArcaneConfig, args: string[]): Promise<number
|
|
|
407
494
|
async function projectStatus(config: ArcaneConfig, args: string[]): Promise<number> {
|
|
408
495
|
if (args.length > 1 || (args[0] !== undefined && !NAME.test(args[0]))) throw new UsageError("project-status [name]");
|
|
409
496
|
return withCli(config, async (cli) => {
|
|
410
|
-
const projects
|
|
411
|
-
const syncs
|
|
497
|
+
const projects: Record<string, unknown>[] = [];
|
|
498
|
+
const syncs: Record<string, unknown>[] = [];
|
|
499
|
+
for (const env of await arcaneEnvs(cli)) {
|
|
500
|
+
const where = env.id === LOCAL_ENV ? {} : { environment: env.name };
|
|
501
|
+
projects.push(...listRows(await cliJson(cli, [...envArgs(env), "projects", "list", "--all"])).map((p) => ({ ...p, ...where })));
|
|
502
|
+
syncs.push(...listRows(await cliJson(cli, [...envArgs(env), "gitops", "list", "--all"])));
|
|
503
|
+
}
|
|
412
504
|
const picked = args[0] === undefined ? projects : projects.filter((p) => p.name === args[0]);
|
|
413
505
|
if (args[0] !== undefined && picked.length !== 1) throw new UsageError(`no single project named ${args[0]}`);
|
|
414
506
|
printJson({
|
|
415
507
|
projects: picked.map((p) => ({
|
|
416
508
|
name: p.name,
|
|
509
|
+
...(p.environment === undefined ? {} : { environment: p.environment }),
|
|
417
510
|
status: p.status,
|
|
418
511
|
running: p.runningCount,
|
|
419
512
|
services: p.serviceCount,
|
|
@@ -448,6 +541,7 @@ async function deploymentSnapshot(config: ArcaneConfig, expect: DeploymentExpect
|
|
|
448
541
|
/** One federated token per check, because a wait can outlive the token's five minutes. */
|
|
449
542
|
async function deployment(config: ArcaneConfig, args: string[]): Promise<number> {
|
|
450
543
|
const expect = parseDeploymentArgs(args);
|
|
544
|
+
targetEnv = await stackEnv(config, expect.project);
|
|
451
545
|
const deadline = Date.now() + expect.timeoutS * 1000;
|
|
452
546
|
for (;;) {
|
|
453
547
|
const snapshot = await deploymentSnapshot(config, expect);
|
|
@@ -750,10 +844,12 @@ async function stackUpdate(config: ArcaneConfig, args: string[]): Promise<number
|
|
|
750
844
|
const holdsPath = join(dir, HOLDS_FILE);
|
|
751
845
|
const holds = existsSync(holdsPath) ? parseHolds(readFileSync(holdsPath, "utf-8")) : {};
|
|
752
846
|
if (holds[stack]) throw new UsageError(`${stack} is held in ${HOLDS_FILE}: ${holds[stack]}`);
|
|
847
|
+
targetEnv = await stackEnv(config, stack);
|
|
753
848
|
|
|
754
849
|
const before = await withCli(config, (cli) => stackState(cli, stack, Date.now()));
|
|
755
850
|
if (before.stopped) throw new UsageError(`${stack} is stopped; a redeploy would start it, so start it on purpose first`);
|
|
756
|
-
|
|
851
|
+
// A held service keeps its version; the rest of the stack still updates.
|
|
852
|
+
const pending = pendingRows(before.rows).filter((r) => !holds[`${stack}/${r.service}`]);
|
|
757
853
|
const answer = (status: StackStatus, extra: Record<string, unknown>) => printJson({ status, stack, ...extra });
|
|
758
854
|
if (pending.length === 0) {
|
|
759
855
|
answer("nothing to do", { unknown: before.rows.filter((r) => r.status === "unknown").map((r) => ({ service: r.service, note: r.note })) });
|
|
@@ -851,6 +947,7 @@ async function stackRollback(config: ArcaneConfig, args: string[]): Promise<numb
|
|
|
851
947
|
const repo = await repoStatus(spec, dir);
|
|
852
948
|
if (repo.changed.length > 0) throw new ServiceError(`the arcane checkout has uncommitted changes: ${repo.changed.join(", ")}`);
|
|
853
949
|
if (repo.ahead > 0) throw new ServiceError(`the arcane checkout is ${repo.ahead} commit(s) ahead of origin`);
|
|
950
|
+
targetEnv = await stackEnv(config, stack);
|
|
854
951
|
const layout = await withCli(config, (cli) => stackLayout(cli, stack));
|
|
855
952
|
if (layout.composePath === null) throw new ServiceError(`${stack} has no single GitOps sync with a Compose path`);
|
|
856
953
|
const shown = (await git(spec, dir, ["show", "--name-only", "--format=%s", commit])).split("\n").filter(Boolean);
|
|
@@ -134,7 +134,7 @@ describe("buildUpdateReport", () => {
|
|
|
134
134
|
"busybox:latest": entry("B", "B"),
|
|
135
135
|
"ghcr.io/astrofoundry/app:main": entry("A1", "A2"),
|
|
136
136
|
};
|
|
137
|
-
const report = await buildUpdateReport({ check, projects, registry, startedAt: NOW, all: false });
|
|
137
|
+
const report = await buildUpdateReport({ sources: [{ check, projects }], registry, startedAt: NOW, all: false });
|
|
138
138
|
expect(report.counts).toEqual({ update: 3, release: 0, rebuild: 0, unknown: 1, "up-to-date": 2 });
|
|
139
139
|
expect(report.rows.map((r) => `${r.stack}/${r.service}:${r.status}`)).toEqual(["media/prowlarr:update", "ours/api:update", "ours/worker:update", "empty/empty:unknown"]);
|
|
140
140
|
expect(report.table).toMatch(/^Fresh check 2026-09-25T18:30:00Z: 6 services in 6 stacks, 4 images\. 1 pinned private images \(1 services\) excluded\./);
|
|
@@ -142,13 +142,23 @@ describe("buildUpdateReport", () => {
|
|
|
142
142
|
expect(report.rows.some((r) => r.stack === "pinned")).toBe(false);
|
|
143
143
|
expect(report.table).toContain("media/prowlarr prowlarr 2.0.5.5160-ls123 2.1.0.5190-ls124 pull");
|
|
144
144
|
expect(report.table).toContain("* empty: Arcane lists no images for this stack");
|
|
145
|
-
const all = await buildUpdateReport({ check, projects, registry, startedAt: NOW, all: true });
|
|
145
|
+
const all = await buildUpdateReport({ sources: [{ check, projects }], registry, startedAt: NOW, all: true });
|
|
146
146
|
expect(all.rows).toHaveLength(6);
|
|
147
147
|
expect(all.table).toContain("* canary: stack is stopped");
|
|
148
148
|
expect(all.rows.find((r) => r.stack === "canary")?.stopped).toBe(true);
|
|
149
149
|
expect(all.rows.find((r) => r.stack === "media")?.stopped).toBeUndefined();
|
|
150
150
|
});
|
|
151
151
|
|
|
152
|
+
it("resolves the same image on each environment's own check and names the remote environment", async () => {
|
|
153
|
+
const registry = new FakeRegistry({ "linuxserver/sonarr": { tags: ["4.0.17.2952-ls300"], digests: { "4.0.17.2952-ls300": "S2" } } });
|
|
154
|
+
const ref = "lscr.io/linuxserver/sonarr:latest";
|
|
155
|
+
const local = { check: { [ref]: entry("S2", "S2") }, projects: { data: [{ name: "media", status: "running", updateInfo: { serviceUpdates: { sonarr: { imageRef: ref } } } }] } };
|
|
156
|
+
const mira = { environment: "mira", check: { [ref]: entry("S1", "S2") }, projects: { data: [{ name: "media-mira", status: "running", updateInfo: { serviceUpdates: { sonarr: { imageRef: ref } } } }] } };
|
|
157
|
+
const report = await buildUpdateReport({ sources: [local, mira], registry, startedAt: NOW, all: true });
|
|
158
|
+
expect(report.rows.map((r) => `${r.stack}:${r.status}:${r.environment ?? "local"}`)).toEqual(["media-mira:update:mira", "media:up-to-date:local"]);
|
|
159
|
+
expect(report.images).toBe(1);
|
|
160
|
+
});
|
|
161
|
+
|
|
152
162
|
it("fetches each repository's tags once", async () => {
|
|
153
163
|
let lists = 0;
|
|
154
164
|
const inner: Registry = {
|
|
@@ -30,6 +30,8 @@ export interface UpdateRow {
|
|
|
30
30
|
latestDigest?: string;
|
|
31
31
|
/** The stack is stopped: `stack-update` refuses it, because a redeploy would start it. */
|
|
32
32
|
stopped?: true;
|
|
33
|
+
/** The Arcane remote environment that runs the stack; absent for the manager's own Docker host. */
|
|
34
|
+
environment?: string;
|
|
33
35
|
}
|
|
34
36
|
|
|
35
37
|
export interface UpdateReport {
|
|
@@ -357,22 +359,35 @@ export function cachingRegistry(inner: Registry): Registry {
|
|
|
357
359
|
* Builds the update report: every service image of every stack, against a check
|
|
358
360
|
* that ran moments before, with release versions read from the registries.
|
|
359
361
|
*/
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
362
|
+
/** One Arcane environment's fresh registry check and stack list; `environment` names a remote one. */
|
|
363
|
+
export interface ReportSource {
|
|
364
|
+
environment?: string;
|
|
365
|
+
check: unknown;
|
|
366
|
+
projects: unknown;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
export async function buildUpdateReport(input: { sources: ReportSource[]; registry: Registry; startedAt: number; all: boolean }): Promise<UpdateReport> {
|
|
370
|
+
// The same image reference runs a different build in each environment, so each one resolves on its own check.
|
|
371
|
+
const entries = input.sources.map((src) => checkEntries(src.check));
|
|
372
|
+
const services = input.sources.flatMap((src, i) => stackServices(src.projects).map((s) => ({ ...s, source: i })));
|
|
373
|
+
const keyOf = (source: number, ref: string) => `${source}\n${ref}`;
|
|
374
|
+
const keys = [...new Set(services.filter((s) => s.ref !== null).map((s) => keyOf(s.source, s.ref as string)))];
|
|
364
375
|
const resolved = new Map<string, Resolution & { kind: RefKind }>();
|
|
365
376
|
const registry = cachingRegistry(input.registry);
|
|
366
|
-
await mapLimit(
|
|
367
|
-
|
|
377
|
+
await mapLimit(keys, CONCURRENCY, async (key) => {
|
|
378
|
+
const [source, ref] = [Number(key.slice(0, key.indexOf("\n"))), key.slice(key.indexOf("\n") + 1)];
|
|
379
|
+
resolved.set(key, await resolveRef(ref, entries[source][ref], registry, input.startedAt));
|
|
368
380
|
});
|
|
369
|
-
const
|
|
381
|
+
const refs = [...new Set(services.map((s) => s.ref).filter((r): r is string => r !== null))];
|
|
382
|
+
const privateImages = refs.filter((ref) => services.some((s) => s.ref === ref && resolved.get(keyOf(s.source, ref))?.private === true));
|
|
370
383
|
const listed = services.filter((s) => s.ref === null || !privateImages.includes(s.ref));
|
|
371
384
|
const rows: UpdateRow[] = listed.map((s) => {
|
|
372
|
-
|
|
373
|
-
const
|
|
385
|
+
const environment = input.sources[s.source].environment;
|
|
386
|
+
const where = environment === undefined ? {} : { environment };
|
|
387
|
+
if (s.ref === null) return { stack: s.stack, service: s.service, ref: "", image: "?", kind: "floating", running: "?", latest: "?", status: "unknown", action: "", note: "Arcane lists no images for this stack", ...where };
|
|
388
|
+
const r = resolved.get(keyOf(s.source, s.ref)) as Resolution & { kind: RefKind };
|
|
374
389
|
const note = [r.note, s.stopped ? "stack is stopped" : ""].filter(Boolean).join("; ");
|
|
375
|
-
return { stack: s.stack, service: s.service, ref: s.ref, image: shortImage(parseImageRef(s.ref)), kind: r.kind, running: r.running, latest: r.latest, status: r.status, action: r.action, note, ...(r.latestDigest ? { latestDigest: r.latestDigest } : {}), ...(s.stopped ? { stopped: true as const } : {}) };
|
|
390
|
+
return { stack: s.stack, service: s.service, ref: s.ref, image: shortImage(parseImageRef(s.ref)), kind: r.kind, running: r.running, latest: r.latest, status: r.status, action: r.action, note, ...(r.latestDigest ? { latestDigest: r.latestDigest } : {}), ...(s.stopped ? { stopped: true as const } : {}), ...where };
|
|
376
391
|
});
|
|
377
392
|
rows.sort((a, b) => STATUS_ORDER[a.status] - STATUS_ORDER[b.status] || a.stack.localeCompare(b.stack) || a.service.localeCompare(b.service));
|
|
378
393
|
const counts: Record<UpdateStatus, number> = { update: 0, release: 0, rebuild: 0, unknown: 0, "up-to-date": 0 };
|
|
@@ -13,13 +13,16 @@ describe("update holds", () => {
|
|
|
13
13
|
it("reads stack names with a reason", () => {
|
|
14
14
|
expect(parseHolds('{"supabase": " coordinated release: review first "}')).toEqual({ supabase: "coordinated release: review first" });
|
|
15
15
|
expect(parseHolds("{}")).toEqual({});
|
|
16
|
+
expect(parseHolds('{"paperless/gotenberg": "Paperless needs Gotenberg 8"}')).toEqual({ "paperless/gotenberg": "Paperless needs Gotenberg 8" });
|
|
16
17
|
});
|
|
17
18
|
|
|
18
19
|
it("refuses a malformed file instead of holding nothing", () => {
|
|
19
20
|
expect(() => parseHolds("not json")).toThrow(/not JSON/);
|
|
20
|
-
expect(() => parseHolds('["supabase"]')).toThrow(/object of stack
|
|
21
|
+
expect(() => parseHolds('["supabase"]')).toThrow(/object of stack/);
|
|
21
22
|
expect(() => parseHolds('{"supabase": ""}')).toThrow(/non-empty reason/);
|
|
22
|
-
expect(() => parseHolds('{"../x": "why"}')).toThrow(/stack name/);
|
|
23
|
+
expect(() => parseHolds('{"../x": "why"}')).toThrow(/stack or stack\/service name/);
|
|
24
|
+
expect(() => parseHolds('{"a/b/c": "why"}')).toThrow(/stack or stack\/service name/);
|
|
25
|
+
expect(() => parseHolds('{"a/../b": "why"}')).toThrow(/stack or stack\/service name/);
|
|
23
26
|
});
|
|
24
27
|
});
|
|
25
28
|
|
|
@@ -2,11 +2,11 @@ import { UsageError } from "./errors.ts";
|
|
|
2
2
|
import { parseTag, type UpdateRow } from "./image-updates.ts";
|
|
3
3
|
import { parseImageRef } from "./registry.ts";
|
|
4
4
|
|
|
5
|
-
/** Stacks whose updates wait for a person, tracked in the root of the GitOps repository: `{ "<stack>": "<reason>" }`. */
|
|
5
|
+
/** Stacks and services whose updates wait for a person, tracked in the root of the GitOps repository: `{ "<stack>": "<reason>", "<stack>/<service>": "<reason>" }`. */
|
|
6
6
|
export const HOLDS_FILE = "update-holds.json";
|
|
7
|
-
const
|
|
7
|
+
const HOLD_KEY = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}(\/[A-Za-z0-9][A-Za-z0-9_.-]{0,127})?$/;
|
|
8
8
|
|
|
9
|
-
/** The held stacks with their reasons; a malformed file is an error, never "nothing held". */
|
|
9
|
+
/** The held stacks and services with their reasons; a malformed file is an error, never "nothing held". */
|
|
10
10
|
export function parseHolds(text: string): Record<string, string> {
|
|
11
11
|
let value: unknown;
|
|
12
12
|
try {
|
|
@@ -14,10 +14,10 @@ export function parseHolds(text: string): Record<string, string> {
|
|
|
14
14
|
} catch {
|
|
15
15
|
throw new UsageError(`${HOLDS_FILE} is not JSON`);
|
|
16
16
|
}
|
|
17
|
-
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new UsageError(`${HOLDS_FILE} must be an object of stack name to reason`);
|
|
17
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new UsageError(`${HOLDS_FILE} must be an object of stack (or stack/service) name to reason`);
|
|
18
18
|
const holds: Record<string, string> = {};
|
|
19
19
|
for (const [stack, reason] of Object.entries(value)) {
|
|
20
|
-
if (!
|
|
20
|
+
if (!HOLD_KEY.test(stack) || typeof reason !== "string" || reason.trim().length === 0) throw new UsageError(`${HOLDS_FILE}: "${stack.slice(0, 40)}" needs a stack or stack/service name and a non-empty reason`);
|
|
21
21
|
holds[stack] = reason.trim().slice(0, 200);
|
|
22
22
|
}
|
|
23
23
|
return holds;
|