@astrofoundry/pi-astro 0.35.0 → 0.36.0

package/specialists/arcane/run.ts

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { dirname, join } from "node:path";
 import { buildHostRemote, compareAudit, HOST_COMMANDS, type AuditFile } from "../lib/arcane-host.ts";
@@ -7,14 +7,16 @@ import { failedChecks, MANAGER_COMPOSE, MANAGER_IMAGE, normalizeVersion, olderTh
 import { readConfig } from "../lib/config.ts";
 import { containerFacts, deploymentChecks, listRows, parseDeploymentArgs, syncFacts, type DeploymentExpectation } from "../lib/deployment.ts";
 import { ServiceError, UsageError } from "../lib/errors.ts";
+import { withLock } from "../lib/lock.ts";
 import { main } from "../lib/main.ts";
-import { buildUpdateReport, parseTag, resolveRef, stackServices, type CheckEntry } from "../lib/image-updates.ts";
+import { buildUpdateReport, parseTag, resolveRef, stackServices, type CheckEntry, type UpdateRow } from "../lib/image-updates.ts";
 import { printJson, printRaw } from "../lib/output.ts";
 import { specialistsHome } from "../lib/paths.ts";
 import { run, type RunResult } from "../lib/proc.ts";
 import { HttpRegistry, parseImageRef } from "../lib/registry.ts";
 import { releaseNotesFor } from "../lib/releases.ts";
-import { ensureCheckout, git, REPO_HELP, repoCommand, repoStatus, type RepoSpec } from "../lib/repo.ts";
+import { ensureCheckout, git, REPO_HELP, repoCommand, repoStatus, safeRepoPath, type RepoSpec } from "../lib/repo.ts";
+import { commitMessage, containersProblem, crossesMajor, HOLDS_FILE, nextImageRef, parseHolds, pendingRows, replaceImageRef, type ServiceResult, type StackContainer, type StackStatus } from "../lib/stack-update.ts";
 import { ARCANE_DENIED_KEYS, filterJsonText, stripKeys } from "../lib/sanitize.ts";
 import { readSecret, secretPath } from "../lib/secrets.ts";
 import { sshFixed } from "../lib/ssh.ts";
@@ -99,8 +101,15 @@ ${Object.entries(HOST_COMMANDS)
     .map(([, c]) => `      ${c.help}`)
     .join("\n")}
   update-report [--all]       fresh image update check for every stack, with running and latest release versions
-                              from the registries; JSON with a ready table (--all also lists up-to-date services).
-                              Checks only: never pulls images or redeploys.
+                              from the registries; JSON with a ready table (--all also lists up-to-date services) and
+                              the held stacks of ${HOLDS_FILE}. Checks only: never pulls images or redeploys.
+  stack-update <stack> [--confirm]
+                              apply every pending update of one stack: pinned tags and digests move in the stack's
+                              Compose file (commit, push, wait for the sync), then the stack is redeployed with a pull
+                              and checked (containers running and healthy, images current). A failed check after a
+                              Compose change reverts it and redeploys; a pull alone cannot be undone. Refuses held
+                              stacks, stopped stacks, and arcane (manager-upgrade). An update across a major version
+                              needs --confirm, which asks an owner for approval.
   release-notes <stack>[/<service>]
                               GitHub release notes between the running and the latest version of one service's image,
                               found through the image's org.opencontainers.image.source label
@@ -160,14 +169,14 @@ function writeCallConfig(config: ArcaneConfig, token: string): string {
     return file;
 }
 
-type Cli = (args: string[]) => Promise<{ stdout: string; stderr: string; code: number }>;
+type Cli = (args: string[], timeoutMs?: number) => Promise<{ stdout: string; stderr: string; code: number }>;
 
 /** One federated token and one private config copy for every arcane-cli call of this invocation. */
 async function withCli<T>(config: ArcaneConfig, fn: (cli: Cli) => Promise<T>): Promise<T> {
     const token = await federatedToken(config);
     const callConfig = writeCallConfig(config, token);
     try {
-        return await fn((args) => run(config.arcaneCli, ["--config", callConfig, "--no-color", ...args], { env: { ARCANE_TOKEN: token }, timeoutMs: TIMEOUT_MS }));
+        return await fn((args, timeoutMs = TIMEOUT_MS) => run(config.arcaneCli, ["--config", callConfig, "--no-color", ...args], { env: { ARCANE_TOKEN: token }, timeoutMs }));
     } finally {
         rmSync(dirname(callConfig), { recursive: true, force: true });
     }
@@ -202,14 +211,30 @@ async function updateReport(config: ArcaneConfig, args: string[]): Promise<numbe
     const extra = args.filter((a) => a !== "--all");
     if (extra.length > 0) throw new UsageError(`update-report takes only --all, got: ${extra.join(" ")}`);
     const startedAt = Date.now();
+    const holds = await holdsFromOrigin(config);
     return withCli(config, async (cli) => {
         const check = await cliJson(cli, ["images", "updates", "check", "--all"]);
         const projects = await cliJson(cli, ["projects", "list", "--all"]);
-        printJson(await buildUpdateReport({ check, projects, registry: new HttpRegistry(), startedAt, all: args.includes("--all") }));
+        const report = await buildUpdateReport({ check, projects, registry: new HttpRegistry(), startedAt, all: args.includes("--all") });
+        printJson({ ...report, holds: holds.holds, ...(holds.error ? { holdsError: holds.error } : {}) });
         return 0;
     });
 }
 
+/** The held stacks as pushed to the GitOps repository, read without touching the checkout's working tree. */
+async function holdsFromOrigin(config: ArcaneConfig): Promise<{ holds: Record<string, string>; error?: string }> {
+    try {
+        const spec = repoSpec(config);
+        const dir = await ensureCheckout(spec);
+        await git(spec, dir, ["fetch", "--quiet", "origin", config.gitopsBranch]);
+        const listed = (await git(spec, dir, ["ls-tree", "--name-only", `origin/${config.gitopsBranch}`, HOLDS_FILE])).trim();
+        if (listed !== HOLDS_FILE) return { holds: {} };
+        return { holds: parseHolds(await git(spec, dir, ["show", `origin/${config.gitopsBranch}:${HOLDS_FILE}`])) };
+    } catch (err) {
+        return { holds: {}, error: `${HOLDS_FILE} unreadable: ${err instanceof Error ? err.message.slice(0, 200) : String(err)}` };
+    }
+}
+
 async function releaseNotes(config: ArcaneConfig, args: string[]): Promise<number> {
     if (args.length !== 1 || !/^[A-Za-z0-9_.-]+(\/[A-Za-z0-9_.-]+)?$/.test(args[0])) throw new UsageError("release-notes <stack>[/<service>]");
     const [stack, service] = args[0].split("/");
@@ -370,6 +395,16 @@ const SYNC_WAIT_MS = 30 * MINUTE;
  * stopped (nothing was restarted). The exit code is 0 in all three so the
  * caller always gets the answer; usage and precondition errors still fail.
  */
+/**
+ * Everything that writes the arcane-gitops checkout takes turns: an update holds it
+ * for up to about two hours (sync and redeploy waits), so the lock outlives that.
+ */
+const CHECKOUT_STALE_MS = 3 * 60 * 60_000;
+
+function checkoutLocked<T>(fn: () => Promise<T>): Promise<T> {
+    return withLock("arcane-gitops", fn, { staleMs: CHECKOUT_STALE_MS });
+}
+
 async function managerUpgrade(config: ArcaneConfig, args: string[]): Promise<number> {
     if (args.length !== 1) throw new UsageError("manager-upgrade <version>");
     const version = normalizeVersion(args[0]);
@@ -480,20 +515,198 @@ async function managerRollback(args: string[]): Promise<number> {
     return 0;
 }
 
+// ---------------------------------------------------------------- stack updates
+
+/** How long a stack's sync may take to report success at a new commit; syncs run every few minutes. */
+const STACK_SYNC_WAIT_MS = 20 * MINUTE;
+/** arcane-cli allows a project redeploy up to 30 minutes for large pulls. */
+const REDEPLOY_TIMEOUT_MS = 35 * MINUTE;
+/** Containers with a health check get this long to leave "starting". */
+const HEALTH_WAIT_MS = 5 * MINUTE;
+
+interface StackState {
+    rows: UpdateRow[];
+    services: number;
+    stopped: boolean;
+    composePath: string | null;
+}
+
+/** The stack's services with a fresh check of each of its images, resolved the way update-report resolves them. */
+async function stackState(cli: Cli, stack: string, startedAt: number): Promise<StackState> {
+    const projects = await cliJson(cli, ["projects", "list", "--all"]);
+    const services = stackServices(projects).filter((s) => s.stack === stack);
+    if (services.length === 0) throw new UsageError(`no stack named ${stack}`);
+    const project = listRows(projects).find((p) => p.name === stack);
+    const sync = listRows(await cliJson(cli, ["gitops", "list", "--all"])).filter((s) => project !== undefined && s.projectId === project.id);
+    const composePath = sync.length === 1 && typeof sync[0].composePath === "string" ? sync[0].composePath : null;
+    const registry = new HttpRegistry();
+    const resolved = new Map<string, Awaited<ReturnType<typeof resolveRef>>>();
+    for (const ref of [...new Set(services.map((s) => s.ref).filter((r): r is string => r !== null))]) {
+        const entry = parseImageRef(ref).digest ? undefined : ((await cliJson(cli, ["images", "updates", "check", ref])) as CheckEntry);
+        resolved.set(ref, await resolveRef(ref, entry, registry, startedAt));
+    }
+    const rows: UpdateRow[] = services.map((s) => {
+        const r = s.ref === null ? undefined : resolved.get(s.ref);
+        if (s.ref === null || !r) return { stack, service: s.service, ref: s.ref ?? "", image: "?", kind: "floating", running: "?", latest: "?", status: "unknown", action: "", note: "Arcane lists no image for this service" };
+        return { stack, service: s.service, ref: s.ref, image: s.ref, kind: r.kind, running: r.running, latest: r.latest, status: r.status, action: r.action, note: r.note, ...(r.latestDigest ? { latestDigest: r.latestDigest } : {}) };
+    });
+    return { rows, services: services.length, stopped: services.some((s) => s.stopped), composePath };
+}
+
+/** The stack's containers as `containers get` reports them, found by the stack's images and kept when Compose says they belong to it. */
+async function stackContainers(cli: Cli, stack: string, refs: readonly string[]): Promise<StackContainer[]> {
+    const listed = listRows(await cliJson(cli, ["containers", "list", "--all"])).filter((c) => typeof c.image === "string" && refs.includes(c.image));
+    const out: StackContainer[] = [];
+    for (const c of listed) {
+        if (typeof c.id !== "string") continue;
+        const name = Array.isArray(c.names) && typeof c.names[0] === "string" ? c.names[0].replace(/^\//, "") : c.id.slice(0, 12);
+        const facts = containerFacts(await cliJson(cli, ["containers", "get", c.id]), name);
+        if (facts.project === stack) out.push({ name, running: facts.running, state: facts.state, exitCode: facts.exitCode, health: facts.health });
+    }
+    return out;
+}
+
+/** Waits until the stack's sync reports success at `commit`; false when it does not within the limit. */
+async function waitForSync(config: ArcaneConfig, stack: string, commit: string): Promise<{ ok: boolean; last: string }> {
+    const deadline = Date.now() + STACK_SYNC_WAIT_MS;
+    for (;;) {
+        const sync = await withCli(config, async (cli) => projectSync(await cliJson(cli, ["projects", "list", "--all"]), await cliJson(cli, ["gitops", "list", "--all"]), stack));
+        const sameCommit = sync.commit !== null && (commit.startsWith(sync.commit) || sync.commit.startsWith(commit));
+        if (sameCommit && sync.status === "success") return { ok: true, last: `${sync.status} at ${sync.commit}` };
+        if (Date.now() > deadline) return { ok: false, last: `${sync.status ?? "none"} at ${sync.commit ?? "none"}` };
+        await new Promise((r) => setTimeout(r, 30_000));
+    }
+}
+
+async function redeploy(config: ArcaneConfig, stack: string): Promise<string | null> {
+    const result = await withCli(config, (cli) => cli(["projects", "redeploy", stack, "--yes"], REDEPLOY_TIMEOUT_MS));
+    return result.code === 0 ? null : (result.stderr.trim().split("\n").pop() ?? "").slice(0, 300) || `projects redeploy exited ${result.code}`;
+}
+
+/** The containers' problem after a redeploy, polled until none remains or the health wait ends. */
+async function settle(config: ArcaneConfig, stack: string, services: number): Promise<string | null> {
+    const deadline = Date.now() + HEALTH_WAIT_MS;
+    for (;;) {
+        const problem = await withCli(config, async (cli) => {
+            const refs = stackServices(await cliJson(cli, ["projects", "list", "--all"]))
+                .filter((s) => s.stack === stack)
+                .map((s) => s.ref)
+                .filter((r): r is string => r !== null);
+            return containersProblem(await stackContainers(cli, stack, refs), services);
+        });
+        if (problem === null || Date.now() > deadline) return problem;
+        await new Promise((r) => setTimeout(r, 15_000));
+    }
+}
+
+/**
+ * One stack's pending updates, end to end and without judgment: pinned
+ * references move in the Compose file through the GitOps repository, then the
+ * stack is redeployed with a pull and checked. Prints one JSON answer whose
+ * `status` is updated, rolled back, failed, or nothing to do; the exit code is
+ * 0 for all of them so the caller always gets the answer.
+ */
+async function stackUpdate(config: ArcaneConfig, args: string[]): Promise<number> {
+    const confirmed = args.includes("--confirm");
+    const rest = args.filter((a) => a !== "--confirm");
+    if (rest.length !== 1 || !NAME.test(rest[0])) throw new UsageError("stack-update <stack> [--confirm]");
+    const stack = rest[0];
+    if (stack === "arcane") throw new UsageError("arcane is the manager itself: use manager-upgrade (the Upgrade Arcane button)");
+    const spec = repoSpec(config);
+    const dir = await ensureCheckout(spec);
+    await git(spec, dir, ["pull", "--ff-only", "origin", config.gitopsBranch]);
+    const repo = await repoStatus(spec, dir);
+    if (repo.changed.length > 0) throw new ServiceError(`the arcane checkout has uncommitted changes: ${repo.changed.join(", ")}`);
+    if (repo.ahead > 0) throw new ServiceError(`the arcane checkout is ${repo.ahead} commit(s) ahead of origin`);
+    const holdsPath = join(dir, HOLDS_FILE);
+    const holds = existsSync(holdsPath) ? parseHolds(readFileSync(holdsPath, "utf-8")) : {};
+    if (holds[stack]) throw new UsageError(`${stack} is held in ${HOLDS_FILE}: ${holds[stack]}`);
+
+    const before = await withCli(config, (cli) => stackState(cli, stack, Date.now()));
+    if (before.stopped) throw new UsageError(`${stack} is stopped; a redeploy would start it, so start it on purpose first`);
+    const pending = pendingRows(before.rows);
+    const answer = (status: StackStatus, extra: Record<string, unknown>) => printJson({ status, stack, ...extra });
+    if (pending.length === 0) {
+        answer("nothing to do", { unknown: before.rows.filter((r) => r.status === "unknown").map((r) => ({ service: r.service, note: r.note })) });
+        return 0;
+    }
+    const majors = pending.filter((r) => crossesMajor(r.running, r.latest));
+    if (majors.length > 0 && !confirmed) {
+        throw new UsageError(`${majors.map((r) => `${r.service} ${r.running} -> ${r.latest}`).join(", ")} crosses a major version: summarise release-notes ${stack}/<service> for the user, then repeat with --confirm after their go`);
+    }
+    const results: ServiceResult[] = pending.map((r) => ({ service: r.service, ref: r.ref, from: r.running, to: r.latest, applied: nextImageRef(r) ?? "pull", result: "unchecked" }));
+    const edits = results.filter((r) => r.applied !== "pull");
+
+    let commit: string | null = null;
+    if (edits.length > 0) {
+        if (before.composePath === null) throw new ServiceError(`${stack} has no single GitOps sync with a Compose path`);
+        const file = safeRepoPath(dir, before.composePath);
+        let text = readFileSync(file, "utf-8");
+        const moves = new Map(edits.map((e) => [e.ref, e.applied]));
+        for (const [from, to] of moves) text = replaceImageRef(text, from, to);
+        writeFileSync(file, text);
+        await git(spec, dir, ["add", before.composePath]);
+        await git(spec, dir, ["commit", "--quiet", "-m", commitMessage(stack, edits.map((e) => ({ service: e.service, from: e.ref, to: e.applied })))]);
+        commit = (await git(spec, dir, ["rev-parse", "HEAD"])).trim();
+        await git(spec, dir, ["push", "origin", config.gitopsBranch]);
+        const synced = await waitForSync(config, stack, commit);
+        if (!synced.ok) {
+            await git(spec, dir, ["revert", "--no-edit", commit]);
+            await git(spec, dir, ["push", "origin", config.gitopsBranch]);
+            answer("failed", { reason: `the sync did not report success at ${commit.slice(0, 12)} within ${STACK_SYNC_WAIT_MS / MINUTE} minutes (last: ${synced.last}); the commit was reverted and nothing was redeployed`, commit, services: results });
+            return 0;
+        }
+    }
+
+    let problem = await redeploy(config, stack);
+    if (problem === null) problem = await settle(config, stack, before.services);
+    if (problem === null) {
+        const after = await withCli(config, (cli) => stackState(cli, stack, Date.now()));
+        for (const r of results) {
+            const now = after.rows.find((x) => x.service === r.service);
+            // Current when nothing newer is pending, or when the target runs and a release appeared meanwhile.
+            r.result = now !== undefined && (now.status === "up-to-date" || (now.status !== "unknown" && now.running === r.to)) ? "current" : "behind";
+        }
+        const behind = results.filter((r) => r.result === "behind").map((r) => r.service);
+        if (behind.length > 0) problem = `still behind after the redeploy: ${behind.join(", ")}`;
+    }
+    if (problem === null) {
+        answer("updated", { services: results, ...(commit ? { commit } : {}) });
+        return 0;
+    }
+    if (commit === null) {
+        answer("failed", { reason: problem, services: results, note: "the stack was redeployed with pulled images; a pull cannot be rolled back, so check the stack with the arcane specialist" });
+        return 0;
+    }
+    await git(spec, dir, ["revert", "--no-edit", commit]);
+    await git(spec, dir, ["push", "origin", config.gitopsBranch]);
+    const revert = (await git(spec, dir, ["rev-parse", "HEAD"])).trim();
+    const synced = await waitForSync(config, stack, revert);
+    const again = synced.ok ? await redeploy(config, stack) : `the revert's sync did not report success (last: ${synced.last})`;
+    const settled = again === null ? await settle(config, stack, before.services) : again;
+    if (settled === null) {
+        answer("rolled back", { reason: problem, services: results, commit, revert, note: "the previous Compose file runs again; images pulled for floating tags stay pulled" });
+        return 0;
+    }
+    answer("failed", { reason: `${problem}; the rollback did not finish either: ${settled}`, services: results, commit, revert });
+    return 0;
+}
+
 export async function command(args: string[]): Promise<number> {
     if (args.length === 0 || args[0] === "--help" || args[0] === "help") {
         printRaw(HELP);
         return 0;
     }
     const config = readConfig<ArcaneConfig>(SERVICE, SHAPE);
-    if (args[0] === "git") return repoCommand(repoSpec(config), args.slice(1));
+    if (args[0] === "git") return checkoutLocked(() => repoCommand(repoSpec(config), args.slice(1)));
     if (args[0] === "update-report") return updateReport(config, args.slice(1));
     if (args[0] === "release-notes") return releaseNotes(config, args.slice(1));
     if (args[0] === "host") return hostCommand(config, args.slice(1));
     if (args[0] === "project-status") return projectStatus(config, args.slice(1));
     if (args[0] === "deployment") return deployment(config, args.slice(1));
-    if (args[0] === "manager-upgrade") return managerUpgrade(config, args.slice(1));
+    if (args[0] === "manager-upgrade") return checkoutLocked(() => managerUpgrade(config, args.slice(1)));
     if (args[0] === "manager-rollback") return managerRollback(args.slice(1));
+    if (args[0] === "stack-update") return checkoutLocked(() => stackUpdate(config, args.slice(1)));
     return passthrough(config, args);
 }
 

package/specialists/lib/stack-update.test.ts

@@ -0,0 +1,101 @@
+import { describe, expect, it } from "vitest";
+import type { UpdateRow } from "./image-updates.ts";
+import { commitMessage, containersProblem, crossesMajor, nextImageRef, parseHolds, pendingRows, replaceImageRef, type StackContainer } from "./stack-update.ts";
+
+const A = `sha256:${"a".repeat(64)}`;
+const B = `sha256:${"b".repeat(64)}`;
+
+function row(over: Partial<UpdateRow>): UpdateRow {
+    return { stack: "s", service: "svc", ref: "nginx:1.31.4-alpine", image: "nginx:1.31.4-alpine", kind: "fixed", running: "1.31.4-alpine", latest: "1.31.6-alpine", status: "release", action: "edit tag", note: "", ...over };
+}
+
+describe("update holds", () => {
+    it("reads stack names with a reason", () => {
+        expect(parseHolds('{"supabase": " coordinated release: review first "}')).toEqual({ supabase: "coordinated release: review first" });
+        expect(parseHolds("{}")).toEqual({});
+    });
+
+    it("refuses a malformed file instead of holding nothing", () => {
+        expect(() => parseHolds("not json")).toThrow(/not JSON/);
+        expect(() => parseHolds('["supabase"]')).toThrow(/object of stack name/);
+        expect(() => parseHolds('{"supabase": ""}')).toThrow(/non-empty reason/);
+        expect(() => parseHolds('{"../x": "why"}')).toThrow(/stack name/);
+    });
+});
+
+describe("pending updates", () => {
+    it("keeps the rows an update changes and spots a major version", () => {
+        const rows = [row({ status: "update" }), row({ status: "release" }), row({ status: "rebuild" }), row({ status: "up-to-date" }), row({ status: "unknown" })];
+        expect(pendingRows(rows).map((r) => r.status)).toEqual(["update", "release", "rebuild"]);
+        expect(crossesMajor("2.14.3", "3.0.0")).toBe(true);
+        expect(crossesMajor("v2.16.1", "v2.17.0")).toBe(false);
+        expect(crossesMajor("older build", "3.0.0")).toBe(false);
+    });
+
+    it("moves a pinned tag or digest and leaves pulls to the redeploy", () => {
+        expect(nextImageRef(row({}))).toBe("nginx:1.31.6-alpine");
+        expect(nextImageRef(row({ ref: "registry.local:5000/app:1.2", latest: "1.3" }))).toBe("registry.local:5000/app:1.3");
+        expect(nextImageRef(row({ ref: `ghcr.io/x/y@${A}`, action: "edit digest", latestDigest: B }))).toBe(`ghcr.io/x/y@${B}`);
+        expect(nextImageRef(row({ ref: `ghcr.io/x/y:v1@${A}`, action: "edit digest", latestDigest: B }))).toBe(`ghcr.io/x/y:v1@${B}`);
+        expect(nextImageRef(row({ action: "pull" }))).toBeNull();
+        expect(() => nextImageRef(row({ latest: "new build" }))).toThrow(/cannot move/);
+        expect(() => nextImageRef(row({ ref: `ghcr.io/x/y@${A}`, action: "edit digest", latestDigest: "sha256:short" }))).toThrow(/new digest/);
+    });
+});
+
+describe("Compose rewrite", () => {
+    const compose = [
+        "services:",
+        "  db:",
+        "    image: postgres:18",
+        "  quoted:",
+        '    image: "postgres:18" # pinned on purpose',
+        "  single:",
+        "    image: 'postgres:18'",
+        "  longer:",
+        "    image: postgres:18.1",
+        "  other:",
+        "    image: mypostgres:18",
+        "  mixed:",
+        "    image: \"postgres:18'",
+        "  env:",
+        "    environment:",
+        "      IMAGE: postgres:18",
+        "      image_note: image: postgres:18",
+        "",
+    ].join("\n");
+
+    it("changes only image lines that name exactly the reference", () => {
+        const next = replaceImageRef(compose, "postgres:18", "postgres:19").split("\n");
+        expect(next[2]).toBe("    image: postgres:19");
+        expect(next[4]).toBe('    image: "postgres:19" # pinned on purpose');
+        expect(next[6]).toBe("    image: 'postgres:19'");
+        expect(next.slice(7)).toEqual(compose.split("\n").slice(7));
+    });
+
+    it("treats the reference as text, not a pattern, and refuses a file without it", () => {
+        expect(replaceImageRef("    image: a.b:1+x\n    image: aXb:1+x\n", "a.b:1+x", "a.b:2")).toBe("    image: a.b:2\n    image: aXb:1+x\n");
+        expect(() => replaceImageRef(compose, "postgres:17", "postgres:18")).toThrow(/no "image: postgres:17" line/);
+    });
+});
+
+describe("stack health after a redeploy", () => {
+    const up = (name: string, over: Partial<StackContainer> = {}): StackContainer => ({ name, running: true, state: "running", exitCode: null, health: "healthy", ...over });
+
+    it("passes running containers and finished one-shot jobs", () => {
+        expect(containersProblem([up("db"), up("app", { health: "none" }), up("init", { running: false, state: "exited", exitCode: 0 })], 3)).toBeNull();
+    });
+
+    it("names missing, stopped, unhealthy, and starting containers", () => {
+        expect(containersProblem([up("db")], 2)).toBe("1 of 2 services have a container");
+        expect(containersProblem([up("db"), up("init", { running: false, state: "exited", exitCode: 1 })], 2)).toBe("not running: init");
+        expect(containersProblem([up("db", { health: "unhealthy" }), up("app", { health: "starting" })], 2)).toBe("health: db (unhealthy), app (starting)");
+    });
+});
+
+describe("commit message", () => {
+    it("names the stack and every moved reference on one short line", () => {
+        expect(commitMessage("media", [{ service: "sonarr", from: "a:1", to: "a:2" }, { service: "radarr", from: "b:1", to: "b:2" }])).toBe("media: sonarr a:1 -> a:2, radarr b:1 -> b:2");
+        expect(commitMessage("s", Array.from({ length: 20 }, () => ({ service: "x", from: "y:1", to: "y:2" }))).length).toBe(200);
+    });
+});

package/specialists/lib/stack-update.ts

@@ -0,0 +1,115 @@
+import { UsageError } from "./errors.ts";
+import { parseTag, type UpdateRow } from "./image-updates.ts";
+import { parseImageRef } from "./registry.ts";
+
+/** Stacks whose updates wait for a person, tracked in the root of the GitOps repository: `{ "<stack>": "<reason>" }`. */
+export const HOLDS_FILE = "update-holds.json";
+const STACK = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/;
+
+/** The held stacks with their reasons; a malformed file is an error, never "nothing held". */
+export function parseHolds(text: string): Record<string, string> {
+    let value: unknown;
+    try {
+        value = JSON.parse(text);
+    } catch {
+        throw new UsageError(`${HOLDS_FILE} is not JSON`);
+    }
+    if (typeof value !== "object" || value === null || Array.isArray(value)) throw new UsageError(`${HOLDS_FILE} must be an object of stack name to reason`);
+    const holds: Record<string, string> = {};
+    for (const [stack, reason] of Object.entries(value)) {
+        if (!STACK.test(stack) || typeof reason !== "string" || reason.trim().length === 0) throw new UsageError(`${HOLDS_FILE}: "${stack.slice(0, 40)}" needs a stack name and a non-empty reason`);
+        holds[stack] = reason.trim().slice(0, 200);
+    }
+    return holds;
+}
+
+/** True when the update moves to a newer major version, read from the first number of both release tags. */
+export function crossesMajor(running: string, latest: string): boolean {
+    const a = parseTag(running);
+    const b = parseTag(latest);
+    return a !== null && b !== null && b.nums[0] > a.nums[0];
+}
+
+/** The rows of one stack that an update changes. */
+export function pendingRows(rows: readonly UpdateRow[]): UpdateRow[] {
+    return rows.filter((r) => r.status === "update" || r.status === "release" || r.status === "rebuild");
+}
+
+/**
+ * The image reference a pinned row moves to: the newer tag for `edit tag`, the
+ * newer digest for `edit digest`; null for rows a redeploy with a pull applies.
+ */
+export function nextImageRef(row: UpdateRow): string | null {
+    if (row.action === "edit tag") {
+        const ref = parseImageRef(row.ref);
+        if (ref.digest || !ref.tag || !parseTag(row.latest)) throw new UsageError(`${row.service}: cannot move ${row.ref} to tag ${row.latest}`);
+        if (!row.ref.endsWith(`:${ref.tag}`)) throw new UsageError(`${row.service}: ${row.ref} does not end with its tag`);
+        return `${row.ref.slice(0, row.ref.length - ref.tag.length)}${row.latest}`;
+    }
+    if (row.action === "edit digest") {
+        const at = row.ref.indexOf("@");
+        if (at < 0 || !row.latestDigest || !/^sha256:[0-9a-f]{64}$/.test(row.latestDigest)) throw new UsageError(`${row.service}: cannot move ${row.ref} to a new digest`);
+        return `${row.ref.slice(0, at)}@${row.latestDigest}`;
+    }
+    return null;
+}
+
+function escapeRegExp(text: string): string {
+    return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+/**
+ * The Compose file with every `image:` line that names exactly `oldRef`
+ * (quoted or not, with an optional trailing comment) changed to `newRef`.
+ * Any other mention of the reference, or a longer reference that starts with
+ * it, stays as it is. Refuses a file without such a line.
+ */
+export function replaceImageRef(compose: string, oldRef: string, newRef: string): string {
+    const line = new RegExp(`^(\\s*image:\\s*)(["']?)${escapeRegExp(oldRef)}\\2(\\s*(?:#.*)?)$`, "gm");
+    let count = 0;
+    const next = compose.replace(line, (_all, head: string, quote: string, tail: string) => {
+        count++;
+        return `${head}${quote}${newRef}${quote}${tail}`;
+    });
+    if (count === 0) throw new UsageError(`the Compose file has no "image: ${oldRef}" line`);
+    return next;
+}
+
+export type StackStatus = "updated" | "rolled back" | "failed" | "nothing to do";
+
+export interface ServiceResult {
+    service: string;
+    ref: string;
+    from: string;
+    to: string;
+    /** `pull` (redeploy with a pull) or the reference written into the Compose file. */
+    applied: string;
+    /** After the update: current, still behind, or not checked. */
+    result: "current" | "behind" | "unchecked";
+}
+
+/** One container of the stack after a redeploy, as `containers get` reports it. */
+export interface StackContainer {
+    name: string;
+    running: boolean | null;
+    state: string | null;
+    exitCode: number | null;
+    health: string;
+}
+
+/**
+ * Why the stack is not healthy yet, or null when every container runs (or, as
+ * a one-shot job, exited with code 0) and none is unhealthy or still starting.
+ */
+export function containersProblem(containers: readonly StackContainer[], expected: number): string | null {
+    if (containers.length < expected) return `${containers.length} of ${expected} services have a container`;
+    const down = containers.filter((c) => c.running !== true && !(c.state === "exited" && c.exitCode === 0)).map((c) => c.name);
+    if (down.length > 0) return `not running: ${down.join(", ")}`;
+    const sick = containers.filter((c) => c.health === "unhealthy" || c.health === "starting").map((c) => `${c.name} (${c.health})`);
+    return sick.length > 0 ? `health: ${sick.join(", ")}` : null;
+}
+
+/** A commit subject that names the stack and every reference it changes. */
+export function commitMessage(stack: string, edits: readonly { service: string; from: string; to: string }[]): string {
+    return `${stack}: ${edits.map((e) => `${e.service} ${e.from} -> ${e.to}`).join(", ")}`.slice(0, 200);
+}

package/specialists/wrappers.test.ts

@@ -44,6 +44,7 @@ describe("arcane wrapper", () => {
         expect(await arcaneCommand(["--help"])).toBe(0);
         expect(String(write.mock.calls[0][0])).toContain("git write");
         expect(String(write.mock.calls[0][0])).toContain("git show");
+        expect(String(write.mock.calls[0][0])).toContain("stack-update <stack> [--confirm]");
         expect(String(write.mock.calls[0][0])).not.toMatch(/^\s+gitops /m);
         write.mockRestore();
     });