@catalyst-cloud/cli 0.16.24 → 0.16.25

package/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.16.25
+
+A self-hosted machine that runs several teams' work from one `compose.json` can now take each approved release by itself. Run `catalyst runner update --fleet --dir <the folder that holds compose.json>`, by hand or on a schedule. It switches only the supervisor to the approved images and leaves work that is already running alone. If the machine's teams are not ready again within four minutes, it puts the previous images back. `--check` shows what would change without changing anything.
+
 ## 0.16.24
 
 One self-hosted machine can now work for several Linear teams. On a machine already set up for one team, run `catalyst setup --runner` with another team selected, and setup adds that team instead of stopping. Each team's own setting still decides whether its work may run there, and the machine asks each of its teams for work in turn. A new machine starts on Catalyst's current approved release, which can serve several teams from its first setup; a machine you already set up gets that release on its next 15-minute update check.

package/dist/args.js

@@ -25,6 +25,7 @@ export const FLAG_TABLES = {
         check: { value: false, help: "read the approved release and say what would change, without changing it" },
         auto: { value: false, help: "one-line output for the scheduled run" },
         dir: { value: true, help: "the runner directory (default: the one setup created)" },
+        fleet: { value: false, help: "update a multi-team supervisor fleet's compose.json in --dir" },
     },
     login: {
         "start-replica": {
@@ -544,7 +545,7 @@ export const FLAG_TABLES = {
 };
 export const VERB_USAGE = {
     hosts: "hosts [list] | hosts remove <name-or-id> [--yes] | hosts rename <name-or-id> <new-name> [--json]",
-    runner: "runner update [--check] [--dir <runner directory>] | runner auto-update [status|on|off]",
+    runner: "runner update [--check] [--dir <runner directory>] [--fleet] | runner auto-update [status|on|off]",
     setup: "setup [onboard options]   (sets up Catalyst step by step; same as catalyst onboard) | setup --engine <file> --engine-sha256 <hex> [-- <install options>]   (run by install.sh; install.sh --help lists the options)",
     login: "login [--base-url <url>] [--start-replica]   (keyless; or --key <personal-key> / CATALYST_CLOUD_TOKEN)",
     onboard: "onboard [--team <ID|key>] [--repo <owner/name>]... [--coding-account <slot>] [--resume-from <step>] [--only <step>] [--local-sync] [--runner|--no-runner] [--no-color] [--yes] [--dry-run] [--json] | onboard --headless [--key-file <path>] [--team ...] [--repo ...]... [--coding-account ...] [--runner yes|no] [--json]",

package/dist/runner-command.js

@@ -3,7 +3,8 @@ import { flagBool, flagString, positionals } from "./args.js";
 import { UsageError } from "./errors.js";
 import { autoUpdateStatus, disableAutoUpdate, enableAutoUpdate, scheduledPath, } from "./runner-auto-update.js";
 import { dockerRunnerUpdateEngine, readReleaseState, runnerDir, runRunnerUpdate, updateResultLines, } from "./runner-update.js";
-const USAGE = "catalyst runner update [--check] [--dir <runner directory>] | runner auto-update [status|on|off]";
+import { dockerFleetUpdateEngine, runFleetUpdate } from "./runner-update-fleet.js";
+const USAGE = "catalyst runner update [--check] [--dir <runner directory>] [--fleet] | runner auto-update [status|on|off]";
 function schedulerDeps(ctx, deps) {
     const cli = deps.scheduler?.cli ?? resolve(process.argv[1] ?? "catalyst");
     return {
@@ -47,8 +48,12 @@ export async function cmdRunner(args, ctx, deps = {}) {
     if (action === "update" && parts.length === 1) {
         const check = flagBool(args, "check");
         const auto = flagBool(args, "auto");
-        const engine = deps.engine ?? dockerRunnerUpdateEngine({ env: ctx.env });
-        const result = await runRunnerUpdate(engine, { dir: dir(), check });
+        const fleet = flagBool(args, "fleet");
+        if (fleet && dirFlag === undefined)
+            throw new UsageError("--fleet needs --dir, the directory that holds compose.json");
+        const result = fleet
+            ? await runFleetUpdate(deps.fleetEngine ?? dockerFleetUpdateEngine({ env: ctx.env }), { dir: dir(), check })
+            : await runRunnerUpdate(deps.engine ?? dockerRunnerUpdateEngine({ env: ctx.env }), { dir: dir(), check });
         if (args.json)
             ctx.stdout(JSON.stringify(result));
         else if (auto)

package/dist/runner-update-fleet.js

@@ -0,0 +1,442 @@
+import { existsSync, readFileSync, unlinkSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { dockerRunnerEngine, prepareRunnerImages, spawnDocker } from "./onboard-runner.js";
+import { decideRelease, DESIRED_RELEASE_SCRIPT, HOST_ID, isRecord, readReleaseState, releaseLock, takeLock, writePrivateAtomic, writeReleaseState, } from "./runner-update.js";
+// `catalyst runner update --fleet <dir>`: the same approved release, applied to a multi-team
+// supervisor fleet (CTC-6530). One supervisor container serves several scopes from a Compose file
+// written as JSON (`compose.json`), under one operator enrollment, and every host credential on the
+// machine reads the same release. The fleet differs from a single-team runner in three ways:
+//
+//   - the images live in `compose.json`, not `.env`: `services.supervisor.image` and its
+//     `environment.CATALYST_RUNNER_IMAGE`. Only those two fields change;
+//   - phase containers are never waited out. The supervisor is recreated alone and re-adopts the
+//     running ones, which keep the image they were granted. So there is no idle gate;
+//   - the deadline watchdog is a separate Compose project and is left alone. The release's watchdog
+//     image is not applied here.
+//
+// The release is read from a one-off container of the current supervisor image that mounts only the
+// credential, read-only. Readiness is per scope: every scope that logged `supervisor_scope_qualified`
+// in the minutes before the update must log it again from the new container, which must stay up with
+// no restart and log no `host_not_ready` for those scopes. Otherwise the previous `compose.json` comes
+// back and the supervisor is recreated on it. That scope set is saved before the recreate and removed
+// once it qualifies or rolls back, so a run killed in between is finished by the next one.
+export const FLEET_COMPOSE_FILE = "compose.json";
+export const PREVIOUS_COMPOSE_FILE = "compose.release-previous.json";
+const CANDIDATE_COMPOSE_FILE = "compose.release-candidate.json";
+/** The scopes the supervisor served before a recreate, kept until that recreate is qualified or rolled
+ *  back. A run that was killed in between finds it and qualifies against it, never against the new
+ *  container's own log, which would read an unqualified supervisor as serving. */
+export const IN_FLIGHT_FILE = "fleet-update-in-flight.json";
+/** How far back a qualification counts as current before the update. The supervisor repeats it
+ *  on every loop while a scope is ready, so a ready scope logs it many times in this window. */
+const QUALIFIED_LOOKBACK_MS = 5 * 60_000;
+const QUALIFY_MIN_MS = 90_000;
+const QUALIFY_DEADLINE_MS = 240_000;
+const QUALIFY_POLL_MS = 10_000;
+const CHILD_NAME_PREFIX = "catalyst-pa_";
+/** The two image fields in a fleet `compose.json`, or null when the file is not a fleet layout. */
+export function fleetImages(compose) {
+    const supervisor = isRecord(compose) && isRecord(compose.services) ? compose.services.supervisor : null;
+    if (!isRecord(supervisor) || typeof supervisor.image !== "string" || !isRecord(supervisor.environment))
+        return null;
+    const runner = supervisor.environment.CATALYST_RUNNER_IMAGE;
+    return typeof runner === "string" ? { supervisor: supervisor.image, runner } : null;
+}
+/** `compose` with only the two image fields replaced. */
+export function withFleetImages(compose, images) {
+    const next = structuredClone(compose);
+    next.services.supervisor.image = images.supervisor;
+    next.services.supervisor.environment.CATALYST_RUNNER_IMAGE = images.runner;
+    return next;
+}
+const logEvents = (text) => text.split("\n").flatMap((line) => {
+    const start = line.indexOf("{");
+    if (start === -1)
+        return [];
+    try {
+        const parsed = JSON.parse(line.slice(start));
+        return isRecord(parsed) ? [parsed] : [];
+    }
+    catch {
+        return [];
+    }
+});
+const scopeKey = (event) => typeof event.tenant === "string" && typeof event.team === "string" ? `${event.tenant}/${event.team}` : null;
+/** The scopes (`tenant/team`) that logged `supervisor_scope_qualified` in this log text. */
+export function qualifiedScopes(text) {
+    const scopes = new Set();
+    for (const event of logEvents(text)) {
+        const key = event.event === "supervisor_scope_qualified" ? scopeKey(event) : null;
+        if (key)
+            scopes.add(key);
+    }
+    return scopes;
+}
+/** A `host_not_ready` line for one of `scopes`, or one that names no scope. Other scopes may be
+ *  unready for their own reasons (a team with no repository yet) and do not stop an update. */
+export function notReadyScope(text, scopes) {
+    for (const event of logEvents(text)) {
+        if (event.event !== "host_not_ready")
+            continue;
+        const key = scopeKey(event);
+        if (key === null || scopes.has(key))
+            return key ?? "unscoped";
+    }
+    return null;
+}
+function readInFlight(dir) {
+    try {
+        const parsed = JSON.parse(readFileSync(join(dir, IN_FLIGHT_FILE), "utf8"));
+        if (!isRecord(parsed) || parsed.version !== 1 || !Array.isArray(parsed.scopes))
+            return null;
+        const scopes = parsed.scopes.filter((scope) => typeof scope === "string");
+        return scopes.length > 0 ? new Set(scopes) : null;
+    }
+    catch {
+        return null;
+    }
+}
+function clearInFlight(dir) {
+    try {
+        unlinkSync(join(dir, IN_FLIGHT_FILE));
+    }
+    catch {
+        // Already gone.
+    }
+}
+export async function runFleetUpdate(engine, options) {
+    const { dir } = options;
+    const now = options.now ?? Date.now;
+    if (!existsSync(join(dir, FLEET_COMPOSE_FILE)))
+        return { outcome: "failed", detail: "fleet_not_installed", generation: null, exitCode: 1 };
+    const lock = options.check ? null : takeLock(dir, now());
+    if (!options.check && lock === null)
+        return { outcome: "retained", detail: "update_running", generation: readReleaseState(dir).generation, exitCode: 0 };
+    try {
+        const result = await update(engine, options, now);
+        if (!options.check) {
+            const state = readReleaseState(dir);
+            writeReleaseState(dir, { ...state, last: { atMs: now(), outcome: result.outcome, detail: result.detail, generation: result.generation } });
+        }
+        return result;
+    }
+    finally {
+        if (lock !== null)
+            releaseLock(dir, lock);
+    }
+}
+async function update(engine, options, now) {
+    const { dir, signal } = options;
+    const composePath = join(dir, FLEET_COMPOSE_FILE);
+    const state = readReleaseState(dir);
+    const done = (outcome, detail, extra = {}) => ({ outcome, detail, generation: state.generation, exitCode: 0, ...extra });
+    const composeText = readFileSync(composePath, "utf8");
+    let compose;
+    try {
+        compose = JSON.parse(composeText);
+    }
+    catch {
+        compose = null;
+    }
+    const current = fleetImages(compose);
+    const project = isRecord(compose) && typeof compose.name === "string" ? compose.name : "";
+    if (!isRecord(compose) || current === null || !/^[a-z0-9][a-z0-9_-]*$/.test(project))
+        return done("failed", "fleet_compose_unrecognized", { exitCode: 1 });
+    const read = await engine.desiredRelease(dir, compose, signal);
+    const decision = decideRelease(state.generation, read);
+    if (decision.action === "retain")
+        return decision.reason === "not_newer"
+            ? done("current", "not_newer")
+            : done("retained", decision.reason, { exitCode: decision.reason === "credential_refused" || decision.reason === "no_credential" ? 1 : 0 });
+    const { release } = decision;
+    const target = { runner: release.images.runner.reference, supervisor: release.images.supervisor.reference };
+    const changed = ["runner", "supervisor"].filter((c) => current[c] !== target[c]);
+    const tagged = { releaseId: release.releaseId };
+    const live = await engine.supervisor(dir, project, signal);
+    if (live === undefined)
+        return done("retained", "docker_unavailable", tagged);
+    const runs = (images) => !!live?.running && live.image === images.supervisor && live.runnerImage === images.runner;
+    const inFlight = readInFlight(dir);
+    if (inFlight === null && decision.action === "verify" && changed.length === 0 && runs(target))
+        return done("current", "not_newer");
+    if (options.check)
+        return done("planned", changed.length > 0 ? `would_update:${changed.join(",")}` : "already_configured", { ...tagged, generation: release.generation });
+    // The scopes the running supervisor qualifies are what the new one has to qualify again, unless an
+    // earlier run stopped mid-recreate: then the scopes it saved are the baseline.
+    let scopes = inFlight;
+    if (scopes === null) {
+        if (!live?.running)
+            return done("retained", "supervisor_not_running", { ...tagged, exitCode: 1 });
+        const recent = await engine.logs(live.id, new Date(now() - QUALIFIED_LOOKBACK_MS).toISOString(), signal);
+        if (recent === null)
+            return done("retained", "docker_unavailable", tagged);
+        scopes = qualifiedScopes(recent);
+        if (scopes.size === 0)
+            return done("retained", "no_qualified_scope", { ...tagged, exitCode: 1 });
+    }
+    const record = (outcome, detail, previous, exitCode = 0) => {
+        writeReleaseState(dir, {
+            ...state,
+            generation: release.generation,
+            releaseId: release.releaseId,
+            appliedAtMs: now(),
+            // The watchdog is not this updater's to move, so it records none.
+            previous: previous ? { ...previous, watchdog: "" } : state.previous,
+        });
+        return { outcome, detail, generation: release.generation, exitCode, ...tagged };
+    };
+    if (inFlight === null && changed.length === 0 && runs(target))
+        return record("recorded", "already_running_release", null);
+    // What a failure goes back to: today's file, or for a half-applied file the one saved before it.
+    let rollbackText = composeText;
+    if (changed.length === 0) {
+        try {
+            rollbackText = readFileSync(join(dir, PREVIOUS_COMPOSE_FILE), "utf8");
+        }
+        catch {
+            rollbackText = null;
+        }
+    }
+    let rollbackImages = null;
+    try {
+        rollbackImages = rollbackText === null ? null : fleetImages(JSON.parse(rollbackText));
+    }
+    catch {
+        rollbackImages = null;
+    }
+    const unprepared = await engine.prepare([target.runner, target.supervisor], signal);
+    if (unprepared !== null)
+        return done("retained", unprepared, tagged);
+    const candidateText = `${JSON.stringify(withFleetImages(compose, target), null, 2)}\n`;
+    const candidatePath = join(dir, CANDIDATE_COMPOSE_FILE);
+    writePrivateAtomic(candidatePath, candidateText);
+    const valid = await engine.configValid(dir, project, candidatePath, signal);
+    try {
+        unlinkSync(candidatePath);
+    }
+    catch {
+        // Already gone.
+    }
+    if (!valid)
+        return done("retained", "compose_config_invalid", { ...tagged, exitCode: 1 });
+    const childrenBefore = await engine.children(signal);
+    if (childrenBefore === null)
+        return done("retained", "docker_unavailable", tagged);
+    // From here the machine changes. The previous file is the rollback.
+    if (changed.length > 0)
+        writePrivateAtomic(join(dir, PREVIOUS_COMPOSE_FILE), composeText);
+    writePrivateAtomic(join(dir, IN_FLIGHT_FILE), `${JSON.stringify({ version: 1, scopes: [...scopes].sort(), atMs: now() })}\n`);
+    writePrivateAtomic(composePath, candidateText);
+    if ((await engine.recreate(dir, project, signal)) && (await qualify(engine, dir, project, target, scopes, options, now))) {
+        clearInFlight(dir);
+        if (changed.length > 0)
+            await engine.removeImages(changed.map((c) => current[c]), signal);
+        // The recreate must leave every running phase container as it was: it is re-adopted, not restarted.
+        const childrenAfter = await engine.children(signal);
+        const disturbed = childrenAfter === null
+            ? -1
+            : Object.entries(childrenBefore).filter(([id, before]) => {
+                const after = childrenAfter[id];
+                return before.running && after?.running === true && after.fingerprint !== before.fingerprint;
+            }).length;
+        const detail = changed.length > 0 ? `updated:${changed.join(",")}` : "reconciled";
+        const previous = changed.length > 0 ? current : null;
+        return disturbed === 0
+            ? record("updated", detail, previous)
+            : record("updated", `${detail};${disturbed < 0 ? "children_unreadable" : `children_changed:${disturbed}`}`, previous, 1);
+    }
+    if (rollbackText === null || rollbackImages === null)
+        return done("failed", "new_supervisor_not_qualified_no_previous", { ...tagged, exitCode: 1 });
+    writePrivateAtomic(composePath, rollbackText);
+    const restored = (await engine.recreate(dir, project, signal)) &&
+        (await qualify(engine, dir, project, rollbackImages, scopes, options, now));
+    // A rollback that did not qualify keeps the baseline for the next run.
+    if (restored)
+        clearInFlight(dir);
+    return done(restored ? "rolled_back" : "failed", restored ? "new_supervisor_not_qualified" : "rollback_not_qualified", { ...tagged, exitCode: 1 });
+}
+/** The recreated supervisor runs `images`, stays up with no restart, logs no `host_not_ready` for a
+ *  scope it must serve, and qualifies every one of `scopes`, held for the minimum window. */
+async function qualify(engine, dir, project, images, scopes, options, now) {
+    const sleep = options.sleep ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+    const start = now();
+    const deadline = start + (options.qualifyDeadlineMs ?? QUALIFY_DEADLINE_MS);
+    const minimum = start + (options.qualifyMinMs ?? QUALIFY_MIN_MS);
+    let id = null;
+    for (;;) {
+        if (options.signal?.aborted)
+            return false;
+        const live = await engine.supervisor(dir, project, options.signal);
+        if (live) {
+            id ??= live.id;
+            if (live.id !== id || !live.running || live.restartCount !== 0)
+                return false;
+            if (live.image !== images.supervisor || live.runnerImage !== images.runner)
+                return false;
+            const text = await engine.logs(live.id, live.startedAt, options.signal);
+            if (text !== null) {
+                if (notReadyScope(text, scopes) !== null)
+                    return false;
+                const seen = qualifiedScopes(text);
+                if (now() >= minimum && [...scopes].every((scope) => seen.has(scope)))
+                    return true;
+            }
+        }
+        if (now() >= deadline)
+            return false;
+        await sleep(QUALIFY_POLL_MS);
+    }
+}
+const CONTAINER_ID = /^[0-9a-f]{12,64}$/;
+/** The one volume or bind in `service.volumes` that holds `file`, as a read-only `--mount` value. */
+export function credentialMount(compose, service, file) {
+    const project = typeof compose.name === "string" ? compose.name : "";
+    const topLevel = isRecord(compose.volumes) ? compose.volumes : {};
+    let best = null;
+    for (const entry of Array.isArray(service.volumes) ? service.volumes : []) {
+        let source;
+        let target;
+        let type;
+        if (typeof entry === "string") {
+            [source, target] = entry.split(":");
+            type = typeof source === "string" && source.startsWith("/") ? "bind" : "volume";
+        }
+        else if (isRecord(entry)) {
+            ({ source, target, type } = entry);
+        }
+        if (typeof source !== "string" || typeof target !== "string" || source === "")
+            continue;
+        if (file !== target && !file.startsWith(`${target.replace(/\/$/, "")}/`))
+            continue;
+        // The deepest mount holding the file is the one the container reads it from.
+        if (best !== null && best.target.length >= target.length)
+            continue;
+        const declared = topLevel[source];
+        const name = isRecord(declared) && typeof declared.name === "string" ? declared.name
+            : isRecord(declared) && declared.external === true ? source : `${project}_${source}`;
+        const mount = type === "bind" ? `type=bind,src=${source},dst=${target},readonly`
+            : type === "volume" && /^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(name) ? `type=volume,src=${name},dst=${target},readonly` : null;
+        best = { target, mount };
+    }
+    return best?.mount ?? null;
+}
+/** The real engine: the docker CLI, and the setup engine's own pull. */
+export function dockerFleetUpdateEngine(deps = {}) {
+    const exec = deps.exec ?? spawnDocker;
+    const env = Object.fromEntries(Object.entries(deps.env ?? process.env)
+        .filter(([key]) => !key.startsWith("CATALYST_") && !key.startsWith("DOCKER_SOCKET_")));
+    const setup = deps.setup ?? dockerRunnerEngine({ exec, env: deps.env });
+    const docker = async (args, options = {}) => {
+        try {
+            return await exec(args, { env, ...options });
+        }
+        catch {
+            return { code: 1, stdout: "", stderr: "" };
+        }
+    };
+    const compose = (dir, project, file = join(dir, FLEET_COMPOSE_FILE)) => ["compose", "--project-name", project, "--file", file];
+    return {
+        // Not `compose run`: that would mount the Docker socket and every scope's state. This container
+        // gets the credential alone, read-only, and the two variables the script reads.
+        async desiredRelease(_dir, file, signal) {
+            const service = isRecord(file.services) && isRecord(file.services.supervisor) ? file.services.supervisor : null;
+            const environment = service && isRecord(service.environment) ? service.environment : null;
+            const credential = environment?.CATALYST_HOST_CREDENTIAL_FILE;
+            const mirror = environment?.CATALYST_MIRROR_URL;
+            if (!service || typeof service.image !== "string" || typeof credential !== "string" || typeof mirror !== "string")
+                return { kind: "no_credential" };
+            const mount = credentialMount(file, service, credential);
+            if (!mount)
+                return { kind: "no_credential" };
+            const user = typeof service.user === "string" && /^[0-9]+(:[0-9]+)?$/.test(service.user) ? ["--user", service.user] : [];
+            const read = await docker(["run", "--rm", "--network", "bridge", "--read-only", "--tmpfs", "/tmp", "--cap-drop", "ALL",
+                "--security-opt", "no-new-privileges", ...user, "--mount", mount,
+                "--env", `CATALYST_HOST_CREDENTIAL_FILE=${credential}`, "--env", `CATALYST_MIRROR_URL=${mirror}`,
+                "--entrypoint", "bun", service.image, "-e", DESIRED_RELEASE_SCRIPT], { signal, timeoutMs: 120_000 });
+            if (read.code !== 0)
+                return { kind: "no_credential" };
+            try {
+                const parsed = JSON.parse(read.stdout);
+                if (!isRecord(parsed))
+                    return { kind: "no_credential" };
+                const hostId = typeof parsed.hostId === "string" && HOST_ID.test(parsed.hostId) ? parsed.hostId : null;
+                if (parsed.kind === "answered" && hostId && typeof parsed.status === "number")
+                    return { kind: "answered", status: parsed.status, body: parsed.body ?? null, hostId };
+                if (parsed.kind === "unreachable" && hostId)
+                    return { kind: "unreachable", detail: String(parsed.detail ?? "fetch failed").slice(0, 80), hostId };
+                return { kind: "no_credential" };
+            }
+            catch {
+                return { kind: "no_credential" };
+            }
+        },
+        async supervisor(dir, project, signal) {
+            const listed = await docker([...compose(dir, project), "ps", "--all", "--quiet", "supervisor"], { cwd: dir, signal });
+            if (listed.code !== 0)
+                return undefined;
+            const id = listed.stdout.trim();
+            if (id === "")
+                return null;
+            if (!CONTAINER_ID.test(id))
+                return undefined;
+            const inspected = await docker(["inspect", id], { signal });
+            try {
+                const [c] = JSON.parse(inspected.stdout);
+                const runner = c.Config.Env.find((line) => line.startsWith("CATALYST_RUNNER_IMAGE="));
+                return {
+                    id: c.Id,
+                    image: c.Config.Image,
+                    runnerImage: runner ? runner.slice("CATALYST_RUNNER_IMAGE=".length) : "",
+                    running: c.State.Running,
+                    restartCount: c.RestartCount,
+                    startedAt: c.State.StartedAt,
+                };
+            }
+            catch {
+                return undefined;
+            }
+        },
+        async logs(id, since, signal) {
+            if (!CONTAINER_ID.test(id))
+                return null;
+            const read = await docker(["logs", "--since", since, id], { signal, timeoutMs: 30_000 });
+            return read.code === 0 ? `${read.stdout}\n${read.stderr ?? ""}` : null;
+        },
+        async children(signal) {
+            const listed = await docker(["ps", "--all", "--quiet", "--no-trunc", "--filter", `name=${CHILD_NAME_PREFIX}`], { signal });
+            if (listed.code !== 0)
+                return null;
+            const ids = listed.stdout.split("\n").map((line) => line.trim()).filter((line) => CONTAINER_ID.test(line));
+            if (ids.length === 0)
+                return {};
+            const inspected = await docker(["inspect", ...ids], { signal });
+            try {
+                const containers = JSON.parse(inspected.stdout);
+                return Object.fromEntries(containers.map((c) => [c.Id, {
+                        running: c.State.Running,
+                        fingerprint: `${c.State.StartedAt}|${c.State.Pid}|${c.Image}|${c.RestartCount}`,
+                    }]));
+            }
+            catch {
+                return null;
+            }
+        },
+        async prepare(images, signal) {
+            const info = await setup.info(signal);
+            if (!info)
+                return "docker_unavailable";
+            const refused = await prepareRunnerImages(setup, images, info.arch, signal);
+            return refused === null ? null : `${refused.reason}${refused.image ? `:${refused.image}` : ""}`;
+        },
+        async configValid(dir, project, file, signal) {
+            return (await docker([...compose(dir, project, file), "config", "--quiet"], { cwd: dirname(file), signal })).code === 0;
+        },
+        async recreate(dir, project, signal) {
+            return (await docker([...compose(dir, project), "up", "--detach", "--no-deps", "--force-recreate", "supervisor"], { cwd: dir, signal, timeoutMs: 120_000 })).code === 0;
+        },
+        async removeImages(images, signal) {
+            for (const ref of images)
+                await docker(["image", "rm", ref], { signal, timeoutMs: 120_000 });
+        },
+    };
+}

package/dist/runner-update.js

@@ -42,7 +42,7 @@ export const RELEASE_ENV_KEYS = {
     supervisor: "CATALYST_SUPERVISOR_IMAGE",
     watchdog: "CATALYST_WATCHDOG_IMAGE",
 };
-const isRecord = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
+export const isRecord = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
 const posInt = (value) => typeof value === "number" && Number.isSafeInteger(value) && value > 0;
 const DIGEST = /^sha256:[0-9a-f]{64}$/;
 const SHA40 = /^[0-9a-f]{40}$/;
@@ -122,12 +122,12 @@ export function readReleaseState(dir) {
     }
 }
 /** Write a private file by rename, so a reader never sees half of it. */
-function writePrivateAtomic(path, text) {
+export function writePrivateAtomic(path, text) {
     const temp = `${path}.${process.pid}.tmp`;
     writeFileSync(temp, text, { mode: 0o600 });
     renameSync(temp, path);
 }
-function writeReleaseState(dir, state) {
+export function writeReleaseState(dir, state) {
     writePrivateAtomic(join(dir, RELEASE_STATE_FILE), `${JSON.stringify(state, null, 2)}\n`);
 }
 /** `.env` with each `KEY=` line for `values` replaced in place (or appended), every other line kept. */
@@ -309,7 +309,7 @@ const alive = (pid) => {
 };
 /** An exclusive lock file holding the owner's pid. It is taken over only when that process is gone,
  *  never by age: one run can legitimately take longer than any fixed limit (three image pulls). */
-function takeLock(dir, nowMs) {
+export function takeLock(dir, nowMs) {
     const path = join(dir, LOCK_FILE);
     const token = `${process.pid} ${nowMs}\n`;
     for (let attempt = 0; attempt < 2; attempt++) {
@@ -333,7 +333,7 @@ function takeLock(dir, nowMs) {
     }
     return null;
 }
-function releaseLock(dir, token) {
+export function releaseLock(dir, token) {
     const path = join(dir, LOCK_FILE);
     try {
         if (readFileSync(path, "utf8") === token)
@@ -345,7 +345,7 @@ function releaseLock(dir, token) {
 }
 /** Runs inside the supervisor image with its own environment. Prints one JSON object, never the
  *  secret: the bearer is built and sent from inside the container. */
-const DESIRED_RELEASE_SCRIPT = `(async () => {
+export const DESIRED_RELEASE_SCRIPT = `(async () => {
   const out = (v) => process.stdout.write(JSON.stringify(v));
   let state;
   try { state = JSON.parse(require("node:fs").readFileSync(process.env.CATALYST_HOST_CREDENTIAL_FILE, "utf8")); }
@@ -367,7 +367,7 @@ const DESIRED_RELEASE_SCRIPT = `(async () => {
     out({ kind: "unreachable", detail: String((error && error.name) || "fetch failed"), hostId: e.hostId });
   }
 })();`;
-const HOST_ID = /^[A-Za-z0-9._:-]{1,128}$/;
+export const HOST_ID = /^[A-Za-z0-9._:-]{1,128}$/;
 /** The real engine: the docker CLI and the setup engine's own pull, custody and Compose steps. */
 export function dockerRunnerUpdateEngine(deps = {}) {
     const exec = deps.exec ?? spawnDocker;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@catalyst-cloud/cli",
-  "version": "0.16.24",
+  "version": "0.16.25",
   "description": "Set up Catalyst Cloud, check your tickets and manage your account from the command line.",
   "license": "MIT",
   "type": "module",

package/skills/catalyst-github/SKILL.md

@@ -4,7 +4,7 @@ description: >-
   Catalyst's GitHub: show a ticket's pull request with its checks, reviews and review threads, say whether it is mergeable under this repository's policy, and explain what a PR accumulates as the ticket moves (the branch, the draft, the rewrite, the force-pushes, the labels, the queue). Use when someone asks "show me the PR", "what are the checks saying", "why hasn't it merged", "what does the queue need", "what does that label mean", or wants the branch and merge conventions Catalyst follows.
 allowed-tools: Bash(catalyst:*) Bash(npx -p @catalyst-cloud/cli catalyst:*)
 ---
-<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.24 — written in this repository for customer accounts -->
+<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.25 — written in this repository for customer accounts -->
 
 # Catalyst's GitHub
 

package/skills/catalyst-linear/SKILL.md

@@ -4,7 +4,7 @@ description: >-
   Catalyst's view of Linear on the customer's own cloud account. Reads a ticket with its comments, relations, labels, linked pull requests and agent sessions inline, from the cloud API by default, with local replica reads available as an explicit opt-in, always naming the source; searches tickets, PRs, projects and initiatives; writes comments, card moves, labels and new tickets through the account's agent proxy as the app actor, with every route, stage id, label id and marker read from the account's contract. Knows what a ticket accumulates as Catalyst works it (phase-outcome comments, the document per phase, the agent session, the labels, the single blocks relation, the bookkeeping marker, the eyes acknowledgement). Use when a person says "show me the ticket", "what did Catalyst write on it", "comment on it", "move it", "label it" or "file a ticket". Not for raising a decision for a human (what-needs-me) and not for pull requests (catalyst-github).
 allowed-tools: Bash(catalyst:*) Bash(npx -p @catalyst-cloud/cli catalyst:*)
 ---
-<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.24 — written in this repository for customer accounts -->
+<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.25 — written in this repository for customer accounts -->
 
 # Catalyst Linear
 

package/skills/catalyst-onboard/SKILL.md

@@ -4,7 +4,7 @@ description: >-
   Set up Catalyst Cloud and say whether setup is working. Walks a person from nothing to their first ticket running, one step at a time: the coding account, the project, the integrations and connected accounts, each repository's settings file, then one real ticket moving. Also logs this machine in, reads the readiness verdict with who can fix each failure, and checks the optional local replica. Use when someone says "set me up", "onboard me", "I just signed up", "get me started", "what do I do first", "log me in", "which account is this machine on", "am I set up", "what is missing", "why does nothing happen" or "is the replica running", when a login expired, or when a skill script exits 2 saying this machine is not connected. Does every step a key can do through the catalyst CLI, hands over the exact page for the steps only a browser can do, and never claims a step it did not watch succeed.
 allowed-tools: Bash(catalyst:*) Bash(npx -p @catalyst-cloud/cli catalyst:*)
 ---
-<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.24 — written in this repository for customer accounts -->
+<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.25 — written in this repository for customer accounts -->
 
 # Onboard me
 

package/skills/run-this-project/SKILL.md

@@ -4,7 +4,7 @@ description: >-
   Own one Catalyst Cloud project end to end until it closes. Use when the person says "run this project for me", "own this until it ships", "keep this moving", or hands you a project id or a set of tickets to drive. Subscribes to the account's event stream for the scope through the catalyst CLI, reacts to each change in the same turn, makes tickets ready and moves them to dispatch, parks what should stop, chases stalls, escalates inward, and keeps one status summary current. Writes to Linear as the app actor; never polls.
 allowed-tools: Bash(catalyst:*) Bash(npx -p @catalyst-cloud/cli catalyst:*)
 ---
-<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.24 — written in this repository for customer accounts -->
+<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.25 — written in this repository for customer accounts -->
 
 # Run this project
 

package/skills/unstick/SKILL.md

@@ -4,7 +4,7 @@ description: >-
   Get a stuck Catalyst Cloud ticket moving again. Use when the person asks "why is this parked and can you release it?", "unpark this", "get things flowing again", "the outage is over, retry what failed", or hands you a ticket that is not moving. Reads why nothing runs and what holds the ticket through the catalyst CLI, decides whether the recorded cause is fixed, previews the release, releases every governor holding the ticket the right way (or one failure class across a team), and raises an ask only for what a person has to do. Never releases a cause it cannot show changed without saying so.
 allowed-tools: Bash(catalyst:*) Bash(npx -p @catalyst-cloud/cli catalyst:*)
 ---
-<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.24 — written in this repository for customer accounts -->
+<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.25 — written in this repository for customer accounts -->
 
 # Unstick
 

package/skills/what-needs-me/SKILL.md

@@ -4,7 +4,7 @@ description: >-
   The human's decision inbox on Catalyst Cloud, and the one way an agent raises a decision on their behalf. Use when the person asks "what needs me?", "what am I blocking?", or when active work is gated on a choice only they can make. Lists open asks ranked by how much open work each one holds, files an ask through the cloud's ask route with the account's own template, and records the answer so the held work releases. Never answers as the human, never duplicates an open ask.
 allowed-tools: Bash(catalyst:*) Bash(npx -p @catalyst-cloud/cli catalyst:*)
 ---
-<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.24 — written in this repository for customer accounts -->
+<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.25 — written in this repository for customer accounts -->
 
 # What needs me
 

package/skills/what-this-repo-needs/SKILL.md

@@ -4,7 +4,7 @@ description: >-
   What environment variable names does this repository need, and where does each one come from? Scans the repository offline (no login, no network) and lists the names in three groups (build/test, deploy-only, bindings), each with where it was found, what uses it, and where a local value would come from. Never reads a value or prints one. Also checks the environment variable table in `.catalyst/catalyst.toml` offline. Use when someone asks "what does catalyst.env.json mean", "what env vars does this repo need", "why does the container need this", or before reviewing or writing the declaration.
 allowed-tools: Bash(catalyst:*) Bash(npx -p @catalyst-cloud/cli catalyst:*)
 ---
-<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.24 — written in this repository for customer accounts -->
+<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.25 — written in this repository for customer accounts -->
 
 # What this repository needs
 

package/skills/whats-happening/SKILL.md

@@ -4,7 +4,7 @@ description: >-
   Use for "what's happening?", "what machines do we have?", "how much capacity do we have?", "where are we?", "why is that stuck?", "what closed?", "what's next?", "how does this work?", "why did it do that?", "how does it prioritise?" and "what does this setting do, where is it set?" in Catalyst Cloud, or asks for something to be done rather than known. Reads the contract via catalyst CLI. Read-only: route work/decisions, never answer as the human.
 allowed-tools: Bash(catalyst:*) Bash(npx -p @catalyst-cloud/cli catalyst:*)
 ---
-<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.24 — written in this repository for customer accounts -->
+<!-- vendored-from: @catalyst-cloud/catalyst-skills@0.16.25 — written in this repository for customer accounts -->
 
 # What's happening