@flame0510/project-aether 1.2.0 → 1.4.0

package/lib/agent-images.ts

@@ -0,0 +1,375 @@
+/**
+ * Agent base images: the OpenClaw versions downloaded here, the agents using each,
+ * the supported versions the registry publishes, and pulling one.
+ *
+ * Images are addressed by version tag, `openclaw-agent-base:<version>`, never by
+ * `:latest`. An agent's OpenClaw version decides its data schema, and a moving shared
+ * tag would move every new or recreated agent with it. Create uses the newest
+ * supported version downloaded; recreate keeps the agent's own version.
+ */
+import { spawn } from 'child_process';
+import { dockerFetch } from '@/lib/docker-socket';
+import { dockerExecNoFail } from '@/lib/docker-exec';
+import {
+  LOCAL_IMAGE_REPOSITORY,
+  compareVersions,
+  isSupportedVersion,
+  localImageRef,
+  parseOpenClawVersion,
+  registryRepository,
+} from '@/lib/agent-versions';
+
+const VERSION_LABEL = 'org.opencontainers.image.version';
+const REMOTE_TTL_MS = 10 * 60 * 1000;
+const REGISTRY_TIMEOUT_MS = 10_000;
+const PULL_TIMEOUT_MS = 30 * 60 * 1000;
+
+export interface LocalAgentImage {
+  id: string;
+  tags: string[];
+  /** From the version label, else from a version tag; null for an unlabelled, untagged image. */
+  version: string | null;
+  sizeBytes: number;
+  /** AGENT_ID of every container, running or stopped, created from this image. */
+  usedBy: string[];
+}
+
+interface DockerImage {
+  Id: string;
+  RepoTags?: string[] | null;
+  Labels?: Record<string, string> | null;
+  Size?: number;
+}
+
+interface DockerContainer {
+  ImageID?: string;
+  Labels?: Record<string, string> | null;
+}
+
+const tagPrefix = `${LOCAL_IMAGE_REPOSITORY}:`;
+
+function versionFromTags(tags: string[]): string | null {
+  for (const tag of tags) {
+    if (!tag.startsWith(tagPrefix)) continue;
+    const version = parseOpenClawVersion(tag.slice(tagPrefix.length));
+    if (version) return version;
+  }
+  return null;
+}
+
+/** Local `openclaw-agent-base` images, through the Docker API. */
+export async function listLocalAgentImages(): Promise<LocalAgentImage[]> {
+  const imageFilter = encodeURIComponent(JSON.stringify({ reference: [LOCAL_IMAGE_REPOSITORY] }));
+  const agentFilter = encodeURIComponent(JSON.stringify({ label: ['AGENT_ID'] }));
+  const [images, containers] = await Promise.all([
+    dockerFetch<DockerImage[]>('GET', `/images/json?filters=${imageFilter}`),
+    dockerFetch<DockerContainer[]>('GET', `/containers/json?all=true&filters=${agentFilter}`),
+  ]);
+
+  const usedBy = new Map<string, string[]>();
+  for (const c of Array.isArray(containers) ? containers : []) {
+    const agentId = c.Labels?.AGENT_ID;
+    if (!c.ImageID || !agentId) continue;
+    usedBy.set(c.ImageID, [...(usedBy.get(c.ImageID) ?? []), agentId]);
+  }
+
+  return (Array.isArray(images) ? images : []).map((img) => {
+    const tags = (img.RepoTags ?? []).filter((t) => t !== '<none>:<none>');
+    return {
+      id: img.Id,
+      tags,
+      version: parseOpenClawVersion(img.Labels?.[VERSION_LABEL]) ?? versionFromTags(tags),
+      sizeBytes: img.Size ?? 0,
+      usedBy: usedBy.get(img.Id) ?? [],
+    };
+  });
+}
+
+/** Supported versions present locally under their version tag, newest first. */
+export function localSupportedVersions(images: LocalAgentImage[]): string[] {
+  const versions = new Set<string>();
+  for (const img of images) {
+    for (const tag of img.tags) {
+      if (tag.startsWith(tagPrefix) && isSupportedVersion(tag.slice(tagPrefix.length))) {
+        versions.add(tag.slice(tagPrefix.length));
+      }
+    }
+  }
+  return [...versions].sort((a, b) => compareVersions(b, a));
+}
+
+export const newestLocalSupportedVersion = (images: LocalAgentImage[]): string | null =>
+  localSupportedVersions(images)[0] ?? null;
+
+/**
+ * Whether an image reference (a tag or an id) exists locally. `dockerFetch` resolves
+ * Docker's error bodies too, so a 404 is told apart by the missing `Id`.
+ */
+async function imageExists(reference: string): Promise<boolean> {
+  try {
+    const info = await dockerFetch<{ Id?: unknown }>('GET', `/images/${encodeURIComponent(reference)}/json`);
+    return typeof info?.Id === 'string';
+  } catch {
+    return false;
+  }
+}
+
+/** Whether `openclaw-agent-base:<version>` exists locally. */
+export const localVersionExists = (version: string): Promise<boolean> => imageExists(localImageRef(version));
+
+/**
+ * Versions read from inside containers, keyed by image id: an image never changes
+ * version. A failed read is remembered for a few minutes too, so a container whose
+ * image is not OpenClaw does not cost a `docker exec` on every agent list poll.
+ */
+const versionByImageId = new Map<string, { version: string | null; at: number }>();
+const UNKNOWN_VERSION_TTL_MS = 5 * 60 * 1000;
+
+/**
+ * The OpenClaw version a container runs: its image's label or version tag, else
+ * `openclaw --version` inside the running container, for images built before the
+ * label existed (2026.7.1-2 published as `:latest` only). Null for a stopped container
+ * whose image carries neither.
+ */
+export async function containerOpenClawVersion(
+  container: { name: string; imageId: string; running: boolean },
+  images: LocalAgentImage[],
+): Promise<string | null> {
+  const fromImage = images.find((img) => img.id === container.imageId)?.version;
+  if (fromImage) return fromImage;
+  const cached = versionByImageId.get(container.imageId);
+  if (cached && (cached.version || Date.now() - cached.at < UNKNOWN_VERSION_TTL_MS)) return cached.version;
+  if (!container.running) return null;
+  const version = parseOpenClawVersion(
+    await dockerExecNoFail(container.name, ['openclaw', '--version'], { timeoutMs: 30_000 }),
+  );
+  versionByImageId.set(container.imageId, { version, at: Date.now() });
+  return version;
+}
+
+// ── Registry ─────────────────────────────────────────────────────────────────
+
+function splitRepository(repository: string): { host: string; path: string } {
+  const [first, ...rest] = repository.split('/');
+  if (rest.length && (first.includes('.') || first.includes(':') || first === 'localhost')) {
+    return { host: first, path: rest.join('/') };
+  }
+  return { host: 'registry-1.docker.io', path: repository.includes('/') ? repository : `library/${repository}` };
+}
+
+const isLocalRegistryHost = (host: string) => /^(localhost|127\.0\.0\.1|\[::1\])(:\d+)?$/.test(host);
+
+/**
+ * Tags of a repository through the registry HTTP API. A public registry such as ghcr
+ * answers 401 with a Bearer challenge; an anonymous token for `pull` is then fetched
+ * from the realm it names. A registry on localhost is plain HTTP.
+ */
+async function fetchRegistryTags(repository: string): Promise<string[]> {
+  const { host, path } = splitRepository(repository);
+  const url = `${isLocalRegistryHost(host) ? 'http' : 'https'}://${host}/v2/${path}/tags/list?n=1000`;
+  const get = (target: string | URL, headers?: Record<string, string>) =>
+    fetch(target, { headers, cache: 'no-store', signal: AbortSignal.timeout(REGISTRY_TIMEOUT_MS) });
+
+  let res = await get(url);
+  if (res.status === 401) {
+    const challenge = res.headers.get('www-authenticate') ?? '';
+    const params = Object.fromEntries([...challenge.matchAll(/(\w+)="([^"]*)"/g)].map((m) => [m[1], m[2]]));
+    if (!params.realm) throw new Error('the registry asked for credentials');
+    const tokenUrl = new URL(params.realm);
+    if (params.service) tokenUrl.searchParams.set('service', params.service);
+    tokenUrl.searchParams.set('scope', params.scope ?? `repository:${path}:pull`);
+    const tokenRes = await get(tokenUrl);
+    if (!tokenRes.ok) throw new Error(`registry token request failed (HTTP ${tokenRes.status})`);
+    const body = (await tokenRes.json()) as { token?: string; access_token?: string };
+    const token = body.token ?? body.access_token;
+    if (!token) throw new Error('the registry returned no token');
+    res = await get(url, { Authorization: `Bearer ${token}` });
+  }
+  if (!res.ok) throw new Error(`registry tag list failed (HTTP ${res.status})`);
+  return ((await res.json()) as { tags?: string[] | null }).tags ?? [];
+}
+
+let remoteCache: { repository: string; versions: string[] | null; at: number } | null = null;
+let remoteInFlight: Promise<string[] | null> | null = null;
+
+/**
+ * Supported versions the registry publishes, newest first; null when it cannot be
+ * reached. Cached for ten minutes, and concurrent callers share one lookup, so a
+ * banner polling every two seconds costs one request per TTL.
+ */
+export async function remoteSupportedVersions(): Promise<string[] | null> {
+  const repository = registryRepository();
+  if (remoteCache && remoteCache.repository === repository && Date.now() - remoteCache.at < REMOTE_TTL_MS) {
+    return remoteCache.versions;
+  }
+  if (remoteInFlight) return remoteInFlight;
+
+  remoteInFlight = (async () => {
+    try {
+      const tags = await fetchRegistryTags(repository);
+      return tags.filter(isSupportedVersion).sort((a, b) => compareVersions(b, a));
+    } catch {
+      return null;
+    }
+  })();
+  try {
+    const versions = await remoteInFlight;
+    remoteCache = { repository, versions, at: Date.now() };
+    return versions;
+  } finally {
+    remoteInFlight = null;
+  }
+}
+
+// ── Pull ─────────────────────────────────────────────────────────────────────
+
+/**
+ * Runs `docker <args>`, streaming output lines; rejects with the last error line.
+ *
+ * `env` is for container environment on `docker run`: each name is inserted as `-e KEY`
+ * right after the subcommand, and the value reaches docker through its own
+ * environment, so secrets stay out of the process table.
+ */
+export function runDocker(
+  args: string[],
+  opts: { onLine?: (line: string) => void; signal?: AbortSignal; timeoutMs: number; env?: Record<string, string> },
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const envNames = Object.keys(opts.env ?? {});
+    const argv = envNames.length ? [args[0], ...envNames.flatMap((k) => ['-e', k]), ...args.slice(1)] : args;
+    const child = spawn('docker', argv, {
+      stdio: ['ignore', 'pipe', 'pipe'],
+      ...(envNames.length ? { env: { ...process.env, ...opts.env } } : {}),
+    });
+    let lastError = '';
+    let firstError = '';
+    const onData = (isErr: boolean) => (chunk: Buffer) => {
+      for (const line of chunk.toString().split(/\r?\n/)) {
+        if (!line.trim()) continue;
+        if (isErr) {
+          // Docker prints the cause first and "Run 'docker run --help' …" last; keep
+          // both ends so the rejection can carry the cause, not the usage pointer.
+          firstError ||= line.trim();
+          lastError = line.trim();
+        }
+        opts.onLine?.(line);
+      }
+    };
+    child.stdout.on('data', onData(false));
+    child.stderr.on('data', onData(true));
+
+    const kill = () => child.kill('SIGKILL');
+    const timer = setTimeout(() => {
+      kill();
+      reject(new Error(`docker ${args[0]} timed out`));
+    }, opts.timeoutMs);
+    opts.signal?.addEventListener('abort', kill, { once: true });
+
+    child.on('error', (err) => {
+      clearTimeout(timer);
+      opts.signal?.removeEventListener('abort', kill);
+      reject(err);
+    });
+    child.on('close', (code) => {
+      clearTimeout(timer);
+      opts.signal?.removeEventListener('abort', kill);
+      if (code === 0) resolve();
+      else reject(new Error(firstError && firstError !== lastError ? `${firstError} (${lastError})` : lastError || `docker ${args[0]} exited with code ${code}`));
+    });
+  });
+}
+
+/**
+ * Pull `<registry>:<version>` and tag it `openclaw-agent-base:<version>`. The registry
+ * reference is removed afterwards (an untag, the layers stay), so local images list
+ * under the version tag only.
+ */
+export async function pullAgentImage(
+  version: string,
+  opts: { onLine?: (line: string) => void; signal?: AbortSignal } = {},
+): Promise<void> {
+  if (!isSupportedVersion(version)) throw new Error(`OpenClaw ${version} is not supported by this Rev4a`);
+  const remote = `${registryRepository()}:${version}`;
+  await runDocker(['pull', remote], { ...opts, timeoutMs: PULL_TIMEOUT_MS });
+  await runDocker(['tag', remote, localImageRef(version)], { ...opts, timeoutMs: 30_000 });
+  await runDocker(['rmi', remote], { timeoutMs: 30_000 }).catch(() => {});
+}
+
+/**
+ * Retention: remove agent images nothing needs.
+ *
+ * Kept: every image a container uses, running or stopped, and the newest and the
+ * previous OpenClaw version present — the previous one is what a rollback runs. Other
+ * images lose all their tags. A kept image loses its extra tags (`latest`,
+ * `2026.9.3-local`) when it has a version tag. Nothing is forced: Docker refuses to
+ * remove an image a container uses, the last safety net. A removed version can be
+ * pulled again while the registry publishes it. Returns the references removed.
+ */
+export async function pruneAgentImages(): Promise<string[]> {
+  const images = await listLocalAgentImages();
+  const versions = [...new Set(images.map((img) => img.version).filter((v): v is string => !!v))]
+    .sort((a, b) => compareVersions(b, a));
+  const keepVersions = new Set(versions.slice(0, 2));
+
+  const removed: string[] = [];
+  for (const img of images) {
+    const keep = img.usedBy.length > 0 || (img.version !== null && keepVersions.has(img.version));
+    const versionTags = img.tags.filter((t) => {
+      if (!t.startsWith(tagPrefix)) return false;
+      const tag = t.slice(tagPrefix.length);
+      return parseOpenClawVersion(tag) === tag;
+    });
+    const targets = keep ? (versionTags.length ? img.tags.filter((t) => !versionTags.includes(t)) : []) : img.tags;
+    for (const tag of targets) {
+      try {
+        await runDocker(['rmi', tag], { timeoutMs: 120_000 });
+        removed.push(tag);
+      } catch {
+        // In use, or already gone.
+      }
+    }
+  }
+  return removed;
+}
+
+/**
+ * The image a recreate uses: the agent's own OpenClaw version under its version tag.
+ * A recreate never changes version.
+ *
+ * When the tag is missing but the container's image is still here (an agent created
+ * from `:latest` before version tags), that image gets the version tag. When the image
+ * is gone too, a supported version is pulled. An agent whose version cannot be read
+ * keeps the exact image id it runs.
+ */
+export async function resolveRecreateImage(container: {
+  Name?: string;
+  Image?: string;
+  Config?: { Image?: string };
+  State?: { Running?: boolean };
+}): Promise<string> {
+  const name = (container.Name ?? '').replace(/^\//, '');
+  const imageId = container.Image ?? '';
+  const images = await listLocalAgentImages();
+  const version = await containerOpenClawVersion(
+    { name, imageId, running: container.State?.Running === true },
+    images,
+  );
+
+  if (version) {
+    const ref = localImageRef(version);
+    if (await localVersionExists(version)) return ref;
+    if (imageId && (await imageExists(imageId))) {
+      await runDocker(['tag', imageId, ref], { timeoutMs: 30_000 });
+      return ref;
+    }
+    if (isSupportedVersion(version)) {
+      await pullAgentImage(version);
+      return ref;
+    }
+    throw new Error(`The image for OpenClaw ${version} is not available locally`);
+  }
+
+  if (imageId && (await imageExists(imageId))) return imageId;
+  if (container.Config?.Image) return container.Config.Image;
+  throw new Error('Cannot tell which image this agent runs');
+}

package/lib/agent-ports-server.ts

@@ -0,0 +1,27 @@
+/**
+ * Server-side host-port lookup for lib/agent-ports.ts. Split out because it uses
+ * `child_process`, which client components cannot import.
+ */
+import { execSync } from 'child_process';
+
+/**
+ * Host ports published by any container, running or not: a stopped container keeps its
+ * port allocations, so `docker run -p` on them fails just the same.
+ *
+ * Parsed in JS rather than piping through grep: the old `grep -oP '\d+(?=->)'` form lost
+ * its backslash inside a template literal, matched nothing, and offered port 3700 to
+ * every agent; `grep -P` is also GNU-only and absent on macOS.
+ */
+export function getUsedHostPorts(): Set<number> {
+  try {
+    const raw = execSync(`docker ps -a --format '{{.Ports}}'`, { encoding: 'utf-8', timeout: 3000 });
+    const used = new Set<number>();
+    for (const [, port] of raw.matchAll(/(\d+)->/g)) {
+      const n = Number(port);
+      if (Number.isInteger(n)) used.add(n);
+    }
+    return used;
+  } catch {
+    return new Set();
+  }
+}

package/lib/agent-ports.ts

@@ -0,0 +1,68 @@
+/**
+ * Host-port helpers shared by agent creation and the edit (rename/ports) flow, so both
+ * validate a range the same way and produce the same error messages. Pure functions only:
+ * this module is imported by client components too. The Docker lookup lives in
+ * `lib/agent-ports-server.ts`.
+ *
+ * The mapping convention is one block starting at the container's gateway port 3000:
+ * `3700` publishes `3700:3000`, and `3700-3709` publishes `3700-3709:3000-3009`.
+ * OpenClaw shifts its derived ports (browser, canvas) from 3000, which is why agents are
+ * created with a block rather than a single mapping.
+ */
+
+/** Rev4a's own port: never offer it to an agent. */
+export const REV4A_PORT = 3740;
+/** Agents publish this many consecutive host ports by default. */
+export const DEFAULT_BLOCK_SIZE = 10;
+
+/** An available host block, from 3700 up, skipping Rev4a's port. */
+export function findAvailablePortBlock(usedPorts: ReadonlySet<number> = new Set(), blockSize = DEFAULT_BLOCK_SIZE): number {
+  for (let base = 3700; base + blockSize - 1 <= 3799; base += DEFAULT_BLOCK_SIZE) {
+    if (base <= REV4A_PORT && REV4A_PORT <= base + blockSize - 1) continue;
+    const block = Array.from({ length: blockSize }, (_, i) => base + i);
+    if (block.every((p) => !usedPorts.has(p))) return base;
+  }
+  let base = 3800;
+  while (true) {
+    const block = Array.from({ length: blockSize }, (_, i) => base + i);
+    if (block.every((p) => !usedPorts.has(p))) return base;
+    base += DEFAULT_BLOCK_SIZE;
+  }
+}
+
+export type PortInputResult =
+  | { valid: true; isBlock: boolean; start: number; end: number }
+  | { valid: false; error: string };
+
+/**
+ * Validate `"3700"` or `"3700-3709"` against the ports already published (pass
+ * `getUsedHostPorts()` on the server; the client passes what it was given).
+ */
+export function validatePortInput(input: string, usedPorts: ReadonlySet<number> = new Set()): PortInputResult {
+  const trimmed = input.trim();
+  if (trimmed.includes('-')) {
+    const m = trimmed.match(/^(\d{1,5})-(\d{1,5})$/);
+    if (!m) return { valid: false, error: 'Invalid format. Use e.g. 3700-3709' };
+    const start = parseInt(m[1], 10);
+    const end = parseInt(m[2], 10);
+    if (start < 1 || end > 65535) return { valid: false, error: 'Ports must be between 1 and 65535' };
+    if (end <= start) return { valid: false, error: 'End port must be greater than start port' };
+    if (start <= REV4A_PORT && REV4A_PORT <= end) return { valid: false, error: `Port ${REV4A_PORT} is reserved for Rev4a` };
+    const conflicts: number[] = [];
+    for (let p = start; p <= end; p++) { if (usedPorts.has(p)) conflicts.push(p); }
+    if (conflicts.length > 0) return { valid: false, error: `Ports already in use: ${conflicts.join(', ')}` };
+    return { valid: true, isBlock: true, start, end };
+  }
+  const m = trimmed.match(/^(\d{1,5})$/);
+  if (!m) return { valid: false, error: 'Use a number (e.g. 3700) or a range (e.g. 3700-3709)' };
+  const port = parseInt(m[1], 10);
+  if (port < 1 || port > 65535) return { valid: false, error: 'Port must be between 1 and 65535' };
+  if (port === REV4A_PORT) return { valid: false, error: `Port ${REV4A_PORT} is reserved for Rev4a` };
+  if (usedPorts.has(port)) return { valid: false, error: `Port ${port} is already in use` };
+  return { valid: true, isBlock: false, start: port, end: port };
+}
+
+/** `docker run` port arguments for a validated range, mapping onto the container 3000. */
+export function portMappingArgs(start: number, end: number): string[] {
+  return ['-p', end === start ? `${start}:3000` : `${start}-${end}:3000-${3000 + (end - start)}`];
+}

package/lib/agent-readiness.ts

@@ -0,0 +1,110 @@
+/**
+ * Waiting for an agent's Gateway to finish starting.
+ *
+ * **Which endpoint.** OpenClaw exposes three, answering three different questions:
+ *
+ *   /health, /healthz    the HTTP server is listening
+ *   /startupz            startup finished and the Gateway is not draining
+ *   /readyz              the above, plus configured channels pass deep checks
+ *
+ * `/startupz` is the one callers need: the create route configures the agent as
+ * soon as this returns. A Gateway that refused readiness — for instance one that
+ * found a legacy session store and wants `doctor --fix` — still answers 200 on
+ * `/health`. `/readyz` would fail on an expired channel token, and the create
+ * route deletes the container when this probe times out, so a false negative
+ * there destroys a healthy agent over an unrelated channel.
+ *
+ * **Two OpenClaw majors.** `/startupz` exists from 9.x: 503 while starting or
+ * draining, 200 with `{"status":"started"}` when done. The 2026.7.1-2 image that
+ * `openclaw-agent-base` currently pins has no `/startupz`; its Gateway answers
+ * every unknown path with 200 and the web UI's HTML. So the probe reads the
+ * status code *and* the body, and falls back to `/health` only for a 200 that is
+ * not JSON — the old image. A 503, or no answer, means "not yet", never "use
+ * `/health`": reading an empty answer as "old image" makes a 9.x Gateway count as
+ * ready the moment it starts listening.
+ *
+ * Delete the `/health` fallback once every image is on 9.x.
+ *
+ * **Async.** A request path must not hold Node's single thread for up to a
+ * minute; see docs/ARCHITECTURE.md §3.1.
+ */
+import { dockerExecNoFail } from '@/lib/docker-exec';
+
+/** Interval between attempts. The Gateway takes seconds to boot, not milliseconds. */
+const POLL_INTERVAL_MS = 2_000;
+
+/** Per-attempt budget. Generous: a loaded host can be slow to answer. */
+const ATTEMPT_TIMEOUT_MS = 5_000;
+
+const sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms));
+
+/**
+ * One HTTP GET from inside the container, with its status code.
+ *
+ * `curl -w` appends the code as the last line. `status` is 0 when nothing usable
+ * came back. A deadline may reject, or resolve with truncated output because
+ * `docker exec` can exit 0 on the SIGTERM (docs/ARCHITECTURE.md, "A timeout does
+ * not reject"); either way a last line that is not a three-digit code counts as
+ * no answer. The image has `curl`, not `wget`.
+ */
+async function probe(container: string, path: string): Promise<{ status: number; body: string }> {
+  const out = await dockerExecNoFail(
+    container,
+    ['curl', '-s', '-o', '-', '-w', '\n%{http_code}', `http://127.0.0.1:3000${path}`],
+    { timeoutMs: ATTEMPT_TIMEOUT_MS },
+  );
+  if (!out) return { status: 0, body: '' };
+  const cut = out.lastIndexOf('\n');
+  const tail = (cut === -1 ? out : out.slice(cut + 1)).trim();
+  if (!/^\d{3}$/.test(tail)) return { status: 0, body: '' };
+  return { status: Number(tail), body: cut === -1 ? '' : out.slice(0, cut) };
+}
+
+function asJson(body: string): Record<string, unknown> | null {
+  if (!body.trimStart().startsWith('{')) return null;
+  try {
+    const v = JSON.parse(body);
+    return v && typeof v === 'object' ? (v as Record<string, unknown>) : null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Poll a container's Gateway until it has finished starting.
+ *
+ * Resolves `true` on the first affirmative answer, `false` on timeout. Never
+ * throws: an unreachable container is indistinguishable from a slow one until the
+ * deadline passes, and every caller treats both the same way.
+ */
+export async function waitForGatewayReady(
+  container: string,
+  timeoutMs = 60_000,
+): Promise<boolean> {
+  const deadline = Date.now() + timeoutMs;
+
+  while (Date.now() < deadline) {
+    const startupz = await probe(container, '/startupz');
+
+    if (startupz.status === 200) {
+      const json = asJson(startupz.body);
+      if (json) {
+        // 9.x: accept only the answer that means startup finished.
+        if (json.status === 'started') return true;
+      } else {
+        // 200 with a non-JSON body: the 2026.7.1-2 web UI catch-all, so this build
+        // has no /startupz. Liveness is the best signal it offers.
+        const health = await probe(container, '/health');
+        const h = health.status === 200 ? asJson(health.body) : null;
+        if (h && (h.ok === true || h.status === 'live')) return true;
+      }
+    }
+    // 503 (starting or draining), any other code, or no answer: not yet.
+
+    // Only sleep if there is still budget left to use afterwards.
+    if (Date.now() + POLL_INTERVAL_MS >= deadline) break;
+    await sleep(POLL_INTERVAL_MS);
+  }
+
+  return false;
+}

package/lib/agent-recreate-state.ts

@@ -0,0 +1,108 @@
+/**
+ * Persisted state of agent recreates: the `agent_recreates` table (lib/db-bootstrap.mjs).
+ *
+ * Kept apart from lib/agent-recreate.ts so busy checks (lib/agent-busy.ts, the provider
+ * sync) can ask which agents are being recreated without importing the recreate action.
+ */
+import { openDb } from '@/lib/db';
+
+export type RecreateStatus = 'backing_up' | 'recreating' | 'done' | 'failed' | 'interrupted';
+
+/** Statuses during which the agent must not be touched by anything else. */
+export const ACTIVE_RECREATE_STATUSES: readonly RecreateStatus[] = ['backing_up', 'recreating'];
+
+export interface AgentRecreateRow {
+  id: number;
+  agent_id: string;
+  status: RecreateStatus;
+  image: string | null;
+  backup_file: string | null;
+  error: string | null;
+  started_at: number;
+  updated_at: number;
+  finished_at: number | null;
+}
+
+const FINAL: readonly RecreateStatus[] = ['done', 'failed', 'interrupted'];
+
+export function insertRecreate(agentId: string, image: string): number {
+  const db = openDb(false);
+  try {
+    const now = Date.now();
+    const info = db.prepare(
+      `INSERT INTO agent_recreates (agent_id, status, image, started_at, updated_at)
+       VALUES (?, 'backing_up', ?, ?, ?)`,
+    ).run(agentId, image, now, now);
+    return Number(info.lastInsertRowid);
+  } finally {
+    db.close();
+  }
+}
+
+type Writable = Partial<Pick<AgentRecreateRow, 'status' | 'backup_file' | 'error'>>;
+
+export function updateRecreateRow(id: number, fields: Writable): void {
+  const entries = Object.entries(fields).filter(([, v]) => v !== undefined);
+  const now = Date.now();
+  const sets = [...entries.map(([k]) => `${k} = ?`), 'updated_at = ?'];
+  const values: unknown[] = [...entries.map(([, v]) => v), now];
+  if (fields.status && FINAL.includes(fields.status)) {
+    sets.push('finished_at = ?');
+    values.push(now);
+  } else if (fields.status) {
+    sets.push('finished_at = NULL');
+  }
+  const db = openDb(false);
+  try {
+    db.prepare(`UPDATE agent_recreates SET ${sets.join(', ')} WHERE id = ?`).run(...values, id);
+  } finally {
+    db.close();
+  }
+}
+
+export function latestRecreate(agentId: string): AgentRecreateRow | null {
+  const db = openDb(true);
+  try {
+    return (db.prepare('SELECT * FROM agent_recreates WHERE agent_id = ? ORDER BY id DESC LIMIT 1').get(agentId) as AgentRecreateRow | undefined) ?? null;
+  } finally {
+    db.close();
+  }
+}
+
+export function isRecreateActive(agentId: string): boolean {
+  const row = latestRecreate(agentId);
+  return !!row && ACTIVE_RECREATE_STATUSES.includes(row.status);
+}
+
+/** AGENT_IDs with a recreate in an active status. */
+export function activeRecreateAgentIds(): Set<string> {
+  const db = openDb(true);
+  try {
+    const placeholders = ACTIVE_RECREATE_STATUSES.map(() => '?').join(', ');
+    const rows = db.prepare(`SELECT DISTINCT agent_id FROM agent_recreates WHERE status IN (${placeholders})`).all(...ACTIVE_RECREATE_STATUSES) as { agent_id: string }[];
+    return new Set(rows.map((r) => r.agent_id));
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * At startup no recreate job can be running: any row still active was cut off by the
+ * restart. It becomes `interrupted`, keeping the step it was in.
+ */
+export function markInterruptedRecreates(): number {
+  const db = openDb(false);
+  try {
+    const placeholders = ACTIVE_RECREATE_STATUSES.map(() => '?').join(', ');
+    const now = Date.now();
+    const info = db.prepare(
+      `UPDATE agent_recreates
+         SET error = COALESCE(error, 'Rev4a restarted while this step was running: ' || status),
+             status = 'interrupted', updated_at = ?, finished_at = ?
+       WHERE status IN (${placeholders})`,
+    ).run(now, now, ...ACTIVE_RECREATE_STATUSES);
+    return info.changes;
+  } finally {
+    db.close();
+  }
+}