@ultimat3/cli 24.0.0 → 25.1.0

package/src/cmd-build.ts

@@ -103,24 +103,94 @@ export function dockerArgs(root: string, tag: string, buildId: string): readonly
 }
 
 /**
- * The define is not optional. A single-file executable carries no `package.json`, so
- * `frameworkVersion()` has nothing to read and throws — which is exactly how this target came to
- * compile an artifact that could never boot. The value is this CLI's own `@ultimat3/core`, which is
- * the app's too: the packages release in lockstep and `x new` pins them together.
+ * What `--platform` may name: Bun's own `--target` spellings, so the flag maps onto Bun's one to
+ * one and an agent who knows Bun's list knows this one. Four, not Bun's every target — these are
+ * the hosts the framework is tested on; a musl or baseline build is `bun build` by hand.
+ */
+export const BINARY_PLATFORMS = [
+  'bun-linux-x64',
+  'bun-linux-arm64',
+  'bun-windows-x64',
+  'bun-darwin-arm64',
+] as const;
+
+export type BinaryPlatform = (typeof BINARY_PLATFORMS)[number];
+
+/** `--platform`, checked before the gate runs: an unknown value is a typo, never a guess. */
+export function readPlatform(raw: string | undefined): BinaryPlatform | undefined {
+  if (raw === undefined) return undefined;
+  const known: readonly string[] = BINARY_PLATFORMS;
+  if (known.includes(raw)) return raw as BinaryPlatform;
+  throw new BadFlagError({
+    flag: 'platform',
+    command: 'build',
+    reason: `${quoteArg(raw)} is not a platform x build compiles for — one of ${BINARY_PLATFORMS.join(', ')}`,
+    fix: 'x build --target binary --platform bun-linux-x64',
+  });
+}
+
+/**
+ * The path the executable really lands at. Bun names a Windows executable `<out>.exe` — whether
+ * `--platform bun-windows-x64` asked for it or the host is Windows and no platform was named — so
+ * the `.exe` is appended HERE and passed to `--outfile`, and the path reported is the path written.
+ */
+export function binaryOutfile(
+  out: string,
+  platform: BinaryPlatform | undefined,
+  host: string = process.platform,
+): string {
+  const windows = platform === undefined ? host === 'win32' : platform.startsWith('bun-windows-');
+  return windows && !out.toLowerCase().endsWith('.exe') ? `${out}.exe` : out;
+}
+
+/**
+ * The define answers `frameworkVersion()` for framework code BUNDLED into an executable, where no
+ * `package.json` exists — the reason this target once compiled an artifact that could never boot,
+ * and why `docker/Dockerfile`'s self-contained `x` still needs it. Here nothing framework is
+ * bundled any more (`LAUNCHER_EXTERNALS`): `@ultimat3/core` loads from the app's `node_modules` and
+ * reads its own manifest, so the define reaches no module of this graph. It stays as a pin on that
+ * one version, this CLI's own — the packages release in lockstep and `x new` pins them together.
  *
  * `externalArgs()` is not optional either, and for the opposite reason: it names the one specifier
  * this graph must NOT resolve. `apps/web/server.ts` reaches `serve.ts`, which reaches the island
  * builder, which reaches `@babel/core` — whose `.cts`-config loader requires a package we
  * deliberately do not install. Bun 1.3 fails the compile on it. See `compile-externals.ts`.
+ *
+ * The two autoload flags are what make the binary a LAUNCHER at all. It imports the app tree it is
+ * started in at run time — `app.config.ts`, then every module under `apps/*` — and a standalone
+ * executable reads neither `package.json` nor `tsconfig.json` at run time by default (Bun:
+ * `--compile-autoload-{package-json,tsconfig}`, both off). With no `package.json` the resolver
+ * ignores `exports`, so every `@ultimat3/*` (`exports` → `./src/index.ts`) was
+ * `Cannot find module '@ultimat3/core' from '<app>/app.config.ts'` and the binary never answered
+ * `/healthz` — on Windows, where CI first ran it, and on Linux alike. With no `tsconfig.json` the
+ * app's own `paths` aliases fail the same way. `e2e/binary-launch.e2e.test.ts` compiles and runs it.
  */
-export function binaryArgs(root: string, out: string): readonly string[] {
+/**
+ * The framework is resolved from the app tree at run time, never bundled into the binary. The app's
+ * own modules import `@ultimat3/*` from its `node_modules` — they are loaded off disk, after the
+ * build — so a bundled copy was a SECOND instance of every package: the app's `configureAuthenticator`
+ * filled the disk copy's slot while the bundled server read its own, empty one (measured:
+ * `X_CONFIG_INVALID … no authenticator is configured` on a scaffold that installs one), and every
+ * registry split the same way. One instance is the one the app's imports reach.
+ */
+const LAUNCHER_EXTERNALS: readonly string[] = ['@ultimat3/*'];
+
+export function binaryArgs(
+  root: string,
+  out: string,
+  platform?: BinaryPlatform,
+): readonly string[] {
   return [
     'bun',
     'build',
     '--compile',
     '--minify',
+    ...(platform === undefined ? [] : ['--target', platform]),
     '--define',
     `${VERSION_DEFINE}=${JSON.stringify(frameworkVersion())}`,
+    '--compile-autoload-package-json',
+    '--compile-autoload-tsconfig',
+    ...LAUNCHER_EXTERNALS.flatMap((specifier) => ['--external', specifier]),
     ...externalArgs(),
     join(root, BUILD_ENTRY.binary),
     '--outfile',
@@ -140,10 +210,12 @@ export function argsFor(
     readonly out: string;
     /** The docker target's `BUILD_ID` build arg — `appManifest(root)`'s own. */
     readonly buildId?: string;
+    /** The binary target's `--platform`, as Bun's `--target`. */
+    readonly platform?: BinaryPlatform;
   },
 ): readonly string[] {
   if (target === 'docker') return dockerArgs(paths.root, paths.tag, paths.buildId ?? 'dev');
-  if (target === 'binary') return binaryArgs(paths.root, paths.out);
+  if (target === 'binary') return binaryArgs(paths.root, paths.out, paths.platform);
   return staticArgs(paths.root, paths.out);
 }
 
@@ -176,6 +248,8 @@ export function buildResult(input: {
   readonly command: readonly string[];
   readonly result: ExecResult;
   readonly report?: StaticReport;
+  /** The binary target's `--platform`, when one was named. */
+  readonly platform?: BinaryPlatform;
 }): CommandResult {
   const { report, result, target } = input;
   const output = result.ok ? '' : execOutput(result);
@@ -195,6 +269,7 @@ export function buildResult(input: {
         ],
     data: {
       target,
+      ...(input.platform === undefined ? {} : { platform: input.platform }),
       artifact: input.artifact,
       durationMs: result.durationMs,
       ...(result.ok ? {} : { output }),
@@ -209,14 +284,14 @@ export function buildResult(input: {
 }
 
 /** The flags only some targets read. */
-const TARGET_SCOPED_FLAGS = ['tag', 'out', 'preflight'] as const;
+const TARGET_SCOPED_FLAGS = ['tag', 'out', 'platform', 'preflight'] as const;
 
 type TargetScopedFlag = (typeof TARGET_SCOPED_FLAGS)[number];
 
 /** The optional flags each target READS. One table, so a flag cannot be read by one and ignored. */
 const TARGET_FLAGS: ReadonlyMap<BuildTarget, readonly TargetScopedFlag[]> = new Map([
   ['docker', ['tag', 'preflight']],
-  ['binary', ['out', 'preflight']],
+  ['binary', ['out', 'platform', 'preflight']],
   ['static', ['out', 'preflight']],
   // The boot reads ONE place, and the image holds no devDependencies to run a gate with.
   ['prebuilt', []],
@@ -226,6 +301,7 @@ const TARGET_FLAGS: ReadonlyMap<BuildTarget, readonly TargetScopedFlag[]> = new
 const UNREAD_BECAUSE: Readonly<Record<TargetScopedFlag, string>> = {
   tag: 'only the docker target tags an image',
   out: 'the docker target writes an image, not a path',
+  platform: 'only the binary target compiles an executable for a platform',
   preflight: 'the prebuilt target runs no gate: `x verify` ran before `docker build` was called',
 };
 
@@ -297,6 +373,7 @@ export const buildCommand: CliCommand = {
     // an agent spends on the wrong question.
     requireEntry(root, target);
     refuseUnreadFlags(ctx.args, target);
+    const platform = readPlatform(flagString(ctx.args, 'platform'));
     if (target === 'prebuilt') return buildPrebuilt(root);
 
     // Run static verify steps before building — unless the caller is a gate that runs the same
@@ -317,10 +394,11 @@ export const buildCommand: CliCommand = {
     // A relative `--out` is a path the caller typed from where they stand, so it resolves against
     // the cwd — against the root it landed somewhere else whenever `x build` ran from `apps/web`.
     const outFlag = flagString(ctx.args, 'out');
-    const out =
+    const requested =
       outFlag === undefined
         ? join(root, '.x', target === 'static' ? 'static' : 'app')
         : resolve(ctx.cwd, outFlag);
+    const out = target === 'binary' ? binaryOutfile(requested, platform) : requested;
     const tag = flagString(ctx.args, 'tag') ?? 'ultimate-app:dev';
     // The docker target stamps the manifest's build id into the image, so no role derives one at
     // boot. The island chunks and stylesheets are the image build's own (`--target prebuilt`).
@@ -331,6 +409,7 @@ export const buildCommand: CliCommand = {
       tag,
       out,
       ...(buildId === undefined ? {} : { buildId }),
+      ...(platform === undefined ? {} : { platform }),
     });
     // Removed BEFORE the builder runs, so a build that writes no inventory can never be reported
     // with the last one's: a stale emitted list is worse than none, because it reads as this run's.
@@ -346,6 +425,7 @@ export const buildCommand: CliCommand = {
       command,
       result,
       ...(report === undefined ? {} : { report }),
+      ...(platform === undefined ? {} : { platform }),
     });
   },
 };

package/src/cmd-db-backfill.ts

@@ -106,7 +106,7 @@ function readShape(args: ParsedArgs): { readonly shape: BackfillShape; readonly
     return refuseShape(
       second.startsWith('--') ? second.slice(2) : 'name',
       `x db backfill was asked for ${asked.join(' and ')} — one shape per invocation: --list, --pending, <name> or --all`,
-      `x db backfill ${asked[0]} --json`,
+      `x db backfill ${quoteArg(asked[0] ?? '--pending')} --json`,
     );
   }
   if (asked.length === 0) {

package/src/cmd-db-branch.ts

@@ -4,7 +4,7 @@
 // The facts (what a branch is, per mode) are `db-branch.ts`; the client lifetime is here.
 
 import { ERROR_DOCS_URL, nearestName } from '@ultimat3/core';
-import { createPostgresClient, type DbClient } from '@ultimat3/db';
+import { type DbClient, postgresClient } from '@ultimat3/db';
 import type { CommandContext } from './command';
 import type { BranchRow } from './db-branch';
 import {
@@ -62,7 +62,7 @@ function branchRetry(word: string, name: string | undefined): string {
  * database routinely outlives the 10s a `web` profile allows.
  */
 async function withBranchClient<T>(url: string, fn: (client: DbClient) => Promise<T>): Promise<T> {
-  const client = createPostgresClient({ url, role: 'migrate', applicationName: 'x-db-branch' });
+  const client = postgresClient({ url, role: 'migrate', applicationName: 'x-db-branch' });
   try {
     return await fn(client);
   } finally {

package/src/cmd-db.ts

@@ -255,8 +255,11 @@ async function runReset(ctx: CommandContext, root: string): Promise<CommandResul
   const services = resolveServices(root, ctx.env);
   if (services.db.mode === 'external') {
     throw new NotImplementedError({
-      cause: 'x db reset against an external Postgres is not implemented in this build',
-      fix: 'drop and recreate the database yourself, then run: x db migrate',
+      // The usual way here is an EXPORTED `DATABASE_URL` (an integration suite's Postgres) that
+      // the shell handed down, so the variable is named and the embedded reset is one command (#674).
+      cause:
+        'DATABASE_URL is set, so x db reset would target an external Postgres — dropping a database this process does not own is not implemented in this build',
+      fix: 'env -u DATABASE_URL x db reset   # resets the embedded database instead — or drop and recreate the external one yourself, then run: x db migrate',
     });
   }
   // A running `x dev` has this directory open: deleting it under a live embedded Postgres is data

package/src/cmd-deploy-compose.ts

@@ -0,0 +1,94 @@
+// The compose half of `x deploy`'s rollout: which serving roles can start their new container
+// BEFORE the old one stops, read off the app's compose file. The sequence that does it is
+// `cmd-deploy-start-first.ts`.
+// `docker compose up -d <role>` alone is stop-first — it recreates, so the role serves nothing
+// between the old container's exit and the new one's healthcheck.
+
+/**
+ * How one serving role is rolled. `start-first` is the zero-downtime path; `stop-first` is
+ * compose's own recreate, kept where two containers of the role cannot coexist, with the reason.
+ */
+export type ComposeStrategy =
+  | { readonly kind: 'start-first'; readonly replicas: number }
+  | { readonly kind: 'stop-first'; readonly why: string };
+
+/** The roles that stay up serving — the only ones a rollout strategy applies to. */
+export const SERVING_ROLES = ['web', 'sync', 'worker', 'scheduler'] as const;
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === 'object' && value !== null && !Array.isArray(value);
+
+/**
+ * The host port a `ports:` entry pins, or `undefined` when Docker picks it. Short syntax is
+ * `[[ip:]host:]container[/proto]`: `3000` and `127.0.0.1::3000` leave the host side to Docker, so
+ * any number of replicas bind; `3000:3000` and `127.0.0.1:3001:3001` name one, which exactly one
+ * container can hold. Long syntax pins one when `published` is said.
+ */
+function pinnedHostPort(entry: unknown): string | undefined {
+  if (isRecord(entry)) {
+    const published = entry['published'];
+    return published === undefined || published === null || published === ''
+      ? undefined
+      : String(published);
+  }
+  if (typeof entry !== 'string') return undefined;
+  const spec = entry.replace(/^\[[^\]]*\]:/, '').split('/')[0] ?? '';
+  const parts = spec.split(':');
+  const host = parts.length === 3 ? parts[1] : parts.length === 2 ? parts[0] : '';
+  return host === undefined || host === '' ? undefined : host;
+}
+
+/** One service's strategy: start-first unless something in its definition forbids a second copy. */
+function strategyOf(service: unknown): ComposeStrategy {
+  if (!isRecord(service)) {
+    return { kind: 'stop-first', why: 'the compose file declares no such service' };
+  }
+  if (service['container_name'] !== undefined) {
+    return {
+      kind: 'stop-first',
+      why: 'container_name names exactly one container, so a second replica cannot be created',
+    };
+  }
+  if (service['network_mode'] === 'host') {
+    return {
+      kind: 'stop-first',
+      why: 'network_mode: host binds the host ports directly, so two replicas collide',
+    };
+  }
+  const ports = Array.isArray(service['ports']) ? service['ports'] : [];
+  const pinned = ports.map(pinnedHostPort).find((port) => port !== undefined);
+  if (pinned !== undefined) {
+    return {
+      kind: 'stop-first',
+      why: `publishes host port ${pinned}, which one container can bind — remove ports: and route through a proxy on the compose network to roll it start-first`,
+    };
+  }
+  const deploy = service['deploy'];
+  const declared = isRecord(deploy) ? deploy['replicas'] : undefined;
+  const replicas =
+    typeof declared === 'number' && Number.isSafeInteger(declared) && declared >= 0 ? declared : 1;
+  return { kind: 'start-first', replicas };
+}
+
+/**
+ * Every serving role's strategy, from the compose file's text. A file that is absent or does not
+ * parse decides nothing: every role keeps stop-first, and compose reports the file's own error
+ * when the step runs — this reader is never a second validator of it.
+ */
+export function composeStrategies(text: string | undefined): ReadonlyMap<string, ComposeStrategy> {
+  let parsed: unknown;
+  try {
+    parsed = text === undefined ? undefined : Bun.YAML.parse(text);
+  } catch {
+    parsed = undefined;
+  }
+  const services = isRecord(parsed) ? parsed['services'] : undefined;
+  if (!isRecord(services)) return new Map();
+  return new Map(SERVING_ROLES.map((role) => [role, strategyOf(services[role])]));
+}
+
+/** The compose file at `path`, or `undefined` when there is none to read. */
+export async function readComposeFile(path: string): Promise<string | undefined> {
+  const file = Bun.file(path);
+  return (await file.exists()) ? file.text() : undefined;
+}

package/src/cmd-deploy-drain.ts

@@ -0,0 +1,48 @@
+// What a helm deploy tells the chart about the drain: `app.config.ts`'s `drain` section — the ONE
+// drain budget since 25.0.0 deleted `http.drainTimeoutMs` — in the seconds `docker/helm` derives
+// each role's terminationGracePeriodSeconds from. Passed on every upgrade, so a raised
+// budget can never ship against a grace period sized for 25 s.
+
+import type { DrainConfig } from '@ultimat3/core';
+import { isUltimateError, READINESS_GRACE_DEFAULT_MS } from '@ultimat3/core';
+import { loadDrainConfig } from './serve-drain';
+
+const seconds = (ms: number): number => Math.ceil(ms / 1000);
+
+/**
+ * The `--set` pairs, or none when the app declares no budget (the chart's defaults are core's).
+ * The chart's `drain.deadlineSeconds` is ONE budget for every role, and so is `drain.deadlineMs`:
+ * the web role drains on it too. The margins stay the chart's.
+ *
+ * The grace is budgeted at no less than core's production default: the value read here is resolved
+ * in THIS process's environment, and an `x deploy` run under `NODE_ENV=development` resolves the
+ * grace to 0 while the pod it deploys runs with 5 s. Over-budgeting costs a pod nothing — it exits
+ * when its drain does — and under-budgeting is a SIGKILL mid-drain.
+ */
+export function helmDrainOverrides(drain: Partial<DrainConfig> | undefined): readonly string[] {
+  const out: string[] = [];
+  if (drain?.deadlineMs !== undefined) {
+    out.push('--set', `drain.deadlineSeconds=${seconds(drain.deadlineMs)}`);
+  }
+  if (drain === undefined) return out;
+  const grace = Math.max(drain.readinessGraceMs ?? 0, READINESS_GRACE_DEFAULT_MS);
+  out.push('--set', `drain.readinessGraceSeconds=${seconds(grace)}`);
+  return out;
+}
+
+/**
+ * The overrides for the app at `root`, off the one config loader. An `app.config.ts` the loader
+ * refuses yields no drain section, and the chart's defaults stand: `X_CONFIG_INVALID`'s own fix for
+ * a helm deploy is `--release <name>`, which must stay a way out, and the pods this upgrade starts
+ * refuse the same config at boot — which `helm upgrade --wait` reports as the failed rollout it is.
+ */
+export async function readHelmDrainOverrides(root: string): Promise<readonly string[]> {
+  let drain: Partial<DrainConfig> | undefined;
+  try {
+    drain = await loadDrainConfig(root);
+  } catch (error) {
+    if (!isUltimateError(error) || error.code !== 'X_CONFIG_INVALID') throw error;
+    return [];
+  }
+  return helmDrainOverrides(drain);
+}

package/src/cmd-deploy-start-first.ts

@@ -0,0 +1,178 @@
+// One compose role rolled start-first: the new containers come up beside the old ones, are waited
+// on until healthy, and only then are the old ones stopped and removed. Converges on the declared
+// replica count, so a rerun after a half-finished deploy finishes it rather than adding replicas.
+
+import type { ExecOptions, Runner } from './exec';
+
+/**
+ * Bounds `--wait`: a new container that never reports healthy fails the step rather than holding
+ * the deploy open. Five minutes is past any `start_period` the scaffold declares (30s) plus a cold
+ * boot, and `--wait` already returns at once when a container turns `unhealthy` or exits.
+ */
+export const COMPOSE_WAIT_TIMEOUT_SECONDS = 300;
+
+/** The label compose stamps on each container: the hash of the service definition it was made from. */
+const CONFIG_HASH = 'com.docker.compose.config-hash';
+const INSPECT_FORMAT = `{{.Id}} {{.Created}} {{index .Config.Labels "${CONFIG_HASH}"}}`;
+
+/** The `up` that starts the new replicas beside the running ones; `--no-recreate` keeps the old. */
+export function startFirstUp(
+  base: readonly string[],
+  role: string,
+  scale: number | undefined,
+): readonly string[] {
+  const wait = ['--wait', '--wait-timeout', String(COMPOSE_WAIT_TIMEOUT_SECONDS)];
+  return scale === undefined
+    ? [...base, 'up', '-d', '--no-deps', ...wait, role]
+    : [
+        ...base,
+        'up',
+        '-d',
+        '--no-deps',
+        '--no-recreate',
+        ...wait,
+        '--scale',
+        `${role}=${scale}`,
+        role,
+      ];
+}
+
+interface Failed {
+  readonly code: number;
+  readonly command: readonly string[];
+}
+
+export type StartFirstOutcome =
+  | { readonly ok: true }
+  | ({
+      readonly ok: false;
+      /**
+       * The cleanup of an unhealthy new replica that ALSO failed — its argv names the containers
+       * still running, which the operator now owns. Absent when the cleanup ran clean.
+       */
+      readonly cleanup?: Failed | undefined;
+    } & Failed);
+
+const linesOf = (stdout: string): readonly string[] =>
+  stdout
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line.length > 0);
+
+/** `docker compose config --hash <role>` prints `<role> <hash>`. */
+const hashOf = (stdout: string): string | undefined => linesOf(stdout)[0]?.split(/\s+/)[1];
+
+/**
+ * The running containers split by the definition they were made from: `current` already runs the
+ * one being deployed (newest first), `stale` anything else. A container `inspect` did not describe
+ * is stale — stopping one too many is a restart, keeping one too many is the old build serving.
+ */
+function partition(
+  running: readonly string[],
+  inspected: string,
+  want: string,
+): { readonly current: readonly string[]; readonly stale: readonly string[] } {
+  const described = new Map(
+    linesOf(inspected).map((line) => {
+      const [id = '', created = '', hash = ''] = line.split(/\s+/);
+      return [id, { created, hash }] as const;
+    }),
+  );
+  const facts = running.map((id) => {
+    const match = [...described].find(([full]) => full === id || full.startsWith(id));
+    return { id, created: match?.[1].created ?? '', hash: match?.[1].hash };
+  });
+  return {
+    current: facts
+      .filter((fact) => fact.hash === want)
+      .sort((a, b) => (a.created < b.created ? 1 : -1))
+      .map((fact) => fact.id),
+    stale: facts.filter((fact) => fact.hash !== want).map((fact) => fact.id),
+  };
+}
+
+/**
+ * Start-first for one role. The containers running now are listed and split by compose's config
+ * hash against the definition being deployed: those already on it are kept (up to `replicas`,
+ * newest first), and only the shortfall is created BESIDE the rest (`--scale running + shortfall
+ * --no-recreate`, so compose builds the extra containers from the new definition and leaves the
+ * others alone). Everything is waited on until its healthcheck — `/readyz` for web and sync, the
+ * scrape port for the others — answers. Only then are the stale containers and any surplus stopped
+ * (SIGTERM, then the `stop_grace_period` compose stamped on each as its stop timeout) and removed.
+ *
+ * Counting the shortfall rather than `running + replicas` is what makes a rerun safe: after a
+ * deploy that brought the new containers up and then failed to stop the old, running it again
+ * creates nothing and finishes the stop.
+ *
+ * A new replica that never turns healthy is stopped and removed, and the old ones keep serving;
+ * a cleanup that fails is reported beside the failure, naming the containers left running. The
+ * same sequence `docker-rollout` performs, because `deploy.update_config.order: start-first` is a
+ * Swarm field `docker compose` ignores.
+ */
+export async function runStartFirst(step: {
+  readonly role: string;
+  readonly base: readonly string[];
+  readonly replicas: number;
+  readonly runner: Runner;
+  readonly options: ExecOptions;
+}): Promise<StartFirstOutcome> {
+  const { role, base, replicas, runner, options } = step;
+  const run = async (command: readonly string[]) => {
+    const result = await runner(command, options);
+    return { result, failed: { ok: false as const, code: result.code, command } };
+  };
+  const listing = await run([...base, 'ps', '-q', role]);
+  if (!listing.result.ok) return listing.failed;
+  const running = linesOf(listing.result.stdout);
+  if (running.length === 0) {
+    const fresh = await run(startFirstUp(base, role, undefined));
+    return fresh.result.ok ? { ok: true } : fresh.failed;
+  }
+  const config = await run([...base, 'config', '--hash', role]);
+  if (!config.result.ok) return config.failed;
+  const inspect = await run(['docker', 'inspect', '--format', INSPECT_FORMAT, ...running]);
+  if (!inspect.result.ok) return inspect.failed;
+  const { current, stale } = partition(
+    running,
+    inspect.result.stdout,
+    hashOf(config.result.stdout) ?? '',
+  );
+  const shortfall = Math.max(0, replicas - current.length);
+  const up = await run(startFirstUp(base, role, running.length + shortfall));
+  if (!up.result.ok) return { ...up.failed, ...(await removeCreated(step, running)) };
+  const retire = [...stale, ...current.slice(replicas)];
+  if (retire.length === 0) return { ok: true };
+  const stopped = await run(['docker', 'stop', ...retire]);
+  if (!stopped.result.ok) return stopped.failed;
+  const removed = await run(['docker', 'rm', ...retire]);
+  return removed.result.ok ? { ok: true } : removed.failed;
+}
+
+/**
+ * After a failed `up`: stop and remove what it created, so the containers that were serving are
+ * the role again. A listing that fails leaves the new replicas to the operator; a stop or rm that
+ * fails is returned as `cleanup`, its argv naming the containers still there.
+ */
+async function removeCreated(
+  step: {
+    readonly role: string;
+    readonly base: readonly string[];
+    readonly runner: Runner;
+    readonly options: ExecOptions;
+  },
+  running: readonly string[],
+): Promise<{ readonly cleanup?: Failed }> {
+  const { role, base, runner, options } = step;
+  const after = await runner([...base, 'ps', '-q', role], options);
+  if (!after.ok) return { cleanup: { code: after.code, command: [...base, 'ps', '-q', role] } };
+  const created = linesOf(after.stdout).filter((id) => !running.includes(id));
+  if (created.length === 0) return {};
+  for (const command of [
+    ['docker', 'stop', ...created],
+    ['docker', 'rm', ...created],
+  ]) {
+    const result = await runner(command, options);
+    if (!result.ok) return { cleanup: { code: result.code, command } };
+  }
+  return {};
+}