deepline 0.3.89 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts

@@ -0,0 +1,1844 @@
+/**
+ * fly-absurd-lane — operate the Absurd runtime worker as immutable, per-release
+ * lanes on Fly Machines.
+ *
+ * A "lane" is one release of the Absurd worker fleet. Every release gets a
+ * stable slug `releaseId`, stamped into each Machine environment and
+ * `config.metadata.release_id`. The worker polls
+ * ONLY its own lane's queue (`runs_<releaseId>`; see the runtime lane contract). New work goes to the new lane; the old lane
+ * drains and self-exits. This is how we ship a breaking shared-plane change
+ * without co-mingling two runtime versions on one queue.
+ *
+ * The verbs:
+ *   push-image  build + push the lane image to the Fly registry (via flyctl).
+ *   plan-image  report whether a lane's registered image still exists, or must
+ *               be rebuilt from its recorded revision before a rollback boots it.
+ *   boot        create K Machines for a release and wait until they pass checks.
+ *   list        show machines for a release (or all lanes).
+ *   drain       flip a lane's machines to DEEPLINE_ABSURD_DRAIN_EXIT=1 so they
+ *               finish in-flight work then exit(0).
+ *   retire      delete a drained/stopped lane's machines (after the queue-empty
+ *               admin gate, or --force).
+ *
+ * Fly Machines API mechanics (cited inline where non-obvious):
+ *   https://fly.io/docs/machines/api/  — base https://api.machines.dev/v1
+ *   https://fly.io/docs/machines/api/machines-resource/
+ * Dependency-free: `fetch` + `node:child_process` only. FLY_API_TOKEN is a
+ * secret and correctly lives in the environment.
+ */
+import { spawnSync } from 'node:child_process';
+import { appendFileSync } from 'node:fs';
+import { RUNTIME_CAPACITY_POLICY } from '../runtime-capacity-policy';
+import { ABSURD_RELEASE_ID_RE } from '../runtime-scheduler-topology';
+import {
+  authenticateFlyRegistry,
+  runtimeImageManifestExists,
+} from './fly-registry';
+
+export const FLY_MACHINES_API_BASE = 'https://api.machines.dev/v1';
+export const FLY_REGISTRY_HOST = 'registry.fly.io';
+
+/** The one release-id shape. See `ABSURD_RELEASE_ID_RE` for why it is shared. */
+export const RELEASE_ID_RE = ABSURD_RELEASE_ID_RE;
+/** Git SHA used for the image label `git-<sha>`. Full or abbreviated hex. */
+export const GIT_SHA_RE = /^[0-9a-f]{7,40}$/;
+
+export const DEFAULT_WORKER_APP = 'deepline-absurd-worker-prod';
+export const DOCKERFILE_ABSURD = 'deploy/fly/runtime-worker/Dockerfile.absurd';
+export const ABSURD_WORKER_CONFIG =
+  'deploy/fly/runtime-worker/absurd-worker.toml';
+
+export const RELEASE_METADATA_KEY = 'release_id';
+/** Content-addressed Vercel ↔ Fly runtime boundary, generated at build time. */
+export const RUNTIME_SURFACE_METADATA_KEY = 'runtime_surface_sha256';
+export const ROLLBACK_HOLD_UNTIL_METADATA_KEY = 'rollback_hold_until_ms';
+export const RUNTIME_DEPLOY_VERSION_ENV = 'DEEPLINE_RUNTIME_DEPLOY_VERSION';
+export const DRAIN_EXIT_ENV = 'DEEPLINE_ABSURD_DRAIN_EXIT';
+// This is an image compatibility contract, not the current source location.
+// Historical images only have this path; new images retain it as a thin entrypoint
+// so both old and new scalers can boot, drain, or recover either generation.
+export const ABSURD_RUNTIME_WORKER_EXEC = [
+  'bun',
+  'scripts/run-absurd-runtime-worker.ts',
+  '--slots',
+  String(RUNTIME_CAPACITY_POLICY.absurd.workerSlotsPerMachine),
+] as const;
+
+/** Default health/boot poll ceilings. Loud timeout, never a silent give-up. */
+export const DEFAULT_BOOT_TIMEOUT_MS = 300_000;
+export const DEFAULT_BOOT_POLL_INTERVAL_MS = 5_000;
+/**
+ * A Machines config update is accepted before a later list necessarily shows
+ * the new version.  Drain is a retirement precondition, so callers must not
+ * race that propagation and mistake the old, still-serving config for a
+ * failed drain.
+ */
+export const DEFAULT_DRAIN_CONVERGENCE_TIMEOUT_MS = 60_000;
+export const DEFAULT_DRAIN_CONVERGENCE_POLL_INTERVAL_MS = 2_000;
+/** Keep workers beside the receipt gateway and scaler in Northern Virginia. */
+export const DEFAULT_ABSURD_FLY_REGION = 'iad';
+export const DEFAULT_IMAGE_PROPAGATION_RETRY_DELAYS_MS = [
+  1_000, 2_000, 4_000, 8_000, 16_000,
+] as const;
+/**
+ * A rate-limited read is safe to repeat. Mutating requests intentionally do
+ * not use this policy: their caller must re-read exact provider state before
+ * deciding whether a timed-out effect needs recovery.
+ */
+export const FLY_READ_RATE_LIMIT_RETRY_DELAYS_MS = [
+  1_000, 2_000, 4_000,
+] as const;
+/** A provider request may not hold the production release fence forever. */
+export const FLY_MACHINES_REQUEST_TIMEOUT_MS = 30_000;
+
+export type FetchLike = (
+  input: string,
+  init?: {
+    method?: string;
+    headers?: Record<string, string>;
+    body?: string;
+    signal?: AbortSignal;
+  },
+) => Promise<{
+  ok: boolean;
+  status: number;
+  text(): Promise<string>;
+  json(): Promise<unknown>;
+  headers?: { get(name: string): string | null };
+}>;
+
+/**
+ * Read a successful response body as JSON, keeping three outcomes distinct.
+ *
+ * - **Absent.** A body with no bytes (or only whitespace) is `null`. An
+ *   endpoint with nothing to say is a legitimate answer, not a failure, and
+ *   "this lane has no machines" must never surface as a parse exception.
+ * - **Parsed.** A body that parses is returned as-is.
+ * - **Malformed.** A body that has bytes and does not parse stays loud, but it
+ *   names the request. `Response.json()` raises a bare `Failed to parse JSON`
+ *   which identifies neither the endpoint, the status, nor the payload; a
+ *   production release once failed on exactly that line with nothing to
+ *   diagnose. Never soften this branch: a non-JSON answer from an endpoint
+ *   whose contract is JSON (an HTML proxy interstitial, a truncated body) is a
+ *   real defect and must not be read as "nothing here".
+ */
+export async function readJsonResponseBody(input: {
+  response: { status: number; text(): Promise<string> };
+  method: string;
+  url: string;
+}): Promise<unknown> {
+  const raw = await input.response.text();
+  if (raw.trim() === '') return null;
+  try {
+    return JSON.parse(raw) as unknown;
+  } catch (error) {
+    throw new Error(
+      `${input.method} ${input.url} returned HTTP ${input.response.status} with a ` +
+        `non-JSON body (${raw.length} bytes): ${raw.slice(0, 300)}` +
+        ` [${error instanceof Error ? error.message : String(error)}]`,
+    );
+  }
+}
+
+/**
+ * Headers for a call to the release-owning application. `x-vercel-protection-bypass`
+ * is not optional decoration: a Vercel deployment URL under Deployment
+ * Protection answers an unbypassed request with an HTML challenge instead of
+ * the route, which is precisely a non-JSON body on a JSON contract.
+ */
+export function adminRequestHeaders(input: {
+  adminToken: string;
+  protectionBypassToken?: string;
+  contentType?: boolean;
+}): Record<string, string> {
+  return {
+    ...(input.contentType ? { 'content-type': 'application/json' } : {}),
+    // Send BOTH auth forms. The deployed internal-token route accepts
+    // `x-deepline-internal-token` and 401s a plain `Authorization: Bearer`;
+    // local and newer routes accept either. Header form is primary.
+    'x-deepline-internal-token': input.adminToken,
+    authorization: `Bearer ${input.adminToken}`,
+    ...(input.protectionBypassToken
+      ? { 'x-vercel-protection-bypass': input.protectionBypassToken }
+      : {}),
+  };
+}
+
+export type FlyMachineCheck = {
+  name?: string;
+  status?: string; // 'passing' | 'warning' | 'critical'
+};
+
+export type FlyMachine = {
+  id: string;
+  name?: string;
+  state?: string; // 'started' | 'stopped' | 'created' | ...
+  /** Fly's machine cordon flag: a cordoned machine is excluded from request
+   * routing. roll-execution-relay drains by cordoning the old generation and
+   * compares this strictly (`!== false` / `!== true`), so absence reads as
+   * "unknown", never as either verdict. */
+  cordoned?: boolean;
+  /** Fly's immutable Machine creation timestamp. Used to distinguish a live
+   * pre-registration deploy candidate from an abandoned candidate. */
+  created_at?: string;
+  /** Provider lifecycle history. Gateway retirement uses the most recent
+   * routing event as an explicit legacy-drain fence. */
+  events?: Array<{ type?: string; timestamp?: number }>;
+  region?: string;
+  config?: {
+    image?: string;
+    metadata?: Record<string, string>;
+    env?: Record<string, string>;
+    init?: {
+      exec?: string[];
+    };
+  };
+  checks?: FlyMachineCheck[];
+};
+
+// ---------------------------------------------------------------------------
+// Arg parsing (pure, unit-tested).
+// ---------------------------------------------------------------------------
+
+export type ParsedArgs = {
+  command: string;
+  flags: Record<string, string | boolean>;
+};
+
+/**
+ * Minimal flag parser. `--flag value` and `--flag=value` set a string;
+ * `--flag` with no following value (or before another `--flag`) is a boolean.
+ * No positional args beyond the leading command.
+ */
+export function parseArgs(argv: readonly string[]): ParsedArgs {
+  const command = argv[0] ?? '';
+  const flags: Record<string, string | boolean> = {};
+  for (let index = 1; index < argv.length; index += 1) {
+    const token = argv[index]!;
+    if (!token.startsWith('--')) {
+      throw new Error(`Unexpected positional argument: ${token}`);
+    }
+    const eq = token.indexOf('=');
+    if (eq !== -1) {
+      flags[token.slice(2, eq)] = token.slice(eq + 1);
+      continue;
+    }
+    const key = token.slice(2);
+    const next = argv[index + 1];
+    if (next === undefined || next.startsWith('--')) {
+      flags[key] = true;
+    } else {
+      flags[key] = next;
+      index += 1;
+    }
+  }
+  return { command, flags };
+}
+
+export function requireStringFlag(
+  flags: Record<string, string | boolean>,
+  name: string,
+): string {
+  const value = flags[name];
+  if (typeof value !== 'string' || value.trim() === '') {
+    throw new Error(`--${name} is required and must be a non-empty string.`);
+  }
+  return value.trim();
+}
+
+export function requireReleaseId(
+  flags: Record<string, string | boolean>,
+): string {
+  const releaseId = requireStringFlag(flags, 'release');
+  if (!RELEASE_ID_RE.test(releaseId)) {
+    throw new Error(
+      `--release "${releaseId}" is not a valid lane id (${RELEASE_ID_RE}). ` +
+        'Use a stable lowercase slug without repeated or trailing hyphens; it becomes the queue suffix runs_<release>.',
+    );
+  }
+  if (
+    (releaseId === 'dev' || releaseId.startsWith('dev-')) &&
+    flags.environment !== 'preview'
+  ) {
+    throw new Error(
+      `--release "${releaseId}" uses the dev/dev-* namespace reserved for preview Absurd lanes; pass --environment preview only for an isolated Preview worker app.`,
+    );
+  }
+  return releaseId;
+}
+
+export function requireSha(flags: Record<string, string | boolean>): string {
+  const sha = requireStringFlag(flags, 'sha').toLowerCase();
+  if (!GIT_SHA_RE.test(sha)) {
+    throw new Error(`--sha "${sha}" is not a git sha (${GIT_SHA_RE}).`);
+  }
+  return sha;
+}
+
+export function requireCount(flags: Record<string, string | boolean>): number {
+  const raw = flags.count;
+  const count = typeof raw === 'string' ? Number(raw) : NaN;
+  if (!Number.isInteger(count) || count < 1 || count > 64) {
+    throw new Error('--count must be an integer between 1 and 64.');
+  }
+  return count;
+}
+
+export function resolveApp(flags: Record<string, string | boolean>): string {
+  const app = flags.app;
+  return typeof app === 'string' && app.trim()
+    ? app.trim()
+    : DEFAULT_WORKER_APP;
+}
+
+// ---------------------------------------------------------------------------
+// Request shaping (pure, unit-tested).
+// ---------------------------------------------------------------------------
+
+export function laneImageRef(app: string, sha: string): string {
+  return `${FLY_REGISTRY_HOST}/${app}:git-${sha}`;
+}
+
+export function laneQueueName(releaseId: string): string {
+  return `runs_${releaseId}`;
+}
+
+/**
+ * Machine create config for a lane worker.
+ * https://fly.io/docs/machines/api/machines-resource/#create-a-machine
+ * - `metadata.release_id` is the lane filter used by `list`/`drain`/`retire`.
+ * - `env.DEEPLINE_RUNTIME_DEPLOY_VERSION` is the worker's runtime identity; it
+ *   also derives the polled queue (runs_<releaseId>), so this per-Machine env
+ *   is the only release identity and makes a source image safely reusable.
+ * - restart.policy = 'on-failure': a crash-looped machine is retried, not left
+ *   silently stopped. (Prod steady-state config uses 'always' in
+ *   absurd-worker.toml; per-lane boots want on-failure so a permanently broken
+ *   image surfaces as a failed boot rather than an infinite restart.)
+ */
+export function buildBootMachineConfig(input: {
+  app: string;
+  releaseId: string;
+  sha: string;
+  runtimeSurfaceHash?: string;
+  region?: string;
+  rollbackHoldUntilMs?: number;
+}): Record<string, unknown> {
+  return {
+    region: input.region ?? DEFAULT_ABSURD_FLY_REGION,
+    config: {
+      image: laneImageRef(input.app, input.sha),
+      metadata: {
+        [RELEASE_METADATA_KEY]: input.releaseId,
+        ...(input.runtimeSurfaceHash
+          ? { [RUNTIME_SURFACE_METADATA_KEY]: input.runtimeSurfaceHash }
+          : {}),
+        ...(input.rollbackHoldUntilMs
+          ? {
+              [ROLLBACK_HOLD_UNTIL_METADATA_KEY]: String(
+                input.rollbackHoldUntilMs,
+              ),
+            }
+          : {}),
+      },
+      env: {
+        [RUNTIME_DEPLOY_VERSION_ENV]: input.releaseId,
+      },
+      // Override the image CMD so rollback and recovery of an older image still
+      // honor today's fleet-wide slot ceiling. Every historical Absurd worker
+      // supports --slots even when its baked default predates this policy.
+      init: {
+        exec: [...ABSURD_RUNTIME_WORKER_EXEC],
+      },
+      restart: {
+        policy: 'on-failure',
+        max_retries: 3,
+      },
+      checks: {
+        worker_alive: {
+          type: 'http',
+          port: 8787,
+          path: '/healthz',
+          interval: '15s',
+          timeout: '5s',
+          grace_period: '30s',
+        },
+      },
+      guest: {
+        cpu_kind: 'shared',
+        cpus: 1,
+        memory_mb: 1024,
+      },
+    },
+  };
+}
+
+/**
+ * A machine belongs to a lane iff its metadata.release_id matches. The Machines
+ * list endpoint has no server-side metadata filter, so we list-then-filter
+ * (https://fly.io/docs/machines/api/machines-resource/#list-machines).
+ */
+export function machineIsInLane(
+  machine: FlyMachine,
+  releaseId: string,
+): boolean {
+  return machine.config?.metadata?.[RELEASE_METADATA_KEY] === releaseId;
+}
+
+export function machineChecksPassing(machine: FlyMachine): boolean {
+  const checks = machine.checks ?? [];
+  if (checks.length === 0) return false;
+  return checks.every((check) => check.status === 'passing');
+}
+
+export function machineIsStarted(machine: FlyMachine): boolean {
+  return machine.state === 'started';
+}
+
+/**
+ * Boot health: a machine is healthy iff it is `started` AND — only if it declares
+ * checks — all of them pass. Machines created through the Machines API do NOT
+ * inherit the app's `[checks]` block, so a healthy lane machine often reports zero
+ * checks; requiring non-empty passing checks (`machineChecksPassing`) would poll to
+ * the boot timeout even for a fully-healthy machine. An empty checks array with a
+ * started state therefore counts as healthy. Injecting a checks block into the
+ * create body was rejected: it couples this script to the worker's healthz shape.
+ */
+export function machineIsHealthyForBoot(machine: FlyMachine): boolean {
+  if (!machineIsStarted(machine)) return false;
+  const checks = machine.checks ?? [];
+  if (checks.length === 0) return true;
+  return checks.every((check) => check.status === 'passing');
+}
+
+/**
+ * A lane is safe to retire when every one of its machines is either already
+ * stopped or explicitly draining (DRAIN_EXIT env set). A `started` machine
+ * without the drain flag means live traffic — refuse to delete it.
+ */
+export function machineIsRetirable(machine: FlyMachine): boolean {
+  if (machine.state === 'stopped' || machine.state === 'destroyed') return true;
+  return machine.config?.env?.[DRAIN_EXIT_ENV] === '1';
+}
+
+/**
+ * In-place env update. The Fly Machines update endpoint REPLACES the machine
+ * config and creates a new machine *version* in place (same id) — it does NOT
+ * require destroy+recreate. We therefore read the machine, merge the drain env
+ * into its existing config, and PUT the whole config back.
+ * https://fly.io/docs/machines/api/machines-resource/#update-a-machine
+ */
+export function buildDrainUpdateConfig(machine: FlyMachine): {
+  config: Record<string, unknown>;
+  skip_launch?: boolean;
+} {
+  const capacityUpdate = buildWorkerSlotCapUpdateConfig(machine);
+  return {
+    ...capacityUpdate,
+    config: {
+      ...capacityUpdate.config,
+      env: {
+        ...(machine.config?.env ?? {}),
+        [DRAIN_EXIT_ENV]: '1',
+      },
+    },
+  };
+}
+
+export function buildRollbackHoldUpdateConfig(
+  machine: FlyMachine,
+  rollbackHoldUntilMs: number,
+): { config: Record<string, unknown>; skip_launch?: boolean } {
+  const capacityUpdate = buildWorkerSlotCapUpdateConfig(machine);
+  return {
+    ...capacityUpdate,
+    config: {
+      ...capacityUpdate.config,
+      // Rollback reactivates this lane. Remove drain mode on every reused
+      // Machine so the fixed-capacity pair is the only serving fleet.
+      env: Object.fromEntries(
+        Object.entries(machine.config?.env ?? {}).filter(
+          ([key]) => key !== DRAIN_EXIT_ENV,
+        ),
+      ),
+      metadata: {
+        ...(machine.config?.metadata ?? {}),
+        [ROLLBACK_HOLD_UNTIL_METADATA_KEY]: String(rollbackHoldUntilMs),
+      },
+    },
+  };
+}
+
+export function buildWorkerSlotCapUpdateConfig(machine: FlyMachine): {
+  config: Record<string, unknown>;
+  skip_launch?: boolean;
+} {
+  const existing = machine.config ?? {};
+  return {
+    // Fly launches a new Machine version after update by default. Preserve a
+    // stopped machine's state so its caller can issue exactly one explicit start.
+    ...(machine.state === 'stopped' ? { skip_launch: true } : {}),
+    config: {
+      ...existing,
+      // Historical images support this CLI override even when their baked
+      // default predates the fixed capacity policy.
+      init: {
+        ...(existing.init ?? {}),
+        exec: [...ABSURD_RUNTIME_WORKER_EXEC],
+      },
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Fly Machines API client (thin; injectable fetch for tests).
+// ---------------------------------------------------------------------------
+
+export function requireFlyToken(
+  env: Readonly<Record<string, string | undefined>> = process.env,
+): string {
+  const token = env.FLY_API_TOKEN?.trim();
+  if (!token) {
+    throw new Error('FLY_API_TOKEN is required for the Fly Machines API.');
+  }
+  return token;
+}
+
+export class FlyMachinesClient {
+  constructor(
+    private readonly app: string,
+    private readonly token: string,
+    private readonly fetchImpl: FetchLike,
+    private readonly rateLimitOptions: Readonly<{
+      wait?: (ms: number) => Promise<void>;
+      retryDelaysMs?: readonly number[];
+      requestTimeoutMs?: number;
+    }> = {},
+  ) {}
+
+  private async request(
+    method: string,
+    path: string,
+    body?: unknown,
+  ): Promise<unknown> {
+    const retryDelaysMs =
+      this.rateLimitOptions.retryDelaysMs ??
+      FLY_READ_RATE_LIMIT_RETRY_DELAYS_MS;
+    const wait = this.rateLimitOptions.wait ?? sleep;
+    for (let attempt = 0; ; attempt += 1) {
+      const requestTimeoutMs =
+        this.rateLimitOptions.requestTimeoutMs ??
+        FLY_MACHINES_REQUEST_TIMEOUT_MS;
+      const signal = AbortSignal.timeout(requestTimeoutMs);
+      let response: Awaited<ReturnType<FetchLike>>;
+      try {
+        response = await this.fetchImpl(
+          `${FLY_MACHINES_API_BASE}/apps/${this.app}${path}`,
+          {
+            method,
+            headers: {
+              authorization: `Bearer ${this.token}`,
+              'content-type': 'application/json',
+            },
+            body: body === undefined ? undefined : JSON.stringify(body),
+            signal,
+          },
+        );
+      } catch (error) {
+        if (signal.aborted) {
+          throw new Error(
+            `Fly Machines API ${method} ${path} timed out after ${requestTimeoutMs}ms.`,
+            { cause: error },
+          );
+        }
+        throw error;
+      }
+      if (
+        method === 'GET' &&
+        response.status === 429 &&
+        attempt < retryDelaysMs.length
+      ) {
+        const retryAfter = response.headers?.get('retry-after');
+        const retryAfterSeconds =
+          typeof retryAfter === 'string' && retryAfter.trim() !== ''
+            ? Number(retryAfter)
+            : Number.NaN;
+        const delayMs = Number.isFinite(retryAfterSeconds)
+          ? Math.max(0, retryAfterSeconds * 1_000)
+          : retryDelaysMs[attempt]!;
+        console.info(
+          `[fly-absurd-lane] Fly Machines API GET ${path} was rate limited; retrying in ${delayMs}ms ` +
+            `(attempt ${attempt + 2}/${retryDelaysMs.length + 1}).`,
+        );
+        await wait(delayMs);
+        continue;
+      }
+      if (!response.ok && method === 'DELETE' && response.status === 404) {
+        return null;
+      }
+      if (!response.ok) {
+        const detail = await response.text().catch(() => '');
+        throw new Error(
+          `Fly Machines API ${method} ${path} failed: ${response.status} ${detail}`,
+        );
+      }
+      if (response.status === 204) return null;
+      return await readJsonResponseBody({
+        response,
+        method,
+        url: `${FLY_MACHINES_API_BASE}/apps/${this.app}${path}`,
+      });
+    }
+  }
+
+  async listMachines(): Promise<FlyMachine[]> {
+    const result = await this.request('GET', '/machines');
+    // An absent body is the Machines API saying "no machines"; that is a
+    // successful empty inventory. Any other non-array shape is a contract
+    // break and must stay loud rather than masquerade as an empty app, which
+    // would let a caller conclude a lane is already gone and skip its work.
+    if (result === null || result === undefined) return [];
+    if (!Array.isArray(result)) {
+      throw new Error(
+        `Fly Machines API GET /apps/${this.app}/machines returned ${typeof result}, expected an array of machines.`,
+      );
+    }
+    return result as FlyMachine[];
+  }
+
+  async getMachine(id: string): Promise<FlyMachine> {
+    return (await this.request('GET', `/machines/${id}`)) as FlyMachine;
+  }
+
+  async createMachine(config: Record<string, unknown>): Promise<FlyMachine> {
+    return (await this.request('POST', '/machines', config)) as FlyMachine;
+  }
+
+  async updateMachine(
+    id: string,
+    config: Record<string, unknown>,
+  ): Promise<FlyMachine> {
+    return (await this.request(
+      'POST',
+      `/machines/${id}`,
+      config,
+    )) as FlyMachine;
+  }
+
+  /**
+   * Metadata has its own Machines API endpoint. Unlike a config update, this
+   * does not create a new Machine version, so it is safe for durable operation
+   * fences on a cordoned candidate.
+   * https://fly.io/docs/machines/api/machines-resource/#add-or-update-machine-metadata
+   */
+  async setMachineMetadata(
+    id: string,
+    key: string,
+    value: string,
+  ): Promise<void> {
+    await this.request(
+      'POST',
+      `/machines/${id}/metadata/${encodeURIComponent(key)}`,
+      { value },
+    );
+  }
+
+  async deleteMachine(id: string, force: boolean): Promise<void> {
+    await this.request(
+      'DELETE',
+      `/machines/${id}?force=${force ? 'true' : 'false'}`,
+    );
+  }
+
+  async startMachine(id: string): Promise<void> {
+    await this.request('POST', `/machines/${id}/start`);
+  }
+
+  /**
+   * Remove a Machine from Fly Proxy admission without stopping accepted work.
+   * Runtime tuple recovery uses this exact provider transition; a config
+   * rewrite would create a new Machine version and lose the retained source
+   * identity needed to undo a cutover.
+   */
+  async cordonMachine(id: string): Promise<void> {
+    await this.request('POST', `/machines/${id}/cordon`);
+  }
+
+  /** Restore a retained Machine to Fly Proxy admission. */
+  async uncordonMachine(id: string): Promise<void> {
+    await this.request('POST', `/machines/${id}/uncordon`);
+  }
+
+  async stopMachine(
+    id: string,
+    options?: { signal?: string; timeout?: string },
+  ): Promise<void> {
+    await this.request('POST', `/machines/${id}/stop`, options);
+  }
+
+  async machinesInLane(releaseId: string): Promise<FlyMachine[]> {
+    const machines = await this.listMachines();
+    return machines.filter((machine) => machineIsInLane(machine, releaseId));
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Verb implementations.
+// ---------------------------------------------------------------------------
+
+function runFlyctl(args: readonly string[]): void {
+  const result = spawnSync('flyctl', args as string[], {
+    stdio: 'inherit',
+    env: process.env,
+  });
+  if (result.error) {
+    throw new Error(`flyctl failed to spawn: ${result.error.message}`);
+  }
+  if (result.status !== 0) {
+    throw new Error(
+      `flyctl ${args.join(' ')} exited with status ${result.status ?? 'null'}.`,
+    );
+  }
+}
+
+function runBun(args: readonly string[]): void {
+  const result = spawnSync('bun', args as string[], {
+    stdio: 'inherit',
+    env: process.env,
+  });
+  if (result.error) {
+    throw new Error(`bun failed to spawn: ${result.error.message}`);
+  }
+  if (result.status !== 0) {
+    throw new Error(
+      `bun ${args.join(' ')} exited with status ${result.status ?? 'null'}.`,
+    );
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Rollback lane image resolution (pure, unit-tested).
+// ---------------------------------------------------------------------------
+
+/** The image tag a production lane carries is always the full commit. */
+export const FULL_GIT_SHA_RE = /^[0-9a-f]{40}$/;
+/** Production release ids are `main-<full sha>` (see the deploy job's env). */
+export const MAIN_RELEASE_ID_SHA_RE = /^main-([0-9a-f]{40})$/;
+
+export type RollbackLaneImageResolution = {
+  /** The immutable revision the lane image must be built from. */
+  sha: string;
+  /** The immutable tag rollback boots from. */
+  imageRef: string;
+  /** Exactly what the registry row recorded, or null when it recorded none. */
+  registeredImageRef: string | null;
+};
+
+/**
+ * Resolve the immutable image a rollback must boot, from the two independent
+ * identities the release carries: the registry row's `imageRef` and the release
+ * id itself. Both are checked, and a disagreement is an error rather than a
+ * silent preference — booting the wrong revision under a release id is exactly
+ * the failure the immutable-tag contract exists to prevent (ADR 0017).
+ */
+export function resolveRollbackLaneImage(input: {
+  app: string;
+  releaseId: string;
+  registeredImageRef?: string | null;
+}): RollbackLaneImageResolution {
+  const prefix = `${FLY_REGISTRY_HOST}/${input.app}:git-`;
+  const registeredImageRef = input.registeredImageRef?.trim() || null;
+  let imageSha: string | null = null;
+  if (registeredImageRef) {
+    if (!registeredImageRef.startsWith(prefix)) {
+      throw new Error(
+        `Registered image ${registeredImageRef} is outside ${prefix}<sha>.`,
+      );
+    }
+    const candidate = registeredImageRef.slice(prefix.length).toLowerCase();
+    if (!GIT_SHA_RE.test(candidate)) {
+      throw new Error(
+        `Registered image ${registeredImageRef} does not carry a git sha tag.`,
+      );
+    }
+    imageSha = candidate;
+  }
+  const releaseSha =
+    MAIN_RELEASE_ID_SHA_RE.exec(input.releaseId)?.[1]?.toLowerCase() ?? null;
+  if (
+    imageSha !== null &&
+    releaseSha !== null &&
+    !imageSha.startsWith(releaseSha) &&
+    !releaseSha.startsWith(imageSha)
+  ) {
+    throw new Error(
+      `Registered image sha ${imageSha} disagrees with release id ${input.releaseId}. ` +
+        'Refusing to guess which revision this lane ran.',
+    );
+  }
+  const sha = [imageSha, releaseSha].find(
+    (candidate): candidate is string =>
+      candidate !== null && FULL_GIT_SHA_RE.test(candidate),
+  );
+  if (!sha) {
+    throw new Error(
+      `Cannot determine a 40-hex revision for release ${input.releaseId} ` +
+        `(registry image: ${registeredImageRef ?? 'none'}). ` +
+        'A lane image can only be rebuilt from its exact recorded revision.',
+    );
+  }
+  return { sha, imageRef: laneImageRef(input.app, sha), registeredImageRef };
+}
+
+export type RollbackLaneImagePlan = RollbackLaneImageResolution & {
+  action: 'reuse' | 'rebuild';
+};
+
+/**
+ * Decide whether a rollback must rebuild the lane image. lane handoff retires the
+ * prior lane within minutes of a release, and a retired lane's image can have
+ * been reaped from the registry; an absent tag is a rebuild, never a refusal.
+ */
+export function planRollbackLaneImage(input: {
+  app: string;
+  releaseId: string;
+  registeredImageRef?: string | null;
+  manifestPresent: boolean;
+}): RollbackLaneImagePlan {
+  const resolution = resolveRollbackLaneImage(input);
+  return {
+    ...resolution,
+    action: input.manifestPresent ? 'reuse' : 'rebuild',
+  };
+}
+
+/**
+ * Legacy argument shape retained for CLI compatibility. New callers route
+ * through ensure-runtime-image.ts, which content-addresses source inputs
+ * before giving the release its immutable `git-<sha>` tag.
+ *
+ * `--config <absurd-worker.toml>` is required alongside `--dockerfile`: without it,
+ * flyctl resolves the dockerfile path relative to the wrong root ("dockerfile not
+ * found") and its config-verification step fails on a machine-less app ("could not
+ * create a fly.toml from any machines"). Passing the lane's app config fixes both.
+ */
+export function pushImageArgs(input: {
+  app: string;
+  sha: string;
+  releaseId: string;
+}): string[] {
+  return [
+    'deploy',
+    '--app',
+    input.app,
+    '--config',
+    ABSURD_WORKER_CONFIG,
+    '--build-only',
+    '--push',
+    '--image-label',
+    `git-${input.sha}`,
+    '--dockerfile',
+    DOCKERFILE_ABSURD,
+    '--remote-only',
+  ];
+}
+
+async function cmdPushImage(
+  flags: Record<string, string | boolean>,
+): Promise<void> {
+  const app = resolveApp(flags);
+  const sha = requireSha(flags);
+  const releaseId = requireReleaseId(flags);
+  console.info(
+    `[fly-absurd-lane] push-image app=${app} sha=${sha} release=${releaseId}`,
+  );
+  runBun([
+    'scripts/ensure-runtime-image.ts',
+    '--kind',
+    'worker',
+    '--app',
+    app,
+    '--config',
+    ABSURD_WORKER_CONFIG,
+    '--dockerfile',
+    DOCKERFILE_ABSURD,
+    '--sha',
+    sha,
+  ]);
+  console.info(`[fly-absurd-lane] pushed ${laneImageRef(app, sha)}`);
+}
+
+/**
+ * Report whether a rollback can boot the lane's registered image as-is, or
+ * whether that tag is gone and must be rebuilt from the recorded revision.
+ * Probing goes through the deploy path's own registry probe so the two cannot
+ * disagree. The rebuild itself is not done here: it must run from a checkout of
+ * the recorded revision, which the caller owns.
+ */
+async function cmdPlanImage(
+  flags: Record<string, string | boolean>,
+): Promise<void> {
+  const app = resolveApp(flags);
+  const releaseId = requireReleaseId(flags);
+  const registeredImageRef =
+    typeof flags['image-ref'] === 'string' ? flags['image-ref'] : null;
+  const resolution = resolveRollbackLaneImage({
+    app,
+    releaseId,
+    registeredImageRef,
+  });
+  authenticateFlyRegistry();
+  const plan = planRollbackLaneImage({
+    app,
+    releaseId,
+    registeredImageRef,
+    manifestPresent: runtimeImageManifestExists(resolution.imageRef),
+  });
+  console.info(
+    `[fly-absurd-lane] plan-image release=${releaseId} image=${plan.imageRef} ` +
+      `sha=${plan.sha} action=${plan.action}`,
+  );
+  if (plan.action === 'rebuild') {
+    console.info(
+      `[fly-absurd-lane] ${plan.imageRef} is absent from the registry; ` +
+        `rebuild it from ${plan.sha} before booting this lane.`,
+    );
+  }
+  const githubOutput = process.env.GITHUB_OUTPUT?.trim();
+  if (githubOutput) {
+    appendFileSync(
+      githubOutput,
+      `action=${plan.action}\nsha=${plan.sha}\nimage=${plan.imageRef}\n`,
+    );
+  }
+}
+
+const sleep = (ms: number): Promise<void> =>
+  new Promise((resolve) => setTimeout(resolve, ms));
+
+function isImageManifestPropagationError(error: unknown): boolean {
+  if (!(error instanceof Error)) return false;
+  return (
+    error.message.includes('failed to get manifest') &&
+    error.message.includes('MANIFEST_UNKNOWN')
+  );
+}
+
+async function createMachineAfterImagePropagation(input: {
+  client: FlyMachinesClient;
+  config: Record<string, unknown>;
+  wait: (ms: number) => Promise<void>;
+  retryDelaysMs: readonly number[];
+}): Promise<FlyMachine> {
+  for (let attempt = 0; ; attempt += 1) {
+    try {
+      return await input.client.createMachine(input.config);
+    } catch (error) {
+      const retryDelayMs = input.retryDelaysMs[attempt];
+      if (
+        retryDelayMs === undefined ||
+        !isImageManifestPropagationError(error)
+      ) {
+        throw error;
+      }
+      console.info(
+        `[fly-absurd-lane] image manifest has not propagated; retrying Machine create in ${retryDelayMs}ms ` +
+          `(attempt ${attempt + 2}/${input.retryDelaysMs.length + 1})`,
+      );
+      await input.wait(retryDelayMs);
+    }
+  }
+}
+
+/**
+ * Boot K machines for a lane, then poll until they are all healthy
+ * (`machineIsHealthyForBoot`: started, plus all checks passing only when the
+ * machine declares checks). Loud timeout: if the wait ceiling lapses before every
+ * machine is healthy, throw with the last observed states so CI fails with a real
+ * diagnostic instead of a silent partial boot.
+ */
+export async function bootLane(input: {
+  client: FlyMachinesClient;
+  app: string;
+  releaseId: string;
+  sha: string;
+  runtimeSurfaceHash?: string;
+  count: number;
+  region?: string;
+  timeoutMs?: number;
+  pollIntervalMs?: number;
+  imagePropagationRetryDelaysMs?: readonly number[];
+  now?: () => number;
+  wait?: (ms: number) => Promise<void>;
+  reuseExisting?: boolean;
+  rollbackHoldUntilMs?: number;
+  rollbackHoldScalerQuiescenceMs?: number;
+}): Promise<FlyMachine[]> {
+  const now = input.now ?? (() => Date.now());
+  const wait = input.wait ?? sleep;
+  const timeoutMs = input.timeoutMs ?? DEFAULT_BOOT_TIMEOUT_MS;
+  const pollIntervalMs = input.pollIntervalMs ?? DEFAULT_BOOT_POLL_INTERVAL_MS;
+  const imagePropagationRetryDelaysMs =
+    input.imagePropagationRetryDelaysMs ??
+    DEFAULT_IMAGE_PROPAGATION_RETRY_DELAYS_MS;
+
+  const expectedImage = laneImageRef(input.app, input.sha);
+  const reusableStates = new Set([
+    'started',
+    'created',
+    'starting',
+    'replacing',
+    'restarting',
+    'stopped',
+  ]);
+  const statePriority = (machine: FlyMachine): number => {
+    if (machineIsHealthyForBoot(machine)) return 0;
+    if (machine.state === 'started') return 1;
+    if (machine.state === 'stopped') return 3;
+    return 2;
+  };
+  const reused = input.reuseExisting
+    ? (await input.client.machinesInLane(input.releaseId))
+        .filter(
+          (machine) =>
+            machine.config?.image === expectedImage &&
+            (!input.runtimeSurfaceHash ||
+              machine.config?.metadata?.[RUNTIME_SURFACE_METADATA_KEY] ===
+                input.runtimeSurfaceHash) &&
+            (input.rollbackHoldUntilMs !== undefined ||
+              machine.config.env?.[DRAIN_EXIT_ENV] !== '1') &&
+            reusableStates.has(machine.state ?? ''),
+        )
+        .sort((left, right) => statePriority(left) - statePriority(right))
+        .slice(0, input.count)
+    : [];
+  if (input.rollbackHoldUntilMs) {
+    for (const machine of reused) {
+      const current = await input.client.getMachine(machine.id);
+      await input.client.updateMachine(
+        machine.id,
+        buildRollbackHoldUpdateConfig(current, input.rollbackHoldUntilMs),
+      );
+    }
+    // A scaler cycle may have planned stops from a pre-hold snapshot. Let one
+    // full scaler interval quiesce, then re-read and restart from current state.
+    await wait(input.rollbackHoldScalerQuiescenceMs ?? 16_000);
+  }
+  for (const machine of reused) {
+    const current = input.rollbackHoldUntilMs
+      ? await input.client.getMachine(machine.id)
+      : machine;
+    if (current.state === 'stopped') {
+      await input.client.startMachine(machine.id);
+      console.info(
+        `[fly-absurd-lane] restarted machine ${machine.id} for release ${input.releaseId}`,
+      );
+    }
+  }
+  const createCount = input.count - reused.length;
+  const created: FlyMachine[] = [];
+  for (let index = 0; index < createCount; index += 1) {
+    const machine = await createMachineAfterImagePropagation({
+      client: input.client,
+      config: buildBootMachineConfig({
+        app: input.app,
+        releaseId: input.releaseId,
+        sha: input.sha,
+        runtimeSurfaceHash: input.runtimeSurfaceHash,
+        region: input.region,
+        rollbackHoldUntilMs: input.rollbackHoldUntilMs,
+      }),
+      wait,
+      retryDelaysMs: imagePropagationRetryDelaysMs,
+    });
+    console.info(
+      `[fly-absurd-lane] created machine ${machine.id} (${index + 1}/${createCount}) for release ${input.releaseId}`,
+    );
+    created.push(machine);
+  }
+
+  const deadline = now() + timeoutMs;
+  const ids = new Set([...reused, ...created].map((machine) => machine.id));
+  for (;;) {
+    const laneMachines = (
+      await input.client.machinesInLane(input.releaseId)
+    ).filter((machine) => ids.has(machine.id));
+    const healthy = laneMachines.filter(machineIsHealthyForBoot);
+    if (healthy.length >= input.count) {
+      const checkless = healthy.filter(
+        (machine) => (machine.checks ?? []).length === 0,
+      );
+      if (checkless.length > 0) {
+        console.info(
+          `[fly-absurd-lane] release ${input.releaseId}: ${checkless.length}/${healthy.length} healthy machines had no machine-level checks ` +
+            `(Machines-API creates do not inherit the app [checks]); treating started state as healthy.`,
+        );
+      }
+      console.info(
+        `[fly-absurd-lane] release ${input.releaseId}: ${healthy.length}/${input.count} machines started + healthy`,
+      );
+      return laneMachines;
+    }
+    if (now() >= deadline) {
+      const states = laneMachines
+        .map((machine) => `${machine.id}=${machine.state ?? 'unknown'}`)
+        .join(', ');
+      throw new Error(
+        `Lane ${input.releaseId} did not reach ${input.count} healthy machines within ` +
+          `${timeoutMs}ms (last: ${states || 'none'}).`,
+      );
+    }
+    console.info(
+      `[fly-absurd-lane] waiting for lane ${input.releaseId}: healthy=${healthy.length}/${input.count}`,
+    );
+    await wait(pollIntervalMs);
+  }
+}
+
+async function cmdBoot(flags: Record<string, string | boolean>): Promise<void> {
+  const app = resolveApp(flags);
+  const releaseId = requireReleaseId(flags);
+  const sha = requireSha(flags);
+  const count = requireCount(flags);
+  const region = typeof flags.region === 'string' ? flags.region : undefined;
+  const reuseExisting = flags['reuse-existing'] === true;
+  const runtimeSurfaceHash =
+    typeof flags['runtime-surface-hash'] === 'string'
+      ? flags['runtime-surface-hash']
+      : undefined;
+  if (runtimeSurfaceHash && !/^[a-f0-9]{64}$/.test(runtimeSurfaceHash)) {
+    throw new Error(
+      '--runtime-surface-hash must be a lowercase SHA-256 hex digest.',
+    );
+  }
+  const rawRollbackHoldUntilMs = flags['rollback-hold-until-ms'];
+  const rollbackHoldUntilMs =
+    typeof rawRollbackHoldUntilMs === 'string'
+      ? Number(rawRollbackHoldUntilMs)
+      : undefined;
+  if (
+    rollbackHoldUntilMs !== undefined &&
+    (!Number.isSafeInteger(rollbackHoldUntilMs) ||
+      rollbackHoldUntilMs <= Date.now())
+  ) {
+    throw new Error(
+      '--rollback-hold-until-ms must be a future Unix timestamp in milliseconds.',
+    );
+  }
+  const client = new FlyMachinesClient(
+    app,
+    requireFlyToken(),
+    globalThis.fetch as FetchLike,
+  );
+  await bootLane({
+    client,
+    app,
+    releaseId,
+    sha,
+    runtimeSurfaceHash,
+    count,
+    region,
+    reuseExisting,
+    rollbackHoldUntilMs,
+  });
+}
+
+export function formatMachineTable(machines: readonly FlyMachine[]): string {
+  if (machines.length === 0) return '(no machines)';
+  const header = ['ID', 'RELEASE', 'STATE', 'REGION', 'CHECKS', 'DRAIN'];
+  const rows = machines.map((machine) => [
+    machine.id,
+    machine.config?.metadata?.[RELEASE_METADATA_KEY] ?? '-',
+    machine.state ?? '-',
+    machine.region ?? '-',
+    machineChecksPassing(machine)
+      ? 'passing'
+      : (machine.checks ?? []).length === 0
+        ? 'none'
+        : 'not-passing',
+    machine.config?.env?.[DRAIN_EXIT_ENV] === '1' ? 'yes' : 'no',
+  ]);
+  const widths = header.map((_, column) =>
+    Math.max(header[column]!.length, ...rows.map((row) => row[column]!.length)),
+  );
+  const line = (cells: string[]) =>
+    cells.map((cell, column) => cell.padEnd(widths[column]!)).join('  ');
+  return [line(header), ...rows.map(line)].join('\n');
+}
+
+async function cmdList(flags: Record<string, string | boolean>): Promise<void> {
+  const app = resolveApp(flags);
+  const client = new FlyMachinesClient(
+    app,
+    requireFlyToken(),
+    globalThis.fetch as FetchLike,
+  );
+  const release = typeof flags.release === 'string' ? flags.release : null;
+  const machines = release
+    ? await client.machinesInLane(release)
+    : await client.listMachines();
+  if (flags.json === true) {
+    console.log(JSON.stringify(machines, null, 2));
+  } else {
+    console.log(formatMachineTable(machines));
+  }
+}
+
+export async function drainLane(input: {
+  client: FlyMachinesClient;
+  releaseId: string;
+  ifPresent?: boolean;
+  /** Test seams; production callers use the bounded defaults above. */
+  timeoutMs?: number;
+  pollIntervalMs?: number;
+  now?: () => number;
+  wait?: (ms: number) => Promise<void>;
+}): Promise<FlyMachine[]> {
+  const now = input.now ?? (() => Date.now());
+  const wait = input.wait ?? sleep;
+  const timeoutMs = input.timeoutMs ?? DEFAULT_DRAIN_CONVERGENCE_TIMEOUT_MS;
+  const pollIntervalMs =
+    input.pollIntervalMs ?? DEFAULT_DRAIN_CONVERGENCE_POLL_INTERVAL_MS;
+  const machines = await input.client.machinesInLane(input.releaseId);
+  if (machines.length === 0) {
+    if (input.ifPresent) {
+      console.info(
+        `[fly-absurd-lane] no machines found for lane ${input.releaseId}; ` +
+          '--if-present treats this as a no-op.',
+      );
+      return [];
+    }
+    throw new Error(
+      `No machines found for lane ${input.releaseId}; nothing to drain.`,
+    );
+  }
+  for (const machine of machines) {
+    const current = await input.client.getMachine(machine.id);
+    await input.client.updateMachine(
+      machine.id,
+      buildDrainUpdateConfig(current),
+    );
+    console.info(
+      `[fly-absurd-lane] drained machine ${machine.id} (release ${input.releaseId}); ` +
+        `${DRAIN_EXIT_ENV}=1 set, worker exits after its quiet window.`,
+    );
+  }
+
+  // Fly accepts a Machine update before a subsequent lane list is guaranteed
+  // to show the replacement config.  The production retire verb makes its
+  // safety decision from that list, so accepting the update response alone
+  // leaves a real drain -> retire race.  Wait only for the observable
+  // precondition (the worker may remain started while it finishes its quiet
+  // window); never wait for a parked durable run to wake.
+  const deadline = now() + timeoutMs;
+  let lastObserved = 'no post-drain Machine listing read';
+  for (;;) {
+    const observed = await input.client.machinesInLane(input.releaseId);
+    const live = observed.filter((machine) => !machineIsRetirable(machine));
+    lastObserved =
+      observed
+        .map(
+          (machine) =>
+            `${machine.id}=${machine.state ?? 'unknown'}:${machine.config?.env?.[DRAIN_EXIT_ENV] === '1' ? 'draining' : 'not-draining'}`,
+        )
+        .join(', ') || 'no Machines';
+    if (live.length === 0) return observed;
+    if (now() >= deadline) {
+      throw new Error(
+        `Drain of lane ${input.releaseId} was accepted but did not converge within ${timeoutMs}ms. ` +
+          `Still active: ${live.map((machine) => machine.id).join(', ')}. Observed: ${lastObserved}.`,
+      );
+    }
+    await wait(pollIntervalMs);
+  }
+}
+
+/**
+ * Remove the exact Machines created for a release that failed before its
+ * runtime pointer could move. This is deliberately not the normal `retire`
+ * path: a pre-pointer candidate was never admitted, so it has no queue to
+ * drain and no registry row to retire. The release controller is the only
+ * caller; it invokes this exclusively before its first public effect.
+ */
+export async function discardPreCommitCandidateLane(input: {
+  client: Pick<FlyMachinesClient, 'machinesInLane' | 'deleteMachine'>;
+  releaseId: string;
+  runtimeSurfaceHash: string;
+}): Promise<FlyMachine[]> {
+  const machines = await input.client.machinesInLane(input.releaseId);
+  const unexpected = machines.filter(
+    (machine) =>
+      machine.config?.metadata?.[RELEASE_METADATA_KEY] !== input.releaseId ||
+      machine.config?.metadata?.[RUNTIME_SURFACE_METADATA_KEY] !==
+        input.runtimeSurfaceHash,
+  );
+  if (unexpected.length > 0) {
+    throw new Error(
+      `Refuse to discard ${unexpected.length} candidate Machine(s) for release ${input.releaseId}: release or runtime-surface metadata does not match.`,
+    );
+  }
+  for (const machine of machines) {
+    await input.client.deleteMachine(machine.id, true);
+    console.info(
+      `[fly-absurd-lane] discarded pre-commit candidate machine ${machine.id} (release ${input.releaseId}).`,
+    );
+  }
+  return machines;
+}
+
+async function cmdDrain(
+  flags: Record<string, string | boolean>,
+): Promise<void> {
+  const app = resolveApp(flags);
+  const releaseId = requireReleaseId(flags);
+  const client = new FlyMachinesClient(
+    app,
+    requireFlyToken(),
+    globalThis.fetch as FetchLike,
+  );
+  await drainLane({
+    client,
+    releaseId,
+    ifPresent: flags['if-present'] === true,
+  });
+}
+
+/**
+ * Queue-emptiness gate. The authoritative "is this lane's queue drained?" check
+ * is an app-side admin endpoint owned by another agent (it can read the
+ * runs_<releaseId> queue depth). If DEEPLINE_ABSURD_ADMIN_URL + token are set we
+ * call it; otherwise we refuse to destroy unless --force is passed, printing the
+ * manual check. Loud, never a silent skip.
+ */
+export type QueueGateResult =
+  | {
+      ok: true;
+      source: 'admin-endpoint';
+      queued: number;
+      basis: 'empty' | 'adopted-parked';
+    }
+  | { ok: false; source: 'admin-endpoint'; queued: number }
+  | { ok: false; source: 'no-endpoint' };
+
+export async function checkLaneQueueEmpty(input: {
+  releaseId: string;
+  fetchImpl: FetchLike;
+  adminUrl?: string;
+  adminToken?: string;
+  protectionBypassToken?: string;
+}): Promise<QueueGateResult> {
+  if (!input.adminUrl || !input.adminToken) {
+    return { ok: false, source: 'no-endpoint' };
+  }
+  const url = `${input.adminUrl.replace(/\/+$/, '')}/api/v2/internal/runtime/lane-queue-depth?release=${encodeURIComponent(input.releaseId)}`;
+  const response = await input.fetchImpl(url, {
+    method: 'GET',
+    headers: adminRequestHeaders({
+      adminToken: input.adminToken,
+      protectionBypassToken: input.protectionBypassToken,
+    }),
+  });
+  if (!response.ok) {
+    throw new Error(
+      `Lane queue-depth admin check failed: ${response.status} ${await response.text().catch(() => '')}`,
+    );
+  }
+  const body = ((await readJsonResponseBody({
+    response,
+    method: 'GET',
+    url,
+  })) ?? {}) as {
+    queued?: number;
+    claiming?: unknown;
+    emptyExceptAdoptableParked?: unknown;
+    adoptableParkedRuns?: unknown;
+  };
+  const queued = typeof body.queued === 'number' ? body.queued : Number.NaN;
+  if (!Number.isFinite(queued)) {
+    throw new Error(
+      'Lane queue-depth admin response did not include a numeric queued count.',
+    );
+  }
+  if (queued === 0) {
+    return { ok: true, source: 'admin-endpoint', queued, basis: 'empty' };
+  }
+  // A lane that has stopped claiming retires without waiting for its parked
+  // durable waits: the newest lane adopts those wakes (lane handoff). Retirement
+  // deletes machines and flips the registry row; the queue tables the adopting
+  // lane claims from are dropped later, by the scaler, behind its own checks.
+  if (
+    body.claiming === false &&
+    body.emptyExceptAdoptableParked === true &&
+    typeof body.adoptableParkedRuns === 'number' &&
+    body.adoptableParkedRuns > 0
+  ) {
+    console.info(
+      `[fly-absurd-lane] lane ${input.releaseId} retires with ` +
+        `${body.adoptableParkedRuns} parked durable wait(s) adopted by the ` +
+        `newest lane; nothing on this lane is claimable or running.`,
+    );
+    return {
+      ok: true,
+      source: 'admin-endpoint',
+      queued,
+      basis: 'adopted-parked',
+    };
+  }
+  return { ok: false, source: 'admin-endpoint', queued };
+}
+
+/**
+ * Flip a registered lane's claiming switch (lane handoff).
+ *
+ * `stop-claiming` is the release controller's demotion step: from that write
+ * the newest lane may adopt this lane's parked durable waits, so drain and
+ * retirement no longer wait for a sleeping run to wake. `resume-claiming`
+ * re-arms a lane that is coming back (rollback, aborted retirement).
+ */
+export async function setLaneClaiming(input: {
+  releaseId: string;
+  claiming: boolean;
+  environment?: 'preview' | 'production';
+  fetchImpl: FetchLike;
+  adminUrl?: string;
+  adminToken?: string;
+  protectionBypassToken?: string;
+}): Promise<void> {
+  if (!input.adminUrl || !input.adminToken) {
+    throw new Error(
+      `Cannot change claiming for lane ${input.releaseId}: no admin endpoint ` +
+        '(set DEEPLINE_ABSURD_ADMIN_URL + DEEPLINE_INTERNAL_TOKEN).',
+    );
+  }
+  const url = `${input.adminUrl.replace(/\/+$/, '')}/api/v2/internal/runtime/releases`;
+  const response = await input.fetchImpl(url, {
+    method: 'POST',
+    headers: adminRequestHeaders({
+      adminToken: input.adminToken,
+      protectionBypassToken: input.protectionBypassToken,
+      contentType: true,
+    }),
+    body: JSON.stringify({
+      action: input.claiming ? 'resume_claiming' : 'stop_claiming',
+      environment: input.environment ?? 'production',
+      schedulerBackend: 'absurd',
+      releaseId: input.releaseId,
+    }),
+  });
+  if (!response.ok) {
+    throw new Error(
+      `Runtime release ${input.claiming ? 'resume_claiming' : 'stop_claiming'} failed: ` +
+        `${response.status} ${await response.text().catch(() => '')}`,
+    );
+  }
+  console.info(
+    `[fly-absurd-lane] lane ${input.releaseId} claiming=${input.claiming}.`,
+  );
+}
+
+async function transitionRuntimeReleaseRetirement(input: {
+  releaseId: string;
+  environment?: 'preview' | 'production';
+  fetchImpl: FetchLike;
+  adminUrl?: string;
+  adminToken?: string;
+  protectionBypassToken?: string;
+  action: 'begin_retire' | 'abort_retire' | 'commit_retire' | 'retire';
+}): Promise<{ retiring: boolean; deleting: boolean; retired: boolean }> {
+  if (!input.adminUrl || !input.adminToken) {
+    throw new Error(
+      `Cannot retire registry release ${input.releaseId}: no admin endpoint ` +
+        '(set DEEPLINE_ABSURD_ADMIN_URL + DEEPLINE_INTERNAL_TOKEN).',
+    );
+  }
+  const url = `${input.adminUrl.replace(/\/+$/, '')}/api/v2/internal/runtime/releases`;
+  const response = await input.fetchImpl(url, {
+    method: 'POST',
+    headers: adminRequestHeaders({
+      adminToken: input.adminToken,
+      protectionBypassToken: input.protectionBypassToken,
+      contentType: true,
+    }),
+    body: JSON.stringify({
+      action: input.action,
+      environment: input.environment ?? 'production',
+      schedulerBackend: 'absurd',
+      releaseId: input.releaseId,
+    }),
+  });
+  if (!response.ok) {
+    throw new Error(
+      `Runtime release registry ${input.action} failed: ${response.status} ${await response.text().catch(() => '')}`,
+    );
+  }
+  const body = ((await readJsonResponseBody({
+    response,
+    method: 'POST',
+    url,
+  })) ?? {}) as {
+    retiring?: unknown;
+    deleting?: unknown;
+    retired?: unknown;
+  };
+  return {
+    retiring: body.retiring === true,
+    deleting: body.deleting === true,
+    retired: body.retired === true,
+  };
+}
+
+/**
+ * `registered: false` means the registry authoritatively has no row for this
+ * release. That is a different answer from "registered, status unreadable",
+ * and retirement branches on the difference: only the registry can own a lane's
+ * retirement, so a lane it has never heard of is not a lane this verb retires.
+ */
+export type RuntimeReleaseRegistryStatus = Readonly<{
+  registered: boolean;
+  status: string | null;
+}>;
+
+async function readRuntimeReleaseRetirementStatus(input: {
+  releaseId: string;
+  environment?: 'preview' | 'production';
+  fetchImpl: FetchLike;
+  adminUrl: string;
+  adminToken: string;
+  protectionBypassToken?: string;
+}): Promise<RuntimeReleaseRegistryStatus> {
+  const url = new URL(
+    '/api/v2/internal/runtime/releases',
+    input.adminUrl.replace(/\/+$/, '') + '/',
+  );
+  url.searchParams.set('environment', input.environment ?? 'production');
+  url.searchParams.set('schedulerBackend', 'absurd');
+  url.searchParams.set('releaseId', input.releaseId);
+  const response = await input.fetchImpl(url.toString(), {
+    method: 'GET',
+    headers: adminRequestHeaders({
+      adminToken: input.adminToken,
+      protectionBypassToken: input.protectionBypassToken,
+    }),
+  });
+  if (!response.ok) {
+    throw new Error(
+      `Runtime release registry status failed: ${response.status} ${await response.text().catch(() => '')}`,
+    );
+  }
+  const body = ((await readJsonResponseBody({
+    response,
+    method: 'GET',
+    url: url.toString(),
+  })) ?? {}) as {
+    release?: { status?: unknown } | null;
+  };
+  const release = body.release ?? null;
+  return {
+    registered: release !== null && typeof release === 'object',
+    status: typeof release?.status === 'string' ? release.status : null,
+  };
+}
+
+export async function beginRuntimeReleaseRetirement(input: {
+  releaseId: string;
+  environment?: 'preview' | 'production';
+  fetchImpl: FetchLike;
+  adminUrl?: string;
+  adminToken?: string;
+  protectionBypassToken?: string;
+}): Promise<{ retiring: boolean; deleting: boolean; retired: boolean }> {
+  return await transitionRuntimeReleaseRetirement({
+    ...input,
+    action: 'begin_retire',
+  });
+}
+
+export async function abortRuntimeReleaseRetirement(input: {
+  releaseId: string;
+  environment?: 'preview' | 'production';
+  fetchImpl: FetchLike;
+  adminUrl?: string;
+  adminToken?: string;
+  protectionBypassToken?: string;
+}): Promise<void> {
+  await transitionRuntimeReleaseRetirement({
+    ...input,
+    action: 'abort_retire',
+  });
+}
+
+export async function commitRuntimeReleaseRetirement(input: {
+  releaseId: string;
+  environment?: 'preview' | 'production';
+  fetchImpl: FetchLike;
+  adminUrl?: string;
+  adminToken?: string;
+  protectionBypassToken?: string;
+}): Promise<{ retiring: boolean; deleting: boolean; retired: boolean }> {
+  return await transitionRuntimeReleaseRetirement({
+    ...input,
+    action: 'commit_retire',
+  });
+}
+
+export async function markRuntimeReleaseRetired(input: {
+  releaseId: string;
+  environment?: 'preview' | 'production';
+  fetchImpl: FetchLike;
+  adminUrl?: string;
+  adminToken?: string;
+  protectionBypassToken?: string;
+}): Promise<void> {
+  await transitionRuntimeReleaseRetirement({ ...input, action: 'retire' });
+}
+
+export async function retireLane(input: {
+  client: FlyMachinesClient;
+  releaseId: string;
+  force: boolean;
+  fetchImpl: FetchLike;
+  adminUrl?: string;
+  adminToken?: string;
+  protectionBypassToken?: string;
+  scalerQuiescenceMs?: number;
+}): Promise<FlyMachine[]> {
+  const machines = await input.client.machinesInLane(input.releaseId);
+  if (machines.length === 0) {
+    console.info(
+      `[fly-absurd-lane] lane ${input.releaseId} has no machines; finalizing registry retirement.`,
+    );
+  }
+
+  if (!input.adminUrl || !input.adminToken) {
+    // Preserve the useful local/operator diagnostic when registry state cannot
+    // be queried. With an endpoint present, the durable deleting/retired
+    // tombstone is consulted first below and overrides this ordinary gate.
+    const live = machines.filter((machine) => !machineIsRetirable(machine));
+    if (live.length > 0 && !input.force) {
+      const ids = live.map((machine) => machine.id).join(', ');
+      throw new Error(
+        `Lane ${input.releaseId} still has active (non-drained, non-stopped) machines: ${ids}. ` +
+          `Drain the lane first, or pass --force.`,
+      );
+    }
+    throw new Error(
+      `Cannot retire lane ${input.releaseId}: no admin endpoint ` +
+        '(set DEEPLINE_ABSURD_ADMIN_URL + DEEPLINE_INTERNAL_TOKEN).',
+    );
+  }
+
+  const registry = await readRuntimeReleaseRetirementStatus({
+    releaseId: input.releaseId,
+    fetchImpl: input.fetchImpl,
+    adminUrl: input.adminUrl,
+    adminToken: input.adminToken,
+    protectionBypassToken: input.protectionBypassToken,
+  });
+  if (!registry.registered) {
+    // The registry is the only owner of a lane's retirement lifecycle. With no
+    // row and no Machines there is nothing left to retire: an operator (or an
+    // earlier attempt of this same command) already reclaimed it, and saying so
+    // is the correct, idempotent answer rather than a failure.
+    if (machines.length === 0) {
+      console.info(
+        `[fly-absurd-lane] lane ${input.releaseId} has no machines and no runtime-release ` +
+          'registry row; nothing to retire.',
+      );
+      return [];
+    }
+    // Machines with no registry row were never admitted, so `retire` is the
+    // wrong verb for them: its gates read registry state this lane does not
+    // have. The release-aware scaler owns unregistered Machines.
+    throw new Error(
+      `Lane ${input.releaseId} has ${machines.length} machine(s) but no runtime-release ` +
+        'registry row, so it was never admitted and cannot be retired. The release-aware ' +
+        'scaler reclaims unregistered Machines after its registration grace.',
+    );
+  }
+  const cleanupOnly =
+    registry.status === 'deleting' || registry.status === 'retired';
+  if (!cleanupOnly) {
+    // Ordinary preconditions must not mutate the registry. A failed drain-first
+    // check leaves rollback and admission untouched.
+    const live = machines.filter((machine) => !machineIsRetirable(machine));
+    if (live.length > 0 && !input.force) {
+      const ids = live.map((machine) => machine.id).join(', ');
+      throw new Error(
+        `Lane ${input.releaseId} still has active (non-drained, non-stopped) machines: ${ids}. ` +
+          `Drain the lane first, or pass --force.`,
+      );
+    }
+  }
+
+  const retirementInput = {
+    releaseId: input.releaseId,
+    fetchImpl: input.fetchImpl,
+    adminUrl: input.adminUrl,
+    adminToken: input.adminToken,
+    protectionBypassToken: input.protectionBypassToken,
+  };
+  const ensureEmptyOrReopen = async () => {
+    const gate = await checkLaneQueueEmpty(retirementInput);
+    if (gate.source === 'no-endpoint') {
+      throw new Error(
+        `Cannot verify lane ${input.releaseId} queue is empty: no admin endpoint ` +
+          `(set DEEPLINE_ABSURD_ADMIN_URL + DEEPLINE_INTERNAL_TOKEN). Manually confirm ` +
+          `${laneQueueName(input.releaseId)} is empty.`,
+      );
+    }
+    if (!gate.ok) {
+      // This is safe before begin (repairs a prior partial abort) and after
+      // begin (reopens descendants when a run raced the first empty check).
+      await abortRuntimeReleaseRetirement(retirementInput);
+      throw new Error(
+        `Lane ${input.releaseId} queue ${laneQueueName(input.releaseId)} is not empty ` +
+          `(${gate.queued} queued). Retirement was aborted and admission reopened.`,
+      );
+    }
+    return gate;
+  };
+  if (!cleanupOnly) await ensureEmptyOrReopen();
+
+  const retirement = await beginRuntimeReleaseRetirement(retirementInput);
+  if (!cleanupOnly && !retirement.deleting && !retirement.retired) {
+    // Recheck after the admission fence. Any child that linearized before the
+    // fence is now visible; aborting advances the epoch and reopens the lane.
+    await ensureEmptyOrReopen();
+    console.info(
+      `[fly-absurd-lane] lane ${input.releaseId} queue empty (admin-verified).`,
+    );
+    await commitRuntimeReleaseRetirement(retirementInput);
+  } else {
+    console.info(
+      `[fly-absurd-lane] registry release ${input.releaseId} is already ${retirement.retired ? 'retired' : 'deleting'}; cleaning remaining machines.`,
+    );
+  }
+
+  const destroyed: FlyMachine[] = [];
+  const destroyedIds = new Set<string>();
+  const deleteCurrentMachines = async () => {
+    const current = await input.client.machinesInLane(input.releaseId);
+    for (const machine of current) {
+      await input.client.deleteMachine(machine.id, true);
+      console.info(
+        `[fly-absurd-lane] destroyed machine ${machine.id} (release ${input.releaseId}).`,
+      );
+      if (!destroyedIds.has(machine.id)) {
+        destroyedIds.add(machine.id);
+        destroyed.push(machine);
+      }
+    }
+  };
+  await deleteCurrentMachines();
+  const quiescenceMs = input.scalerQuiescenceMs ?? 16_000;
+  if (quiescenceMs > 0) {
+    await new Promise((resolve) => setTimeout(resolve, quiescenceMs));
+  }
+  await deleteCurrentMachines();
+  if (!retirement.retired) {
+    await markRuntimeReleaseRetired(retirementInput);
+    console.info(
+      `[fly-absurd-lane] registry release ${input.releaseId} marked retired.`,
+    );
+  }
+  return destroyed;
+}
+
+async function cmdRetire(
+  flags: Record<string, string | boolean>,
+): Promise<void> {
+  const app = resolveApp(flags);
+  const releaseId = requireReleaseId(flags);
+  const force = flags.force === true;
+  const client = new FlyMachinesClient(
+    app,
+    requireFlyToken(),
+    globalThis.fetch as FetchLike,
+  );
+  await retireLane({
+    client,
+    releaseId,
+    force,
+    fetchImpl: globalThis.fetch as FetchLike,
+    adminUrl: process.env.DEEPLINE_ABSURD_ADMIN_URL?.trim() || undefined,
+    adminToken: process.env.DEEPLINE_INTERNAL_TOKEN?.trim() || undefined,
+    protectionBypassToken:
+      process.env.VERCEL_PROTECTION_BYPASS_TOKEN?.trim() || undefined,
+  });
+}
+
+async function cmdSetClaiming(
+  flags: Record<string, string | boolean>,
+  claiming: boolean,
+): Promise<void> {
+  const releaseId = requireReleaseId(flags);
+  await setLaneClaiming({
+    releaseId,
+    claiming,
+    environment:
+      typeof flags.environment === 'string' && flags.environment === 'preview'
+        ? 'preview'
+        : 'production',
+    fetchImpl: globalThis.fetch as FetchLike,
+    adminUrl: process.env.DEEPLINE_ABSURD_ADMIN_URL?.trim() || undefined,
+    adminToken: process.env.DEEPLINE_INTERNAL_TOKEN?.trim() || undefined,
+    protectionBypassToken:
+      process.env.VERCEL_PROTECTION_BYPASS_TOKEN?.trim() || undefined,
+  });
+}
+
+const USAGE = `fly-absurd-lane <command> [flags]
+
+Commands:
+  push-image --sha <sha> --release <id> [--app <app>]
+  plan-image --release <id> [--image-ref <ref>] [--app <app>]
+  boot       --release <id> --sha <sha> --count <K> [--reuse-existing] [--rollback-hold-until-ms <ms>] [--app <app>] [--region <r>]
+  list       [--release <id>] [--json] [--app <app>]
+  drain          --release <id> [--if-present] [--app <app>]
+  stop-claiming  --release <id> [--environment production|preview]
+  resume-claiming --release <id> [--environment production|preview]
+  retire         --release <id> [--force] [--app <app>]
+
+Env:
+  FLY_API_TOKEN              (required) Fly Machines API token.
+  DEEPLINE_ABSURD_ADMIN_URL  (retire, *-claiming) app base url.
+  DEEPLINE_INTERNAL_TOKEN    (retire, *-claiming) bearer token.
+  VERCEL_PROTECTION_BYPASS_TOKEN
+                             (optional) required when DEEPLINE_ABSURD_ADMIN_URL
+                             is an immutable Vercel deployment URL behind
+                             Deployment Protection, which otherwise answers with
+                             an HTML challenge instead of the JSON route.
+
+stop-claiming hands this lane's parked durable waits to the newest lane, so
+drain and retire no longer wait for a sleeping run to wake (lane handoff). It is
+refused for the active lane. resume-claiming re-arms a lane that is coming back.
+`;
+
+export async function main(argv: readonly string[]): Promise<void> {
+  const { command, flags } = parseArgs(argv);
+  switch (command) {
+    case 'push-image':
+      return cmdPushImage(flags);
+    case 'plan-image':
+      return cmdPlanImage(flags);
+    case 'boot':
+      return cmdBoot(flags);
+    case 'list':
+      return cmdList(flags);
+    case 'drain':
+      return cmdDrain(flags);
+    case 'stop-claiming':
+      return cmdSetClaiming(flags, false);
+    case 'resume-claiming':
+      return cmdSetClaiming(flags, true);
+    case 'retire':
+      return cmdRetire(flags);
+    case '':
+    case 'help':
+    case '--help':
+    case '-h':
+      console.log(USAGE);
+      return;
+    default:
+      throw new Error(`Unknown command: ${command}\n\n${USAGE}`);
+  }
+}