deepline 0.3.90 → 0.3.92

package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-receipt-gateway.ts

@@ -0,0 +1,1378 @@
+#!/usr/bin/env bun
+
+/**
+ * Small, Fly-native receipt gateway rollout.
+ *
+ * Fly is the operation record: an immutable image tag plus deterministic,
+ * cordoned Machine names and metadata. There is no second controller state
+ * machine to reconcile with the provider.
+ *
+ * prepare: build image, clone the serving fleet as cordoned Machines, and
+ *          wait for Fly's service checks to prove boot readiness.
+ * prove: run the signed internal fixture through every still-cordoned green
+ *        Machine. The fixture's transport asserts that Fly did not fall back
+ *        to another gateway.
+ * activate: uncordon an already-proven green fleet, verify normal routing,
+ *           then cordon blue. It never starts a Play or retires blue.
+ * drain: retire exact, already-cordoned blue sources only after their own
+ *        live-work metric reads zero.
+ * cleanup: remove only still-cordoned candidates for this SHA.
+ */
+import { spawn, spawnSync } from 'node:child_process';
+import { resolve } from 'node:path';
+import { FlyMachinesClient, requireFlyToken } from '../fly-machines';
+import {
+  RECEIPT_GATEWAY_DRAIN_TIMEOUT,
+  stopAndDestroyCordonedMachines,
+  type FlyMachine,
+} from '../../deployment/fly-machine-lifecycle';
+import {
+  assertGatewayRouteProbe,
+  exactFlyMachineRouteHeaders,
+  formatGatewayRouteProbeRetry,
+  waitForGatewayRouteProbe,
+} from '../fly-gateway-route-probe';
+import { reconcileGatewaySourceDrain } from '../../deployment/runtime-gateway-drain';
+import {
+  gatewayMachineConfig,
+  productionGatewayTopology,
+  RUNTIME_TOPOLOGY,
+} from '../../deployment/runtime-topology';
+
+const topology = productionGatewayTopology('receipt');
+const fleet = {
+  app: RUNTIME_TOPOLOGY.production.components.receiptGateway.app,
+  config: RUNTIME_TOPOLOGY.production.components.receiptGateway.image.config,
+  dockerfile:
+    RUNTIME_TOPOLOGY.production.components.receiptGateway.image.dockerfile,
+  minimumMachines: topology.minimumMachines,
+  primaryRegion: topology.region,
+  processGroup: topology.processGroup,
+};
+const HEALTH_TIMEOUT_MS = 10 * 60_000;
+
+const reportGatewayRouteProbeRetry = (
+  retry: Parameters<typeof formatGatewayRouteProbeRetry>[0],
+): void => {
+  console.warn(formatGatewayRouteProbeRetry(retry));
+};
+const POLL_MS = 5_000;
+const CANDIDATE_WAIT_PROGRESS_INTERVAL_MS = 30_000;
+const FLY_MACHINE_INVENTORY_TIMEOUT_MS = 30_000;
+const RECEIPT_GATEWAY_OPERATION_PHASE_METADATA_KEY =
+  'deepline_gateway_operation_phase';
+
+/**
+ * This is persisted on every candidate in Fly metadata before the next
+ * externally-visible transition. The string is only the Fly boundary codec;
+ * controller code consumes this enum rather than an unvalidated state string.
+ */
+export enum ReceiptGatewayOperationPhase {
+  Prepared = 'prepared',
+  Proven = 'proven',
+  Activating = 'activating',
+}
+
+/** Provider-observed routing state for a single SHA-fenced candidate fleet. */
+export enum ReceiptGatewayCandidateRouting {
+  Cordoned = 'cordoned',
+  Mixed = 'mixed',
+  Public = 'public',
+}
+
+export enum ReceiptGatewayPreparationAction {
+  Prepare = 'prepare',
+  ReconcileActivation = 'reconcile-activation',
+  ResumeActivation = 'resume-activation',
+}
+
+type Flags = Map<string, string | boolean>;
+
+function flags(argv: string[]): Flags {
+  const result = new Map<string, string | boolean>();
+  for (let index = 0; index < argv.length; index += 1) {
+    const token = argv[index]!;
+    if (!token.startsWith('--')) throw new Error(`Unexpected ${token}.`);
+    const key = token.slice(2);
+    const value = argv[index + 1];
+    if (!value || value.startsWith('--')) result.set(key, true);
+    else {
+      result.set(key, value);
+      index += 1;
+    }
+  }
+  return result;
+}
+
+function sha(input: Flags): string {
+  const value = input.get('sha');
+  if (typeof value !== 'string' || !/^[a-f0-9]{7,40}$/i.test(value)) {
+    throw new Error('--sha must be a 7-40 character git SHA.');
+  }
+  return value.toLowerCase();
+}
+
+function requireApply(input: Flags): void {
+  if (
+    input.get('environment') !== 'production' ||
+    input.get('apply') !== true
+  ) {
+    throw new Error('Requires --environment production --apply.');
+  }
+}
+
+type ReceiptGatewayCommandResult = Readonly<{
+  error?: Error & { code?: string };
+  status: number | null;
+  stdout?: string | null;
+  stderr?: string | null;
+}>;
+
+type ReceiptGatewayCommandSpawn = (
+  command: string,
+  args: readonly string[],
+  options: Readonly<{
+    encoding: 'utf8';
+    env: NodeJS.ProcessEnv;
+    timeout?: number;
+  }>,
+) => ReceiptGatewayCommandResult;
+
+export function runReceiptGatewayCommand(
+  command: string,
+  args: string[],
+  env = process.env,
+  options: {
+    echoStdout?: boolean;
+    timeoutMs?: number;
+    spawnCommand?: ReceiptGatewayCommandSpawn;
+  } = {},
+): string {
+  const spawnCommand: ReceiptGatewayCommandSpawn =
+    options.spawnCommand ??
+    ((executable, commandArgs, spawnOptions) =>
+      spawnSync(executable, commandArgs, spawnOptions));
+  const result = spawnCommand(command, args, {
+    encoding: 'utf8',
+    env,
+    ...(options.timeoutMs === undefined ? {} : { timeout: options.timeoutMs }),
+  });
+  if (result.stdout && options.echoStdout !== false)
+    process.stdout.write(result.stdout);
+  if (result.stderr) process.stderr.write(result.stderr);
+  if (result.error) {
+    if (result.error.code === 'ETIMEDOUT' && options.timeoutMs !== undefined) {
+      throw new Error(
+        `${command} ${args.join(' ')} timed out after ${options.timeoutMs}ms.`,
+      );
+    }
+    throw result.error;
+  }
+  if (result.status !== 0) {
+    throw new Error(
+      `${command} ${args.join(' ')} failed with ${result.status}.`,
+    );
+  }
+  return result.stdout ?? '';
+}
+
+function run(
+  command: string,
+  args: string[],
+  env = process.env,
+  options: { echoStdout?: boolean } = {},
+): string {
+  return runReceiptGatewayCommand(command, args, env, options);
+}
+
+function runAsync(
+  command: string,
+  args: string[],
+  env = process.env,
+  cwd?: string,
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, { cwd, env, stdio: 'inherit' });
+    child.once('error', reject);
+    child.once('close', (status) => {
+      if (status === 0) {
+        resolve();
+        return;
+      }
+      reject(
+        new Error(`${command} failed with ${status ?? 'no'} exit status.`),
+      );
+    });
+  });
+}
+
+function receiptGatewayImage(operationId: string): string {
+  return `registry.fly.io/${fleet.app}:git-${operationId}`;
+}
+
+/**
+ * One bounded line per Machine. Echoing the raw `flyctl machines list --json`
+ * payload printed thousands of lines of config, image labels, and event
+ * history on every poll, which buried the actual failure underneath it. The
+ * full config is never printed.
+ */
+let lastPrintedInventorySignature: string | null = null;
+
+export function formatReceiptGatewayInventory(
+  inventory: readonly FlyMachine[],
+): string[] {
+  return inventory.map((machine) => {
+    const checks = machine.checks ?? [];
+    const rendered = checks.length
+      ? checks.map((check) => formatFlyMachineCheck(check)).join(' ')
+      : 'checks=none';
+    return [
+      machine.id ?? '<no-id>',
+      machine.name ?? '<no-name>',
+      `state=${machine.state ?? '<no-state>'}`,
+      `cordoned=${machine.cordoned ?? '<unknown>'}`,
+      `image=${imageTag(machine)}`,
+      rendered,
+    ].join(' ');
+  });
+}
+
+function receiptGatewayCandidateWaitDetails(
+  candidates: readonly FlyMachine[],
+): string {
+  const details = formatReceiptGatewayInventory(candidates).join(' | ');
+  return details.length <= 2_000 ? details : `${details.slice(0, 2_000)}…`;
+}
+
+function formatFlyMachineCheck(check: {
+  name?: string;
+  status?: string;
+  output?: string;
+}): string {
+  const output = check.output?.trim();
+  return `check ${check.name ?? '<unnamed>'}=${check.status ?? '<no-status>'}${
+    output ? ` "${output}"` : ''
+  }`;
+}
+
+/** Only the image tag; the registry path is identical on every Machine. */
+function imageTag(machine: FlyMachine): string {
+  const image = machine.config?.image;
+  if (!image) return '<no-image>';
+  const tag = image.slice(image.lastIndexOf(':') + 1);
+  return tag || image;
+}
+
+function machines(): FlyMachine[] {
+  const output = runReceiptGatewayCommand(
+    'flyctl',
+    ['machines', 'list', '-a', fleet.app, '--json'],
+    process.env,
+    {
+      echoStdout: false,
+      timeoutMs: FLY_MACHINE_INVENTORY_TIMEOUT_MS,
+    },
+  );
+  const parsed: unknown = JSON.parse(output);
+  if (!Array.isArray(parsed))
+    throw new Error('Fly Machine inventory is invalid.');
+  const inventory = parsed as FlyMachine[];
+  // Printed before anything can throw on this inventory, so the failure that
+  // follows is the last thing in the log rather than the first. Polls repeat
+  // every few seconds; only a changed fleet state is worth a new line.
+  const lines = formatReceiptGatewayInventory(inventory);
+  const signature = lines.join('\n');
+  if (signature !== lastPrintedInventorySignature) {
+    lastPrintedInventorySignature = signature;
+    for (const line of lines) console.info(line);
+  }
+  return inventory;
+}
+
+function active(machine: FlyMachine): boolean {
+  return !['destroyed', 'destroying'].includes(machine.state ?? '');
+}
+
+function healthy(machine: FlyMachine): boolean {
+  return (
+    machine.state === 'started' &&
+    (machine.checks?.length ?? 0) > 0 &&
+    machine.checks!.every((check) => check.status === 'passing')
+  );
+}
+
+function candidateName(operationId: string, index: number): string {
+  return `receipt-${operationId.slice(0, 12)}-${index + 1}`;
+}
+
+function candidatesFor(
+  operationId: string,
+  inventory = machines(),
+): FlyMachine[] {
+  return inventory.filter(
+    (machine) =>
+      active(machine) &&
+      machine.config?.metadata?.deepline_gateway_role === 'receipt' &&
+      machine.config.metadata.deepline_gateway_rollout === operationId,
+  );
+}
+
+/**
+ * A prepared candidate may be activated only against the exact serving
+ * generation it cloned. Another release can prepare concurrently, but once a
+ * different release has cut over, this candidate is stale and must be cleaned
+ * up or re-prepared. This prevents a late job from uncordoning an older image.
+ */
+export function assertCandidateSourceGeneration(input: {
+  candidates: FlyMachine[];
+  serving: FlyMachine[];
+}): string[] {
+  const sourceIds = candidateSourceIds(input.candidates);
+  const servingIds = input.serving
+    .map((machine) => machine.id)
+    .filter((id): id is string => Boolean(id));
+  if (
+    sourceIds.length !== servingIds.length ||
+    sourceIds.some((id) => !servingIds.includes(id))
+  ) {
+    throw new Error(
+      'Receipt candidates were prepared against a superseded serving generation; clean them up and prepare again.',
+    );
+  }
+  return sourceIds;
+}
+
+export function candidateSourceIds(candidates: FlyMachine[]): string[] {
+  const sourceIds = [
+    ...new Set(
+      candidates
+        .map(
+          (machine) =>
+            machine.config?.metadata?.deepline_gateway_source_machine,
+        )
+        .filter((id): id is string => Boolean(id)),
+    ),
+  ];
+  if (sourceIds.length !== candidates.length) {
+    throw new Error(
+      'Receipt candidates do not carry an exact source Machine identity.',
+    );
+  }
+  return sourceIds;
+}
+
+function candidatePhase(machine: FlyMachine): ReceiptGatewayOperationPhase {
+  const stored =
+    machine.config?.metadata?.[RECEIPT_GATEWAY_OPERATION_PHASE_METADATA_KEY];
+  if (stored === undefined && machine.cordoned === true) {
+    // Existing cordoned candidates predate the durable phase marker. They are
+    // compatible with a fresh proof, but never with activation or recovery.
+    return ReceiptGatewayOperationPhase.Prepared;
+  }
+  if (
+    !Object.values(ReceiptGatewayOperationPhase).includes(
+      stored as ReceiptGatewayOperationPhase,
+    )
+  ) {
+    throw new Error(
+      `Receipt candidate ${machine.id ?? '<missing>'} has an invalid durable operation phase.`,
+    );
+  }
+  return stored as ReceiptGatewayOperationPhase;
+}
+
+export function receiptGatewayCandidateRouting(
+  candidates: readonly FlyMachine[],
+): ReceiptGatewayCandidateRouting {
+  if (candidates.length === 0) {
+    throw new Error('Receipt rollout has no Fly candidates.');
+  }
+  const cordoned = candidates.filter((machine) => machine.cordoned === true);
+  if (cordoned.length === candidates.length) {
+    return ReceiptGatewayCandidateRouting.Cordoned;
+  }
+  if (cordoned.length === 0) {
+    return ReceiptGatewayCandidateRouting.Public;
+  }
+  return ReceiptGatewayCandidateRouting.Mixed;
+}
+
+function assertCandidatePhase(input: {
+  candidates: readonly FlyMachine[];
+  allowed: readonly ReceiptGatewayOperationPhase[];
+}): ReceiptGatewayOperationPhase {
+  const phases = candidatePhases(input.candidates);
+  if (phases.size !== 1) {
+    throw new Error(
+      'Receipt candidates disagree on their durable operation phase.',
+    );
+  }
+  const [phase] = phases;
+  if (!phase || !input.allowed.includes(phase)) {
+    throw new Error(
+      `Receipt candidates are not in an allowed durable operation phase: ${phase ?? 'missing'}.`,
+    );
+  }
+  return phase;
+}
+
+function candidatePhases(
+  candidates: readonly FlyMachine[],
+): ReadonlySet<ReceiptGatewayOperationPhase> {
+  return new Set(candidates.map(candidatePhase));
+}
+
+export type ReceiptGatewayCandidateOperation = Readonly<{
+  phase: ReceiptGatewayOperationPhase;
+  routing: ReceiptGatewayCandidateRouting;
+}>;
+
+/**
+ * Decode Fly's candidate metadata and admission state once. A caller can only
+ * resume a non-cordoned fleet when every candidate carries the durable
+ * `Activating` fence written before the first uncordon.
+ */
+export function classifyReceiptGatewayCandidateOperation(
+  candidates: readonly FlyMachine[],
+): ReceiptGatewayCandidateOperation {
+  const phase = assertCandidatePhase({
+    candidates,
+    allowed: Object.values(ReceiptGatewayOperationPhase),
+  });
+  const routing = receiptGatewayCandidateRouting(candidates);
+  if (
+    routing !== ReceiptGatewayCandidateRouting.Cordoned &&
+    phase !== ReceiptGatewayOperationPhase.Activating
+  ) {
+    throw new Error(
+      `Receipt candidates are not in an allowed durable operation phase for ${routing} routing: ${phase}.`,
+    );
+  }
+  return { phase, routing };
+}
+
+export function receiptGatewayPreparationAction(
+  candidates: readonly FlyMachine[],
+): ReceiptGatewayPreparationAction {
+  const routing = receiptGatewayCandidateRouting(candidates);
+  const phases = candidatePhases(candidates);
+  const hasOnly = (...expected: ReceiptGatewayOperationPhase[]): boolean =>
+    [...phases].every((phase) => expected.includes(phase));
+  if (routing !== ReceiptGatewayCandidateRouting.Cordoned) {
+    if (
+      phases.size === 1 &&
+      phases.has(ReceiptGatewayOperationPhase.Activating)
+    ) {
+      return ReceiptGatewayPreparationAction.ResumeActivation;
+    }
+    throw new Error(
+      'Public receipt candidates require a complete durable activation fence.',
+    );
+  }
+  if (
+    phases.size === 1 &&
+    phases.has(ReceiptGatewayOperationPhase.Activating)
+  ) {
+    return ReceiptGatewayPreparationAction.ResumeActivation;
+  }
+  if (
+    phases.has(ReceiptGatewayOperationPhase.Activating) &&
+    hasOnly(
+      ReceiptGatewayOperationPhase.Proven,
+      ReceiptGatewayOperationPhase.Activating,
+    )
+  ) {
+    // A metadata POST may have succeeded for only part of the cordoned fleet.
+    // Complete the same intent before considering any other transition.
+    return ReceiptGatewayPreparationAction.ReconcileActivation;
+  }
+  if (
+    hasOnly(
+      ReceiptGatewayOperationPhase.Prepared,
+      ReceiptGatewayOperationPhase.Proven,
+    )
+  ) {
+    // Partial proof markers carry no activation authority. Re-run the exact
+    // cordoned fixture proof rather than inferring success from a subset.
+    return ReceiptGatewayPreparationAction.Prepare;
+  }
+  throw new Error(
+    'Receipt candidates have an unrecoverable durable phase set.',
+  );
+}
+
+function assertCandidateFleet(input: {
+  candidates: readonly FlyMachine[];
+  operationId: string;
+  count: number;
+  routing?: ReceiptGatewayCandidateRouting;
+}): 'ready' | 'starting' {
+  if (input.candidates.length !== input.count) return 'starting';
+  for (const machine of input.candidates) {
+    if (!machine.id) throw new Error('Receipt candidate has no identity.');
+    if (
+      !machine.name?.startsWith(`receipt-${input.operationId.slice(0, 12)}-`)
+    ) {
+      throw new Error(
+        `Receipt candidate ${machine.id} has no deterministic operation name.`,
+      );
+    }
+    if (
+      ['failed', 'stopped', 'suspended', 'destroying', 'destroyed'].includes(
+        machine.state ?? '',
+      )
+    ) {
+      throw new Error(
+        `Receipt candidate ${machine.id} entered terminal state ${machine.state}.`,
+      );
+    }
+    if (machine.config?.image !== receiptGatewayImage(input.operationId)) {
+      throw new Error(
+        `Receipt candidate ${machine.id} has the wrong immutable image.`,
+      );
+    }
+    if (!healthy(machine)) return 'starting';
+  }
+  if (input.routing !== undefined) {
+    const observed = receiptGatewayCandidateRouting(input.candidates);
+    if (observed !== input.routing) return 'starting';
+  }
+  return 'ready';
+}
+
+/**
+ * "not healthy" alone cannot be acted on. Name every serving Machine that is
+ * unhealthy and the exact check that says so, so the release log answers
+ * "which Machine, which check, and what did it report?" without a second Fly
+ * query.
+ */
+export function describeUnhealthyServingFleet(input: {
+  serving: readonly FlyMachine[];
+  minimumMachines: number;
+}): string | null {
+  const reasons = input.serving
+    .filter((machine) => !healthy(machine))
+    .map((machine) => {
+      const identity = `${machine.id ?? '<no-id>'} (${machine.name ?? '<no-name>'})`;
+      const checks = machine.checks ?? [];
+      if (machine.state !== 'started') {
+        return `${identity} state=${machine.state ?? '<no-state>'}`;
+      }
+      if (checks.length === 0) return `${identity} reported no health checks`;
+      return `${identity} ${checks
+        .filter((check) => check.status !== 'passing')
+        .map((check) => formatFlyMachineCheck(check))
+        .join(' ')}`;
+    });
+  if (input.serving.length < input.minimumMachines) {
+    reasons.unshift(
+      `expected at least ${input.minimumMachines} serving Machine(s), found ${input.serving.length}`,
+    );
+  }
+  return reasons.length > 0 ? reasons.join('; ') : null;
+}
+
+function serving(inventory = machines()): FlyMachine[] {
+  const result = inventory.filter(
+    (machine) => active(machine) && machine.cordoned === false,
+  );
+  const unhealthy = describeUnhealthyServingFleet({
+    serving: result,
+    minimumMachines: fleet.minimumMachines,
+  });
+  if (unhealthy) {
+    throw new Error(
+      `Receipt gateway serving fleet is not healthy: ${unhealthy}`,
+    );
+  }
+  const images = new Set(
+    result.map((machine) => machine.config?.image).filter(Boolean),
+  );
+  if (images.size !== 1)
+    throw new Error('Receipt gateway serving fleet has mixed images.');
+  return result;
+}
+
+/**
+ * A single Fly check flapping on one blue Machine for a few seconds must not
+ * fail a release. Poll the serving generation until it is coherent or the
+ * health budget is spent, then surface the last actionable reason. Only a
+ * transient fleet condition is retried; every other failure is immediate.
+ */
+const RETRYABLE_SERVING_FAILURE =
+  /serving fleet is not healthy|serving fleet has mixed images/;
+
+export async function waitForServingGeneration(input: {
+  serving: () => FlyMachine[];
+  sleep: (milliseconds: number) => Promise<void>;
+  now?: () => number;
+  timeoutMs?: number;
+  pollMs?: number;
+}): Promise<FlyMachine[]> {
+  const now = input.now ?? Date.now;
+  const pollMs = input.pollMs ?? POLL_MS;
+  const deadline = now() + (input.timeoutMs ?? HEALTH_TIMEOUT_MS);
+  for (;;) {
+    try {
+      return input.serving();
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      if (!RETRYABLE_SERVING_FAILURE.test(message)) throw error;
+      if (now() >= deadline) throw error;
+      await input.sleep(pollMs);
+    }
+  }
+}
+
+export function readinessOfCordonedCandidates(input: {
+  candidates: FlyMachine[];
+  operationId: string;
+  count: number;
+}): 'ready' | 'starting' {
+  return assertCandidateFleet({
+    ...input,
+    routing: ReceiptGatewayCandidateRouting.Cordoned,
+  });
+}
+
+/**
+ * Fly only reports candidate health through its Machine inventory. That
+ * inventory may remain unchanged while a Machine is still booting, so emit a
+ * bounded heartbeat with the latest reason instead of leaving a release log
+ * silent for the full health budget.
+ */
+export async function waitForReceiptGatewayCandidates(input: {
+  operationId: string;
+  count: number;
+  candidates: () => FlyMachine[];
+  readiness: (candidates: FlyMachine[]) => 'ready' | 'starting';
+  sleep: (milliseconds: number) => Promise<void>;
+  now?: () => number;
+  timeoutMs?: number;
+  pollMs?: number;
+  progressIntervalMs?: number;
+  reportProgress?: (message: string) => void;
+  shouldFailImmediately?: (failure: string) => boolean;
+}): Promise<FlyMachine[]> {
+  const now = input.now ?? Date.now;
+  const timeoutMs = input.timeoutMs ?? HEALTH_TIMEOUT_MS;
+  const pollMs = input.pollMs ?? POLL_MS;
+  const progressIntervalMs =
+    input.progressIntervalMs ?? CANDIDATE_WAIT_PROGRESS_INTERVAL_MS;
+  const startedAt = now();
+  const deadline = startedAt + timeoutMs;
+  let lastProgressAt: number | undefined;
+  let observedCandidates: FlyMachine[] = [];
+  let observedCount = 0;
+  let failure = 'starting';
+
+  while (now() < deadline) {
+    try {
+      const candidates = input.candidates();
+      observedCandidates = candidates;
+      observedCount = candidates.length;
+      if (candidates.length !== input.count) {
+        failure = `expected ${input.count} candidates; found ${candidates.length}`;
+      } else if (input.readiness(candidates) === 'starting') {
+        failure = 'receipt candidate is still starting';
+      } else {
+        return candidates;
+      }
+    } catch (error) {
+      failure = error instanceof Error ? error.message : String(error);
+      if (input.shouldFailImmediately?.(failure)) throw error;
+    }
+
+    const current = now();
+    if (
+      lastProgressAt === undefined ||
+      current - lastProgressAt >= progressIntervalMs
+    ) {
+      const elapsedSeconds = Math.floor((current - startedAt) / 1_000);
+      const remainingSeconds = Math.max(
+        0,
+        Math.ceil((deadline - current) / 1_000),
+      );
+      input.reportProgress?.(
+        `[receipt-gateway] still waiting for candidates: operation=${input.operationId} elapsed=${elapsedSeconds}s remaining=${remainingSeconds}s candidates=${observedCount}/${input.count} reason=${JSON.stringify(failure)} details=${JSON.stringify(receiptGatewayCandidateWaitDetails(observedCandidates))}`,
+      );
+      lastProgressAt = current;
+    }
+    await input.sleep(pollMs);
+  }
+
+  throw new Error(
+    `Receipt candidates did not become ready after ${Math.ceil(timeoutMs / 1_000)}s: ${failure} (last observed ${observedCount}/${input.count} candidates).`,
+  );
+}
+
+async function waitForCandidates(
+  operationId: string,
+  count: number,
+): Promise<FlyMachine[]> {
+  return waitForReceiptGatewayCandidates({
+    operationId,
+    count,
+    candidates: () => candidatesFor(operationId),
+    readiness: (candidates) =>
+      readinessOfCordonedCandidates({
+        candidates,
+        operationId,
+        count,
+      }),
+    sleep: (milliseconds) =>
+      new Promise((resolve) => setTimeout(resolve, milliseconds)),
+    reportProgress: console.info,
+    shouldFailImmediately: (failure) =>
+      /wrong immutable image|became public|terminal state|has no identity|deterministic operation name|did not serve its public route probe/.test(
+        failure,
+      ),
+  });
+}
+
+async function waitForCandidateFleet(input: {
+  operationId: string;
+  count: number;
+  routing: ReceiptGatewayCandidateRouting;
+  allowedPhases: readonly ReceiptGatewayOperationPhase[];
+}): Promise<FlyMachine[]> {
+  return waitForReceiptGatewayCandidates({
+    operationId: input.operationId,
+    count: input.count,
+    candidates: () => candidatesFor(input.operationId),
+    readiness: (candidates) => {
+      const readiness = assertCandidateFleet({
+        candidates,
+        operationId: input.operationId,
+        count: input.count,
+        routing: input.routing,
+      });
+      if (readiness === 'ready') {
+        assertCandidatePhase({
+          candidates,
+          allowed: input.allowedPhases,
+        });
+      }
+      return readiness;
+    },
+    sleep: (milliseconds) =>
+      new Promise((resolve) => setTimeout(resolve, milliseconds)),
+    reportProgress: console.info,
+    shouldFailImmediately: (failure) =>
+      /wrong immutable image|terminal state|has no identity|deterministic operation name|invalid durable operation phase|disagree on their durable operation phase|not in an allowed durable operation phase/.test(
+        failure,
+      ),
+  });
+}
+
+async function persistCandidatePhase(input: {
+  candidates: readonly FlyMachine[];
+  phase: ReceiptGatewayOperationPhase;
+}): Promise<void> {
+  const client = new FlyMachinesClient(fleet.app, requireFlyToken(), fetch);
+  await Promise.all(
+    input.candidates.map(async (candidate) => {
+      if (!candidate.id) throw new Error('Receipt candidate has no identity.');
+      await client.setMachineMetadata(
+        candidate.id,
+        RECEIPT_GATEWAY_OPERATION_PHASE_METADATA_KEY,
+        input.phase,
+      );
+    }),
+  );
+}
+
+async function assertPublic(machineIds: readonly string[]): Promise<void> {
+  const url = `https://${fleet.app}.fly.dev/readyz`;
+  for (const machineId of machineIds) {
+    const response = await fetch(url, {
+      headers: exactFlyMachineRouteHeaders(machineId),
+      signal: AbortSignal.timeout(10_000),
+    });
+    assertGatewayRouteProbe({
+      status: response.status,
+      body: await response.text(),
+      expectedMachineId: machineId,
+      actualMachineId: response.headers.get('x-deepline-gateway-machine-id'),
+    });
+  }
+  const response = await fetch(url, { signal: AbortSignal.timeout(10_000) });
+  assertGatewayRouteProbe({
+    status: response.status,
+    body: await response.text(),
+    actualMachineId: response.headers.get('x-deepline-gateway-machine-id'),
+  });
+}
+
+async function removeCordoned(operationId: string): Promise<void> {
+  const targets = candidatesFor(operationId);
+  if (targets.some((machine) => !machine.id)) {
+    throw new Error(
+      'Receipt candidate cleanup requires exact Fly Machine identities.',
+    );
+  }
+  if (targets.some((machine) => machine.cordoned !== true)) {
+    throw new Error(
+      'Refusing to remove a receipt candidate that received public traffic.',
+    );
+  }
+  const client = new FlyMachinesClient(fleet.app, requireFlyToken(), fetch);
+  await stopAndDestroyCordonedMachines({
+    machineIds: targets.map((machine) => machine.id!).filter(Boolean),
+    readMachines: machines,
+    listProviderMachineIds: async () =>
+      (await client.listMachines()).map((machine) => machine.id),
+    stopMachine: (machineId) =>
+      client.stopMachine(machineId, {
+        signal: 'SIGTERM',
+        timeout: RECEIPT_GATEWAY_DRAIN_TIMEOUT,
+      }),
+    deleteMachine: (machineId) => client.deleteMachine(machineId, false),
+  });
+}
+
+/** The public green candidates are the durable Fly operation record. Their
+ * source metadata is the only authority for destructive blue retirement. */
+async function drain(operationId: string): Promise<void> {
+  const client = new FlyMachinesClient(fleet.app, requireFlyToken(), fetch);
+  const green = candidatesFor(operationId, await client.listMachines());
+  if (
+    green.length < fleet.minimumMachines ||
+    green.some(
+      (machine) =>
+        !machine.id || !healthy(machine) || machine.cordoned !== false,
+    )
+  ) {
+    throw new Error(
+      'Receipt gateway drain requires the exact rollout to remain a complete healthy public generation.',
+    );
+  }
+  const outcome = await reconcileGatewaySourceDrain({
+    app: fleet.app,
+    client,
+    sourceMachineIds: candidateSourceIds(green),
+  });
+  console.info(JSON.stringify({ operationId, ...outcome }));
+}
+
+async function prepare(operationId: string): Promise<void> {
+  // Different immutable releases may prepare cordoned candidates in parallel.
+  // Never let one release delete another release's fenced evidence; the
+  // topology cleanup command owns that exact-SHA operation explicitly.
+  const existing = candidatesFor(operationId);
+  if (existing.length > 0) {
+    const action = receiptGatewayPreparationAction(existing);
+    if (action !== ReceiptGatewayPreparationAction.Prepare) {
+      const routing = receiptGatewayCandidateRouting(existing);
+      // The durable activation fence is written before any uncordon. A retry
+      // must observe and resume this exact fleet, not re-evaluate the current
+      // public generation, create replacements, or attempt compensation.
+      if (action === ReceiptGatewayPreparationAction.ReconcileActivation) {
+        await persistCandidatePhase({
+          candidates: existing,
+          phase: ReceiptGatewayOperationPhase.Activating,
+        });
+      }
+      const resumed = await waitForCandidateFleet({
+        operationId,
+        count: existing.length,
+        routing,
+        allowedPhases: [ReceiptGatewayOperationPhase.Activating],
+      });
+      candidateSourceIds(resumed);
+      console.info(
+        `Resuming receipt gateway activation for ${operationId} from ${routing}.`,
+      );
+      return;
+    }
+  }
+  run('bun', [
+    'scripts/ensure-runtime-image.ts',
+    '--kind',
+    'gateway',
+    '--app',
+    fleet.app,
+    '--config',
+    fleet.config,
+    '--dockerfile',
+    fleet.dockerfile,
+    '--sha',
+    operationId,
+  ]);
+  const blue = serving();
+  if (existing.length > 0) {
+    assertCandidateSourceGeneration({ candidates: existing, serving: blue });
+  }
+  const client = new FlyMachinesClient(fleet.app, requireFlyToken(), fetch);
+  for (const [index, source] of blue.entries()) {
+    const name = candidateName(operationId, index);
+    if (existing.some((machine) => machine.name === name)) continue;
+    if (!source.id) throw new Error('Serving Machine has no identity.');
+    await client.createMachine({
+      name,
+      region: fleet.primaryRegion,
+      skip_service_registration: true,
+      config: gatewayMachineConfig({
+        profile: 'production',
+        role: 'receipt',
+        image: receiptGatewayImage(operationId),
+        metadata: {
+          deepline_gateway_role: 'receipt',
+          deepline_gateway_rollout: operationId,
+          deepline_gateway_source_machine: source.id,
+          [RECEIPT_GATEWAY_OPERATION_PHASE_METADATA_KEY]:
+            ReceiptGatewayOperationPhase.Prepared,
+          fly_process_group: fleet.processGroup,
+        },
+      }),
+    });
+  }
+  const ready = await waitForCandidates(operationId, blue.length);
+  await assertPublic(blue.map((machine) => machine.id!));
+  console.info(
+    JSON.stringify({
+      operationId,
+      candidates: ready.map((machine) => machine.id),
+    }),
+  );
+}
+
+/**
+ * Gateway policy code can live in a separately checked-out revision from the
+ * candidate application. The fixture is application code, so its imports and
+ * generated artifacts must resolve from the application checkout instead of
+ * from the policy command's current directory.
+ */
+export function resolveReceiptFixtureApplicationDirectory(
+  applicationDirectory?: string,
+): string {
+  return applicationDirectory?.trim()
+    ? resolve(applicationDirectory)
+    : process.cwd();
+}
+
+export function receiptFixtureCanaryInvocation(input: {
+  url: string;
+  machineId: string;
+  applicationDirectory?: string;
+}): Readonly<{
+  command: 'bun';
+  args: readonly string[];
+  cwd: string;
+}> {
+  return {
+    command: 'bun',
+    args: [
+      'scripts/run-production-release-canary.ts',
+      '--host',
+      input.url,
+      '--label',
+      `receipt-${input.machineId}`,
+      '--mode',
+      'receipt-candidate',
+    ],
+    cwd: resolveReceiptFixtureApplicationDirectory(input.applicationDirectory),
+  };
+}
+
+function canary(
+  url: string,
+  machineId: string,
+  applicationDirectory?: string,
+): Promise<void> {
+  const invocation = receiptFixtureCanaryInvocation({
+    url,
+    machineId,
+    applicationDirectory,
+  });
+  return runAsync(
+    invocation.command,
+    [...invocation.args],
+    {
+      ...process.env,
+      DEEPLINE_PREFERRED_RECEIPT_GATEWAY_MACHINE_ID: machineId,
+    },
+    invocation.cwd,
+  );
+}
+
+/**
+ * The proof boundary deliberately has no routing mutation.  A fixture is
+ * force-routed only by the already-authorized internal runtime transport; the
+ * helper re-reads the cordon invariant before and after every exact-Machine
+ * proof so a concurrent or accidental uncordon cannot be mistaken for a
+ * cordoned proof.
+ */
+export async function proveCordonedReceiptCandidates(input: {
+  greenMachineIds: readonly string[];
+  assertCordoned: (machineIds: readonly string[]) => Promise<void>;
+  canary: (machineId: string) => void | Promise<void>;
+}): Promise<void> {
+  await input.assertCordoned(input.greenMachineIds);
+  const outcomes = await Promise.allSettled(
+    input.greenMachineIds.map(
+      async (machineId) => await input.canary(machineId),
+    ),
+  );
+  await input.assertCordoned(input.greenMachineIds);
+  const failed = outcomes.find(
+    (outcome): outcome is PromiseRejectedResult =>
+      outcome.status === 'rejected',
+  );
+  if (failed) {
+    throw failed.reason;
+  }
+}
+
+export async function activateCordonedReceiptCandidates(input: {
+  greenMachineIds: readonly string[];
+  blueMachineIds: readonly string[];
+  uncordonMachineIds?: readonly string[];
+  uncordon: (machineId: string) => void | Promise<void>;
+  cordon: (machineId: string) => void | Promise<void>;
+  assertPublic: (machineIds: readonly string[]) => Promise<void>;
+  sleep: (milliseconds: number) => Promise<void>;
+}): Promise<void> {
+  try {
+    for (const machineId of input.uncordonMachineIds ?? input.greenMachineIds)
+      await input.uncordon(machineId);
+    await input.sleep(POLL_MS);
+    await waitForGatewayRouteProbe({
+      description: 'Receipt gateway green generation',
+      probe: () => input.assertPublic(input.greenMachineIds),
+      sleep: input.sleep,
+      onRetry: reportGatewayRouteProbeRetry,
+    });
+  } catch (error) {
+    for (const machineId of input.greenMachineIds)
+      await input.cordon(machineId);
+    await input.sleep(POLL_MS);
+    await waitForGatewayRouteProbe({
+      description: 'Receipt gateway rollback generation',
+      probe: () => input.assertPublic(input.blueMachineIds),
+      sleep: input.sleep,
+      onRetry: reportGatewayRouteProbeRetry,
+    });
+    throw error;
+  }
+}
+
+/**
+ * Restore the exact source generation after a public candidate failed a later
+ * tuple check.  Source Machines are uncordoned and proven first; only then is
+ * the failed generation removed from admission.  This is intentionally the
+ * inverse of `activateCordonedReceiptCandidates`, not a new deployment.
+ */
+export async function restoreReceiptGatewaySourceGeneration(input: {
+  greenMachineIds: readonly string[];
+  blueMachineIds: readonly string[];
+  uncordon: (machineId: string) => void | Promise<void>;
+  cordon: (machineId: string) => void | Promise<void>;
+  assertPublic: (machineIds: readonly string[]) => Promise<void>;
+  sleep: (milliseconds: number) => Promise<void>;
+}): Promise<void> {
+  if (input.greenMachineIds.length === 0 || input.blueMachineIds.length === 0) {
+    throw new Error(
+      'Gateway rollback requires exact green and blue Machine identities.',
+    );
+  }
+  for (const machineId of input.blueMachineIds) await input.uncordon(machineId);
+  await input.sleep(POLL_MS);
+  await waitForGatewayRouteProbe({
+    description: 'Receipt gateway rollback source generation',
+    probe: () => input.assertPublic(input.blueMachineIds),
+    sleep: input.sleep,
+    onRetry: reportGatewayRouteProbeRetry,
+  });
+  for (const machineId of input.greenMachineIds) await input.cordon(machineId);
+  await input.sleep(POLL_MS);
+  await waitForGatewayRouteProbe({
+    description: 'Receipt gateway restored source generation',
+    probe: () => input.assertPublic(input.blueMachineIds),
+    sleep: input.sleep,
+    onRetry: reportGatewayRouteProbeRetry,
+  });
+}
+
+function assertExactCordonedCandidates(input: {
+  operationId: string;
+  expectedMachineIds: readonly string[];
+}): Promise<void> {
+  const candidates = candidatesFor(input.operationId);
+  const candidateIds = candidates
+    .map((machine) => machine.id)
+    .filter((machineId): machineId is string => Boolean(machineId));
+  if (
+    candidateIds.length !== input.expectedMachineIds.length ||
+    input.expectedMachineIds.some(
+      (machineId) => !candidateIds.includes(machineId),
+    )
+  ) {
+    throw new Error(
+      'Receipt proof candidates no longer match the SHA-fenced Fly operation.',
+    );
+  }
+  if (candidates.some((machine) => machine.cordoned !== true)) {
+    throw new Error(
+      'Receipt candidate became public before the proof completed.',
+    );
+  }
+  return Promise.resolve();
+}
+
+async function prove(
+  operationId: string,
+  stagedUrl: string,
+  fixtureApplicationDirectory?: string,
+): Promise<void> {
+  let green = candidatesFor(operationId);
+  if (green.length === 0) {
+    throw new Error('Receipt rollout has no Fly candidates to prove.');
+  }
+  const greenMachineIds = green
+    .map((machine) => machine.id)
+    .filter((machineId): machineId is string => Boolean(machineId));
+  if (greenMachineIds.length !== green.length) {
+    throw new Error('Receipt proof candidate has no Fly Machine identity.');
+  }
+  const routing = receiptGatewayCandidateRouting(green);
+  if (routing === ReceiptGatewayCandidateRouting.Cordoned) {
+    const action = receiptGatewayPreparationAction(green);
+    if (action !== ReceiptGatewayPreparationAction.Prepare) {
+      // A worker died after recording activation intent but before the first
+      // uncordon. That durable intent is stronger than a vanished runner-local
+      // receipt; the later commit will resume the same exact candidates.
+      if (action === ReceiptGatewayPreparationAction.ReconcileActivation) {
+        await persistCandidatePhase({
+          candidates: green,
+          phase: ReceiptGatewayOperationPhase.Activating,
+        });
+      }
+      await waitForCandidateFleet({
+        operationId,
+        count: greenMachineIds.length,
+        routing: ReceiptGatewayCandidateRouting.Cordoned,
+        allowedPhases: [ReceiptGatewayOperationPhase.Activating],
+      });
+      return;
+    }
+    assertCandidateSourceGeneration({
+      candidates: green,
+      serving: await waitForServingGeneration({
+        serving: () => serving(),
+        sleep: (milliseconds) =>
+          new Promise((resolve) => setTimeout(resolve, milliseconds)),
+      }),
+    });
+    green = await waitForCandidates(operationId, greenMachineIds.length);
+    await proveCordonedReceiptCandidates({
+      greenMachineIds,
+      assertCordoned: (machineIds) =>
+        assertExactCordonedCandidates({
+          operationId,
+          expectedMachineIds: machineIds,
+        }),
+      canary: (machineId) =>
+        canary(stagedUrl, machineId, fixtureApplicationDirectory),
+    });
+    await persistCandidatePhase({
+      candidates: green,
+      phase: ReceiptGatewayOperationPhase.Proven,
+    });
+    await waitForCandidateFleet({
+      operationId,
+      count: greenMachineIds.length,
+      routing: ReceiptGatewayCandidateRouting.Cordoned,
+      allowedPhases: [ReceiptGatewayOperationPhase.Proven],
+    });
+    return;
+  }
+
+  // A public or partially-public green fleet can only be resumed when every
+  // candidate carries the durable activation intent written before its first
+  // uncordon. Never pretend this is a fresh cordoned proof; it is recovery of
+  // the exact provider operation after a runner/network interruption.
+  await waitForCandidateFleet({
+    operationId,
+    count: greenMachineIds.length,
+    routing,
+    allowedPhases: [ReceiptGatewayOperationPhase.Activating],
+  });
+  console.info(
+    `Resuming interrupted receipt gateway activation for ${operationId} from ${routing}.`,
+  );
+}
+
+async function activate(operationId: string): Promise<void> {
+  let green = candidatesFor(operationId);
+  if (green.length === 0)
+    throw new Error('Receipt rollout has no Fly candidates.');
+  let candidateOperation = classifyReceiptGatewayCandidateOperation(green);
+  let routing = candidateOperation.routing;
+  // Once green is public, retries complete this same Fly topology. They never
+  // invent a replacement fleet or silently roll back an ambiguous cutover.
+  let sourceIds: string[];
+  if (routing === ReceiptGatewayCandidateRouting.Cordoned) {
+    const phase = candidateOperation.phase;
+    if (
+      ![
+        ReceiptGatewayOperationPhase.Proven,
+        ReceiptGatewayOperationPhase.Activating,
+      ].includes(phase)
+    ) {
+      throw new Error(
+        `Cordoned receipt candidates are not in an allowed durable operation phase: ${phase}.`,
+      );
+    }
+    sourceIds = assertCandidateSourceGeneration({
+      candidates: green,
+      serving: serving(),
+    });
+    if (phase === ReceiptGatewayOperationPhase.Proven) {
+      // Write the externally durable intent before the first admission
+      // mutation. A new runner can now resume this exact candidate set even
+      // when its temporary proof receipt has disappeared.
+      await persistCandidatePhase({
+        candidates: green,
+        phase: ReceiptGatewayOperationPhase.Activating,
+      });
+    }
+    green = await waitForCandidateFleet({
+      operationId,
+      count: sourceIds.length,
+      routing: ReceiptGatewayCandidateRouting.Cordoned,
+      allowedPhases: [ReceiptGatewayOperationPhase.Activating],
+    });
+    candidateOperation = classifyReceiptGatewayCandidateOperation(green);
+    routing = candidateOperation.routing;
+  } else {
+    green = await waitForCandidateFleet({
+      operationId,
+      count: green.length,
+      routing,
+      allowedPhases: [ReceiptGatewayOperationPhase.Activating],
+    });
+    sourceIds = candidateSourceIds(green);
+  }
+
+  // There is deliberately no release-SHA fence here. The workflow's
+  // release-train concurrency group runs one production release at a time and
+  // debounces queued pushes to the newest main commit, so a cutover that has
+  // started finishes even when main moves; the newer commit releases next.
+  // Aborting here used to strand every release once merges landed faster than
+  // one release could complete.
+
+  if (routing !== ReceiptGatewayCandidateRouting.Public) {
+    await activateCordonedReceiptCandidates({
+      greenMachineIds: green.map((machine) => machine.id!),
+      blueMachineIds: sourceIds,
+      uncordonMachineIds: green
+        .filter((machine) => machine.cordoned === true)
+        .map((machine) => machine.id!),
+      uncordon: (machineId) => {
+        run('flyctl', ['machine', 'uncordon', machineId, '-a', fleet.app]);
+      },
+      cordon: (machineId) => {
+        run('flyctl', ['machine', 'cordon', machineId, '-a', fleet.app]);
+      },
+      assertPublic,
+      sleep: (milliseconds) =>
+        new Promise((resolve) => setTimeout(resolve, milliseconds)),
+    });
+  } else {
+    // An interrupted commit resumes its exact public green fleet. Do not run
+    // a hidden fixture here: proof is an explicit, pre-commit operation.
+    await waitForGatewayRouteProbe({
+      description: 'Receipt gateway resumed serving generation',
+      probe: () => assertPublic(green.map((machine) => machine.id!)),
+      onRetry: reportGatewayRouteProbeRetry,
+    });
+  }
+
+  const blue = machines().filter(
+    (machine) => active(machine) && sourceIds.includes(machine.id ?? ''),
+  );
+  for (const machine of blue) {
+    if (machine.cordoned === false) {
+      run('flyctl', ['machine', 'cordon', machine.id!, '-a', fleet.app]);
+    }
+  }
+  await new Promise((resolve) => setTimeout(resolve, POLL_MS));
+  await waitForGatewayRouteProbe({
+    description: 'Receipt gateway serving generation after blue cordon',
+    probe: () => assertPublic(green.map((machine) => machine.id!)),
+    onRetry: reportGatewayRouteProbeRetry,
+  });
+  // Blue remains cordoned as the rollback fleet. Its separately authorized
+  // retirement must use exact source IDs and provider-confirmed deletion; an
+  // activation retry must never destroy the only rollback path.
+}
+
+async function rollback(operationId: string): Promise<void> {
+  const green = candidatesFor(operationId);
+  const greenMachineIds = green
+    .map((machine) => machine.id)
+    .filter((machineId): machineId is string => Boolean(machineId));
+  if (greenMachineIds.length !== green.length || green.length === 0) {
+    throw new Error(
+      'Receipt gateway rollback requires a complete candidate generation.',
+    );
+  }
+  const sourceIds = candidateSourceIds(green);
+  const inventory = machines();
+  const blue = inventory.filter((machine) =>
+    sourceIds.includes(machine.id ?? ''),
+  );
+  if (
+    blue.length !== sourceIds.length ||
+    blue.some((machine) => !machine.id || !active(machine) || !healthy(machine))
+  ) {
+    throw new Error(
+      'Receipt gateway rollback source generation is not a complete healthy retained fleet.',
+    );
+  }
+  await restoreReceiptGatewaySourceGeneration({
+    greenMachineIds,
+    blueMachineIds: sourceIds,
+    uncordon: (machineId) => {
+      run('flyctl', ['machine', 'uncordon', machineId, '-a', fleet.app]);
+    },
+    cordon: (machineId) => {
+      run('flyctl', ['machine', 'cordon', machineId, '-a', fleet.app]);
+    },
+    assertPublic,
+    sleep: (milliseconds) =>
+      new Promise((resolve) => setTimeout(resolve, milliseconds)),
+  });
+}
+
+export async function main(): Promise<void> {
+  const command = process.argv[2];
+  const input = flags(process.argv.slice(3));
+  requireApply(input);
+  const operationId = sha(input);
+  if (command === 'prepare') return await prepare(operationId);
+  if (command === 'prove') {
+    const stagedUrl = input.get('staged-url');
+    if (typeof stagedUrl !== 'string' || !stagedUrl.startsWith('https://')) {
+      throw new Error('prove requires --staged-url https://…');
+    }
+    const fixtureApplicationDirectory = input.get('fixture-application-dir');
+    if (
+      fixtureApplicationDirectory !== undefined &&
+      typeof fixtureApplicationDirectory !== 'string'
+    ) {
+      throw new Error('prove --fixture-application-dir requires a directory.');
+    }
+    return await prove(operationId, stagedUrl, fixtureApplicationDirectory);
+  }
+  if (command === 'activate') {
+    return await activate(operationId);
+  }
+  if (command === 'rollback') return await rollback(operationId);
+  if (command === 'drain') return await drain(operationId);
+  if (command === 'cleanup') return await removeCordoned(operationId);
+  throw new Error(
+    'Usage: roll-receipt-gateway.ts prepare|prove|activate|rollback|drain|cleanup --environment production --sha <sha> [--staged-url <url>] [--fixture-application-dir <dir>] --apply',
+  );
+}