deepline 0.3.90 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/runtime-topology.ts

@@ -0,0 +1,1024 @@
+#!/usr/bin/env bun
+
+/**
+ * Typed, declarative release executor for the fixed production Fly runtime.
+ *
+ * The manifest owns identities and machine shapes. This command owns the one
+ * fenced transition between receipt gateway revisions. GitHub Actions passes
+ * credentials and the immutable release SHA; it does not encode the state
+ * machine in YAML.
+ */
+import {
+  spawn,
+  spawnSync,
+  type SpawnSyncOptionsWithStringEncoding,
+  type SpawnSyncReturns,
+} from 'node:child_process';
+import {
+  appendFileSync,
+  renameSync,
+  readFileSync,
+  writeFileSync,
+} from 'node:fs';
+import { productionGatewayTopology } from '../../deployment/runtime-topology';
+import {
+  shouldCreateDetachedProcessGroup,
+  terminateChildProcessTree,
+} from '../process-tree';
+
+export enum TopologyOperation {
+  ReconcilePlatform = 'reconcile-platform',
+  StageGatewaySecrets = 'stage-gateway-secrets',
+  EnsureExecutionRelay = 'ensure-execution-relay',
+  PrepareReceiptCandidates = 'prepare-receipt-candidates',
+  ProveReceiptCandidates = 'prove-receipt-candidates',
+  ActivateReceiptCandidates = 'activate-receipt-candidates',
+  ProveActiveGatewayFleet = 'prove-active-gateway-fleet',
+  CleanupReceiptCandidates = 'cleanup-receipt-candidates',
+}
+
+/**
+ * The only release transitions are prepare → prove → commit.  Candidate
+ * identities live in Fly metadata and the typed phase prevents an in-process
+ * caller from activating an unproven topology.
+ */
+export enum ProductionTopologyControllerPhase {
+  Prepare = 'prepare',
+  Prove = 'prove',
+  Commit = 'commit',
+}
+
+enum RuntimeTopologyCliCommand {
+  Prepare = 'prepare',
+  Prove = 'prove',
+  Commit = 'commit',
+  Rollback = 'rollback',
+  Cleanup = 'cleanup',
+}
+
+export type ProductionTopologyPlan = {
+  environment: 'production';
+  releaseSha: string;
+  operations: readonly TopologyOperation[];
+  receiptGatewayApp: string;
+  executionRelayApp: string;
+};
+
+const operations: readonly TopologyOperation[] = [
+  TopologyOperation.ReconcilePlatform,
+  TopologyOperation.StageGatewaySecrets,
+  TopologyOperation.EnsureExecutionRelay,
+  TopologyOperation.PrepareReceiptCandidates,
+  TopologyOperation.ProveReceiptCandidates,
+  TopologyOperation.ActivateReceiptCandidates,
+  TopologyOperation.ProveActiveGatewayFleet,
+];
+
+export function planProductionTopologyRelease(input: {
+  releaseSha: string;
+}): ProductionTopologyPlan {
+  if (!/^[a-f0-9]{40}$/i.test(input.releaseSha)) {
+    throw new Error('The topology plan requires a full 40-character git SHA.');
+  }
+  return {
+    environment: 'production',
+    releaseSha: input.releaseSha.toLowerCase(),
+    operations,
+    receiptGatewayApp: productionGatewayTopology('receipt').app,
+    executionRelayApp: productionGatewayTopology('execution').app,
+  };
+}
+
+export function planProductionTopologyCleanup(input: {
+  releaseSha: string;
+}): ProductionTopologyPlan {
+  const release = planProductionTopologyRelease(input);
+  return {
+    ...release,
+    operations: [TopologyOperation.CleanupReceiptCandidates],
+  };
+}
+
+export type RuntimeTopologyCommand = (input: {
+  command: string;
+  args: string[];
+  environment: Record<string, string>;
+  description: string;
+}) => void | Promise<void>;
+
+/**
+ * Candidate preparation for the two gateway apps is independent once their
+ * shared secrets have been staged. The asynchronous form lets the production
+ * CLI boot them together, while the synchronous command remains useful for
+ * deterministic unit tests and one-at-a-time recovery paths.
+ */
+export type RuntimeTopologyParallelCommand = (input: {
+  command: string;
+  args: string[];
+  environment: Record<string, string>;
+  description: string;
+}) => Promise<void>;
+
+export type RuntimeTopologySpawn = (
+  command: string,
+  args: readonly string[],
+  options: SpawnSyncOptionsWithStringEncoding,
+) => Pick<SpawnSyncReturns<string>, 'error' | 'signal' | 'status'>;
+
+type RuntimeTopologyCancellationSignal = 'SIGINT' | 'SIGTERM';
+type RuntimeTopologySignalProcess = {
+  on: (
+    signal: RuntimeTopologyCancellationSignal,
+    listener: () => void,
+  ) => unknown;
+  removeListener: (
+    signal: RuntimeTopologyCancellationSignal,
+    listener: () => void,
+  ) => unknown;
+};
+type ActiveRuntimeTopologyCommand = Readonly<{
+  cancel: (signal: RuntimeTopologyCancellationSignal) => void;
+}>;
+
+const activeRuntimeTopologyCommands = new Set<ActiveRuntimeTopologyCommand>();
+const runtimeTopologyCancellationSignals: readonly RuntimeTopologyCancellationSignal[] =
+  ['SIGINT', 'SIGTERM'];
+
+function cancelActiveRuntimeTopologyCommands(
+  signal: RuntimeTopologyCancellationSignal,
+): number {
+  for (const command of activeRuntimeTopologyCommands) command.cancel(signal);
+  return activeRuntimeTopologyCommands.size;
+}
+
+/**
+ * `prepare` owns two parallel gateway child processes. Propagate GitHub's
+ * cancellation signal to both so the parent can compensate their cordoned
+ * candidate state instead of leaving an orphaned health wait behind.
+ */
+export function installRuntimeTopologyCancellationHandlers(
+  input: {
+    signalProcess?: RuntimeTopologySignalProcess;
+    cancelActiveCommands?: (
+      signal: RuntimeTopologyCancellationSignal,
+    ) => number;
+    reportCancellation?: (message: string) => void;
+  } = {},
+): () => void {
+  const signalProcess = input.signalProcess ?? process;
+  const cancelActiveCommands =
+    input.cancelActiveCommands ?? cancelActiveRuntimeTopologyCommands;
+  const reportCancellation = input.reportCancellation ?? console.error;
+  const listeners = new Map<RuntimeTopologyCancellationSignal, () => void>();
+  for (const signal of runtimeTopologyCancellationSignals) {
+    const listener = () => {
+      const active = cancelActiveCommands(signal);
+      reportCancellation(
+        `[runtime-topology] cancellation requested by ${signal}; stopping ${active} active topology command(s) before compensation.`,
+      );
+    };
+    listeners.set(signal, listener);
+    signalProcess.on(signal, listener);
+  }
+  return () => {
+    for (const [signal, listener] of listeners) {
+      signalProcess.removeListener(signal, listener);
+    }
+  };
+}
+
+type TopologyExecutionInput = {
+  plan: ProductionTopologyPlan;
+  environment: Record<string, string>;
+  command: RuntimeTopologyCommand;
+  parallelCommand?: RuntimeTopologyParallelCommand;
+};
+
+export type PreparedProductionTopology = Readonly<{
+  phase: ProductionTopologyControllerPhase.Prepare;
+  plan: ProductionTopologyPlan;
+}>;
+
+export type ProvenProductionTopology = Readonly<{
+  phase: ProductionTopologyControllerPhase.Prove;
+  plan: ProductionTopologyPlan;
+  stagedVercelUrl: string;
+}>;
+
+/**
+ * The workflow-scoped proof receipt joins two separate shell steps without
+ * weakening the typed prepare → prove → commit boundary. It contains no
+ * credentials and is intentionally retained after commit: an interrupted
+ * commit must resume its exact SHA-fenced provider operation, not invent a
+ * replacement topology.
+ */
+type ProvenTopologyReceipt = Readonly<{
+  phase: ProductionTopologyControllerPhase.Prove;
+  releaseSha: string;
+  stagedVercelUrl: string;
+}>;
+
+export type CommittedProductionTopology = Readonly<{
+  phase: ProductionTopologyControllerPhase.Commit;
+  plan: ProductionTopologyPlan;
+  stagedVercelUrl: string;
+}>;
+
+export function runRuntimeTopologyCommand(
+  input: Parameters<RuntimeTopologyCommand>[0],
+  command: RuntimeTopologySpawn = spawnSync,
+): void {
+  const result = command(input.command, input.args, {
+    encoding: 'utf8',
+    stdio: 'inherit',
+    env: { ...process.env, ...input.environment },
+  });
+  if (result.error) {
+    throw new Error(
+      `${input.description} could not start: ${result.error.message}`,
+      { cause: result.error },
+    );
+  }
+  if (result.status !== 0) {
+    // Secret staging passes values as Fly CLI args. Do not echo args on error.
+    throw new Error(
+      result.signal
+        ? `${input.description} was terminated by ${result.signal}.`
+        : `${input.description} failed with exit code ${result.status}.`,
+    );
+  }
+}
+
+export async function runRuntimeTopologyAsyncCommand(input: {
+  command: string;
+  args: string[];
+  environment: Record<string, string>;
+  description: string;
+}): Promise<void> {
+  await new Promise<void>((resolve, reject) => {
+    const child = spawn(input.command, input.args, {
+      env: { ...process.env, ...input.environment },
+      stdio: 'inherit',
+      detached: shouldCreateDetachedProcessGroup(),
+    });
+    const activeCommand: ActiveRuntimeTopologyCommand = {
+      cancel: (signal) => {
+        if (child.exitCode === null && child.signalCode === null) {
+          try {
+            terminateChildProcessTree(child, signal);
+          } catch (error) {
+            console.error(
+              `[runtime-topology] could not stop ${input.description} process tree: ${error instanceof Error ? error.message : String(error)}`,
+            );
+          }
+        }
+      },
+    };
+    activeRuntimeTopologyCommands.add(activeCommand);
+    let settled = false;
+    const settle = (): boolean => {
+      if (settled) return false;
+      settled = true;
+      activeRuntimeTopologyCommands.delete(activeCommand);
+      return true;
+    };
+    child.once('error', (error) => {
+      if (!settle()) return;
+      reject(
+        new Error(`${input.description} could not start: ${error.message}`, {
+          cause: error,
+        }),
+      );
+    });
+    child.once('close', (status, signal) => {
+      if (!settle()) return;
+      if (status === 0) {
+        resolve();
+        return;
+      }
+      reject(
+        new Error(
+          signal
+            ? `${input.description} was terminated by ${signal}.`
+            : `${input.description} failed with exit code ${status ?? '<unknown>'}.`,
+        ),
+      );
+    });
+  });
+}
+
+const runCommand: RuntimeTopologyCommand = runRuntimeTopologyAsyncCommand;
+
+export async function runRuntimeTopologyParallelCommand(input: {
+  command: string;
+  args: string[];
+  environment: Record<string, string>;
+  description: string;
+}): Promise<void> {
+  const startedAt = Date.now();
+  console.info(`[runtime-topology] ▶ ${input.description}`);
+  await runRuntimeTopologyAsyncCommand(input);
+  console.info(
+    `[runtime-topology] ✓ ${input.description} (${Math.max(0, Math.round((Date.now() - startedAt) / 1000))}s)`,
+  );
+}
+
+async function runIndependentTopologyCommands(
+  input: TopologyExecutionInput,
+  commands: readonly Parameters<RuntimeTopologyCommand>[0][],
+): Promise<void> {
+  const run =
+    input.parallelCommand ??
+    (async (command: Parameters<RuntimeTopologyCommand>[0]) => {
+      await input.command(command);
+    });
+  const results = await Promise.allSettled(
+    commands.map((command) => run(command)),
+  );
+  const failures = results
+    .filter(
+      (result): result is PromiseRejectedResult => result.status === 'rejected',
+    )
+    .map((result) => result.reason);
+  if (failures.length === 1) throw failures[0];
+  if (failures.length > 1) {
+    throw new AggregateError(
+      failures,
+      'Independent gateway preparation operations failed.',
+    );
+  }
+}
+
+async function stageSecrets(input: {
+  plan: ProductionTopologyPlan;
+  environment: Record<string, string>;
+  command: RuntimeTopologyCommand;
+  parallelCommand?: RuntimeTopologyParallelCommand;
+}): Promise<void> {
+  const commands = [
+    input.plan.receiptGatewayApp,
+    input.plan.executionRelayApp,
+  ].map((app) => ({
+    command: 'flyctl',
+    args: [
+      'secrets',
+      'set',
+      '--app',
+      app,
+      '--stage',
+      `WORKFLOW_RUNTIME_POSTGRES=${input.environment.WORKFLOW_RUNTIME_POSTGRES}`,
+      `DEEPLINE_INTERNAL_TOKEN=${input.environment.DEEPLINE_INTERNAL_TOKEN}`,
+    ],
+    environment: input.environment,
+    description: `Stage secrets for ${app}`,
+  }));
+  await runIndependentTopologyCommands(input, commands);
+}
+
+/**
+ * Refuse ambiguous destinations before touching receipt routing. The Vercel
+ * gate resolves this URL from Vercel's deployment record; `commit` accepts it
+ * as an explicit immutable input so a later step never waits or discovers a
+ * different release on its own.
+ */
+export function assertStagedVercelUrl(stagedVercelUrl: string): string {
+  if (/[\r\n]/.test(stagedVercelUrl)) {
+    throw new Error('--staged-url must be an absolute HTTPS URL.');
+  }
+  let parsed: URL;
+  try {
+    parsed = new URL(stagedVercelUrl);
+  } catch {
+    throw new Error('--staged-url must be an absolute HTTPS URL.');
+  }
+  if (
+    parsed.protocol !== 'https:' ||
+    !parsed.hostname ||
+    parsed.username ||
+    parsed.password ||
+    parsed.hash ||
+    parsed.search
+  ) {
+    throw new Error('--staged-url must be an absolute HTTPS URL.');
+  }
+  return stagedVercelUrl;
+}
+
+export async function cleanupProductionGatewayCandidates(input: {
+  plan: ProductionTopologyPlan;
+  environment: Record<string, string>;
+  command: RuntimeTopologyCommand;
+  description?: string;
+}): Promise<void> {
+  await input.command({
+    command: 'bun',
+    args: [
+      'scripts/roll-receipt-gateway.ts',
+      'cleanup',
+      '--environment',
+      'production',
+      '--sha',
+      input.plan.releaseSha,
+      '--apply',
+    ],
+    environment: {
+      ...input.environment,
+      GITHUB_SHA: input.plan.releaseSha,
+    },
+    description:
+      input.description ?? 'Clean up still-cordoned receipt gateway candidates',
+  });
+  await input.command({
+    command: 'bun',
+    args: [
+      'scripts/roll-execution-relay.ts',
+      'cleanup',
+      '--environment',
+      'production',
+      '--sha',
+      input.plan.releaseSha,
+      '--apply',
+    ],
+    environment: {
+      ...input.environment,
+      GITHUB_SHA: input.plan.releaseSha,
+    },
+    description: 'Clean up still-cordoned execution relay candidates',
+  });
+}
+
+function releaseEnvironment(
+  input: TopologyExecutionInput,
+): Record<string, string> {
+  // A manual dispatch may name an older immutable main SHA. Every child fence
+  // must observe that target rather than the workflow file checkout SHA.
+  return {
+    ...input.environment,
+    GITHUB_SHA: input.plan.releaseSha,
+  };
+}
+
+function writeGithubStepOutput(name: string, value: string): void {
+  if (process.env.GITHUB_OUTPUT) {
+    appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`);
+  }
+}
+
+async function compensatePreparedGatewayCandidates(input: {
+  plan: ProductionTopologyPlan;
+  environment: Record<string, string>;
+  command: RuntimeTopologyCommand;
+}): Promise<void> {
+  try {
+    await cleanupProductionGatewayCandidates({
+      ...input,
+      description: 'Compensate still-cordoned gateway candidates',
+    });
+  } catch (cleanupError) {
+    console.error(
+      `[runtime-topology] compensation failed; retaining the fenced candidates: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`,
+    );
+  }
+}
+
+/**
+ * Idempotently create the cordoned Fly topology. This phase deliberately does
+ * not wait for Vercel or change receipt traffic; it can run beside the staged
+ * Vercel build and is safe to retry after a runner interruption.
+ */
+export async function prepareProductionTopologyPlan(
+  input: TopologyExecutionInput,
+): Promise<PreparedProductionTopology> {
+  const environment = releaseEnvironment(input);
+  let gatewayPrepareStarted = false;
+  try {
+    await input.command({
+      command: 'bun',
+      args: ['scripts/reconcile-runtime-topology.ts', 'ensure-production-apps'],
+      environment,
+      description: 'Reconcile declared Fly apps',
+    });
+    await stageSecrets({ ...input, environment, command: input.command });
+    const executionRelayArgs = [
+      'scripts/roll-execution-relay.ts',
+      'prepare',
+      '--environment',
+      'production',
+      '--sha',
+      input.plan.releaseSha,
+      '--apply',
+    ];
+    // Set the fence immediately before the first provider mutation: an
+    // interrupted relay create can leave a partial cordoned candidate, while
+    // app/secrets reconciliation has nothing candidate-scoped to clean up.
+    gatewayPrepareStarted = true;
+    await runIndependentTopologyCommands(input, [
+      {
+        command: 'bun',
+        args: executionRelayArgs,
+        environment,
+        description: 'Ensure execution relay fleet',
+      },
+      {
+        command: 'bun',
+        args: [
+          'scripts/roll-receipt-gateway.ts',
+          'prepare',
+          '--environment',
+          'production',
+          '--sha',
+          input.plan.releaseSha,
+          '--apply',
+        ],
+        environment,
+        description: 'Prepare receipt gateway candidates',
+      },
+    ]);
+    return {
+      phase: ProductionTopologyControllerPhase.Prepare,
+      plan: input.plan,
+    };
+  } catch (error) {
+    if (gatewayPrepareStarted) {
+      await compensatePreparedGatewayCandidates({
+        plan: input.plan,
+        environment,
+        command: input.command,
+      });
+    }
+    throw error;
+  }
+}
+
+/**
+ * Prove the exact SHA-fenced candidates through the signed internal fixture
+ * before any Fly public-route mutation. A failed proof can safely compensate
+ * because the gateway command refuses to uncordon during this phase.
+ */
+export async function proveProductionTopologyPlan(input: {
+  prepared: PreparedProductionTopology;
+  stagedVercelUrl: string;
+  fixtureApplicationDirectory?: string;
+  environment: Record<string, string>;
+  command: RuntimeTopologyCommand;
+}): Promise<ProvenProductionTopology> {
+  const stagedVercelUrl = assertStagedVercelUrl(input.stagedVercelUrl);
+  const environment = {
+    ...input.environment,
+    GITHUB_SHA: input.prepared.plan.releaseSha,
+  };
+  try {
+    await input.command({
+      command: 'bun',
+      args: [
+        'scripts/roll-receipt-gateway.ts',
+        'prove',
+        '--environment',
+        'production',
+        '--sha',
+        input.prepared.plan.releaseSha,
+        '--staged-url',
+        stagedVercelUrl,
+        ...(input.fixtureApplicationDirectory
+          ? ['--fixture-application-dir', input.fixtureApplicationDirectory]
+          : []),
+        '--apply',
+      ],
+      environment,
+      description: 'Prove cordoned receipt gateway candidates',
+    });
+    return {
+      phase: ProductionTopologyControllerPhase.Prove,
+      plan: input.prepared.plan,
+      stagedVercelUrl,
+    };
+  } catch (error) {
+    await compensatePreparedGatewayCandidates({
+      plan: input.prepared.plan,
+      environment,
+      command: input.command,
+    });
+    throw error;
+  }
+}
+
+function topologyProofReceipt(
+  input: ProvenProductionTopology,
+): ProvenTopologyReceipt {
+  return {
+    phase: ProductionTopologyControllerPhase.Prove,
+    releaseSha: input.plan.releaseSha,
+    stagedVercelUrl: input.stagedVercelUrl,
+  };
+}
+
+function proofReceiptPath(path: string): string {
+  if (!path || /[\r\n]/.test(path)) {
+    throw new Error('--proof-file must be a non-empty filesystem path.');
+  }
+  return path;
+}
+
+function isObject(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+export function parseProvenTopologyReceipt(input: {
+  contents: string;
+  plan: ProductionTopologyPlan;
+}): ProvenProductionTopology {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(input.contents);
+  } catch {
+    throw new Error('Topology proof receipt is not valid JSON.');
+  }
+  if (
+    !isObject(parsed) ||
+    parsed.phase !== ProductionTopologyControllerPhase.Prove ||
+    parsed.releaseSha !== input.plan.releaseSha ||
+    typeof parsed.stagedVercelUrl !== 'string'
+  ) {
+    throw new Error('Topology proof receipt does not match this release SHA.');
+  }
+  return {
+    phase: ProductionTopologyControllerPhase.Prove,
+    plan: input.plan,
+    stagedVercelUrl: assertStagedVercelUrl(parsed.stagedVercelUrl),
+  };
+}
+
+function writeProvenTopologyReceipt(input: {
+  file: string;
+  proven: ProvenProductionTopology;
+}): void {
+  const file = proofReceiptPath(input.file);
+  const temporary = `${file}.tmp-${process.pid}`;
+  writeFileSync(
+    temporary,
+    `${JSON.stringify(topologyProofReceipt(input.proven))}\n`,
+    {
+      mode: 0o600,
+    },
+  );
+  renameSync(temporary, file);
+}
+
+function readProvenTopologyReceipt(input: {
+  file: string;
+  plan: ProductionTopologyPlan;
+}): ProvenProductionTopology {
+  const file = proofReceiptPath(input.file);
+  return parseProvenTopologyReceipt({
+    contents: readFileSync(file, 'utf8'),
+    plan: input.plan,
+  });
+}
+
+/**
+ * Commit accepts only the typed result of a completed proof. It never waits,
+ * creates a replacement, runs a Play, or compensates an ambiguous cutover.
+ * The exact fenced Fly operation remains available for an idempotent retry.
+ */
+export async function commitProductionTopologyPlan(input: {
+  proven: ProvenProductionTopology;
+  environment: Record<string, string>;
+  command: RuntimeTopologyCommand;
+}): Promise<CommittedProductionTopology> {
+  const environment = {
+    ...input.environment,
+    GITHUB_SHA: input.proven.plan.releaseSha,
+  };
+  await input.command({
+    command: 'bun',
+    args: [
+      'scripts/roll-receipt-gateway.ts',
+      'activate',
+      '--environment',
+      'production',
+      '--sha',
+      input.proven.plan.releaseSha,
+      '--apply',
+    ],
+    environment,
+    description: 'Activate proven receipt gateway candidates',
+  });
+  await input.command({
+    command: 'bun',
+    args: [
+      'scripts/roll-execution-relay.ts',
+      'activate',
+      '--environment',
+      'production',
+      '--sha',
+      input.proven.plan.releaseSha,
+      '--apply',
+    ],
+    environment,
+    description: 'Activate proven execution relay generation',
+  });
+  // A receipt fixture can prove a cordoned candidate path, but it cannot show
+  // that Fly's serving route has switched every Machine. Promotion is blocked
+  // until both request-serving roles prove the exact candidate image and the
+  // canonical reader capability through forced-instance readiness probes.
+  await input.command({
+    command: 'bun',
+    args: [
+      'scripts/runtime-gateway-fleet.ts',
+      'verify-reader-capability',
+      '--environment',
+      'production',
+      '--role',
+      'all',
+      '--sha',
+      input.proven.plan.releaseSha,
+      '--apply',
+    ],
+    environment,
+    description: 'Prove active receipt and execution gateway fleet',
+  });
+  return {
+    phase: ProductionTopologyControllerPhase.Commit,
+    plan: input.proven.plan,
+    stagedVercelUrl: input.proven.stagedVercelUrl,
+  };
+}
+
+/**
+ * Revert only the exact Fly operation identified by the failed candidate SHA.
+ * Each role restores its retained source generation before cordoning green;
+ * this command never creates, destroys, or guesses a replacement Machine.
+ */
+export async function rollbackProductionTopologyPlan(input: {
+  plan: ProductionTopologyPlan;
+  environment: Record<string, string>;
+  command: RuntimeTopologyCommand;
+}): Promise<void> {
+  const environment = releaseEnvironment(input);
+  await input.command({
+    command: 'bun',
+    args: [
+      'scripts/roll-receipt-gateway.ts',
+      'rollback',
+      '--environment',
+      'production',
+      '--sha',
+      input.plan.releaseSha,
+      '--apply',
+    ],
+    environment,
+    description: 'Restore retained receipt gateway generation',
+  });
+  await input.command({
+    command: 'bun',
+    args: [
+      'scripts/roll-execution-relay.ts',
+      'rollback',
+      '--environment',
+      'production',
+      '--sha',
+      input.plan.releaseSha,
+      '--apply',
+    ],
+    environment,
+    description: 'Restore retained execution relay generation',
+  });
+}
+
+type Flags = Map<string, string | boolean>;
+
+function flags(argv: string[]): Flags {
+  const result = new Map<string, string | boolean>();
+  for (let index = 0; index < argv.length; index += 1) {
+    const token = argv[index]!;
+    if (!token.startsWith('--'))
+      throw new Error(`Unexpected argument ${token}.`);
+    const key = token.slice(2);
+    const value = argv[index + 1];
+    if (!value || value.startsWith('--')) result.set(key, true);
+    else {
+      result.set(key, value);
+      index += 1;
+    }
+  }
+  return result;
+}
+
+function requireValue(input: Flags, key: string): string {
+  const value = input.get(key);
+  if (typeof value !== 'string') throw new Error(`--${key} is required.`);
+  return value;
+}
+
+function requirePrepareEnvironment(): Record<string, string> {
+  return requireEnvironmentValues([
+    'FLY_API_TOKEN',
+    'FLY_ORG',
+    'WORKFLOW_RUNTIME_POSTGRES',
+    'DEEPLINE_INTERNAL_TOKEN',
+  ]);
+}
+
+function requireCommitEnvironment(): Record<string, string> {
+  return requireEnvironmentValues([
+    'FLY_API_TOKEN',
+    'GITHUB_REPOSITORY',
+    'GH_TOKEN',
+  ]);
+}
+
+function requireRollbackEnvironment(): Record<string, string> {
+  return requireEnvironmentValues(['FLY_API_TOKEN']);
+}
+
+function requireProofEnvironment(): Record<string, string> {
+  return requireEnvironmentValues([
+    'FLY_API_TOKEN',
+    'DEEPLINE_API_KEY',
+    'DEEPLINE_INTERNAL_TOKEN',
+    'VERCEL_PROTECTION_BYPASS_TOKEN',
+    'GITHUB_REPOSITORY',
+    'GH_TOKEN',
+  ]);
+}
+
+function requireEnvironmentValues(
+  names: readonly string[],
+): Record<string, string> {
+  const missing = names.filter((name) => !process.env[name]?.trim());
+  if (missing.length)
+    throw new Error(`Topology apply is missing: ${missing.join(', ')}.`);
+  return Object.fromEntries(
+    names.map((name) => [name, process.env[name]!]),
+  ) as Record<string, string>;
+}
+
+function requireCleanupEnvironment(): Record<string, string> {
+  if (!process.env.FLY_API_TOKEN?.trim()) {
+    throw new Error('Topology cleanup is missing FLY_API_TOKEN.');
+  }
+  return { FLY_API_TOKEN: process.env.FLY_API_TOKEN };
+}
+
+function assertOnlyFlags(input: Flags, allowed: readonly string[]): void {
+  for (const key of input.keys()) {
+    if (!allowed.includes(key)) {
+      throw new Error(`--${key} is not supported for this topology command.`);
+    }
+  }
+}
+
+function requireProductionApply(input: Flags): void {
+  if (
+    input.get('environment') !== 'production' ||
+    input.get('apply') !== true
+  ) {
+    throw new Error('Requires --environment production --apply.');
+  }
+}
+
+function outputTopologyResult(
+  result:
+    | PreparedProductionTopology
+    | ProvenProductionTopology
+    | CommittedProductionTopology,
+): void {
+  writeGithubStepOutput('topology_phase', result.phase);
+  writeGithubStepOutput('topology_release_sha', result.plan.releaseSha);
+  if (
+    result.phase === ProductionTopologyControllerPhase.Prove ||
+    result.phase === ProductionTopologyControllerPhase.Commit
+  ) {
+    writeGithubStepOutput('topology_staged_vercel_url', result.stagedVercelUrl);
+  }
+}
+
+async function main(): Promise<void> {
+  const command = process.argv[2] as RuntimeTopologyCliCommand | undefined;
+  const input = flags(process.argv.slice(3));
+  if (!command || !Object.values(RuntimeTopologyCliCommand).includes(command)) {
+    throw new Error(
+      'Usage: runtime-topology.ts prepare|prove|commit|rollback|cleanup --environment production --sha <full-sha> --apply',
+    );
+  }
+  const commonFlags = ['environment', 'sha'] as const;
+  const allowedFlags: Record<RuntimeTopologyCliCommand, readonly string[]> = {
+    [RuntimeTopologyCliCommand.Prepare]: [...commonFlags, 'apply'],
+    [RuntimeTopologyCliCommand.Prove]: [
+      ...commonFlags,
+      'apply',
+      'staged-url',
+      'proof-file',
+      'fixture-application-dir',
+    ],
+    [RuntimeTopologyCliCommand.Commit]: [...commonFlags, 'apply', 'proof-file'],
+    [RuntimeTopologyCliCommand.Rollback]: [...commonFlags, 'apply'],
+    [RuntimeTopologyCliCommand.Cleanup]: [...commonFlags, 'apply', 'intent'],
+  };
+  assertOnlyFlags(input, allowedFlags[command]);
+  if (input.get('environment') !== 'production') {
+    throw new Error('Only --environment production is supported.');
+  }
+  const release = {
+    releaseSha: requireValue(input, 'sha'),
+  };
+  const intent = input.get('intent');
+  if (intent !== undefined && intent !== 'release' && intent !== 'cleanup') {
+    throw new Error('--intent must be release or cleanup.');
+  }
+  const plan =
+    intent === 'cleanup'
+      ? planProductionTopologyCleanup(release)
+      : planProductionTopologyRelease(release);
+  requireProductionApply(input);
+  if (command === RuntimeTopologyCliCommand.Cleanup) {
+    if (intent === 'release') {
+      throw new Error('cleanup cannot use --intent release.');
+    }
+    await cleanupProductionGatewayCandidates({
+      plan: planProductionTopologyCleanup(release),
+      environment: requireCleanupEnvironment(),
+      command: runCommand,
+    });
+    return;
+  }
+  if (command === RuntimeTopologyCliCommand.Prepare && intent !== undefined) {
+    throw new Error('prepare does not support --intent.');
+  }
+  if (command === RuntimeTopologyCliCommand.Prove && intent !== undefined) {
+    throw new Error('prove does not support --intent.');
+  }
+  if (command === RuntimeTopologyCliCommand.Commit && intent !== undefined) {
+    throw new Error('commit does not support --intent.');
+  }
+  if (intent === 'cleanup') {
+    throw new Error('Use the cleanup command for --intent cleanup.');
+  }
+  if (command === RuntimeTopologyCliCommand.Prepare) {
+    outputTopologyResult(
+      await prepareProductionTopologyPlan({
+        plan,
+        environment: requirePrepareEnvironment(),
+        command: runCommand,
+        parallelCommand: runRuntimeTopologyParallelCommand,
+      }),
+    );
+    return;
+  }
+  if (command === RuntimeTopologyCliCommand.Commit) {
+    outputTopologyResult(
+      await commitProductionTopologyPlan({
+        proven: readProvenTopologyReceipt({
+          file: requireValue(input, 'proof-file'),
+          plan,
+        }),
+        environment: requireCommitEnvironment(),
+        command: runCommand,
+      }),
+    );
+    return;
+  }
+  if (command === RuntimeTopologyCliCommand.Rollback) {
+    await rollbackProductionTopologyPlan({
+      plan,
+      environment: requireRollbackEnvironment(),
+      command: runCommand,
+    });
+    return;
+  }
+  if (command === RuntimeTopologyCliCommand.Prove) {
+    const fixtureApplicationDirectory = input.get('fixture-application-dir');
+    if (
+      fixtureApplicationDirectory !== undefined &&
+      typeof fixtureApplicationDirectory !== 'string'
+    ) {
+      throw new Error('prove --fixture-application-dir requires a directory.');
+    }
+    const proven = await proveProductionTopologyPlan({
+      prepared: {
+        phase: ProductionTopologyControllerPhase.Prepare,
+        plan,
+      },
+      stagedVercelUrl: requireValue(input, 'staged-url'),
+      fixtureApplicationDirectory,
+      environment: requireProofEnvironment(),
+      command: runCommand,
+    });
+    writeProvenTopologyReceipt({
+      file: requireValue(input, 'proof-file'),
+      proven,
+    });
+    outputTopologyResult(proven);
+    return;
+  }
+}
+
+export async function runRuntimeTopologyCli(): Promise<void> {
+  const removeCancellationHandlers =
+    installRuntimeTopologyCancellationHandlers();
+  try {
+    await main();
+  } finally {
+    removeCancellationHandlers();
+  }
+}