deepline 0.3.89 → 0.3.91

package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-execution-relay.ts

@@ -0,0 +1,521 @@
+#!/usr/bin/env bun
+
+/**
+ * Controller-owned execution-relay generation rollout.
+ *
+ * Fly's blue/green deploy has a fixed kill timeout. That is not a valid drain
+ * contract for billable provider requests: an accepted request may outlive
+ * that timeout. This controller instead creates a cordoned green generation,
+ * switches only new ingress to it, and leaves blue started until each exact
+ * blue Machine proves zero live provider dispatches through its metrics port.
+ */
+import { spawnSync } from 'node:child_process';
+import { FlyMachinesClient, requireFlyToken } from '../fly-machines';
+import {
+  RECEIPT_GATEWAY_DRAIN_TIMEOUT,
+  stopAndDestroyCordonedMachines,
+} from '../../deployment/fly-machine-lifecycle';
+import {
+  exactFlyMachineRouteHeaders,
+  formatGatewayRouteProbeRetry,
+  GatewayRouteProbeHttpError,
+  waitForGatewayRouteProbe,
+} from '../fly-gateway-route-probe';
+import {
+  gatewayMachineConfig,
+  productionGatewayTopology,
+  RUNTIME_TOPOLOGY,
+} from '../../deployment/runtime-topology';
+
+export const EXECUTION_RELAY_ROLLOUT_METADATA =
+  'deepline_execution_relay_rollout';
+export const EXECUTION_RELAY_SOURCE_METADATA =
+  'deepline_execution_relay_source_machine';
+const app = RUNTIME_TOPOLOGY.production.components.executionRelay.app;
+const topology = productionGatewayTopology('execution');
+const healthUrl = `https://${app}.fly.dev/readyz`;
+const pollMs = 5_000;
+const timeoutMs = 10 * 60_000;
+
+const reportGatewayRouteProbeRetry = (
+  retry: Parameters<typeof formatGatewayRouteProbeRetry>[0],
+): void => {
+  console.warn(formatGatewayRouteProbeRetry(retry));
+};
+
+type FlyMachine = Awaited<
+  ReturnType<FlyMachinesClient['listMachines']>
+>[number] & {
+  // The Fly API returns this field, but the generated client type has not yet
+  // caught up. Keep it optional so older API responses remain valid too.
+  cordoned?: boolean;
+};
+
+function parseFlags(argv: string[]): Map<string, string | boolean> {
+  const values = new Map<string, string | boolean>();
+  for (let i = 0; i < argv.length; i += 1) {
+    const token = argv[i]!;
+    if (!token.startsWith('--')) throw new Error(`Unexpected ${token}.`);
+    const key = token.slice(2);
+    const value = argv[i + 1];
+    if (!value || value.startsWith('--')) values.set(key, true);
+    else {
+      values.set(key, value);
+      i += 1;
+    }
+  }
+  return values;
+}
+
+function requireApply(flags: Map<string, string | boolean>): void {
+  if (
+    flags.get('environment') !== 'production' ||
+    flags.get('apply') !== true
+  ) {
+    throw new Error('Requires --environment production --apply.');
+  }
+}
+
+function requireSha(flags: Map<string, string | boolean>): string {
+  const sha = flags.get('sha');
+  if (typeof sha !== 'string' || !/^[a-f0-9]{7,40}$/i.test(sha)) {
+    throw new Error('--sha must be a 7-40 character git SHA.');
+  }
+  return sha.toLowerCase();
+}
+
+function image(sha: string): string {
+  return `registry.fly.io/${app}:git-${sha}`;
+}
+function active(machine: FlyMachine): boolean {
+  return machine.state !== 'destroyed' && machine.state !== 'destroying';
+}
+function healthy(machine: FlyMachine): boolean {
+  return (
+    machine.state === 'started' &&
+    (machine.checks?.length ?? 0) > 0 &&
+    machine.checks!.every((check) => check.status === 'passing')
+  );
+}
+function candidateName(sha: string, index: number): string {
+  return `execution-${sha.slice(0, 12)}-${index + 1}`;
+}
+
+export function planExecutionRelayCutover(input: {
+  blue: FlyMachine[];
+  green: FlyMachine[];
+  sha: string;
+}): { sourceMachineIds: string[]; greenMachineIds: string[] } {
+  if (
+    input.blue.length < topology.minimumMachines ||
+    input.blue.some(
+      (machine) =>
+        !healthy(machine) || machine.cordoned !== false || !machine.id,
+    )
+  ) {
+    throw new Error(
+      'Execution relay serving generation is not a healthy uncordoned fleet.',
+    );
+  }
+  const images = new Set(
+    input.blue.map((machine) => machine.config?.image).filter(Boolean),
+  );
+  if (images.size !== 1)
+    throw new Error('Execution relay serving generation has mixed images.');
+  if (
+    input.green.length !== input.blue.length ||
+    input.green.some(
+      (machine) =>
+        !healthy(machine) ||
+        machine.cordoned !== true ||
+        machine.config?.image !== image(input.sha) ||
+        !machine.id,
+    )
+  ) {
+    throw new Error(
+      'Execution relay green generation is not a complete cordoned healthy fleet.',
+    );
+  }
+  const sources = input.green.map(
+    (machine) => machine.config?.metadata?.[EXECUTION_RELAY_SOURCE_METADATA],
+  );
+  if (
+    new Set(sources).size !== input.blue.length ||
+    sources.some(
+      (id) => !id || !input.blue.some((machine) => machine.id === id),
+    )
+  ) {
+    throw new Error(
+      'Execution relay candidates do not identify the exact blue generation.',
+    );
+  }
+  return {
+    sourceMachineIds: sources as string[],
+    greenMachineIds: input.green.map((machine) => machine.id!),
+  };
+}
+
+/**
+ * Green must prove its full runtime contract before blue is cordoned.  Exact
+ * readiness is only reachable through Fly Proxy after uncordon, so if that
+ * proof fails, immediately remove every green candidate from admission and
+ * leave blue serving.  This makes an interrupted/retried activation converge
+ * on the same safe state instead of stranding a bad green fleet in traffic.
+ */
+export async function validateRelayCandidatesBeforeBlueCordon(input: {
+  candidateMachineIds: readonly string[];
+  machineIdsToUncordon: readonly string[];
+  uncordon: (machineId: string) => Promise<void>;
+  assertReady: (machineId: string) => Promise<void>;
+  cordon: (machineId: string) => Promise<void>;
+}): Promise<void> {
+  try {
+    for (const machineId of input.machineIdsToUncordon) {
+      await input.uncordon(machineId);
+    }
+    await Promise.all(input.candidateMachineIds.map(input.assertReady));
+  } catch (error) {
+    const rollback = await Promise.allSettled(
+      input.candidateMachineIds.map(input.cordon),
+    );
+    const rollbackFailures = rollback
+      .filter(
+        (entry): entry is PromiseRejectedResult => entry.status === 'rejected',
+      )
+      .map((entry) =>
+        entry.reason instanceof Error
+          ? entry.reason.message
+          : String(entry.reason),
+      );
+    if (rollbackFailures.length > 0) {
+      throw new Error(
+        `Execution relay candidate validation failed and green rollback failed: ${rollbackFailures.join('; ')}`,
+        { cause: error },
+      );
+    }
+    throw error;
+  }
+}
+
+/**
+ * Restore the retained source generation after a later tuple check fails.
+ * Make blue reachable and prove every exact Machine before cordoning green so
+ * a failed Fly operation cannot turn a bad candidate into a gateway outage.
+ */
+export async function restoreRelaySourceGeneration(input: {
+  greenMachineIds: readonly string[];
+  blueMachineIds: readonly string[];
+  uncordon: (machineId: string) => Promise<void>;
+  cordon: (machineId: string) => Promise<void>;
+  assertReady: (machineId: string) => Promise<void>;
+}): Promise<void> {
+  if (input.greenMachineIds.length === 0 || input.blueMachineIds.length === 0) {
+    throw new Error(
+      'Execution relay rollback requires exact green and blue Machine identities.',
+    );
+  }
+  for (const machineId of input.blueMachineIds) {
+    await input.uncordon(machineId);
+  }
+  await Promise.all(input.blueMachineIds.map(input.assertReady));
+  for (const machineId of input.greenMachineIds) {
+    await input.cordon(machineId);
+  }
+  await Promise.all(input.blueMachineIds.map(input.assertReady));
+}
+
+function run(command: string, args: string[]): void {
+  const result = spawnSync(command, args, {
+    encoding: 'utf8',
+    stdio: 'inherit',
+  });
+  if (result.status !== 0)
+    throw new Error(
+      `${command} ${args.join(' ')} failed with ${result.status}.`,
+    );
+}
+async function inventory(client: FlyMachinesClient): Promise<FlyMachine[]> {
+  return await client.listMachines();
+}
+function candidates(machines: FlyMachine[], sha: string): FlyMachine[] {
+  return machines.filter(
+    (machine) =>
+      active(machine) &&
+      machine.config?.metadata?.[EXECUTION_RELAY_ROLLOUT_METADATA] === sha,
+  );
+}
+function serving(machines: FlyMachine[]): FlyMachine[] {
+  return machines.filter(
+    (machine) => active(machine) && machine.cordoned === false,
+  );
+}
+async function waitForGreen(
+  client: FlyMachinesClient,
+  sha: string,
+  count: number,
+): Promise<FlyMachine[]> {
+  const deadline = Date.now() + timeoutMs;
+  let last = 'waiting for candidates';
+  while (Date.now() < deadline) {
+    const green = candidates(await inventory(client), sha);
+    try {
+      if (green.length !== count)
+        last = `expected ${count} candidates; found ${green.length}`;
+      else if (
+        green.some((machine) => !healthy(machine) || machine.cordoned !== true)
+      )
+        last = 'candidate health or cordon state is not ready';
+      else return green;
+    } catch (error) {
+      last = error instanceof Error ? error.message : String(error);
+    }
+    await new Promise((resolve) => setTimeout(resolve, pollMs));
+  }
+  throw new Error(`Execution relay candidates did not become ready: ${last}`);
+}
+async function assertExactReadyOnce(machineId: string): Promise<void> {
+  const response = await fetch(healthUrl, {
+    headers: exactFlyMachineRouteHeaders(machineId),
+    signal: AbortSignal.timeout(10_000),
+  });
+  if (!response.ok) {
+    const body = await response.text();
+    throw new GatewayRouteProbeHttpError({
+      status: response.status,
+      body,
+      expectedMachineId: machineId,
+      actualMachineId: response.headers.get('x-deepline-gateway-machine-id'),
+    });
+  }
+  if (response.headers.get('x-deepline-gateway-machine-id') !== machineId) {
+    throw new Error(
+      `Execution relay Machine ${machineId} failed its exact readiness probe.`,
+    );
+  }
+}
+
+async function assertExactReady(machineId: string): Promise<void> {
+  await waitForGatewayRouteProbe({
+    description: `Execution relay Machine ${machineId}`,
+    probe: () => assertExactReadyOnce(machineId),
+    onRetry: reportGatewayRouteProbeRetry,
+  });
+}
+async function prepare(sha: string): Promise<void> {
+  run('bun', [
+    'scripts/ensure-runtime-image.ts',
+    '--kind',
+    'gateway',
+    '--app',
+    app,
+    '--config',
+    RUNTIME_TOPOLOGY.production.components.executionRelay.image.config,
+    '--dockerfile',
+    RUNTIME_TOPOLOGY.production.components.executionRelay.image.dockerfile,
+    '--sha',
+    sha,
+  ]);
+  const client = new FlyMachinesClient(app, requireFlyToken(), fetch);
+  const initial = await inventory(client);
+  const existing = candidates(initial, sha);
+  const blue = serving(initial);
+  if (existing.length > 0 && existing.every((machine) => healthy(machine))) {
+    const alreadyPublic = existing.every(
+      (machine) => machine.cordoned === false,
+    );
+    const stillCordoned = existing.every(
+      (machine) => machine.cordoned === true,
+    );
+    if (!alreadyPublic && !stillCordoned) {
+      throw new Error(
+        'Execution relay candidates have mixed routing state; resume activation instead of preparing a replacement.',
+      );
+    }
+    if (alreadyPublic) {
+      await Promise.all(
+        existing.map((machine) => assertExactReady(machine.id!)),
+      );
+      return;
+    }
+  }
+  if (existing.length === 0) {
+    if (
+      blue.length < topology.minimumMachines ||
+      blue.some((machine) => !healthy(machine) || !machine.id)
+    )
+      throw new Error(
+        'Cannot prepare an execution relay generation without a healthy serving fleet.',
+      );
+    for (const [index, source] of blue.entries()) {
+      await client.createMachine({
+        name: candidateName(sha, index),
+        region: topology.region,
+        skip_service_registration: true,
+        config: gatewayMachineConfig({
+          profile: 'production',
+          role: 'execution',
+          image: image(sha),
+          metadata: {
+            deepline_gateway_role: 'execution',
+            [EXECUTION_RELAY_ROLLOUT_METADATA]: sha,
+            [EXECUTION_RELAY_SOURCE_METADATA]: source.id!,
+            fly_process_group: topology.processGroup,
+          },
+        }),
+      });
+    }
+  }
+  await waitForGreen(client, sha, blue.length);
+  // Candidate proof before activation is the Fly health check alone. Exact
+  // readiness runs immediately after uncordon in `activate`, before the blue
+  // generation is cordoned or any later release promotion may proceed.
+  //
+  // That leaves a window in which unproven green briefly takes public ingress.
+  // The receipt gateway does not have this window: it force-routes a fixture
+  // through each still-cordoned candidate with `fly-force-instance-id` and
+  // re-asserts the cordon on both sides of the probe. Forced routing does
+  // reach a cordoned Machine -- that path passes on every production release
+  // -- so this rollout can adopt the same pre-uncordon proof. Tracked
+  // separately; do not assume the header is the blocker.
+}
+async function activate(sha: string): Promise<void> {
+  const client = new FlyMachinesClient(app, requireFlyToken(), fetch);
+  const machines = await inventory(client);
+  const green = candidates(machines, sha);
+  if (
+    green.length < topology.minimumMachines ||
+    green.some(
+      (machine) =>
+        !healthy(machine) ||
+        machine.config?.image !== image(sha) ||
+        !machine.id,
+    )
+  ) {
+    throw new Error(
+      'Execution relay activation has no complete healthy green generation.',
+    );
+  }
+  const sourceMachineIds = green.map(
+    (machine) => machine.config?.metadata?.[EXECUTION_RELAY_SOURCE_METADATA],
+  );
+  if (
+    new Set(sourceMachineIds).size !== green.length ||
+    sourceMachineIds.some((id) => !id)
+  ) {
+    throw new Error('Execution relay activation has no exact blue generation.');
+  }
+  await validateRelayCandidatesBeforeBlueCordon({
+    candidateMachineIds: green.map((machine) => machine.id!),
+    machineIdsToUncordon: green
+      .filter((machine) => machine.cordoned === true)
+      .map((machine) => machine.id!),
+    uncordon: async (machineId) => {
+      run('flyctl', ['machine', 'uncordon', machineId, '-a', app]);
+    },
+    assertReady: assertExactReady,
+    cordon: async (machineId) => {
+      run('flyctl', ['machine', 'cordon', machineId, '-a', app]);
+    },
+  });
+  const refreshed = await inventory(client);
+  for (const machineId of sourceMachineIds as string[]) {
+    const blue = refreshed.find((machine) => machine.id === machineId);
+    if (!blue) continue;
+    if (!active(blue)) {
+      throw new Error(
+        `Execution relay source Machine ${machineId} is no longer active.`,
+      );
+    }
+    if (blue.cordoned === false) {
+      run('flyctl', ['machine', 'cordon', machineId, '-a', app]);
+    }
+  }
+}
+
+async function rollback(sha: string): Promise<void> {
+  const client = new FlyMachinesClient(app, requireFlyToken(), fetch);
+  const current = await inventory(client);
+  const green = candidates(current, sha);
+  if (
+    green.length < topology.minimumMachines ||
+    green.some((machine) => !machine.id)
+  ) {
+    throw new Error(
+      'Execution relay rollback requires a complete candidate generation.',
+    );
+  }
+  const sourceMachineIds = green.map(
+    (machine) => machine.config?.metadata?.[EXECUTION_RELAY_SOURCE_METADATA],
+  );
+  if (
+    new Set(sourceMachineIds).size !== green.length ||
+    sourceMachineIds.some((machineId) => !machineId)
+  ) {
+    throw new Error('Execution relay rollback has no exact source generation.');
+  }
+  const blue = current.filter((machine) =>
+    (sourceMachineIds as string[]).includes(machine.id ?? ''),
+  );
+  if (
+    blue.length !== sourceMachineIds.length ||
+    blue.some((machine) => !machine.id || !active(machine) || !healthy(machine))
+  ) {
+    throw new Error(
+      'Execution relay rollback source generation is not a complete healthy retained fleet.',
+    );
+  }
+  await restoreRelaySourceGeneration({
+    greenMachineIds: green.map((machine) => machine.id!),
+    blueMachineIds: sourceMachineIds as string[],
+    uncordon: async (machineId) => {
+      run('flyctl', ['machine', 'uncordon', machineId, '-a', app]);
+    },
+    cordon: async (machineId) => {
+      run('flyctl', ['machine', 'cordon', machineId, '-a', app]);
+    },
+    assertReady: assertExactReady,
+  });
+}
+/** Remove only a candidate that never received public traffic. This mirrors
+ * receipt cleanup so a failed pre-commit proof cannot strand relay Machines. */
+async function cleanup(sha: string): Promise<void> {
+  const client = new FlyMachinesClient(app, requireFlyToken(), fetch);
+  const targets = candidates(await inventory(client), sha);
+  if (targets.some((machine) => !machine.id)) {
+    throw new Error(
+      'Execution relay candidate cleanup requires exact Fly Machine identities.',
+    );
+  }
+  if (targets.some((machine) => machine.cordoned !== true)) {
+    throw new Error(
+      'Refusing to remove an execution relay candidate that received public traffic.',
+    );
+  }
+  await stopAndDestroyCordonedMachines({
+    machineIds: targets.map((machine) => machine.id!),
+    readMachines: () => inventory(client),
+    listProviderMachineIds: async () =>
+      (await client.listMachines())
+        .map((machine) => machine.id)
+        .filter((id): id is string => Boolean(id)),
+    stopMachine: (machineId) =>
+      client.stopMachine(machineId, {
+        signal: 'SIGTERM',
+        timeout: RECEIPT_GATEWAY_DRAIN_TIMEOUT,
+      }),
+    deleteMachine: (machineId) => client.deleteMachine(machineId, false),
+  });
+}
+export async function main(): Promise<void> {
+  const command = process.argv[2];
+  const flags = parseFlags(process.argv.slice(3));
+  requireApply(flags);
+  const sha = requireSha(flags);
+  if (command === 'prepare') return await prepare(sha);
+  if (command === 'activate') return await activate(sha);
+  if (command === 'rollback') return await rollback(sha);
+  if (command === 'cleanup') return await cleanup(sha);
+  throw new Error(
+    'Usage: roll-execution-relay.ts prepare|activate|rollback|cleanup --environment production --sha <sha> --apply',
+  );
+}