deepline 0.3.90 → 0.3.92

package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-gateway-route-probe.ts

@@ -0,0 +1,193 @@
+export function exactFlyMachineRouteHeaders(
+  machineId: string,
+): Record<'fly-force-instance-id', string> {
+  return { 'fly-force-instance-id': machineId };
+}
+
+/**
+ * Fly Proxy can briefly lose an exact-instance route while an uncordon is
+ * propagating. A one-shot timeout is not evidence that an already-proven
+ * candidate is bad. Keep the bounded convergence policy here so receipt and
+ * execution cutovers do not invent different retry semantics.
+ */
+export const GATEWAY_ROUTE_PROBE_TIMEOUT_MS = 120_000;
+export const GATEWAY_ROUTE_PROBE_RETRY_MS = 5_000;
+
+export type GatewayRouteProbeRetry = Readonly<{
+  description: string;
+  attempt: number;
+  retryInMs: number;
+  remainingMs: number;
+  error: Error;
+}>;
+
+/**
+ * Retry lines deliberately carry the current cause and remaining budget. They
+ * are concise enough for the normal release timeline and let an operator see
+ * a transient Fly route converge instead of seeing a silent two-minute wait.
+ */
+export function formatGatewayRouteProbeRetry(
+  input: GatewayRouteProbeRetry,
+): string {
+  return `[gateway-route-probe] ${input.description}: retry ${input.attempt}; next probe in ${input.retryInMs}ms; ${input.remainingMs}ms remaining; cause=${input.error.name}: ${input.error.message}`;
+}
+
+export class GatewayRouteProbeHttpError extends Error {
+  readonly status: number;
+  readonly actualMachineId: string | null;
+  readonly expectedMachineId: string | undefined;
+
+  constructor(input: {
+    status: number;
+    body: string;
+    actualMachineId: string | null;
+    expectedMachineId?: string;
+  }) {
+    super(
+      `Runtime gateway health check failed: HTTP ${input.status}, body=${JSON.stringify(input.body.slice(0, 200))}.`,
+    );
+    this.name = 'GatewayRouteProbeHttpError';
+    this.status = input.status;
+    this.actualMachineId = input.actualMachineId;
+    this.expectedMachineId = input.expectedMachineId;
+  }
+}
+
+export function isRetryableGatewayRouteProbeError(error: unknown): boolean {
+  if (!(error instanceof Error)) return false;
+  if (error.name === 'TimeoutError' || error.name === 'AbortError') return true;
+  // Fetch surfaces a dropped Fly Proxy connection as an ordinary Error/TypeError
+  // before the request timeout can fire. Keep this list narrow: contract,
+  // identity, and application failures must still roll back immediately.
+  if (
+    /^(fetch failed|unable to connect\.|connection (?:reset|refused)|network error$)/i.test(
+      error.message,
+    )
+  ) {
+    return true;
+  }
+  if (error instanceof GatewayRouteProbeHttpError) {
+    const transientStatus =
+      error.status === 408 ||
+      error.status === 425 ||
+      error.status === 429 ||
+      error.status >= 500;
+    if (!transientStatus) return false;
+    return (
+      error.actualMachineId === null ||
+      (error.expectedMachineId !== undefined &&
+        error.actualMachineId !== error.expectedMachineId)
+    );
+  }
+  return (
+    /^Replacement Machine .+ did not serve its public route probe/.test(
+      error.message,
+    ) ||
+    /^Execution relay Machine .+ failed its exact readiness probe\.$/.test(
+      error.message,
+    )
+  );
+}
+
+export async function waitForGatewayRouteProbe(input: {
+  description: string;
+  probe: () => Promise<void>;
+  now?: () => number;
+  sleep?: (milliseconds: number) => Promise<void>;
+  timeoutMs?: number;
+  retryMs?: number;
+  onRetry?: (retry: GatewayRouteProbeRetry) => void | Promise<void>;
+}): Promise<void> {
+  const now = input.now ?? Date.now;
+  const sleep =
+    input.sleep ??
+    ((milliseconds: number) =>
+      new Promise<void>((resolve) => setTimeout(resolve, milliseconds)));
+  const timeoutMs = input.timeoutMs ?? GATEWAY_ROUTE_PROBE_TIMEOUT_MS;
+  const retryMs = input.retryMs ?? GATEWAY_ROUTE_PROBE_RETRY_MS;
+  if (!Number.isSafeInteger(timeoutMs) || timeoutMs < 0) {
+    throw new Error(
+      'Gateway route probe timeout must be a non-negative integer.',
+    );
+  }
+  if (!Number.isSafeInteger(retryMs) || retryMs <= 0) {
+    throw new Error(
+      'Gateway route probe retry interval must be a positive integer.',
+    );
+  }
+
+  const deadline = now() + timeoutMs;
+  let lastRetryableError: Error | undefined;
+  let attempt = 0;
+  for (;;) {
+    if (lastRetryableError && now() >= deadline) {
+      throw new Error(
+        `${input.description} did not become routeable within ${timeoutMs} ms: ${lastRetryableError.message}`,
+        { cause: lastRetryableError },
+      );
+    }
+    try {
+      await input.probe();
+      return;
+    } catch (error) {
+      if (
+        !isRetryableGatewayRouteProbeError(error) ||
+        !(error instanceof Error)
+      ) {
+        throw error;
+      }
+      lastRetryableError = error;
+      if (now() >= deadline) {
+        throw new Error(
+          `${input.description} did not become routeable within ${timeoutMs} ms: ${lastRetryableError.message}`,
+          { cause: lastRetryableError },
+        );
+      }
+      attempt += 1;
+      const remainingMs = Math.max(0, deadline - now());
+      const retryInMs = Math.min(retryMs, remainingMs);
+      // Report the first retry, then sampled checkpoints, and the final
+      // budget-consuming retry. This proves active convergence without
+      // turning a transient route delay into a wall of log lines.
+      if (
+        input.onRetry &&
+        (attempt === 1 || attempt % 5 === 0 || retryInMs === remainingMs)
+      ) {
+        await input.onRetry({
+          description: input.description,
+          attempt,
+          retryInMs,
+          remainingMs,
+          error: lastRetryableError,
+        });
+      }
+      await sleep(retryInMs);
+    }
+  }
+}
+
+export function assertGatewayRouteProbe(input: {
+  status: number;
+  body: string;
+  expectedMachineId?: string;
+  actualMachineId: string | null;
+  allowMissingExpectedIdentity?: boolean;
+}): void {
+  if (
+    !Number.isInteger(input.status) ||
+    input.status < 200 ||
+    input.status >= 300 ||
+    !/\bok\b/i.test(input.body)
+  ) {
+    throw new GatewayRouteProbeHttpError(input);
+  }
+  if (!input.expectedMachineId) return;
+  if (input.allowMissingExpectedIdentity && input.actualMachineId === null) {
+    return;
+  }
+  if (input.actualMachineId !== input.expectedMachineId) {
+    throw new Error(
+      `Replacement Machine ${input.expectedMachineId} did not serve its public route probe (served=${input.actualMachineId ?? '<missing>'}).`,
+    );
+  }
+}

package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machine-lifecycle.ts

@@ -0,0 +1,104 @@
+/** Shared Fly Machine identity and provider-confirmed deletion primitives. */
+export type FlyMachine = {
+  id?: string;
+  name?: string;
+  state?: string;
+  cordoned?: boolean;
+  region?: string;
+  /**
+   * `name` and `output` are what make a failing check actionable in a release
+   * log; Fly always sends them alongside `status`.
+   */
+  checks?: Array<{ name?: string; status?: string; output?: string }>;
+  events?: Array<{ type?: string; timestamp?: number }>;
+  config?: {
+    image?: string;
+    metadata?: Record<string, string | undefined>;
+    guest?: {
+      cpu_kind?: string;
+      cpus?: number;
+      memory_mb?: number;
+    };
+  };
+};
+
+// Fly Machines API expects a Go duration string. Never send bare seconds.
+export const RECEIPT_GATEWAY_DRAIN_TIMEOUT = '300s';
+
+export async function stopAndDestroyCordonedMachines(input: {
+  machineIds: string[];
+  readMachines: () => FlyMachine[] | Promise<FlyMachine[]>;
+  listProviderMachineIds: () => Promise<string[]>;
+  stopMachine: (machineId: string) => Promise<void>;
+  deleteMachine: (machineId: string) => Promise<void>;
+  timeoutMs?: number;
+  pollIntervalMs?: number;
+  now?: () => number;
+  sleep?: (ms: number) => Promise<void>;
+}): Promise<void> {
+  const timeoutMs = input.timeoutMs ?? 10 * 60_000;
+  const pollIntervalMs = input.pollIntervalMs ?? 5_000;
+  const now = input.now ?? Date.now;
+  const sleep =
+    input.sleep ??
+    ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  const deadline = now() + timeoutMs;
+  const readTargets = async (): Promise<FlyMachine[]> =>
+    (await input.readMachines()).filter((machine) =>
+      input.machineIds.includes(machine.id ?? ''),
+    );
+  const assertCordoned = (machines: FlyMachine[]): void => {
+    for (const machine of machines) {
+      if (machine.cordoned !== true) {
+        throw new Error(
+          `Refusing to delete Machine ${machine.id ?? '<missing>'} because it is not cordoned.`,
+        );
+      }
+    }
+  };
+
+  const initial = await readTargets();
+  assertCordoned(initial);
+  for (const machine of initial) {
+    if (!['stopped', 'suspended'].includes(machine.state ?? '')) {
+      await input.stopMachine(machine.id!);
+    }
+  }
+
+  let stopped: FlyMachine[] = [];
+  while (true) {
+    const machines = await readTargets();
+    assertCordoned(machines);
+    if (
+      machines.length === 0 ||
+      machines.every((machine) =>
+        ['stopped', 'suspended'].includes(machine.state ?? ''),
+      )
+    ) {
+      stopped = machines;
+      break;
+    }
+    if (now() >= deadline) {
+      throw new Error(
+        `Cordoned Machines did not stop within ${timeoutMs} ms: ${machines.map((machine) => `${machine.id ?? '<missing>'}=${machine.state ?? 'unknown'}`).join(', ')}.`,
+      );
+    }
+    await sleep(Math.min(pollIntervalMs, Math.max(0, deadline - now())));
+  }
+
+  for (const machine of stopped) await input.deleteMachine(machine.id!);
+  while (true) {
+    const providerMachineIds = await input.listProviderMachineIds();
+    if (
+      input.machineIds.every(
+        (machineId) => !providerMachineIds.includes(machineId),
+      )
+    ) {
+      return;
+    }
+    if (now() >= deadline) {
+      throw new Error('Cordoned Machine deletion is not provider-confirmed.');
+    }
+    await sleep(Math.min(pollIntervalMs, Math.max(0, deadline - now())));
+  }
+}