deepline 0.3.90 → 0.3.92

package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-tuple-recovery.ts

@@ -0,0 +1,519 @@
+/**
+ * The one recovery implementation for a failed Runtime Release Bundle.
+ *
+ * Both the publishing command and the independent scaler call this Module.
+ * The publication row is the canonical desired state; a caller may only make
+ * one external recovery step while it still holds that row's lease.
+ */
+import { promoteVercelProductionDeployment } from './vercel-production-gate';
+
+export type RuntimeTupleMember = Readonly<{
+  releaseId: string;
+  sha: string;
+  deploymentId: string;
+  deploymentUrl: string;
+}>;
+
+export type RuntimeTuplePanicPublication = Readonly<{
+  operationId: string;
+  state: 'panic_reverting';
+  candidate: RuntimeTupleMember;
+  baseline: RuntimeTupleMember;
+}>;
+
+export type RuntimeTupleActivePublication = Readonly<{
+  operationId: string;
+  state: 'publishing' | 'panic_reverting';
+  candidate: RuntimeTupleMember;
+  baseline: RuntimeTupleMember;
+}>;
+
+export type RuntimeTupleRecoveryCommand = (input: {
+  executable: string;
+  args: readonly string[];
+  environment: Record<string, string>;
+  description: string;
+}) => Promise<void>;
+
+export type RuntimeTuplePanicRecovery = Readonly<{
+  reconcile: (input: { owner: string }) => Promise<boolean>;
+  activePublication: () => Promise<RuntimeTupleActivePublication | null>;
+}>;
+
+export class RuntimeTupleRecoveryLeaseLostError extends Error {
+  constructor(operationId: string) {
+    super(
+      `Runtime tuple publication ${operationId} recovery lease is no longer held; stopping before another external mutation.`,
+    );
+    this.name = 'RuntimeTupleRecoveryLeaseLostError';
+  }
+}
+
+const RECOVERY_LEASE_RENEW_INTERVAL_MS = 15_000;
+const DEFAULT_ALIAS_WAIT_MS = 120_000;
+const ALIAS_POLL_INTERVAL_MS = 2_000;
+// This is deliberately an optional capability hint, not a new control
+// operation. Older control applications ignore it and still understand the
+// idempotent `claim_panic` handshake used below.
+const RENEWING_RECOVERY_PROTOCOL = 'renewing-v1';
+
+function required(value: string | undefined, label: string): string {
+  const normalized = value?.trim();
+  if (!normalized) {
+    throw new Error(`${label} is required for runtime tuple recovery.`);
+  }
+  return normalized;
+}
+
+function tupleMember(value: unknown, label: string): RuntimeTupleMember {
+  if (!value || typeof value !== 'object') {
+    throw new Error(`Runtime tuple publication ${label} is missing.`);
+  }
+  const source = value as Record<string, unknown>;
+  const field = (name: string) =>
+    required(
+      typeof source[name] === 'string' ? source[name] : undefined,
+      `runtime tuple publication ${label}.${name}`,
+    );
+  return {
+    releaseId: field('releaseId'),
+    sha: field('sha').toLowerCase(),
+    deploymentId: field('deploymentId'),
+    deploymentUrl: field('deploymentUrl'),
+  };
+}
+
+function panicPublication(value: unknown): RuntimeTuplePanicPublication {
+  if (!value || typeof value !== 'object') {
+    throw new Error('Runtime tuple panic claim returned no publication.');
+  }
+  const source = value as Record<string, unknown>;
+  const state = source.state;
+  if (state !== 'panic_reverting') {
+    throw new Error(
+      `Runtime tuple panic claim returned ${typeof state === 'string' ? state : 'an invalid'} publication state.`,
+    );
+  }
+  return {
+    operationId: required(
+      typeof source.operationId === 'string' ? source.operationId : undefined,
+      'runtime tuple publication operationId',
+    ),
+    state,
+    candidate: tupleMember(source.candidate, 'candidate'),
+    baseline: tupleMember(source.baseline, 'baseline'),
+  };
+}
+
+function activePublication(value: unknown): RuntimeTupleActivePublication {
+  if (!value || typeof value !== 'object') {
+    throw new Error(
+      'Runtime tuple control read returned an invalid publication.',
+    );
+  }
+  const source = value as Record<string, unknown>;
+  const state = source.state;
+  if (state !== 'publishing' && state !== 'panic_reverting') {
+    throw new Error(
+      `Runtime tuple control read returned ${typeof state === 'string' ? state : 'an invalid'} active publication state.`,
+    );
+  }
+  return {
+    operationId: required(
+      typeof source.operationId === 'string' ? source.operationId : undefined,
+      'runtime tuple publication operationId',
+    ),
+    state,
+    candidate: tupleMember(source.candidate, 'candidate'),
+    baseline: tupleMember(source.baseline, 'baseline'),
+  };
+}
+
+async function errorWithResponse(
+  description: string,
+  response: Response,
+): Promise<never> {
+  const body = await response.text();
+  throw new Error(
+    `${description} failed with HTTP ${response.status}: ${body.slice(0, 500)}`,
+  );
+}
+
+function isAlreadyCurrentProductionDeploymentConflict(
+  response: Response,
+  body: string,
+): boolean {
+  if (response.status !== 409) return false;
+  try {
+    const payload = JSON.parse(body) as {
+      error?: { code?: unknown; message?: unknown };
+    };
+    return (
+      payload.error?.code === 'conflict' &&
+      typeof payload.error.message === 'string' &&
+      payload.error.message.includes(
+        'is already the current production deployment.',
+      )
+    );
+  } catch {
+    return false;
+  }
+}
+
+function delay(milliseconds: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, milliseconds));
+}
+
+/**
+ * Build the concrete recovery Adapter once. There is deliberately no separate
+ * guardian version: callers supply logging and command presentation only.
+ */
+export function createRuntimeTuplePanicRecovery(input: {
+  controlUrl: string;
+  internalToken: string;
+  vercelProtectionBypassToken?: string;
+  vercel: Readonly<{
+    token: string;
+    teamId: string;
+    projectId: string;
+    productionDomain?: string;
+  }>;
+  gatewayFlyToken: string;
+  command: RuntimeTupleRecoveryCommand;
+  fetchImpl?: typeof fetch;
+  now?: () => number;
+  delay?: (milliseconds: number) => Promise<void>;
+  aliasWaitMs?: number;
+  leaseRenewIntervalMs?: number;
+  log?: (event: string, fields: Record<string, unknown>) => void;
+}): RuntimeTuplePanicRecovery {
+  const controlUrl = required(
+    input.controlUrl,
+    'RUNTIME_TUPLE_CONTROL_URL',
+  ).replace(/\/$/, '');
+  const internalToken = required(
+    input.internalToken,
+    'DEEPLINE_INTERNAL_TOKEN',
+  );
+  const fetchImpl = input.fetchImpl ?? fetch;
+  const now = input.now ?? Date.now;
+  const wait = input.delay ?? delay;
+  const aliasWaitMs = input.aliasWaitMs ?? DEFAULT_ALIAS_WAIT_MS;
+  const leaseRenewIntervalMs =
+    input.leaseRenewIntervalMs ?? RECOVERY_LEASE_RENEW_INTERVAL_MS;
+  const log = input.log ?? (() => {});
+  const productionDomain =
+    input.vercel.productionDomain?.trim() || 'code.deepline.com';
+
+  const internalHeaders = (): Record<string, string> => ({
+    'content-type': 'application/json',
+    'x-deepline-internal-token': internalToken,
+    ...(input.vercelProtectionBypassToken?.trim()
+      ? {
+          'x-vercel-protection-bypass':
+            input.vercelProtectionBypassToken.trim(),
+        }
+      : {}),
+  });
+
+  const publicationRequest = async <T>(
+    body: Record<string, unknown>,
+  ): Promise<T> => {
+    const response = await fetchImpl(
+      `${controlUrl}/api/v2/internal/runtime/tuple-publications`,
+      {
+        method: 'POST',
+        headers: internalHeaders(),
+        body: JSON.stringify(body),
+      },
+    );
+    if (!response.ok) {
+      return await errorWithResponse('Runtime tuple control request', response);
+    }
+    return (await response.json()) as T;
+  };
+
+  const readActivePublication =
+    async (): Promise<RuntimeTupleActivePublication | null> => {
+      const response = await fetchImpl(
+        `${controlUrl}/api/v2/internal/runtime/tuple-publications`,
+        { headers: internalHeaders() },
+      );
+      if (!response.ok) {
+        return await errorWithResponse(
+          'Read runtime tuple control state',
+          response,
+        );
+      }
+      const body = (await response.json()) as { publication?: unknown };
+      if (body.publication === null || body.publication === undefined)
+        return null;
+      return activePublication(body.publication);
+    };
+
+  const renew = async (
+    publication: RuntimeTuplePanicPublication,
+    owner: string,
+  ): Promise<boolean> => {
+    // Claiming again with the same owner has always extended the reconciliation
+    // lease. Reuse that operation instead of introducing a new required
+    // action: a newly deployed guardian must be able to recover through the
+    // previously serving control application.
+    const body = await publicationRequest<{ publication?: unknown }>({
+      action: 'claim_panic',
+      operationId: publication.operationId,
+      owner,
+      recoveryProtocol: RENEWING_RECOVERY_PROTOCOL,
+    });
+    if (body.publication === null || body.publication === undefined) {
+      return false;
+    }
+    const renewed = panicPublication(body.publication);
+    return (
+      renewed.operationId === publication.operationId &&
+      renewed.candidate.releaseId === publication.candidate.releaseId &&
+      renewed.candidate.sha === publication.candidate.sha &&
+      renewed.candidate.deploymentId === publication.candidate.deploymentId &&
+      renewed.baseline.releaseId === publication.baseline.releaseId &&
+      renewed.baseline.sha === publication.baseline.sha &&
+      renewed.baseline.deploymentId === publication.baseline.deploymentId
+    );
+  };
+
+  const restoreAdmissionPointer = async (
+    publication: RuntimeTuplePanicPublication,
+  ): Promise<void> => {
+    const response = await fetchImpl(
+      `${publication.baseline.deploymentUrl}/api/v2/internal/runtime/releases`,
+      {
+        method: 'POST',
+        headers: internalHeaders(),
+        body: JSON.stringify({
+          action: 'rollback',
+          environment: 'production',
+          schedulerBackend: 'absurd',
+          releaseId: publication.baseline.releaseId,
+        }),
+      },
+    );
+    if (!response.ok) {
+      return await errorWithResponse(
+        'Restore runtime admission pointer',
+        response,
+      );
+    }
+  };
+
+  const restoreAndVerifyPublicApplication = async (
+    publication: RuntimeTuplePanicPublication,
+  ): Promise<void> => {
+    const activeDeploymentId = async (): Promise<string | null> => {
+      const alias = await fetchImpl(
+        `https://api.vercel.com/v4/aliases/${encodeURIComponent(productionDomain)}?teamId=${encodeURIComponent(input.vercel.teamId)}`,
+        { headers: { authorization: `Bearer ${input.vercel.token}` } },
+      );
+      if (!alias.ok) {
+        await errorWithResponse('Read Vercel Production alias', alias);
+      }
+      const body = (await alias.json()) as {
+        deploymentId?: unknown;
+        deployment?: { id?: unknown };
+      };
+      return typeof body.deploymentId === 'string'
+        ? body.deploymentId
+        : typeof body.deployment?.id === 'string'
+          ? body.deployment.id
+          : null;
+    };
+
+    const activeBeforeRestore = await activeDeploymentId();
+    if (activeBeforeRestore === publication.baseline.deploymentId) return;
+    if (activeBeforeRestore !== publication.candidate.deploymentId) {
+      // Recovery owns only the failed candidate. A normal Vercel Git deployment
+      // may advance the alias while the candidate is canarying; restoring a
+      // captured baseline here would roll that newer deployment back.
+      log('preserving Vercel Production deployment outside the failed tuple', {
+        operationId: publication.operationId,
+        activeDeploymentId: activeBeforeRestore,
+        candidateDeploymentId: publication.candidate.deploymentId,
+        baselineDeploymentId: publication.baseline.deploymentId,
+      });
+      return;
+    }
+
+    const response = await promoteVercelProductionDeployment({
+      deploymentId: publication.baseline.deploymentId,
+      fetchImpl,
+      projectId: input.vercel.projectId,
+      teamId: input.vercel.teamId,
+      token: input.vercel.token,
+    });
+    if (!response.ok && response.status !== 202) {
+      const body = await response.text();
+      if (!isAlreadyCurrentProductionDeploymentConflict(response, body)) {
+        throw new Error(
+          `Restore Vercel Production deployment failed with HTTP ${response.status}: ${body.slice(0, 500)}`,
+        );
+      }
+    }
+
+    const deadline = now() + aliasWaitMs;
+    while (now() < deadline) {
+      if ((await activeDeploymentId()) === publication.baseline.deploymentId) {
+        return;
+      }
+      await wait(ALIAS_POLL_INTERVAL_MS);
+    }
+    throw new Error(
+      `Timed out waiting for Vercel Production alias ${productionDomain} to restore deployment ${publication.baseline.deploymentId}.`,
+    );
+  };
+
+  const restoreGatewayTuple = async (
+    publication: RuntimeTuplePanicPublication,
+  ): Promise<void> => {
+    await input.command({
+      executable: 'bun',
+      args: [
+        'scripts/runtime-topology.ts',
+        'rollback',
+        '--environment',
+        'production',
+        '--sha',
+        publication.candidate.sha,
+        '--apply',
+      ],
+      environment: { FLY_API_TOKEN: input.gatewayFlyToken },
+      description: 'Restore retained runtime gateway tuple',
+    });
+  };
+
+  /**
+   * A reverted candidate's gateway Machines are cordoned again by the tuple
+   * rollback but were never anyone's "source", so the ordinary generation
+   * drain would keep them forever. Remove them through the exact-SHA cleanup
+   * the release controller uses for a pre-publication failure.
+   */
+  const discardCandidateGateways = async (
+    publication: RuntimeTuplePanicPublication,
+  ): Promise<void> => {
+    await input.command({
+      executable: 'bun',
+      args: [
+        'scripts/runtime-topology.ts',
+        'cleanup',
+        '--environment',
+        'production',
+        '--sha',
+        publication.candidate.sha,
+        '--apply',
+      ],
+      environment: { FLY_API_TOKEN: input.gatewayFlyToken },
+      description: 'Discard reverted runtime gateway candidates',
+    });
+  };
+
+  const reconcile = async ({ owner }: { owner: string }): Promise<boolean> => {
+    const claimed = await publicationRequest<{ publication?: unknown }>({
+      action: 'claim_panic',
+      operationId: 'scaler',
+      owner,
+      recoveryProtocol: RENEWING_RECOVERY_PROTOCOL,
+    });
+    if (claimed.publication === null || claimed.publication === undefined) {
+      return false;
+    }
+    const publication = panicPublication(claimed.publication);
+    log('PANIC: restoring baseline tuple', {
+      operationId: publication.operationId,
+      candidateReleaseId: publication.candidate.releaseId,
+      baselineReleaseId: publication.baseline.releaseId,
+    });
+
+    let leaseFailure: unknown = null;
+    let closed = false;
+    const renewLease = async (): Promise<void> => {
+      if (closed || leaseFailure) return;
+      if (!(await renew(publication, owner))) {
+        leaseFailure = new RuntimeTupleRecoveryLeaseLostError(
+          publication.operationId,
+        );
+      }
+    };
+    const assertLease = async (): Promise<void> => {
+      if (leaseFailure) throw leaseFailure;
+      try {
+        await renewLease();
+      } catch (error) {
+        leaseFailure = error;
+      }
+      if (leaseFailure) throw leaseFailure;
+    };
+    const heartbeat = setInterval(() => {
+      void renewLease().catch((error) => {
+        leaseFailure = error;
+      });
+    }, leaseRenewIntervalMs);
+
+    const recoverStep = async (
+      name: string,
+      effect: () => Promise<void>,
+    ): Promise<void> => {
+      await assertLease();
+      log('recovery step started', {
+        operationId: publication.operationId,
+        step: name,
+      });
+      await effect();
+      await assertLease();
+      log('recovery step verified', {
+        operationId: publication.operationId,
+        step: name,
+      });
+    };
+
+    try {
+      await recoverStep(
+        'restore_admission',
+        async () => await restoreAdmissionPointer(publication),
+      );
+      await recoverStep(
+        'restore_public_application',
+        async () => await restoreAndVerifyPublicApplication(publication),
+      );
+      await recoverStep(
+        'restore_gateway_tuple',
+        async () => await restoreGatewayTuple(publication),
+      );
+      await assertLease();
+      await publicationRequest({
+        action: 'reverted',
+        operationId: publication.operationId,
+        owner,
+      });
+      log('baseline tuple restored', { operationId: publication.operationId });
+      // The baseline is durably restored above; candidate removal must never
+      // hold the publication in panic_reverting and block the next release.
+      // A failure here is logged and the scaler's orphan sweep finishes it.
+      try {
+        await discardCandidateGateways(publication);
+        log('candidate gateways discarded', {
+          operationId: publication.operationId,
+          candidateSha: publication.candidate.sha,
+        });
+      } catch (error) {
+        log('candidate gateway cleanup deferred to the scaler orphan sweep', {
+          operationId: publication.operationId,
+          candidateSha: publication.candidate.sha,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+      return true;
+    } finally {
+      closed = true;
+      clearInterval(heartbeat);
+    }
+  };
+
+  return { reconcile, activePublication: readActivePublication };
+}