p-dev-harness 0.3.1 → 0.4.0

package/workspace-snapshot/files/src/setup/target-workflow-finalization.ts

@@ -21,8 +21,10 @@ import {
   TARGET_WORKFLOW_PATH,
   type RemoteWorkflowStatus,
 } from "./remote-actions.js";
+import { randomUUID } from "node:crypto";
 import {
   buildTargetWorkflowBranchName,
+  buildTargetWorkflowPrBody,
   buildTargetWorkflowPrTitle,
   compareTargetWorkflowContent,
   previewTargetWorkflowSetup,
@@ -31,28 +33,54 @@ import {
   blockedCategoryMessage,
   classifyWorkflowInstallMergeRejection,
 } from "./workflow-install-merge-errors.js";
+import { shouldAttemptMerge } from "./workflow-install-merge-gate.js";
+import {
+  countOpenPullRequestsOnBranch,
+  ensureOpenInstallPullRequest,
+  hashWorkflowContentSha256,
+  isInstallBranchAlreadyClean,
+  isStaleHarnessInstallBranch,
+  recoverHarnessInstallBranch,
+  validateInstallBranchRecoveryProof,
+} from "./workflow-install-branch-recovery.js";
 import type { GitHubRemoteSetupProvider } from "./github-remote-provider.js";
 import {
   buildFinalizationLockKey,
   withTargetWorkflowFinalizationLock,
 } from "./target-workflow-finalization-lock.js";
+import {
+  clearTargetWorkflowFinalizationProgress,
+  readTargetWorkflowFinalizationProgress,
+  writeTargetWorkflowFinalizationProgressAtomic,
+} from "./target-workflow-finalization-progress.js";
 import type {
   TargetWorkflowFinalizeInput,
   TargetWorkflowFinalizationResult,
   WorkflowInstallBlockedCategory,
+  WorkflowInstallErrorCode,
   WorkflowInstallLifecycle,
+  WorkflowInstallUiPhase,
 } from "./target-workflow-finalization-types.js";
 import {
+  WORKFLOW_INSTALL_BASE_RETRY_MS,
   WORKFLOW_INSTALL_CHECK_POLL_TIMEOUT_MS,
+  WORKFLOW_INSTALL_MAX_RETRY_MS,
   WORKFLOW_INSTALL_VERIFICATION_TIMEOUT_MS,
+  errorCodeForBlockedCategory,
+  isRetryableBlockedCategory,
+  lifecycleToUiPhase,
 } from "./target-workflow-finalization-types.js";
 
 interface FinalizationSession {
+  operationId: string;
   checksPendingSince?: number;
   verificationStartedAt?: number;
   lastValidatedHeadSha?: string;
   mergeAttemptedForHeadSha?: string;
   branchUpdateAttemptedForHeadSha?: string;
+  recoveryAttemptedForHeadSha?: string;
+  startedAt: string;
+  supersededPrNumber?: number;
 }
 
 const sessions = new Map<string, FinalizationSession>();
@@ -61,6 +89,17 @@ function sessionKey(targetRepoSlug: string, repoConfigId: string): string {
   return `${targetRepoSlug}:${repoConfigId}`;
 }
 
+function computeRetryAfterMs(retryable: boolean): number | undefined {
+  if (!retryable) {
+    return undefined;
+  }
+  const jitter = Math.floor(Math.random() * WORKFLOW_INSTALL_BASE_RETRY_MS);
+  return Math.min(
+    WORKFLOW_INSTALL_MAX_RETRY_MS,
+    WORKFLOW_INSTALL_BASE_RETRY_MS + jitter,
+  );
+}
+
 function blockedResult(input: {
   repoConfigId: string;
   targetRepo: string;
@@ -69,35 +108,53 @@ function blockedResult(input: {
   branchName: string;
   category: WorkflowInstallBlockedCategory;
   workflowStatus: RemoteWorkflowStatus;
+  operationId: string;
   prUrl?: string;
   prNumber?: number;
+  supersededPrNumber?: number;
   validatedHeadSha?: string;
   advancedThisRequest: boolean;
   lockContended: boolean;
   customMessage?: string;
+  lastSafeCheckpoint?: string;
 }): TargetWorkflowFinalizationResult {
+  const retryable =
+    input.lockContended || isRetryableBlockedCategory(input.category);
+  const errorCode: WorkflowInstallErrorCode = input.lockContended
+    ? "lock_contended"
+    : errorCodeForBlockedCategory(input.category);
+  const lifecycle: WorkflowInstallLifecycle = "blocked";
   return {
     repoConfigId: input.repoConfigId,
     targetRepo: input.targetRepo,
     targetRepoSlug: input.targetRepoSlug,
     productionBranch: input.productionBranch,
     branchName: input.branchName,
-    lifecycle: "blocked",
+    lifecycle,
+    phase: lifecycleToUiPhase(lifecycle),
+    operationId: input.operationId,
     blockedCategory: input.category,
     message: input.customMessage ?? blockedCategoryMessage(input.category),
     prUrl: input.prUrl,
     prNumber: input.prNumber,
+    supersededPrNumber: input.supersededPrNumber,
     validatedHeadSha: input.validatedHeadSha,
     workflowStatus: input.workflowStatus,
-    canRetry: input.category === "verification-failed",
+    canRetry: retryable || input.category === "verification-failed",
+    retryable,
+    retryAfterMs: computeRetryAfterMs(retryable),
+    lastSafeCheckpoint: input.lastSafeCheckpoint,
+    errorCode,
     requiresGitHubIntervention: ![
       "checks-pending",
       "mergeability-pending",
       "branch-behind",
       "verification-failed",
+      "transient-github-unavailable",
     ].includes(input.category),
     advancedThisRequest: input.advancedThisRequest,
     lockContended: input.lockContended,
+    updatedAt: new Date().toISOString(),
   };
 }
 
@@ -108,17 +165,31 @@ function progressResult(input: {
   productionBranch: string;
   branchName: string;
   lifecycle: WorkflowInstallLifecycle;
+  phase?: WorkflowInstallUiPhase;
+  operationId: string;
   workflowStatus: RemoteWorkflowStatus;
   message: string;
   prUrl?: string;
   prNumber?: number;
+  supersededPrNumber?: number;
   validatedHeadSha?: string;
   advancedThisRequest: boolean;
   lockContended: boolean;
   blockedCategory?: WorkflowInstallBlockedCategory;
   canRetry?: boolean;
+  retryable?: boolean;
   requiresGitHubIntervention?: boolean;
+  lastSafeCheckpoint?: string;
+  errorCode?: WorkflowInstallErrorCode;
 }): TargetWorkflowFinalizationResult {
+  const retryable =
+    input.retryable ??
+    (input.lockContended ||
+      input.lifecycle === "waiting-for-checks" ||
+      input.lifecycle === "updating-branch" ||
+      input.lifecycle === "verifying" ||
+      input.lifecycle === "merging" ||
+      input.lifecycle === "preparing");
   return {
     repoConfigId: input.repoConfigId,
     targetRepo: input.targetRepo,
@@ -126,16 +197,24 @@ function progressResult(input: {
     productionBranch: input.productionBranch,
     branchName: input.branchName,
     lifecycle: input.lifecycle,
+    phase: input.phase ?? lifecycleToUiPhase(input.lifecycle),
+    operationId: input.operationId,
     blockedCategory: input.blockedCategory,
     message: input.message,
     prUrl: input.prUrl,
     prNumber: input.prNumber,
+    supersededPrNumber: input.supersededPrNumber,
     validatedHeadSha: input.validatedHeadSha,
     workflowStatus: input.workflowStatus,
-    canRetry: input.canRetry ?? false,
+    canRetry: input.canRetry ?? retryable,
+    retryable,
+    retryAfterMs: computeRetryAfterMs(retryable),
+    lastSafeCheckpoint: input.lastSafeCheckpoint ?? input.lifecycle,
+    errorCode: input.errorCode ?? (input.lockContended ? "lock_contended" : "none"),
     requiresGitHubIntervention: input.requiresGitHubIntervention ?? false,
     advancedThisRequest: input.advancedThisRequest,
     lockContended: input.lockContended,
+    updatedAt: new Date().toISOString(),
   };
 }
 
@@ -145,8 +224,10 @@ function completeResult(input: {
   targetRepoSlug: string;
   productionBranch: string;
   branchName: string;
+  operationId: string;
   prUrl?: string;
   prNumber?: number;
+  supersededPrNumber?: number;
   validatedHeadSha?: string;
   advancedThisRequest: boolean;
   lockContended: boolean;
@@ -155,10 +236,15 @@ function completeResult(input: {
   return progressResult({
     ...input,
     lifecycle: "complete",
+    phase: "verifying-production-workflow",
     workflowStatus: "present",
     message: "Workflow installed on the production branch.",
     advancedThisRequest: input.advancedThisRequest,
     lockContended: input.lockContended,
+    retryable: false,
+    canRetry: false,
+    errorCode: "none",
+    lastSafeCheckpoint: "complete",
   });
 }
 
@@ -213,6 +299,516 @@ function validatePullRequestFiles(
   return files[0]?.path === workflowPath;
 }
 
+const REFRESHING_BRANCH_MESSAGE =
+  "Refreshing the workflow install branch…";
+
+interface AttemptStaleInstallBranchRecoveryInput {
+  client: GitHubClient;
+  input: TargetWorkflowFinalizeInput;
+  targetRepoSlug: string;
+  branchName: string;
+  productionStatus: { workflowStatus: RemoteWorkflowStatus };
+  intendedWorkflowContent: string;
+  inspection: Awaited<ReturnType<typeof inspectPullRequestForMerge>>;
+  parsedPr: NonNullable<ReturnType<typeof parsePrUrl>>;
+  prUrl: string;
+  prNumber: number;
+  validatedHeadSha: string;
+  session: FinalizationSession;
+  lockContended: boolean;
+  filesValidationPassed: boolean;
+}
+
+async function attemptStaleInstallBranchRecovery(
+  recoveryInput: AttemptStaleInstallBranchRecoveryInput & {
+    harnessDispatchRepoSlug: string;
+    cwd?: string;
+    inputFingerprint: string;
+  },
+): Promise<TargetWorkflowFinalizationResult | null> {
+  const {
+    client,
+    input,
+    targetRepoSlug,
+    branchName,
+    productionStatus,
+    intendedWorkflowContent,
+    inspection,
+    parsedPr,
+    prUrl,
+    prNumber,
+    validatedHeadSha,
+    session,
+    lockContended,
+    filesValidationPassed,
+    harnessDispatchRepoSlug,
+    cwd,
+    inputFingerprint,
+  } = recoveryInput;
+
+  if (session.recoveryAttemptedForHeadSha === validatedHeadSha) {
+    return null;
+  }
+
+  const headWorkflowContent = await readWorkflowAtRef(
+    client,
+    targetRepoSlug,
+    TARGET_WORKFLOW_PATH,
+    inspection.headSha,
+  );
+  const headWorkflowMatchesIntended =
+    headWorkflowContent !== null &&
+    compareTargetWorkflowContent(headWorkflowContent, intendedWorkflowContent) ===
+      "present";
+
+  const [owner, repo] = targetRepoSlug.split("/");
+  let compareStatus: string | null = null;
+  try {
+    const compare = await client.compareCommits(
+      owner,
+      repo,
+      input.productionBranch,
+      branchName,
+    );
+    compareStatus = compare.status;
+  } catch {
+    compareStatus = null;
+  }
+
+  if (
+    !isStaleHarnessInstallBranch({
+      changedFiles: inspection.changedFiles,
+      workflowPath: TARGET_WORKFLOW_PATH,
+      mergeableState: inspection.mergeableState,
+      compareStatus,
+      headWorkflowMatchesIntended,
+      filesValidationPassed,
+    })
+  ) {
+    return null;
+  }
+
+  const openPullCount = await countOpenPullRequestsOnBranch(client, {
+    targetRepoSlug,
+    productionBranch: input.productionBranch,
+    branchName,
+  });
+  const allowZeroOpenPullRequests =
+    openPullCount === 0 ||
+    inspection.changedFiles.length === 0 ||
+    compareStatus === "identical";
+  const proof = validateInstallBranchRecoveryProof({
+    configuredTargetRepoSlug: targetRepoSlug,
+    observedTargetRepoSlug: targetRepoSlug,
+    configuredRepoConfigId: input.repoConfigId,
+    reservedBranchName: branchName,
+    observedBranchName: inspection.branch,
+    configuredProductionBranch: input.productionBranch,
+    observedProductionBranch: inspection.baseBranch,
+    configuredWorkflowPath: TARGET_WORKFLOW_PATH,
+    pullRequestOwner: parsedPr.owner,
+    pullRequestRepo: parsedPr.repo,
+    openPullRequestsOnBranch: openPullCount,
+    allowZeroOpenPullRequests,
+  });
+  if (!proof.ok) {
+    session.recoveryAttemptedForHeadSha = validatedHeadSha;
+    sessions.set(sessionKey(targetRepoSlug, input.repoConfigId), session);
+    return blockedResult({
+      repoConfigId: input.repoConfigId,
+      targetRepo: input.targetRepo,
+      targetRepoSlug,
+      productionBranch: input.productionBranch,
+      branchName,
+      category: "unexpected-pr-content",
+      workflowStatus: productionStatus.workflowStatus,
+      operationId: session.operationId,
+      prUrl,
+      prNumber,
+      validatedHeadSha,
+      advancedThisRequest: true,
+      lockContended,
+      customMessage: proof.reason,
+      lastSafeCheckpoint: "recovery-proof-failed",
+    });
+  }
+
+  const alreadyClean = await isInstallBranchAlreadyClean({
+    client,
+    targetRepoSlug,
+    productionBranch: input.productionBranch,
+    branchName,
+    workflowPath: TARGET_WORKFLOW_PATH,
+    intendedWorkflowContent,
+  });
+  if (alreadyClean) {
+    session.recoveryAttemptedForHeadSha = validatedHeadSha;
+    sessions.set(sessionKey(targetRepoSlug, input.repoConfigId), session);
+    return null;
+  }
+
+  await persistFinalizationProgress({
+    cwd,
+    session,
+    input,
+    targetRepoSlug,
+    branchName,
+    intendedWorkflowContent,
+    harnessDispatchRepoSlug,
+    inputFingerprint,
+    phase: "creating-or-refreshing-install-branch",
+    prUrl,
+    prNumber,
+    observedInstallHeadSha: validatedHeadSha,
+    lastSafeCheckpoint: "before-branch-recovery",
+  });
+
+  const recovery = await recoverHarnessInstallBranch({
+    client,
+    targetRepoSlug,
+    productionBranch: input.productionBranch,
+    branchName,
+    workflowPath: TARGET_WORKFLOW_PATH,
+    workflowContent: intendedWorkflowContent,
+    expectedReservedBranchHeadSha: validatedHeadSha,
+  });
+
+  if (!recovery.recovered) {
+    if (recovery.needsReconciliation) {
+      return progressResult({
+        repoConfigId: input.repoConfigId,
+        targetRepo: input.targetRepo,
+        targetRepoSlug,
+        productionBranch: input.productionBranch,
+        branchName,
+        lifecycle: "updating-branch",
+        phase: "creating-or-refreshing-install-branch",
+        operationId: session.operationId,
+        workflowStatus: productionStatus.workflowStatus,
+        message: REFRESHING_BRANCH_MESSAGE,
+        prUrl,
+        prNumber,
+        validatedHeadSha: recovery.observedHeadSha,
+        advancedThisRequest: true,
+        lockContended,
+        blockedCategory: "branch-behind",
+        retryable: true,
+        lastSafeCheckpoint: "branch-head-changed-reconcile",
+        errorCode: "branch_behind",
+      });
+    }
+    session.recoveryAttemptedForHeadSha = validatedHeadSha;
+    sessions.set(sessionKey(targetRepoSlug, input.repoConfigId), session);
+    return blockedResult({
+      repoConfigId: input.repoConfigId,
+      targetRepo: input.targetRepo,
+      targetRepoSlug,
+      productionBranch: input.productionBranch,
+      branchName,
+      category: "unexpected-pr-content",
+      workflowStatus: productionStatus.workflowStatus,
+      operationId: session.operationId,
+      prUrl,
+      prNumber,
+      validatedHeadSha,
+      advancedThisRequest: true,
+      lockContended,
+      customMessage: recovery.reason,
+      lastSafeCheckpoint: "recovery-failed",
+    });
+  }
+
+  const prEnsure = await ensureOpenInstallPullRequest({
+    client,
+    targetRepoSlug,
+    productionBranch: input.productionBranch,
+    branchName,
+    repoConfigId: input.repoConfigId,
+    prTitle: buildTargetWorkflowPrTitle(),
+    prBody: buildTargetWorkflowPrBody({
+      repoConfigId: input.repoConfigId,
+      productionBranch: input.productionBranch,
+      harnessDispatchRepo: harnessDispatchRepoSlug,
+    }),
+    verifiedHeadSha: recovery.headSha,
+    harnessDispatchRepo: harnessDispatchRepoSlug,
+  });
+
+  if (prEnsure.supersededPrNumber) {
+    session.supersededPrNumber = prEnsure.supersededPrNumber;
+  }
+  session.recoveryAttemptedForHeadSha = validatedHeadSha;
+  session.lastValidatedHeadSha = recovery.headSha;
+  sessions.set(sessionKey(targetRepoSlug, input.repoConfigId), session);
+
+  await persistFinalizationProgress({
+    cwd,
+    session,
+    input,
+    targetRepoSlug,
+    branchName,
+    intendedWorkflowContent,
+    harnessDispatchRepoSlug,
+    inputFingerprint,
+    phase: "creating-or-refreshing-install-branch",
+    prUrl: prEnsure.prUrl,
+    prNumber: prEnsure.prNumber,
+    supersededPrNumber: session.supersededPrNumber,
+    observedInstallHeadSha: recovery.headSha,
+    observedProductionHeadSha: recovery.productionSha,
+    lastSafeCheckpoint: "branch-recovered",
+  });
+
+  return progressResult({
+    repoConfigId: input.repoConfigId,
+    targetRepo: input.targetRepo,
+    targetRepoSlug,
+    productionBranch: input.productionBranch,
+    branchName,
+    lifecycle: "updating-branch",
+    phase: "creating-or-refreshing-install-branch",
+    operationId: session.operationId,
+    workflowStatus: productionStatus.workflowStatus,
+    message: REFRESHING_BRANCH_MESSAGE,
+    prUrl: prEnsure.prUrl,
+    prNumber: prEnsure.prNumber,
+    supersededPrNumber: session.supersededPrNumber,
+    validatedHeadSha: recovery.headSha,
+    advancedThisRequest: true,
+    lockContended,
+    blockedCategory: "branch-behind",
+    retryable: true,
+    lastSafeCheckpoint: "branch-recovered",
+    errorCode: "branch_behind",
+  });
+}
+
+async function persistFinalizationProgress(input: {
+  cwd?: string;
+  session: FinalizationSession;
+  input: TargetWorkflowFinalizeInput;
+  targetRepoSlug: string;
+  branchName: string;
+  intendedWorkflowContent: string;
+  harnessDispatchRepoSlug: string;
+  inputFingerprint: string;
+  phase: TargetWorkflowFinalizationResult["phase"];
+  prUrl?: string;
+  prNumber?: number;
+  supersededPrNumber?: number;
+  observedInstallHeadSha?: string;
+  observedProductionHeadSha?: string;
+  lastSafeCheckpoint: string;
+  lastRedactedError?: string;
+  retryCount?: number;
+}): Promise<void> {
+  const now = new Date().toISOString();
+  await writeTargetWorkflowFinalizationProgressAtomic(
+    {
+      operationId: input.session.operationId,
+      repoConfigId: input.input.repoConfigId,
+      inputFingerprint: input.inputFingerprint,
+      intendedWorkflowSha256: hashWorkflowContentSha256(
+        input.intendedWorkflowContent,
+      ),
+      harnessDispatchRepo: input.harnessDispatchRepoSlug,
+      targetRepo: input.input.targetRepo,
+      targetRepoSlug: input.targetRepoSlug,
+      productionBranch: input.input.productionBranch,
+      installBranch: input.branchName,
+      observedProductionHeadSha: input.observedProductionHeadSha,
+      observedInstallHeadSha: input.observedInstallHeadSha,
+      prNumber: input.prNumber,
+      prUrl: input.prUrl,
+      supersededPrNumber: input.supersededPrNumber,
+      phase: input.phase,
+      phaseStartedAt: now,
+      startedAt: input.session.startedAt,
+      checksDeadlineAt: input.session.checksPendingSince
+        ? new Date(
+            input.session.checksPendingSince +
+              WORKFLOW_INSTALL_CHECK_POLL_TIMEOUT_MS,
+          ).toISOString()
+        : undefined,
+      verificationDeadlineAt: input.session.verificationStartedAt
+        ? new Date(
+            input.session.verificationStartedAt +
+              WORKFLOW_INSTALL_VERIFICATION_TIMEOUT_MS,
+          ).toISOString()
+        : undefined,
+      lastVerifiedRemoteHead: input.observedInstallHeadSha,
+      lastSafeCheckpoint: input.lastSafeCheckpoint,
+      retryCount: input.retryCount ?? 0,
+      lastRedactedError: input.lastRedactedError
+        ? redactSecretsString(input.lastRedactedError)
+        : undefined,
+    },
+    input.cwd,
+  );
+}
+
+async function attemptEmptyInstallRecovery(input: {
+  client: GitHubClient;
+  cwd?: string;
+  finalizeInput: TargetWorkflowFinalizeInput;
+  targetRepoSlug: string;
+  branchName: string;
+  productionStatus: { workflowStatus: RemoteWorkflowStatus };
+  intendedWorkflowContent: string;
+  harnessDispatchRepoSlug: string;
+  inputFingerprint: string;
+  session: FinalizationSession;
+  lockContended: boolean;
+  supersededPrNumber?: number;
+}): Promise<TargetWorkflowFinalizationResult | null> {
+  if (
+    input.productionStatus.workflowStatus !== "missing" &&
+    input.productionStatus.workflowStatus !== "differs"
+  ) {
+    return null;
+  }
+
+  await persistFinalizationProgress({
+    cwd: input.cwd,
+    session: input.session,
+    input: input.finalizeInput,
+    targetRepoSlug: input.targetRepoSlug,
+    branchName: input.branchName,
+    intendedWorkflowContent: input.intendedWorkflowContent,
+    harnessDispatchRepoSlug: input.harnessDispatchRepoSlug,
+    inputFingerprint: input.inputFingerprint,
+    phase: "creating-or-refreshing-install-branch",
+    supersededPrNumber: input.supersededPrNumber,
+    lastSafeCheckpoint: "before-empty-pr-recovery",
+  });
+
+  const [owner, repo] = input.targetRepoSlug.split("/");
+  let expectedHead: string | null = null;
+  try {
+    const ref = await input.client.getBranchRef(owner, repo, input.branchName);
+    expectedHead = ref.object.sha;
+  } catch (error) {
+    if (!(error instanceof GitHubApiError && error.status === 404)) {
+      throw error;
+    }
+  }
+
+  const recovery = await recoverHarnessInstallBranch({
+    client: input.client,
+    targetRepoSlug: input.targetRepoSlug,
+    productionBranch: input.finalizeInput.productionBranch,
+    branchName: input.branchName,
+    workflowPath: TARGET_WORKFLOW_PATH,
+    workflowContent: input.intendedWorkflowContent,
+    expectedReservedBranchHeadSha: expectedHead,
+  });
+
+  if (!recovery.recovered) {
+    if (recovery.needsReconciliation) {
+      return progressResult({
+        repoConfigId: input.finalizeInput.repoConfigId,
+        targetRepo: input.finalizeInput.targetRepo,
+        targetRepoSlug: input.targetRepoSlug,
+        productionBranch: input.finalizeInput.productionBranch,
+        branchName: input.branchName,
+        lifecycle: "updating-branch",
+        phase: "creating-or-refreshing-install-branch",
+        operationId: input.session.operationId,
+        workflowStatus: input.productionStatus.workflowStatus,
+        message: REFRESHING_BRANCH_MESSAGE,
+        validatedHeadSha: recovery.observedHeadSha,
+        advancedThisRequest: true,
+        lockContended: input.lockContended,
+        retryable: true,
+        lastSafeCheckpoint: "empty-pr-reconcile",
+        errorCode: "branch_behind",
+      });
+    }
+    return blockedResult({
+      repoConfigId: input.finalizeInput.repoConfigId,
+      targetRepo: input.finalizeInput.targetRepo,
+      targetRepoSlug: input.targetRepoSlug,
+      productionBranch: input.finalizeInput.productionBranch,
+      branchName: input.branchName,
+      category: "unexpected-pr-content",
+      workflowStatus: input.productionStatus.workflowStatus,
+      operationId: input.session.operationId,
+      advancedThisRequest: true,
+      lockContended: input.lockContended,
+      customMessage: recovery.reason,
+      lastSafeCheckpoint: "empty-pr-recovery-failed",
+    });
+  }
+
+  const prEnsure = await ensureOpenInstallPullRequest({
+    client: input.client,
+    targetRepoSlug: input.targetRepoSlug,
+    productionBranch: input.finalizeInput.productionBranch,
+    branchName: input.branchName,
+    repoConfigId: input.finalizeInput.repoConfigId,
+    prTitle: buildTargetWorkflowPrTitle(),
+    prBody: buildTargetWorkflowPrBody({
+      repoConfigId: input.finalizeInput.repoConfigId,
+      productionBranch: input.finalizeInput.productionBranch,
+      harnessDispatchRepo: input.harnessDispatchRepoSlug,
+    }),
+    verifiedHeadSha: recovery.headSha,
+    harnessDispatchRepo: input.harnessDispatchRepoSlug,
+  });
+
+  if (prEnsure.supersededPrNumber) {
+    input.session.supersededPrNumber = prEnsure.supersededPrNumber;
+  } else if (input.supersededPrNumber) {
+    input.session.supersededPrNumber = input.supersededPrNumber;
+  }
+  input.session.lastValidatedHeadSha = recovery.headSha;
+  sessions.set(
+    sessionKey(input.targetRepoSlug, input.finalizeInput.repoConfigId),
+    input.session,
+  );
+
+  await persistFinalizationProgress({
+    cwd: input.cwd,
+    session: input.session,
+    input: input.finalizeInput,
+    targetRepoSlug: input.targetRepoSlug,
+    branchName: input.branchName,
+    intendedWorkflowContent: input.intendedWorkflowContent,
+    harnessDispatchRepoSlug: input.harnessDispatchRepoSlug,
+    inputFingerprint: input.inputFingerprint,
+    phase: "creating-or-refreshing-install-branch",
+    prUrl: prEnsure.prUrl,
+    prNumber: prEnsure.prNumber,
+    supersededPrNumber: input.session.supersededPrNumber,
+    observedInstallHeadSha: recovery.headSha,
+    observedProductionHeadSha: recovery.productionSha,
+    lastSafeCheckpoint: "empty-pr-recovered",
+  });
+
+  return progressResult({
+    repoConfigId: input.finalizeInput.repoConfigId,
+    targetRepo: input.finalizeInput.targetRepo,
+    targetRepoSlug: input.targetRepoSlug,
+    productionBranch: input.finalizeInput.productionBranch,
+    branchName: input.branchName,
+    lifecycle: "updating-branch",
+    phase: "creating-or-refreshing-install-branch",
+    operationId: input.session.operationId,
+    workflowStatus: input.productionStatus.workflowStatus,
+    message: REFRESHING_BRANCH_MESSAGE,
+    prUrl: prEnsure.prUrl,
+    prNumber: prEnsure.prNumber,
+    supersededPrNumber: input.session.supersededPrNumber,
+    validatedHeadSha: recovery.headSha,
+    advancedThisRequest: true,
+    lockContended: input.lockContended,
+    blockedCategory: "branch-behind",
+    retryable: true,
+    lastSafeCheckpoint: "empty-pr-recovered",
+    errorCode: "branch_behind",
+  });
+}
+
 export interface AdvanceTargetWorkflowFinalizationOptions {
   cwd?: string;
   input: TargetWorkflowFinalizeInput;
@@ -226,6 +822,13 @@ export async function advanceTargetWorkflowFinalizationStep(
 ): Promise<TargetWorkflowFinalizationResult> {
   const { input, provider, client, lockContended = false } = options;
   const targetRepoSlug = targetRepoSlugFromUrl(input.targetRepo);
+  const durable = await readTargetWorkflowFinalizationProgress(
+    input.repoConfigId,
+    options.cwd,
+  );
+  const operationId =
+    input.operationId ?? durable?.operationId ?? randomUUID();
+
   if (!targetRepoSlug) {
     return blockedResult({
       repoConfigId: input.repoConfigId,
@@ -235,6 +838,7 @@ export async function advanceTargetWorkflowFinalizationStep(
       branchName: buildTargetWorkflowBranchName(input.repoConfigId),
       category: "unexpected-pr-content",
       workflowStatus: "unknown",
+      operationId,
       advancedThisRequest: true,
       lockContended,
       customMessage: `Invalid target repo URL: ${input.targetRepo}`,
@@ -255,6 +859,7 @@ export async function advanceTargetWorkflowFinalizationStep(
   });
   const intendedWorkflowContent = preview.workflowContent;
   const harnessDispatchRepoSlug = formatHarnessDispatchRepo(harnessDispatchRepo);
+  const inputFingerprint = preview.fingerprint;
 
   const productionStatus = await provider.checkTargetWorkflowStatus({
     targetRepoSlug,
@@ -264,23 +869,47 @@ export async function advanceTargetWorkflowFinalizationStep(
   });
 
   if (productionStatus.workflowStatus === "present") {
+    await clearTargetWorkflowFinalizationProgress(
+      input.repoConfigId,
+      options.cwd,
+    );
     return completeResult({
       repoConfigId: input.repoConfigId,
       targetRepo: input.targetRepo,
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId,
       advancedThisRequest: true,
       lockContended,
     });
   }
 
-  const session =
-    sessions.get(sessionKey(targetRepoSlug, input.repoConfigId)) ?? {};
+  const existingSession = sessions.get(
+    sessionKey(targetRepoSlug, input.repoConfigId),
+  );
+  const session: FinalizationSession = existingSession ?? {
+    operationId,
+    startedAt: durable?.startedAt ?? new Date().toISOString(),
+    supersededPrNumber: durable?.supersededPrNumber,
+    checksPendingSince: durable?.checksDeadlineAt
+      ? Date.parse(durable.checksDeadlineAt) -
+        WORKFLOW_INSTALL_CHECK_POLL_TIMEOUT_MS
+      : undefined,
+    verificationStartedAt: durable?.verificationDeadlineAt
+      ? Date.parse(durable.verificationDeadlineAt) -
+        WORKFLOW_INSTALL_VERIFICATION_TIMEOUT_MS
+      : undefined,
+  };
+  session.operationId = operationId;
+  if (durable?.supersededPrNumber) {
+    session.supersededPrNumber = durable.supersededPrNumber;
+  }
+  sessions.set(sessionKey(targetRepoSlug, input.repoConfigId), session);
 
-  let prUrl = input.prUrl;
-  let prNumber: number | undefined;
-  let validatedHeadSha: string | undefined;
+  let prUrl = input.prUrl ?? durable?.prUrl;
+  let prNumber: number | undefined = durable?.prNumber;
+  let validatedHeadSha: string | undefined = durable?.observedInstallHeadSha;
 
   if (prUrl) {
     const parsed = parsePrUrl(prUrl);
@@ -303,6 +932,23 @@ export async function advanceTargetWorkflowFinalizationStep(
   }
 
   if (!prUrl || !prNumber) {
+    const emptyRecovery = await attemptEmptyInstallRecovery({
+      client,
+      cwd: options.cwd,
+      finalizeInput: input,
+      targetRepoSlug,
+      branchName,
+      productionStatus,
+      intendedWorkflowContent,
+      harnessDispatchRepoSlug,
+      inputFingerprint,
+      session,
+      lockContended,
+      supersededPrNumber: durable?.prNumber ?? session.supersededPrNumber,
+    });
+    if (emptyRecovery) {
+      return emptyRecovery;
+    }
     return blockedResult({
       repoConfigId: input.repoConfigId,
       targetRepo: input.targetRepo,
@@ -311,10 +957,12 @@ export async function advanceTargetWorkflowFinalizationStep(
       branchName,
       category: "unexpected-pr-content",
       workflowStatus: productionStatus.workflowStatus,
+      operationId: session.operationId,
       advancedThisRequest: true,
       lockContended,
       customMessage:
         "No open workflow install PR was found for the deterministic install branch.",
+      lastSafeCheckpoint: durable?.lastSafeCheckpoint,
     });
   }
 
@@ -326,6 +974,7 @@ export async function advanceTargetWorkflowFinalizationStep(
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       category: "unexpected-pr-content",
       workflowStatus: productionStatus.workflowStatus,
       prUrl,
@@ -354,6 +1003,7 @@ export async function advanceTargetWorkflowFinalizationStep(
         targetRepoSlug,
         productionBranch: input.productionBranch,
         branchName,
+      operationId: session.operationId,
         prUrl,
         prNumber,
         advancedThisRequest: true,
@@ -372,6 +1022,7 @@ export async function advanceTargetWorkflowFinalizationStep(
         targetRepoSlug,
         productionBranch: input.productionBranch,
         branchName,
+      operationId: session.operationId,
         category: "verification-failed",
         workflowStatus: reverified.workflowStatus,
         prUrl,
@@ -387,6 +1038,7 @@ export async function advanceTargetWorkflowFinalizationStep(
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       lifecycle: "verifying",
       workflowStatus: reverified.workflowStatus,
       message: "Verifying workflow on the production branch.",
@@ -404,6 +1056,7 @@ export async function advanceTargetWorkflowFinalizationStep(
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       category: "unexpected-pr-content",
       workflowStatus: productionStatus.workflowStatus,
       prUrl,
@@ -426,6 +1079,7 @@ export async function advanceTargetWorkflowFinalizationStep(
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       category: "unexpected-pr-content",
       workflowStatus: productionStatus.workflowStatus,
       prUrl,
@@ -451,13 +1105,40 @@ export async function advanceTargetWorkflowFinalizationStep(
   validatedHeadSha = inspection.headSha;
   session.lastValidatedHeadSha = validatedHeadSha;
 
-  if (!validatePullRequestFiles(inspection.changedFiles, TARGET_WORKFLOW_PATH)) {
+  const filesValidationPassed = validatePullRequestFiles(
+    inspection.changedFiles,
+    TARGET_WORKFLOW_PATH,
+  );
+  if (!filesValidationPassed) {
+    const recoveryResult = await attemptStaleInstallBranchRecovery({
+      client,
+      input,
+      targetRepoSlug,
+      branchName,
+      productionStatus,
+      intendedWorkflowContent,
+      inspection,
+      parsedPr,
+      prUrl,
+      prNumber,
+      validatedHeadSha,
+      session,
+      lockContended,
+      filesValidationPassed,
+      harnessDispatchRepoSlug,
+      cwd: options.cwd,
+      inputFingerprint,
+    });
+    if (recoveryResult) {
+      return recoveryResult;
+    }
     return blockedResult({
       repoConfigId: input.repoConfigId,
       targetRepo: input.targetRepo,
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       category: "unexpected-pr-content",
       workflowStatus: productionStatus.workflowStatus,
       prUrl,
@@ -478,12 +1159,35 @@ export async function advanceTargetWorkflowFinalizationStep(
     compareTargetWorkflowContent(headWorkflowContent, intendedWorkflowContent) !==
     "present"
   ) {
+    const recoveryResult = await attemptStaleInstallBranchRecovery({
+      client,
+      input,
+      targetRepoSlug,
+      branchName,
+      productionStatus,
+      intendedWorkflowContent,
+      inspection,
+      parsedPr,
+      prUrl,
+      prNumber,
+      validatedHeadSha,
+      session,
+      lockContended,
+      filesValidationPassed: true,
+      harnessDispatchRepoSlug,
+      cwd: options.cwd,
+      inputFingerprint,
+    });
+    if (recoveryResult) {
+      return recoveryResult;
+    }
     return blockedResult({
       repoConfigId: input.repoConfigId,
       targetRepo: input.targetRepo,
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       category: "unexpected-pr-content",
       workflowStatus: productionStatus.workflowStatus,
       prUrl,
@@ -506,6 +1210,7 @@ export async function advanceTargetWorkflowFinalizationStep(
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       category: "unexpected-pr-content",
       workflowStatus: productionStatus.workflowStatus,
       prUrl,
@@ -529,6 +1234,7 @@ export async function advanceTargetWorkflowFinalizationStep(
         targetRepoSlug,
         productionBranch: input.productionBranch,
         branchName,
+      operationId: session.operationId,
         category: "checks-failing",
         workflowStatus: productionStatus.workflowStatus,
         prUrl,
@@ -551,6 +1257,7 @@ export async function advanceTargetWorkflowFinalizationStep(
         targetRepoSlug,
         productionBranch: input.productionBranch,
         branchName,
+      operationId: session.operationId,
         category: "checks-pending",
         workflowStatus: productionStatus.workflowStatus,
         prUrl,
@@ -568,6 +1275,7 @@ export async function advanceTargetWorkflowFinalizationStep(
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       lifecycle: "waiting-for-checks",
       workflowStatus: productionStatus.workflowStatus,
       message: "Waiting for GitHub checks on the workflow install PR.",
@@ -589,6 +1297,7 @@ export async function advanceTargetWorkflowFinalizationStep(
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       lifecycle: "waiting-for-checks",
       workflowStatus: productionStatus.workflowStatus,
       message: "Waiting for GitHub mergeability on the workflow install PR.",
@@ -618,6 +1327,7 @@ export async function advanceTargetWorkflowFinalizationStep(
           targetRepoSlug,
           productionBranch: input.productionBranch,
           branchName,
+      operationId: session.operationId,
           lifecycle: "updating-branch",
           workflowStatus: productionStatus.workflowStatus,
           message: "Updating the workflow install branch.",
@@ -629,12 +1339,38 @@ export async function advanceTargetWorkflowFinalizationStep(
         });
       } catch (error) {
         const classified = classifyWorkflowInstallMergeRejection({ error });
+        const recoveryResult = await attemptStaleInstallBranchRecovery({
+          client,
+          input,
+          targetRepoSlug,
+          branchName,
+          productionStatus,
+          intendedWorkflowContent,
+          inspection,
+          parsedPr,
+          prUrl,
+          prNumber,
+          validatedHeadSha,
+          session,
+          lockContended,
+          filesValidationPassed: validatePullRequestFiles(
+            inspection.changedFiles,
+            TARGET_WORKFLOW_PATH,
+          ),
+          harnessDispatchRepoSlug,
+          cwd: options.cwd,
+          inputFingerprint,
+        });
+        if (recoveryResult) {
+          return recoveryResult;
+        }
         return blockedResult({
           repoConfigId: input.repoConfigId,
           targetRepo: input.targetRepo,
           targetRepoSlug,
           productionBranch: input.productionBranch,
           branchName,
+      operationId: session.operationId,
           category: classified.category,
           workflowStatus: productionStatus.workflowStatus,
           prUrl,
@@ -646,12 +1382,38 @@ export async function advanceTargetWorkflowFinalizationStep(
         });
       }
     }
+    const recoveryResult = await attemptStaleInstallBranchRecovery({
+      client,
+      input,
+      targetRepoSlug,
+      branchName,
+      productionStatus,
+      intendedWorkflowContent,
+      inspection,
+      parsedPr,
+      prUrl,
+      prNumber,
+      validatedHeadSha,
+      session,
+      lockContended,
+      filesValidationPassed: validatePullRequestFiles(
+        inspection.changedFiles,
+        TARGET_WORKFLOW_PATH,
+      ),
+      harnessDispatchRepoSlug,
+      cwd: options.cwd,
+      inputFingerprint,
+    });
+    if (recoveryResult) {
+      return recoveryResult;
+    }
     return blockedResult({
       repoConfigId: input.repoConfigId,
       targetRepo: input.targetRepo,
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       category: "branch-behind",
       workflowStatus: productionStatus.workflowStatus,
       prUrl,
@@ -662,16 +1424,15 @@ export async function advanceTargetWorkflowFinalizationStep(
     });
   }
 
-  if (mergeableState === "dirty" || mergeableState === "blocked") {
-    const category: WorkflowInstallBlockedCategory =
-      mergeableState === "blocked" ? "review-required" : "merge-conflict";
+  if (mergeableState === "dirty") {
     return blockedResult({
       repoConfigId: input.repoConfigId,
       targetRepo: input.targetRepo,
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
-      category,
+      operationId: session.operationId,
+      category: "merge-conflict",
       workflowStatus: productionStatus.workflowStatus,
       prUrl,
       prNumber,
@@ -681,13 +1442,19 @@ export async function advanceTargetWorkflowFinalizationStep(
     });
   }
 
-  if (inspection.mergeable === false) {
+  if (
+    !shouldAttemptMerge({
+      mergeableState: inspection.mergeableState,
+      mergeable: inspection.mergeable,
+    })
+  ) {
     return blockedResult({
       repoConfigId: input.repoConfigId,
       targetRepo: input.targetRepo,
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       category: "merge-conflict",
       workflowStatus: productionStatus.workflowStatus,
       prUrl,
@@ -706,6 +1473,7 @@ export async function advanceTargetWorkflowFinalizationStep(
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       lifecycle: "merging",
       workflowStatus: productionStatus.workflowStatus,
       message: "Waiting for workflow install PR merge to complete.",
@@ -740,6 +1508,7 @@ export async function advanceTargetWorkflowFinalizationStep(
         error,
         mergeableState: inspection.mergeableState,
         message: error instanceof Error ? error.message : String(error),
+        checkPolicy,
       });
       if (classified.waiting) {
         sessions.set(sessionKey(targetRepoSlug, input.repoConfigId), session);
@@ -749,6 +1518,7 @@ export async function advanceTargetWorkflowFinalizationStep(
           targetRepoSlug,
           productionBranch: input.productionBranch,
           branchName,
+      operationId: session.operationId,
           lifecycle:
             classified.category === "checks-pending"
               ? "waiting-for-checks"
@@ -770,6 +1540,7 @@ export async function advanceTargetWorkflowFinalizationStep(
           targetRepoSlug,
           productionBranch: input.productionBranch,
           branchName,
+      operationId: session.operationId,
           category: "permission-denied",
           workflowStatus: productionStatus.workflowStatus,
           prUrl,
@@ -785,6 +1556,7 @@ export async function advanceTargetWorkflowFinalizationStep(
         targetRepoSlug,
         productionBranch: input.productionBranch,
         branchName,
+      operationId: session.operationId,
         category: classified.category,
         workflowStatus: productionStatus.workflowStatus,
         prUrl,
@@ -811,6 +1583,7 @@ export async function advanceTargetWorkflowFinalizationStep(
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       prUrl,
       prNumber,
       validatedHeadSha,
@@ -831,6 +1604,7 @@ export async function advanceTargetWorkflowFinalizationStep(
       targetRepoSlug,
       productionBranch: input.productionBranch,
       branchName,
+      operationId: session.operationId,
       category: "verification-failed",
       workflowStatus: postMergeStatus.workflowStatus,
       prUrl,
@@ -850,6 +1624,7 @@ export async function advanceTargetWorkflowFinalizationStep(
     targetRepoSlug,
     productionBranch: input.productionBranch,
     branchName,
+      operationId: session.operationId,
     lifecycle: "verifying",
     workflowStatus: postMergeStatus.workflowStatus,
     message: "Verifying workflow on the production branch.",
@@ -886,10 +1661,26 @@ export async function finalizeTargetWorkflowRemote(options: {
       }),
   );
 
-  return {
+  const merged = {
     ...result,
     lockContended: lockContended || result.lockContended,
+    retryable: lockContended ? true : result.retryable,
+    errorCode: lockContended
+      ? ("lock_contended" as const)
+      : result.errorCode,
+    retryAfterMs: lockContended
+      ? computeRetryAfterMs(true)
+      : result.retryAfterMs,
   };
+
+  if (merged.lifecycle === "complete") {
+    await clearTargetWorkflowFinalizationProgress(
+      options.input.repoConfigId,
+      options.cwd,
+    );
+  }
+
+  return merged;
 }
 
 export function resetTargetWorkflowFinalizationSessionsForTests(): void {