@opengeni/worker-bundle 1.0.0-canary.0 → 1.0.0

package/src/activities/agent-turn/failure-settlement.ts

@@ -1,35 +1,58 @@
+import { environmentsEncryptionKeyBytes } from "@opengeni/config";
 import {
   requestSessionTurnRecovery,
   getSessionGoal,
   armXaiCapacityWait,
+  armClaudeCapacityWait,
+  reconcileClaudeCapacityWait,
+  recordClaudeAccountUsage,
+  resolveClaudeAccountCredential,
+  loadClaudeAccountCredential,
+  ClaudeSubscriptionConnectionChanged,
   reconcileXaiCapacityWait,
   listCodexAccountStatuses,
   quarantineCodexCredentialForLease,
   recordUsageEvent,
   getActiveSessionHistoryItemsPaged,
+  recheckCodexCredentialPlan,
   settleCodexCredentialLeaseLoss,
   settleCodexCredentialFailover,
   readLease,
   SandboxLeaseSupersededError,
   isSessionEventPersistenceError,
+  SANDBOX_SETUP_RECOVERY_LIMIT,
   type CodexLeaseAccountStatus,
 } from "@opengeni/db";
 import { publishDurableSessionEvents } from "@opengeni/events";
-import { maxTurnsExceededRunState } from "@opengeni/runtime";
-import { CancelledFailure } from "@temporalio/activity";
+import {
+  maxTurnsExceededRunState,
+  isModalTaskExecStartPreDispatchUnavailableError,
+  isModalCommandStartOutcomeUnknownError,
+  isProviderCommandObservationUnavailableError,
+} from "@opengeni/runtime";
+import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
 import {
   authoritativeCodexCapacityResetAt,
   classifyCodexPin,
   selectCodexCredentialLeaseForTurn,
   type CodexRotationStrategy,
 } from "../codex-rotation";
-import type { Settings } from "@opengeni/config";
+import { TurnExecutionPolicyDefinitionMismatchError, type Settings } from "@opengeni/config";
 import {
   classifyCodexEncryptedArtifactRejection,
+  classifyCodexEntitlementRejection,
   classifyCodexUsageLimitError,
   isCodexTransportError,
   type CodexUsageHeaderSnapshot,
 } from "@opengeni/codex";
+import {
+  assessCodexPlanEntitlement,
+  codexAccountDisplayLabel,
+  codexPlanEntitlementFailurePayload,
+  codexRequestRejectedFailurePayload,
+  type CodexPlanEntitlementFailurePayload,
+  type CodexRequestRejectedFailurePayload,
+} from "./codex-plan-entitlement";
 import { TurnAttemptFencedError } from "../turn-attempt-fenced";
 import { deliverFailedChildTurnToParent } from "../parent-wake";
 import type {
@@ -53,10 +76,13 @@ import {
   sandboxRouteTransitionCode,
   safeErrorDiagnostic,
   classifyXaiCredentialFailure,
+  classifyClaudeCredentialFailure,
   agentRunFailurePayload,
+  agentRunRecoveryFailurePayload,
   codexCredentialCooldownUntil,
   classifyCodexCredentialFailure,
   codexUsageLimitFailurePayload,
+  type CodexCredentialFailure,
 } from "./errors";
 import { selectRejectedProviderArtifactHistoryIds } from "./history";
 import { waitForTurnFinalizerStep, turnFinalizerCancellationSignal } from "./quiescence";
@@ -122,7 +148,7 @@ type CodexCapacityWaitFailurePayload = {
  * recover the same durable turn immediately.
  */
 export function codexDefinitiveFailureDisposition(input: {
-  failureKind: "auth" | "forbidden" | "rate_limit" | "quota";
+  failureKind: CodexCredentialFailure["kind"];
   rotationEnabled: boolean;
   pinDisposition: "manual" | "sharded" | "clearStale" | "unpinned";
   decisionKind: "active" | "allCapped" | "none";
@@ -136,6 +162,17 @@ export function codexDefinitiveFailureDisposition(input: {
     input.decisionCredentialId !== null &&
     input.decisionCredentialId !== input.servingCredentialId;
   if (alternateAvailable) return "failover";
+  // A plan that no longer includes the model does not recover by itself, and
+  // a manual pin or rotation-off pointer names exactly that account. Only a
+  // rotation-on pool whose OTHER accounts are temporarily capped is worth a
+  // durable wait; everything else fails the turn with typed copy.
+  if (input.failureKind === "plan_entitlement") {
+    return input.decisionKind === "allCapped" &&
+      input.rotationEnabled &&
+      input.pinDisposition !== "manual"
+      ? "wait"
+      : "terminal";
+  }
   if (
     input.failureKind === "quota" ||
     input.failureKind === "rate_limit" ||
@@ -167,12 +204,23 @@ export function codexCredentialFailoverLimit(
 
 /** Build the durable waiter payload without collapsing quota refusals into 403. */
 export function codexCapacityWaitFailurePayload(input: {
-  failureKind: "auth" | "forbidden" | "rate_limit" | "quota";
+  failureKind: CodexCredentialFailure["kind"];
   usageLimit: { resetsInSeconds: number | null } | null;
   cooldownSeconds: number | null;
   detail: string;
   allAccounts: boolean;
+  planEntitlement?: CodexPlanEntitlementFailurePayload | null;
 }): CodexCapacityWaitFailurePayload {
+  if (input.failureKind === "plan_entitlement") {
+    return {
+      error:
+        input.planEntitlement?.error ??
+        "The serving ChatGPT account's plan does not include this model. Opengeni is waiting for another connected account to become available.",
+      code: "codex_plan_entitlement",
+      detail: input.detail,
+      retryable: false,
+    };
+  }
   if (input.failureKind === "quota") {
     return codexUsageLimitFailurePayload(
       input.usageLimit ?? { resetsInSeconds: input.cooldownSeconds },
@@ -225,6 +273,42 @@ function acceptedCodexPolicySnapshot(
 }
 
 export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgentTurnResult> {
+  try {
+    return await settleTurnFailureInAttempt(deps);
+  } catch (error) {
+    // Connectivity can disappear while settling an unrelated run error too.
+    // Do not overwrite a possibly committed settlement; the control lane
+    // re-reads exact ownership and becomes a stale no-op if it already closed.
+    if (deps.attempt.turnId && deps.attempt.triggerEventId) {
+      const recovery = postClaimDatabaseRecoveryFailure({
+        error,
+        turnId: deps.attempt.turnId,
+        triggerEventId: deps.attempt.triggerEventId,
+        executionGeneration: deps.attempt.executionGeneration,
+        requireDatabaseProvenance: true,
+      });
+      if (recovery) {
+        deps.control.activityStatus = "recovering";
+        deps.control.turnMetricOutcome = "recovering";
+        deps.control.activityError = error;
+        throw recovery;
+      }
+    }
+    throw error;
+  }
+}
+
+async function settleTurnFailureInAttempt(deps: TurnFailureDeps): Promise<RunAgentTurnResult> {
+  if (deps.settings.environment === "local" && deps.error instanceof Error) {
+    // Keep local startup failures diagnosable without logging error messages,
+    // absolute host paths, prompts, credentials, or provider response bodies.
+    const locations = (deps.error.stack ?? "")
+      .split("\n")
+      .slice(1)
+      .flatMap((line) => line.match(/(?:apps|packages)\/[A-Za-z0-9_./-]+:\d+:\d+/g) ?? [])
+      .slice(0, 8);
+    console.error(JSON.stringify({ message: "Local turn failure source locations", locations }));
+  }
   const {
     error,
     input,
@@ -249,6 +333,36 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
     acknowledgeLostAttemptOwnership,
     acknowledgeRecoveryQuiescence,
   } = deps;
+  // Capture before any recovery/checkpoint DB operation can fail again.
+  if (isSessionEventPersistenceError(error)) {
+    try {
+      const diagnosticId = observability.recordFailureDiagnostic({
+        code: error.details.code,
+        stage:
+          error.details.stage === "session_events.append_generic"
+            ? "session_events.append_generic"
+            : error.details.stage === "session_events.append_for_turn_attempt"
+              ? "session_events.append_for_turn_attempt"
+              : error.details.stage === "session_attempts.claim"
+                ? "session_attempts.claim"
+                : "failure_settlement",
+        retryDecision: error.details.retryOutcome,
+        error,
+        sessionId: input.sessionId,
+        ...(attempt.turnId ? { turnId: attempt.turnId } : {}),
+        attemptId: input.attemptId,
+        attempts: error.details.attempts,
+        eventTypes: error.details.eventTypes,
+        sqlState: error.details.sqlState,
+        ...(error.details.database.constraint
+          ? { constraint: error.details.database.constraint }
+          : {}),
+      });
+      observability.error("session event persistence failed", { correlationId: diagnosticId });
+    } catch {
+      // Failure settlement must not depend on telemetry availability.
+    }
+  }
   // Graceful worker shutdown (deploy / rollout restart): checkpoint the
   // same current inference for a new fenced attempt instead of failing the
   // session. Conversation truth is already persisted per model response;
@@ -262,6 +376,69 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
   // back to the workflow-claimed turn when the local lookup had not
   // finished yet.
   const recoveryTurnId = attempt.turnId;
+  // Unlike proven pre-dispatch failure, a genuine SDK Start/Wait uncertainty
+  // cannot reconstruct setup on a replacement attempt. The exact command and
+  // writer remain retained by sandbox-runtime; the logical turn is parked as
+  // recovering with a durable no-replay marker, not failed or completed.
+  const observationUnavailable = isProviderCommandObservationUnavailableError(error);
+  if (
+    (isModalCommandStartOutcomeUnknownError(error) || observationUnavailable) &&
+    recoveryTurnId &&
+    attempt.triggerEventId &&
+    attempt.executionGeneration > 0
+  ) {
+    let recovery: Awaited<ReturnType<typeof requestSessionTurnRecovery>>;
+    try {
+      if (eventing.turnStartedPublished) {
+        await flushRuntimeBatcher();
+        await historySink.reconcileConversationTruth({ requireDurable: true });
+      }
+      recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
+        sessionId: input.sessionId,
+        turnId: recoveryTurnId,
+        triggerEventId: attempt.triggerEventId,
+        attemptId: input.attemptId,
+        reason: observationUnavailable
+          ? "sandbox_command_observation_unavailable"
+          : "sandbox_command_start_outcome_unknown",
+        sandboxSetupOutcomeUnknown: true,
+        detail: {
+          code: observationUnavailable
+            ? "sandbox_command_observation_unavailable"
+            : "sandbox_command_start_outcome_unknown",
+          retryable: false,
+          setupOutcome: "unknown",
+          replay: "blocked",
+          providerRecoveryCount: attempt.providerRecoveryCount,
+        },
+      });
+    } catch (checkpointError) {
+      const databaseRecovery = postClaimDatabaseRecoveryFailure({
+        error: checkpointError,
+        turnId: recoveryTurnId,
+        triggerEventId: attempt.triggerEventId,
+        executionGeneration: attempt.executionGeneration,
+        sandboxSetupOutcomeUnknown: true,
+      });
+      if (!databaseRecovery) throw checkpointError;
+      control.activityStatus = "recovering";
+      control.turnMetricOutcome = "recovering";
+      control.activityError = error;
+      throw databaseRecovery;
+    }
+    if (recovery.action === "stale") {
+      acknowledgeLostAttemptOwnership();
+      control.activityStatus = "cancelled";
+      control.turnMetricOutcome = "cancelled";
+      return claimedResult({ status: "cancelled" });
+    }
+    acknowledgeRecoveryQuiescence();
+    await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
+    control.activityStatus = "recovering";
+    control.turnMetricOutcome = "recovering";
+    control.activityError = error;
+    return claimedResult({ status: "recovering", deferredUntilWake: true });
+  }
   // A true epoch supersession and a provider lifecycle transition are both
   // recoverable control-plane states, never session failures. A rotation
   // persists an exact group/epoch wait marker so the workflow parks before
@@ -340,13 +517,11 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
       throw recoveryError;
     }
   }
-  // A managed-home session can start directly on a Connected Machine without
-  // creating or leasing its cloud home. If sandbox_attach/sandbox_swap then
-  // clears the durable pointer to home, the commit is valid but this exact
-  // attempt cannot serve a later home operation. Preserve the completed attach
+  // A route change can require a different home, filesystem root, or native
+  // capability set than this attempt established. Preserve the completed attach
   // and every preceding model/tool receipt, close only the unresolved suffix,
   // and continue the SAME logical turn in a fresh attempt. That next attempt
-  // starts from the now-null pointer and establishes home normally.
+  // starts from the committed pointer and establishes its route normally.
   const routeTransitionCode = sandboxRouteTransitionCode(error);
   if (routeTransitionCode && recoveryTurnId && eventing.publish && eventing.turnStartedPublished) {
     try {
@@ -370,7 +545,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
         return claimedResult({ status: "cancelled" });
       }
       if (recovery.action !== "recovering") {
-        throw new Error("Home sandbox route transition could not recover the current turn");
+        throw new Error("Sandbox route transition could not recover the current turn");
       }
       acknowledgeRecoveryQuiescence();
       await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
@@ -504,6 +679,27 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
     control.turnMetricOutcome = "cancelled";
     throw cancellationFailure;
   }
+  if (attempt.turnId && attempt.triggerEventId && attempt.executionGeneration > 0) {
+    const recoveryFailure = postClaimDatabaseRecoveryFailure({
+      error,
+      turnId: attempt.turnId,
+      triggerEventId: attempt.triggerEventId,
+      executionGeneration: attempt.executionGeneration,
+      requireDatabaseProvenance: eventing.turnStartedPublished || attempt.modelRequestStarted,
+    });
+    if (recoveryFailure) {
+      // Stop this non-retryable activity without terminal logical settlement.
+      // Do not retry a delta/tool-ledger mutation with an unknown commit, nor
+      // claim that DB-down cleanup proved writer exit. Finally still drains
+      // the physical writers; the existing DB-only workflow lane closes this
+      // exact owner after connectivity returns and admission waits for its
+      // ordinary durable quiescence/writer fences.
+      control.activityStatus = "recovering";
+      control.turnMetricOutcome = "recovering";
+      control.activityError = error;
+      throw recoveryFailure;
+    }
+  }
   // The SDK's per-segment turn cap is a pacing valve, not a failure: end
   // the turn gracefully and idle the session so an active goal continues
   // via a synthesized continuation turn (or a user message resumes work).
@@ -642,13 +838,13 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
       leases.codex.generation,
     );
   }
-  if (
-    leases.xai.lost &&
-    billingState.isXaiTurn &&
-    eventing.publish &&
-    attempt.turnId &&
-    eventing.turnStartedPublished
-  ) {
+  const scopedLeaseLost =
+    billingState.isClaudeTurn && leases.claude.lost
+      ? "claude"
+      : billingState.isXaiTurn && leases.xai.lost
+        ? "xai"
+        : null;
+  if (scopedLeaseLost && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
     await flushRuntimeBatcher();
     await historySink.reconcileConversationTruth({ requireDurable: true });
     const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
@@ -656,8 +852,10 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
       turnId: attempt.turnId,
       triggerEventId: attempt.triggerEventId!,
       attemptId: input.attemptId,
-      reason: "xai_lease_lost",
-      detail: { provider: "supergrok-subscription" },
+      reason: scopedLeaseLost + "_lease_lost",
+      detail: {
+        provider: scopedLeaseLost === "claude" ? "claude-subscription" : "supergrok-subscription",
+      },
     });
     if (recovery.action === "stale") {
       acknowledgeLostAttemptOwnership();
@@ -678,10 +876,107 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
   // 5xx does not classify here and therefore cannot consume another
   // subscription or duplicate a side effect.
   const usageLimit = isCodexTransportError(error) ? classifyCodexUsageLimitError(error) : null;
-  const codexCredentialFailure =
+  let codexCredentialFailure: CodexCredentialFailure | null =
     billingState.isCodexTurn && providerTurn.effectiveCodexCredentialId
       ? classifyCodexCredentialFailure(error)
       : null;
+  // Plan entitlement evidence (an explicit plan refusal, or an HTTP 400 with no
+  // body) is ambiguous until the serving account's CURRENT plan is re-read. A
+  // proven loss becomes a definitive `plan_entitlement` refusal for this model
+  // only and walks the pool through the same checkpointed failover below; an
+  // unexplained rejection stays terminal with typed copy. Nothing retries the
+  // rejected request itself.
+  let codexTerminalFailure:
+    | CodexPlanEntitlementFailurePayload
+    | CodexRequestRejectedFailurePayload
+    | null = null;
+  let codexPlanEntitlement: {
+    modelId: string;
+    planType: string | null;
+    planObserved: boolean;
+    credentialVersion: number | null;
+    waitPayload: CodexPlanEntitlementFailurePayload;
+  } | null = null;
+  const codexEntitlementRejection =
+    billingState.isCodexTurn && providerTurn.effectiveCodexCredentialId && !codexCredentialFailure
+      ? classifyCodexEntitlementRejection(error)
+      : null;
+  if (
+    codexEntitlementRejection &&
+    providerTurn.effectiveCodexCredentialId &&
+    eventing.publish &&
+    attempt.turnId &&
+    eventing.turnStartedPublished &&
+    leases.codex.holderId &&
+    leases.codex.generation !== null
+  ) {
+    const servingCredentialId = providerTurn.effectiveCodexCredentialId;
+    const servingAccount = (
+      await listCodexAccountStatuses(db, input.workspaceId, attempt.turnId).catch(() => [])
+    ).find((account) => account.id === servingCredentialId);
+    const accountLabel = codexAccountDisplayLabel(servingAccount);
+    const recheck = await recheckCodexCredentialPlan(
+      db,
+      settings,
+      input.workspaceId,
+      servingCredentialId,
+      {
+        turnId: attempt.turnId,
+        holderId: leases.codex.holderId,
+        generation: leases.codex.generation,
+      },
+    ).catch(() => null);
+    const modelId = providerTurn.codexProductModelId ?? null;
+    const assessment = recheck
+      ? assessCodexPlanEntitlement(codexEntitlementRejection, recheck, modelId)
+      : codexEntitlementRejection.evidence === "plan_entitlement"
+        ? {
+            kind: "entitlement_lost" as const,
+            planType: servingAccount?.planType ?? null,
+            planObserved: false,
+            planChanged: false,
+            credentialVersion: null,
+          }
+        : { kind: "unexplained" as const, planType: null };
+    observability.incrementCounter({
+      name: "opengeni_codex_plan_rechecks_total",
+      help: "Codex plan re-checks after an entitlement-shaped rejection, by outcome.",
+      labels: {
+        workspace_key: codexWorkspaceKey,
+        evidence: codexEntitlementRejection.evidence,
+        outcome: assessment.kind,
+        source: recheck?.source ?? "none",
+      },
+    });
+    if (assessment.kind === "entitlement_lost") {
+      const payloadInput = {
+        accountLabel,
+        // Name a plan only when the provider just reported it; a recorded
+        // plan may be the very one that changed.
+        planType: assessment.planObserved ? assessment.planType : null,
+        planChanged: assessment.planChanged,
+        modelId,
+        rejection: codexEntitlementRejection,
+      };
+      codexTerminalFailure = codexPlanEntitlementFailurePayload(payloadInput);
+      if (modelId) {
+        codexCredentialFailure = { kind: "plan_entitlement", cooldownSeconds: null };
+        codexPlanEntitlement = {
+          modelId,
+          planType: assessment.planType,
+          planObserved: assessment.planObserved,
+          credentialVersion: assessment.credentialVersion,
+          waitPayload: codexPlanEntitlementFailurePayload({ ...payloadInput, waiting: true }),
+        };
+      }
+    } else {
+      codexTerminalFailure = codexRequestRejectedFailurePayload({
+        accountLabel,
+        planType: recheck?.planType ?? null,
+        rejection: codexEntitlementRejection,
+      });
+    }
+  }
   if (
     codexCredentialFailure &&
     providerTurn.effectiveCodexCredentialId &&
@@ -724,7 +1019,9 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
         labels: { workspace_key: codexWorkspaceKey, outcome: "completed" },
       });
       const now = new Date();
-      const before = await listCodexAccountStatuses(db, input.workspaceId).catch(() => []);
+      const before = await listCodexAccountStatuses(db, input.workspaceId, attempt.turnId).catch(
+        () => [],
+      );
       const servingCached = before.find(
         (account) => account.id === providerTurn.effectiveCodexCredentialId,
       );
@@ -743,10 +1040,14 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
           }
         : null;
       const cooldownUntil = codexCredentialCooldownUntil(codexCredentialFailure, serving, now);
+      // A plan re-check may itself have rotated tokens (same family, version
+      // CAS-advanced by this holder); fence the quarantine on that version.
+      const quarantineCredentialVersion =
+        codexPlanEntitlement?.credentialVersion ?? providerTurn.effectiveCodexCredentialVersion;
       const quarantineResult =
         leases.codex.holderId &&
         leases.codex.generation !== null &&
-        providerTurn.effectiveCodexCredentialVersion !== null
+        quarantineCredentialVersion !== null
           ? await quarantineCodexCredentialForLease(db, {
               accountId: input.accountId,
               workspaceId: input.workspaceId,
@@ -759,7 +1060,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
               dispatchId: attempt.dispatchId,
               expectedRedispatches: attempt.redispatchesAtDispatch,
               credentialId: providerTurn.effectiveCodexCredentialId,
-              credentialVersion: providerTurn.effectiveCodexCredentialVersion,
+              credentialVersion: quarantineCredentialVersion,
               holderId: leases.codex.holderId,
               generation: leases.codex.generation,
               maxFailovers: providerTurn.codexCredentialFailoverLimit,
@@ -776,11 +1077,18 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
                         status: "error",
                         lastError: "model request was forbidden for this credential",
                       }
-                    : {
-                        kind: "cooldown",
-                        until: cooldownUntil!,
-                        cooldownKind: codexCredentialFailure.kind,
-                      },
+                    : codexCredentialFailure.kind === "plan_entitlement"
+                      ? {
+                          kind: "plan_entitlement",
+                          modelId: codexPlanEntitlement!.modelId,
+                          planType: codexPlanEntitlement!.planType,
+                          planObserved: codexPlanEntitlement!.planObserved,
+                        }
+                      : {
+                          kind: "cooldown",
+                          until: cooldownUntil!,
+                          cooldownKind: codexCredentialFailure.kind,
+                        },
             })
           : null;
       if (
@@ -892,7 +1200,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
       }
       let accounts: Awaited<ReturnType<typeof listCodexAccountStatuses>>;
       try {
-        accounts = await listCodexAccountStatuses(db, input.workspaceId);
+        accounts = await listCodexAccountStatuses(db, input.workspaceId, attempt.turnId);
       } catch (metadataError) {
         // Current account health/cooldown metadata is still required after
         // quarantine. Operational database failures re-enter the existing
@@ -921,6 +1229,9 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
           rotationStrategy: acceptedPolicy.rotationStrategy,
           existingCredentialId: null,
           failedCredentialIds: [providerTurn.effectiveCodexCredentialId],
+          ...(providerTurn.codexProductModelId
+            ? { modelId: providerTurn.codexProductModelId }
+            : {}),
           policyScope: null,
           unavailableDiagnostics: [],
         },
@@ -1079,9 +1390,10 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
         const capacityAccounts = policyCredentialId
           ? accounts.filter((account) => account.id === policyCredentialId)
           : accounts;
-        const authoritativeResetAt = exactProviderReset
-          ? (authoritativeCodexCapacityResetAt(capacityAccounts, now) ?? cooldownUntil)
-          : null;
+        const authoritativeResetAt =
+          exactProviderReset || codexCredentialFailure.kind === "plan_entitlement"
+            ? (authoritativeCodexCapacityResetAt(capacityAccounts, now) ?? cooldownUntil)
+            : null;
         const allAccounts =
           acceptedPolicy.rotationEnabled &&
           pinDisposition !== "manual" &&
@@ -1097,6 +1409,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
                 : String(error)
               : "the same accepted turn is waiting for eligible credential capacity",
           allAccounts,
+          planEntitlement: codexPlanEntitlement?.waitPayload ?? null,
         });
         const evaluated = await armAndReconcileCodexCapacityWait(
           { db, bus },
@@ -1110,6 +1423,8 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
             goalId: activeGoal?.id ?? null,
             goalVersion: activeGoal?.version ?? null,
             earliestResetAt: authoritativeResetAt,
+            // plan_entitlement waits only on OTHER capped accounts, so it keeps
+            // their reset/bounded-refresh cadence rather than a mutation-only wait.
             resetKind: authoritativeResetAt
               ? "authoritative"
               : codexCredentialFailure.kind === "auth" ||
@@ -1126,6 +1441,11 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
           { onArmed: () => (leases.codex.held = false) },
         );
         control.activityError = error;
+        if (evaluated.action === "stopped") {
+          control.activityStatus = evaluated.sessionStatus === "queued" ? "idle" : "failed";
+          control.turnMetricOutcome = "failed";
+          return claimedResult({ status: control.activityStatus });
+        }
         if (evaluated.action === "resumed") {
           control.activityStatus = "recovering";
           control.turnMetricOutcome = "recovering";
@@ -1152,83 +1472,265 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
       }
     }
   }
-  const xaiCredentialFailure =
+  const xaiFailure =
     billingState.isXaiTurn && providerTurn.effectiveXaiCredentialId
       ? classifyXaiCredentialFailure(error)
       : null;
+  const claudeFailure =
+    billingState.isClaudeTurn && providerTurn.effectiveClaudeCredentialId
+      ? classifyClaudeCredentialFailure(error)
+      : null;
+  const scopedFailure = claudeFailure ?? xaiFailure;
+  const scopedProvider = claudeFailure ? ("claude" as const) : ("xai" as const);
+  const scopedName = claudeFailure ? "Claude" : "SuperGrok";
+  const scopedCredentialId = claudeFailure
+    ? providerTurn.effectiveClaudeCredentialId
+    : providerTurn.effectiveXaiCredentialId;
+  const scopedAuthority = claudeFailure
+    ? providerTurn.claudeAuthoritySnapshot
+    : providerTurn.xaiAuthoritySnapshot;
+  const scopedLease = claudeFailure ? leases.claude : leases.xai;
+  const armScopedWait = claudeFailure ? armClaudeCapacityWait : armXaiCapacityWait;
+  const reconcileScopedWait = claudeFailure
+    ? reconcileClaudeCapacityWait
+    : reconcileXaiCapacityWait;
   if (
-    xaiCredentialFailure &&
-    providerTurn.effectiveXaiCredentialId &&
-    providerTurn.xaiAuthoritySnapshot &&
-    leases.xai.subjectId &&
-    leases.xai.holderId &&
-    leases.xai.generation !== null &&
+    scopedFailure &&
+    scopedCredentialId &&
+    scopedAuthority &&
+    scopedLease.subjectId &&
+    scopedLease.holderId &&
+    scopedLease.generation !== null &&
     eventing.publish &&
     attempt.turnId &&
     eventing.turnStartedPublished
   ) {
     await flushRuntimeBatcher();
     await historySink.reconcileConversationTruth({ requireDurable: true });
+    const recoverChangedAccount = async () => {
+      const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
+        sessionId: input.sessionId,
+        turnId: attempt.turnId!,
+        triggerEventId: attempt.triggerEventId!,
+        attemptId: input.attemptId,
+        reason: "claude_credential_changed",
+        detail: { code: "claude_credential_changed", retryable: true },
+      });
+      if (recovery.action === "stale") {
+        acknowledgeLostAttemptOwnership();
+        control.activityStatus = "cancelled";
+        control.turnMetricOutcome = "cancelled";
+        return claimedResult({ status: "cancelled" });
+      }
+      acknowledgeRecoveryQuiescence();
+      await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
+      control.activityStatus = "recovering";
+      control.turnMetricOutcome = "recovering";
+      control.activityError = error;
+      return claimedResult({ status: "recovering" });
+    };
     const goal = await getSessionGoal(db, input.workspaceId, input.sessionId).catch(() => null);
     const activeGoal = goal?.status === "active" ? goal : null;
     const now = new Date();
+    let claudeTokenFence: { encryptionKey: Uint8Array; observedAccessToken: string } | undefined;
     const cooldownUntil =
-      xaiCredentialFailure.kind === "rate_limit"
-        ? new Date(now.getTime() + Math.max(1, xaiCredentialFailure.cooldownMs ?? 60_000))
+      scopedFailure.kind === "rate_limit"
+        ? new Date(now.getTime() + Math.max(1, scopedFailure.cooldownMs ?? 60_000))
         : null;
+    if (claudeFailure) {
+      const key = environmentsEncryptionKeyBytes(settings);
+      const matchingReceipts = [...providerTurn.latestClaudeUsage.values()].filter(
+        (value) =>
+          value.expectedConnectionId === scopedCredentialId &&
+          value.expectedCredentialVersion === providerTurn.effectiveClaudeCredentialVersion &&
+          value.upstreamModelId === providerTurn.claudeUpstreamModelId &&
+          (value.responseStatus === (scopedFailure.kind === "rate_limit" ? 429 : 401) ||
+            (value.responseStatus === 200 &&
+              !!claudeFailure.requestId &&
+              value.requestId === claudeFailure.requestId)) &&
+          (!claudeFailure.requestId || value.requestId === claudeFailure.requestId),
+      );
+      const receipt = matchingReceipts.length === 1 ? matchingReceipts[0] : undefined;
+      if (
+        !key ||
+        !providerTurn.claudeUpstreamModelId ||
+        providerTurn.effectiveClaudeCredentialVersion === null
+      )
+        throw new Error("Claude refused request has no exact serving account");
+      if (!receipt) {
+        // Refresh may discover a revoked grant before any physical model call.
+        // Verify durable reconnect evidence instead of inventing a response receipt.
+        const current = await loadClaudeAccountCredential(
+          db,
+          {
+            accountId: input.accountId,
+            workspaceId: input.workspaceId,
+            subjectId: scopedLease.subjectId,
+            credentialId: scopedCredentialId,
+            authoritySnapshot: scopedAuthority,
+          },
+          key,
+        );
+        if (current.version !== providerTurn.effectiveClaudeCredentialVersion)
+          return await recoverChangedAccount();
+        if (scopedFailure.kind !== "auth" || current.usage.refreshStatus !== "reconnect")
+          throw new Error("Claude refused request has no exact account receipt");
+        claudeTokenFence = { encryptionKey: key, observedAccessToken: current.secret.token };
+      }
+      if (receipt) claudeTokenFence = { encryptionKey: key, observedAccessToken: receipt.token };
+      if (
+        receipt &&
+        scopedFailure.kind === "auth" &&
+        !(
+          attempt.claudeAuthRecovery?.credentialId === scopedCredentialId &&
+          attempt.claudeAuthRecovery.credentialVersion === receipt.expectedCredentialVersion
+        )
+      ) {
+        const credential = await resolveClaudeAccountCredential(
+          db,
+          settings,
+          {
+            accountId: input.accountId,
+            workspaceId: input.workspaceId,
+            subjectId: scopedLease.subjectId,
+            credentialId: scopedCredentialId,
+            authoritySnapshot: scopedAuthority,
+          },
+          {
+            expectedCredentialVersion: receipt.expectedCredentialVersion,
+            forceRefresh: true,
+            observedAccessToken: receipt.token,
+          },
+        ).catch((refreshError) => {
+          if (refreshError instanceof ClaudeSubscriptionConnectionChanged) return null;
+          throw refreshError;
+        });
+        if (!credential) return await recoverChangedAccount();
+        if (!("reconnectRequired" in credential) && credential.secret.token !== receipt.token) {
+          const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
+            sessionId: input.sessionId,
+            turnId: attempt.turnId,
+            triggerEventId: attempt.triggerEventId!,
+            attemptId: input.attemptId,
+            reason: "claude_token_renewed",
+            claudeAuthRecovery: {
+              credentialId: scopedCredentialId,
+              credentialVersion: receipt.expectedCredentialVersion,
+            },
+            detail: { code: "claude_token_renewed", retryable: true },
+          });
+          if (recovery.action === "stale") {
+            acknowledgeLostAttemptOwnership();
+            control.activityStatus = "cancelled";
+            control.turnMetricOutcome = "cancelled";
+            return claimedResult({ status: "cancelled" });
+          }
+          acknowledgeRecoveryQuiescence();
+          await publishDurableSessionEvents(
+            bus,
+            input.workspaceId,
+            input.sessionId,
+            recovery.events,
+          );
+          control.activityStatus = "recovering";
+          control.turnMetricOutcome = "recovering";
+          control.activityError = error;
+          return claimedResult({ status: "recovering" });
+        }
+      }
+      const recorded = receipt
+        ? await recordClaudeAccountUsage(
+            db,
+            {
+              accountId: input.accountId,
+              workspaceId: input.workspaceId,
+              subjectId: scopedLease.subjectId,
+              credentialId: scopedCredentialId,
+              authoritySnapshot: scopedAuthority,
+            },
+            {
+              encryptionKey: key,
+              token: receipt.token,
+              expectedCredentialVersion: receipt.expectedCredentialVersion,
+              ...(receipt.observation ? { observation: receipt.observation } : {}),
+              ...(receipt.refresh ? { refresh: receipt.refresh } : {}),
+              ...(cooldownUntil
+                ? {
+                    modelCooldown: {
+                      upstreamModelId:
+                        receipt.upstreamModelId ?? providerTurn.claudeUpstreamModelId,
+                      until: cooldownUntil,
+                    },
+                  }
+                : {}),
+            },
+          )
+        : true;
+      if (!recorded) return await recoverChangedAccount();
+    }
     const failurePayload = {
       error:
-        xaiCredentialFailure.kind === "auth"
-          ? "The serving SuperGrok account requires reconnection"
-          : xaiCredentialFailure.kind === "forbidden"
-            ? "The serving SuperGrok account is not authorized for this request"
-            : "The serving SuperGrok account is temporarily rate limited",
+        scopedFailure.kind === "auth"
+          ? "The serving " + scopedName + " account requires reconnection"
+          : scopedFailure.kind === "forbidden"
+            ? "The serving " + scopedName + " account is not authorized for this request"
+            : "The serving " + scopedName + " account is temporarily rate limited",
       code:
-        xaiCredentialFailure.kind === "auth"
-          ? "xai_relogin_required"
-          : xaiCredentialFailure.kind === "forbidden"
-            ? "xai_account_forbidden"
-            : "xai_account_rate_limited",
+        scopedFailure.kind === "auth"
+          ? scopedProvider + "_relogin_required"
+          : scopedFailure.kind === "forbidden"
+            ? scopedProvider + "_account_forbidden"
+            : scopedProvider + "_account_rate_limited",
       detail: "the same accepted turn is waiting for another eligible account",
     };
-    const armed = await armXaiCapacityWait(db, {
+    const armed = await armScopedWait(db, {
       accountId: input.accountId,
       workspaceId: input.workspaceId,
-      subjectId: leases.xai.subjectId,
+      subjectId: scopedLease.subjectId,
       sessionId: input.sessionId,
       turnId: attempt.turnId,
       attemptId: input.attemptId,
       workflowId: input.workflowId,
-      authoritySnapshot: providerTurn.xaiAuthoritySnapshot,
+      authoritySnapshot: scopedAuthority,
       goalId: activeGoal?.id ?? null,
       goalVersion: activeGoal?.version ?? null,
       earliestResetAt: cooldownUntil,
       failurePayload,
       leaseFence: {
-        holderId: leases.xai.holderId,
-        generation: leases.xai.generation,
+        holderId: scopedLease.holderId,
+        generation: scopedLease.generation,
       },
-      credentialQuarantine:
-        xaiCredentialFailure.kind === "auth"
-          ? {
-              kind: "status",
-              status: "needs_relogin",
-              lastError: "model request remained unauthorized after refresh",
-            }
-          : xaiCredentialFailure.kind === "forbidden"
-            ? {
-                kind: "status",
-                status: "error",
-                lastError: "model request was forbidden for this credential",
-              }
-            : { kind: "cooldown", until: cooldownUntil! },
+      ...(claudeFailure
+        ? {
+            expectedCredentialVersion: providerTurn.effectiveClaudeCredentialVersion!,
+            ...(claudeTokenFence ? { credentialTokenFence: claudeTokenFence } : {}),
+          }
+        : {}),
+      ...(!claudeFailure || scopedFailure.kind !== "rate_limit"
+        ? {
+            credentialQuarantine:
+              scopedFailure.kind === "auth"
+                ? {
+                    kind: "status",
+                    status: "needs_relogin",
+                    lastError: "model request remained unauthorized after refresh",
+                  }
+                : scopedFailure.kind === "forbidden"
+                  ? {
+                      kind: "status",
+                      status: "error",
+                      lastError: "model request was forbidden for this credential",
+                    }
+                  : { kind: "cooldown", until: cooldownUntil! },
+          }
+        : {}),
       now,
     });
     if (armed.action === "waiting") {
-      leases.xai.held = false;
-      providerTurn.xaiCredentialQuarantined = true;
+      scopedLease.held = false;
+      if (!claudeFailure) providerTurn.xaiCredentialQuarantined = true;
       await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, armed.events);
-      const evaluated = await reconcileXaiCapacityWait(db, {
+      const evaluated = await reconcileScopedWait(db, {
         accountId: input.accountId,
         workspaceId: input.workspaceId,
         sessionId: input.sessionId,
@@ -1256,7 +1758,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
         return claimedResult({
           status: "waiting_capacity",
           capacityWait: {
-            provider: "xai",
+            provider: scopedProvider,
             waiterId: evaluated.waiter.id,
             generation: evaluated.waiter.generation,
             nextCheckAt: evaluated.waiter.nextCheckAt.toISOString(),
@@ -1275,7 +1777,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
       turnId: attempt.turnId,
       triggerEventId: attempt.triggerEventId!,
       attemptId: input.attemptId,
-      reason: "xai_credential_recheck",
+      reason: scopedProvider + "_credential_recheck",
       detail: failurePayload,
     });
     if (recovery.action === "stale") {
@@ -1343,12 +1845,16 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
     if (
       !(await eventing.settle!({
         events: [
+          ...(error.allowance
+            ? [{ type: "usage.exhausted" as const, payload: error.allowance }]
+            : []),
           {
             type: "turn.completed",
             payload: {
               output: "",
               segmentLimit: "budget_exhausted",
               detail: error.message,
+              ...(error.allowance ?? {}),
             },
           },
           { type: "session.status.changed", payload: { status: "idle" } },
@@ -1356,6 +1862,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
         turnStatus: "completed",
         sessionStatus: "idle",
         activeTurnId: null,
+        ...(error.allowance ? { allowanceGoalPause: { rationale: error.allowance.message } } : {}),
       }))
     ) {
       return claimedResult({ status: "cancelled" });
@@ -1452,46 +1959,87 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
   // truth, recover this SAME accepted turn, then let the workflow re-claim
   // it after a pacing delay. This is independent of goal state and never
   // relies on a synthetic continuation prompt.
-  let failure = agentRunFailurePayload(error, {
-    isCodexTurn: billingState.isCodexTurn,
-  }) as ReturnType<typeof agentRunFailurePayload>;
-  if (isSessionEventPersistenceError(error)) {
-    // Preserve the exact source message in the internal runtime diagnostic;
-    // SQLSTATE/catalog facts remain separate classification attributes.
-    observability.error("session event persistence failed", {
-      accountId: input.accountId,
-      workspaceId: input.workspaceId,
-      sessionId: input.sessionId,
-      turnId: attempt.turnId,
-      attemptId: input.attemptId,
-      code: error.details.code,
-      sqlState: error.details.sqlState ?? "unknown",
-      stage: error.details.stage,
-      eventTypes: error.details.eventTypes.join(","),
-      correlationId: error.details.correlationId,
-      attempts: error.details.attempts,
-      retryOutcome: error.details.retryOutcome,
-      dbSeverity: error.details.database.severity,
-      dbSchema: error.details.database.schema,
-      dbTable: error.details.database.table,
-      dbColumn: error.details.database.column,
-      dbDataType: error.details.database.dataType,
-      dbConstraint: error.details.database.constraint,
-      dbRoutine: error.details.database.routine,
-      error: error.message,
-    });
-  }
-  if (failure.retryable && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
+  // A rolling-deployment definition mismatch is a separate configuration
+  // class: only the exact typed setup error can use this checkpoint before
+  // eventing exists. No generic setup/credential failure gains retry authority.
+  const earlyDefinitionMismatch =
+    error instanceof TurnExecutionPolicyDefinitionMismatchError &&
+    !attempt.modelRequestStarted &&
+    !eventing.turnStartedPublished &&
+    !!attempt.turnId &&
+    !!attempt.triggerEventId &&
+    attempt.executionGeneration > 0;
+  const earlyCommandStartUnavailable =
+    isModalTaskExecStartPreDispatchUnavailableError(error) &&
+    !attempt.modelRequestStarted &&
+    !eventing.turnStartedPublished &&
+    !!attempt.turnId &&
+    !!attempt.triggerEventId &&
+    attempt.executionGeneration > 0;
+  const earlyRecoverableSetup = earlyDefinitionMismatch || earlyCommandStartUnavailable;
+  let failure = (
+    earlyDefinitionMismatch
+      ? { error: error.message, code: error.code, retryable: true }
+      : (codexTerminalFailure ??
+        agentRunFailurePayload(error, {
+          isCodexTurn: billingState.isCodexTurn,
+        }))
+  ) as ReturnType<typeof agentRunFailurePayload>;
+  if (
+    attempt.turnId &&
+    (earlyRecoverableSetup ||
+      (failure.retryable && eventing.publish && eventing.turnStartedPublished))
+  ) {
     const nextProviderRecoveryCount = attempt.providerRecoveryCount + 1;
     const recoveryResult = providerRecoveryResult({
       failureCode: failure.code,
       attemptNumber: nextProviderRecoveryCount,
       retryAfterMs: providerRetryAfterMs(error),
     });
+    const setupRecoveryExhausted =
+      earlyCommandStartUnavailable &&
+      recoveryResult.status === "exhausted" &&
+      attempt.providerRecoveryCount === SANDBOX_SETUP_RECOVERY_LIMIT;
     try {
+      if (setupRecoveryExhausted) {
+        // This is positive pre-dispatch proof, not an ambiguous command. Park
+        // the SAME accepted turn without resetting or advancing its budget.
+        const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
+          sessionId: input.sessionId,
+          turnId: attempt.turnId,
+          triggerEventId: attempt.triggerEventId!,
+          attemptId: input.attemptId,
+          reason: "sandbox_command_start_recovery_exhausted",
+          sandboxSetupRecoveryExhausted: true,
+          detail: {
+            code: failure.code,
+            error:
+              "Automatic sandbox setup recovery exhausted after five retries; the accepted turn remains parked without starting a command.",
+            retryable: false,
+            setupOutcome: "not_started",
+            replay: "blocked",
+            recoveryExhausted: true,
+            providerRecoveryCount: SANDBOX_SETUP_RECOVERY_LIMIT,
+          },
+        });
+        if (recovery.action === "stale") {
+          acknowledgeLostAttemptOwnership();
+          control.activityStatus = "cancelled";
+          control.turnMetricOutcome = "cancelled";
+          return claimedResult({ status: "cancelled" });
+        }
+        acknowledgeRecoveryQuiescence();
+        await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
+        control.turnMetricOutcome = "recovering";
+        control.activityStatus = "recovering";
+        control.activityError = error;
+        return claimedResult({ status: "recovering" });
+      }
       if (recoveryResult.status === "recovering") {
-        await flushRuntimeBatcher();
-        await historySink.reconcileConversationTruth({ requireDurable: true });
+        if (!earlyRecoverableSetup) {
+          await flushRuntimeBatcher();
+          await historySink.reconcileConversationTruth({ requireDurable: true });
+        }
         const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
           sessionId: input.sessionId,
           turnId: attempt.turnId,
@@ -1500,7 +2048,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
           reason: failure.code ?? "provider_unavailable",
           providerRecoveryCount: nextProviderRecoveryCount,
           detail: {
-            ...failure,
+            ...agentRunRecoveryFailurePayload(error, failure),
             continueDelayMs: recoveryResult.continueDelayMs,
             providerRecoveryCount: nextProviderRecoveryCount,
           },
@@ -1519,6 +2067,22 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
         return claimedResult(recoveryResult);
       }
       failure = providerRecoveryExhaustedFailure(failure, recoveryResult);
+      if (earlyRecoverableSetup) {
+        // Setup has no eventing sink yet. Carry only the fixed, safe diagnostic
+        // through Temporal into exact-attempt workflow failure settlement.
+        control.activityStatus = "failed";
+        control.turnMetricOutcome = "failed";
+        control.activityError = error;
+        throw ApplicationFailure.create({
+          message: earlyDefinitionMismatch
+            ? `${error.message}. Automatic same-turn configuration recovery exhausted after ${recoveryResult.providerRecoveryCount} retries.`
+            : failure.error,
+          type: earlyDefinitionMismatch
+            ? "TurnExecutionPolicyDefinitionMismatchError"
+            : "SandboxCommandStartUnavailableError",
+          nonRetryable: true,
+        });
+      }
     } catch (recoveryError) {
       const escaped =
         recoveryResult.status === "recovering"
@@ -1552,7 +2116,15 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
                 providerRecoveryCount: nextProviderRecoveryCount,
               },
             })
-          : null;
+          : setupRecoveryExhausted
+            ? postClaimDatabaseRecoveryFailure({
+                error: recoveryError,
+                turnId: attempt.turnId,
+                triggerEventId: attempt.triggerEventId!,
+                executionGeneration: attempt.executionGeneration,
+                sandboxSetupRecoveryExhausted: true,
+              })
+            : null;
       if (postClaimRecovery) {
         control.activityStatus = "recovering";
         control.turnMetricOutcome = "recovering";
@@ -1562,26 +2134,6 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
       throw recoveryError;
     }
   }
-  if (
-    attempt.turnId &&
-    attempt.triggerEventId &&
-    attempt.executionGeneration > 0 &&
-    !eventing.turnStartedPublished &&
-    !attempt.modelRequestStarted
-  ) {
-    const recoveryFailure = postClaimDatabaseRecoveryFailure({
-      error,
-      turnId: attempt.turnId,
-      triggerEventId: attempt.triggerEventId,
-      executionGeneration: attempt.executionGeneration,
-    });
-    if (recoveryFailure) {
-      control.activityStatus = "recovering";
-      control.turnMetricOutcome = "recovering";
-      control.activityError = error;
-      throw recoveryFailure;
-    }
-  }
   control.activityStatus = "failed";
   control.activityError = error;
   if (!attempt.turnId) {