@opengeni/worker-bundle 1.0.0-canary.1 → 1.0.0

package/src/activities/agent-turn/errors.ts

@@ -1,7 +1,13 @@
+import { ClaudeSubscriptionReconnectRequired } from "@opengeni/db";
+import { DrizzleQueryError } from "drizzle-orm";
+import { SandboxCapabilitiesChangedError } from "./provider-dispatch-barrier";
+import { ClaudeSubscriptionConnectionUnavailable } from "./claude-usage-observer";
 import {
   ActiveSessionHistoryLimitExceededError,
   ApprovalRunStateLimitExceededError,
   nestedPostgresSqlState,
+  databaseFailureCode,
+  isRetryablePersistenceSqlState,
   safeDatabaseErrorFacts,
   isRetryableDatabaseTransportFailure,
   isSessionEventPersistenceError,
@@ -9,14 +15,31 @@ import {
 } from "@opengeni/db";
 import {
   ActiveBackendUnresolvableError,
+  AnthropicProviderRejection,
   CompactionProviderResponseError,
   EmptyCompactionSummaryError,
+  compactionProviderRejection,
+  describeCompactionProviderRejection,
+  type CompactionProviderRejection,
   isMcpRequestTimeoutError,
   isMcpTransportConnectivityError,
-  isModalTaskExecStartDnsResolutionError,
+  isModalTaskExecStartPreDispatchUnavailableError,
+  isModalCommandStartOutcomeUnknownError,
+  isProviderCommandObservationUnavailableError,
+  isRoutingMutationOutcomeUnknownError,
   RoutingWorkspaceRootChangedError,
+  ResponsesStreamingTerminalError,
+  SandboxMaterializationVerificationError,
+  materializationVerificationDiagnostic,
+  type MaterializationVerificationDiagnostic,
+  PROVIDER_QUOTA_EXHAUSTED_CODE,
+  type ProviderQuotaExhaustion,
+  type ProviderQuotaScope,
+  classifyProviderQuotaError,
+  providerQuotaExhaustedMessage,
   SelfhostedWorkspaceRootChangedError,
   UNKNOWN_MODEL_FINISH_REASON_CODE,
+  AnthropicRequestError,
 } from "@opengeni/runtime";
 import {
   mcpTransportRequestFailureDiagnostic,
@@ -27,11 +50,19 @@ import { CODEX_USAGE_EXHAUSTED_PCT } from "../codex-rotation";
 import { RetainedAttachmentTransportLimitError } from "../run-input";
 import type { CodexAccountStatus } from "@opengeni/db";
 import {
+  CODEX_USAGE_LIMIT_ERROR_TYPE,
   CodexReloginRequired,
+  classifyCodexEncryptedArtifactRejection,
+  classifyCodexEntitlementRejection,
   classifyCodexResponseTimeoutError,
   classifyCodexUsageLimitError,
   isCodexTransportError,
 } from "@opengeni/codex";
+import {
+  CodexPlanEntitlementError,
+  codexPlanEntitlementFailurePayload,
+  codexRequestRejectedFailurePayload,
+} from "./codex-plan-entitlement";
 import {
   classifyXaiSubscriptionStreamingTerminalError,
   classifyXaiSubscriptionStreamIdleTimeoutError,
@@ -63,6 +94,12 @@ import {
 export const PROVIDER_BACKPRESSURE_DELAY_MS = 60_000;
 export const PROVIDER_CONNECTIVITY_BACKOFF_MS = [2_000, 5_000, 15_000, 30_000, 60_000] as const;
 export const MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_MS.length;
+/**
+ * Minimum wait per rate-limited recovery. Providers such as Azure OpenAI often
+ * answer a per-minute token limit with a `retry-after` of about a second, which
+ * alone would spend every automatic recovery before the window resets.
+ */
+export const PROVIDER_RATE_LIMIT_BACKOFF_MS = [10_000, 20_000, 40_000, 60_000, 120_000] as const;
 export const POST_COMPACTION_CONTINUATION_EMPTY_CODE = "post_compaction_continuation_empty";
 
 export class PostCompactionContinuationEmptyError extends Error {
@@ -106,12 +143,21 @@ export function providerRecoveryResult(input: {
       : null;
   const continueDelayMs =
     input.failureCode === "provider_rate_limited"
-      ? (providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS)
+      ? Math.max(
+          providerDelay ?? PROVIDER_BACKPRESSURE_DELAY_MS,
+          PROVIDER_RATE_LIMIT_BACKOFF_MS[
+            Math.min(
+              Math.max(Math.trunc(input.attemptNumber) - 1, 0),
+              PROVIDER_RATE_LIMIT_BACKOFF_MS.length - 1,
+            )
+          ]!,
+        )
       : input.failureCode === "provider_unavailable" ||
           input.failureCode === "upstream_connectivity_unavailable" ||
           input.failureCode === "sandbox_command_start_unavailable" ||
           input.failureCode === "mcp_transport_timeout" ||
           input.failureCode === "mcp_transport_unavailable" ||
+          input.failureCode === "turn_execution_policy_definition_mismatch" ||
           input.failureCode === POST_COMPACTION_CONTINUATION_EMPTY_CODE
         ? Math.max(
             providerDelay ?? 0,
@@ -187,6 +233,13 @@ export function providerRetryAfterMs(error: unknown, nowMs = Date.now()): number
       value.error && typeof value.error === "object"
         ? (value.error as Record<string, unknown>)
         : null;
+    const milliseconds = Number(
+      headerValue(value.headers, "retry-after-ms") ??
+        headerValue(value.responseHeaders, "retry-after-ms") ??
+        headerValue(body?.headers, "retry-after-ms") ??
+        undefined,
+    );
+    if (Number.isFinite(milliseconds) && milliseconds > 0) return Math.ceil(milliseconds);
     const directSeconds = Number(
       value.retry_after_seconds ?? body?.retry_after_seconds ?? value.retryAfterSeconds,
     );
@@ -269,22 +322,38 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
  * Convert the atomic claim transaction's failure into a small, stable
  * Temporal wire contract. The original error remains in activity diagnostics,
  * but SQL text, parameters, and arbitrary invariant messages never enter
- * workflow history. Every database failure is safe to re-read after backoff
- * because the claim transaction contains no model/tool effects. Database
- * constraints and authorization guards can also be repaired by a deployment or
- * authority refresh, so they must preserve pending work. Only a non-database
- * claim invariant is terminal.
+ * workflow history. Operational failures retry; other persistence rejections
+ * retain accepted work behind a durable explicit-recheck fence. Repeating the
+ * same rejected transaction without changed conditions is not recovery.
  */
 export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
   const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
   const retryableCode = retryableDatabaseFailureCode(error);
-  // The workflow retries after a durable re-read and bounded delay. A
-  // SessionEventPersistenceError proves the failure happened inside the atomic
-  // claim boundary, before provider or tool work; retaining it is safer than
-  // destroying accepted machine input on an application SQLSTATE.
+  const rejectedClaim =
+    persistenceFailure?.details.stage === "session_attempts.claim" &&
+    persistenceFailure.details.retryOutcome === "not_retryable" &&
+    !retryableCode;
   const detail: PreClaimFailureDetail = {
-    disposition: persistenceFailure || retryableCode ? "retryable" : "permanent",
+    disposition: rejectedClaim
+      ? "blocked"
+      : persistenceFailure || retryableCode
+        ? "retryable"
+        : "permanent",
     code: retryableCode ?? persistenceFailure?.details.code ?? "claim_invariant",
+    ...(persistenceFailure && rejectedClaim
+      ? {
+          sqlState: persistenceFailure.details.sqlState,
+          reason:
+            persistenceFailure.details.stage === "session_attempts.claim" &&
+            persistenceFailure.details.sqlState === "OG001"
+              ? ("initiator_membership_required" as const)
+              : persistenceFailure.details.stage === "session_attempts.claim" &&
+                  persistenceFailure.details.sqlState === "OG002"
+                ? ("personal_resource_grant_required" as const)
+                : ("database_claim_rejected" as const),
+          retryPolicy: "explicit_recheck" as const,
+        }
+      : {}),
   };
   return ApplicationFailure.create({
     message: PRE_CLAIM_FAILURE_MESSAGE,
@@ -294,20 +363,177 @@ export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
   });
 }
 
+/** Complete structured-cause graph or no recovery authority. Budget distinct
+ * objects (including array containers), not queue positions/duplicate refs.
+ * The separate link ceiling bounds huge duplicate arrays without truncating
+ * them into permission. Cycles are harmless; overflow/unreadable edges hold. */
+function databaseRecoveryCauseGraph(error: unknown): Map<object, Set<object>> | null {
+  const graph = new Map<object, Set<object>>();
+  const queue: object[] = [];
+  let links = 0;
+  const add = (value: unknown): boolean => {
+    if (!value || typeof value !== "object" || graph.has(value)) return true;
+    if (graph.size >= 64) return false;
+    graph.set(value, new Set());
+    queue.push(value);
+    return true;
+  };
+  if (!add(error)) return null;
+  try {
+    for (const current of queue) {
+      const record = current as Record<string, unknown>;
+      const children = Array.isArray(current)
+        ? current
+        : ["cause", "original", "driverError", "error", "errors"].map((key) => record[key]);
+      for (const child of children) {
+        if (++links > 4096 || !add(child)) return null;
+        if (child && typeof child === "object") graph.get(current)!.add(child);
+      }
+    }
+  } catch {
+    return null;
+  }
+  return graph;
+}
+
 function retryableDatabaseFailureCode(
   error: unknown,
+  requireDatabaseProvenance = false,
 ): PostClaimDatabaseRecoveryDetail["code"] | null {
-  const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
-  const sqlState = persistenceFailure?.details.sqlState ?? null;
-  if (sqlState === null && isRetryableDatabaseTransportFailure(error)) {
-    return "db_failure";
+  try {
+    const graph = databaseRecoveryCauseGraph(error);
+    if (!graph) return null;
+    const transports = new Set<object>();
+    const boundaries = new Set<object>();
+    const ownDatabaseNodes = new Set<object>();
+    const codes = new Set<PostClaimDatabaseRecoveryDetail["code"]>();
+    // Ask the canonical transport predicate about ONLY this node's facts. Its
+    // recursive search must not pair a DB sibling with an unrelated provider.
+    for (const node of graph.keys()) {
+      if (isRoutingMutationOutcomeUnknownError(node)) return null;
+      const record = node as Record<string, unknown>;
+      if (
+        requireDatabaseProvenance
+          ? isRunningTurnDatabaseTransportFailure(record)
+          : isRetryableDatabaseTransportFailure({ code: record.code, errno: record.errno })
+      )
+        transports.add(node);
+      const sqlState = isSessionEventPersistenceError(node)
+        ? node.details.sqlState
+        : record.name === "PostgresError" && typeof record.code === "string"
+          ? record.code
+          : null;
+      if (
+        requireDatabaseProvenance
+          ? isRunningTurnDatabaseConnectionSqlState(sqlState)
+          : isDatabaseConnectionSqlState(sqlState)
+      )
+        transports.add(node);
+      if (node instanceof DrizzleQueryError || isSessionEventPersistenceError(node)) {
+        // Only actual errors raised at our ORM/typed persistence boundary own
+        // their driver subtree. A PostgresError name, SDK wrapper or provider
+        // socket by itself is never own-client provenance for a running turn.
+        const queue: object[] = [node];
+        for (const source of queue) {
+          if (ownDatabaseNodes.has(source)) continue;
+          ownDatabaseNodes.add(source);
+          queue.push(...graph.get(source)!);
+        }
+      }
+    }
+    const hasOwnTransport = (boundary: object): boolean => {
+      const seen = new Set<object>();
+      const queue = [boundary];
+      for (const node of queue) {
+        if (seen.has(node)) continue;
+        seen.add(node);
+        if (transports.has(node)) return true;
+        queue.push(...graph.get(node)!);
+      }
+      return false;
+    };
+    // Inspect ALL trusted DB evidence before returning a permit. A permanent,
+    // auth or uncertain SQLSTATE vetoes every sibling regardless of traversal
+    // order; a deeper transport/reset cannot override it either.
+    for (const node of graph.keys()) {
+      const record = node as Record<string, unknown>;
+      if (node instanceof DrizzleQueryError) {
+        boundaries.add(node);
+        if (hasOwnTransport(node)) codes.add("db_failure");
+      }
+      const typed = isSessionEventPersistenceError(node);
+      if (!typed && record.name !== "PostgresError") continue;
+      boundaries.add(node);
+      const sqlState = typed
+        ? node.details.sqlState
+        : typeof record.code === "string"
+          ? record.code
+          : null;
+      const connectionOutage = requireDatabaseProvenance
+        ? isRunningTurnDatabaseConnectionSqlState(sqlState)
+        : isDatabaseConnectionSqlState(sqlState);
+      const code = typed
+        ? retryablePersistenceFailureCode(sqlState)
+        : connectionOutage || isRetryablePersistenceSqlState(sqlState)
+          ? databaseFailureCode(sqlState)
+          : null;
+      if (
+        !code ||
+        (requireDatabaseProvenance &&
+          !connectionOutage &&
+          !(sqlState === null && hasOwnTransport(node)))
+      )
+        return null;
+      if (!requireDatabaseProvenance || ownDatabaseNodes.has(node)) codes.add(code);
+    }
+    // Preserve the legacy pre-execution transport-only allowance, but never
+    // borrow it across an explicit DB boundary with no eligible own evidence.
+    if (!requireDatabaseProvenance && boundaries.size === 0 && transports.size > 0)
+      codes.add("db_failure");
+    // Stable classification for multiple positive DB siblings too.
+    for (const code of ["db_failure", "db_deadlock", "db_serialization_failure"] as const)
+      if (codes.has(code)) return code;
+    return null;
+  } catch {
+    // Unreadable structured facts are no more authority than unreadable edges.
+    return null;
   }
+}
+
+const RUNNING_TURN_DATABASE_TRANSPORT_CODES = new Set([
+  "ECONNREFUSED",
+  "ECONNRESET",
+  "CONNECT_TIMEOUT",
+]);
+
+function isRunningTurnDatabaseTransportFailure(record: Record<string, unknown>): boolean {
+  return [record.code, record.errno].some(
+    (code) => typeof code === "string" && RUNNING_TURN_DATABASE_TRANSPORT_CODES.has(code),
+  );
+}
+
+function isRunningTurnDatabaseConnectionSqlState(sqlState: string | null): boolean {
+  return (
+    sqlState !== null &&
+    (/^08[0-9A-Z]{3}$/.test(sqlState) || ["57P01", "57P02", "57P03"].includes(sqlState))
+  );
+}
+
+function isDatabaseConnectionSqlState(sqlState: string | null): boolean {
+  return (
+    sqlState !== null &&
+    (sqlState.startsWith("08") || ["57P01", "57P02", "57P03"].includes(sqlState))
+  );
+}
+
+function retryablePersistenceFailureCode(
+  sqlState: string | null,
+): PostClaimDatabaseRecoveryDetail["code"] | null {
   if (
-    !persistenceFailure ||
     !(
       sqlState === null ||
       sqlState.startsWith("08") ||
-      sqlState.startsWith("40") ||
+      isRetryablePersistenceSqlState(sqlState) ||
       sqlState.startsWith("53") ||
       sqlState === "55P03" ||
       sqlState === "57014" ||
@@ -319,26 +545,43 @@ function retryableDatabaseFailureCode(
   ) {
     return null;
   }
-  return persistenceFailure.details.code;
+  return databaseFailureCode(sqlState);
 }
 
 /**
- * Carry one exact claimed-but-not-started attempt into the workflow's DB-only
- * recovery lane. Permanent database/state failures remain terminal; only the
- * same operational outage classes that are safe before claim are admitted.
+ * Carry one exact claimed attempt into the workflow's DB-only
+ * recovery lane. Permanent database/state failures remain terminal. The
+ * running-turn lane additionally requires a closed own-client outage class;
+ * existing pre-execution recovery classifications are unchanged.
  */
 export function postClaimDatabaseRecoveryFailure(input: {
   error: unknown;
   turnId: string;
   triggerEventId: string;
   executionGeneration: number;
+  /** Require an actual own-client boundary and the closed running-turn allowlist. */
+  requireDatabaseProvenance?: boolean;
+  sandboxSetupOutcomeUnknown?: true;
+  sandboxSetupRecoveryExhausted?: true;
   providerRecovery?: {
     failureCode: string;
     providerRecoveryCount: number;
   };
 }): ApplicationFailure | null {
-  const code = retryableDatabaseFailureCode(input.error);
-  if (!code || input.executionGeneration < 1) return null;
+  const code = retryableDatabaseFailureCode(input.error, input.requireDatabaseProvenance);
+  if (
+    !code ||
+    input.executionGeneration < 1 ||
+    (input.requireDatabaseProvenance && code !== "db_failure")
+  )
+    return null;
+  if (
+    (input.sandboxSetupOutcomeUnknown && input.sandboxSetupRecoveryExhausted) ||
+    ((input.sandboxSetupOutcomeUnknown || input.sandboxSetupRecoveryExhausted) &&
+      input.providerRecovery)
+  ) {
+    return null;
+  }
   if (
     input.providerRecovery &&
     (!Number.isSafeInteger(input.providerRecovery.providerRecoveryCount) ||
@@ -353,6 +596,8 @@ export function postClaimDatabaseRecoveryFailure(input: {
     triggerEventId: input.triggerEventId,
     executionGeneration: input.executionGeneration,
     code,
+    ...(input.sandboxSetupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {}),
+    ...(input.sandboxSetupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {}),
     ...(input.providerRecovery
       ? {
           providerFailureCode: input.providerRecovery.failureCode,
@@ -439,8 +684,25 @@ export function sandboxLifecycleTransitionDiagnostic(
   return null;
 }
 
+export function modelPreparationFailureEventPayload(error: unknown, durationMs: number) {
+  const transition = sandboxLifecycleTransitionDiagnostic(error);
+  return {
+    phase: "model_preparation",
+    durationMs: Math.max(0, Math.round(durationMs)),
+    expectedTransition: transition !== null,
+    ...(transition
+      ? {
+          failureCategory: "drain_capture_wait",
+          failureStage: "lifecycle_wait",
+          failureCode: transition.reason,
+          retryable: true,
+        }
+      : {}),
+  };
+}
+
 /**
- * Recognize the one active-route transition that cannot finish inside its
+ * Recognize active-route transitions that cannot finish inside their
  * originating attempt. A Modal-home session may start on a Connected Machine
  * without creating or leasing its managed home box. When an explicit attach
  * clears the active pointer back to home, the pointer commit is authoritative,
@@ -453,7 +715,11 @@ export function sandboxLifecycleTransitionDiagnostic(
  */
 export function sandboxRouteTransitionCode(
   error: unknown,
-): "home_unavailable_this_turn" | "workspace_root_changed_this_turn" | null {
+):
+  | "home_unavailable_this_turn"
+  | "workspace_root_changed_this_turn"
+  | "native_capabilities_changed_this_attempt"
+  | null {
   const pending: unknown[] = [error];
   const seen = new WeakSet<object>();
   let inspected = 0;
@@ -466,6 +732,13 @@ export function sandboxRouteTransitionCode(
 
     try {
       const record = current as Record<string, unknown>;
+      if (
+        (current instanceof SandboxCapabilitiesChangedError ||
+          record.name === "SandboxCapabilitiesChangedError") &&
+        record.code === "native_capabilities_changed_this_attempt"
+      ) {
+        return "native_capabilities_changed_this_attempt";
+      }
       if (
         (current instanceof ActiveBackendUnresolvableError ||
           record.name === "ActiveBackendUnresolvableError") &&
@@ -574,7 +847,29 @@ export function safeErrorForTelemetry(error: unknown): Error {
   return safe;
 }
 
+/**
+ * Guidance appended to a definitive provider rejection. Repeating the exact
+ * request cannot succeed, so the generic "send another message to retry"
+ * advice is wrong here: a new prompt re-sends the same rejected history.
+ */
+export const COMPACTION_PROVIDER_REJECTION_GUIDANCE =
+  "The provider refused this exact request, so repeating it fails the same way until the conversation changes. If a new message fails again, start a new session.";
+
 export function compactionFailureReasonFromError(error: unknown): string {
+  const rejection = compactionProviderRejection(error);
+  if (rejection) {
+    return compactionFailureReason(
+      `the model provider rejected the compaction request (${describeCompactionProviderRejection(rejection)}). Active history was preserved. ${COMPACTION_PROVIDER_REJECTION_GUIDANCE}`,
+    );
+  }
+  // An exhausted provider quota is not retried (see agentRunFailurePayload),
+  // so name the refusal plainly instead of the raw diagnostic envelope.
+  const quota = classifyProviderQuotaExhaustionError(error);
+  if (quota) {
+    return compactionFailureReason(
+      `${providerQuotaExhaustedMessage(quota.scope)} Active history was preserved.`,
+    );
+  }
   if (
     error instanceof CompactionProviderResponseError ||
     error instanceof EmptyCompactionSummaryError
@@ -585,6 +880,39 @@ export function compactionFailureReasonFromError(error: unknown): string {
   return compactionFailureReason(`unexpected ${errorName}`);
 }
 
+/**
+ * Exact `turn.failed` payload for a terminal compaction failure. A definitive
+ * provider rejection additionally carries its closed identifier record so the
+ * timeline, API consumers, and operators can name the rejected field without
+ * parsing the message.
+ */
+export function compactionFailureTurnEventPayload(
+  error: unknown,
+  overrides: { error?: string } = {},
+): {
+  error: string;
+  code: "context_compaction_failed";
+  retryable: false;
+  recovery: "user_message";
+  compacted: false;
+  providerRejection?: CompactionProviderRejection;
+  quotaScope?: ProviderQuotaScope;
+} {
+  const rejection = compactionProviderRejection(error);
+  // The same closed marker as a `provider_quota_exhausted` turn failure, so
+  // clients can name the exhausted limit and offer another model here too.
+  const quota = rejection ? null : classifyProviderQuotaExhaustionError(error);
+  return {
+    error: overrides.error ?? compactionFailureReasonFromError(error),
+    code: "context_compaction_failed",
+    retryable: false,
+    recovery: "user_message",
+    compacted: false,
+    ...(rejection ? { providerRejection: rejection } : {}),
+    ...(quota ? { quotaScope: quota.scope } : {}),
+  };
+}
+
 export function isCompactionSummaryFailure(error: unknown): boolean {
   return (
     error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError
@@ -594,12 +922,24 @@ export function isCompactionSummaryFailure(error: unknown): boolean {
 export function shouldRecoverCompactionProviderFailure(error: unknown): boolean {
   if (!(error instanceof CompactionProviderResponseError)) return false;
   if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
+  // Codex may reject an opaque artifact it minted itself on the compaction
+  // request exactly as it can on an ordinary request. Failure settlement
+  // invalidates only the exact participating artifacts and recovers the same
+  // logical turn; when nothing can be invalidated it fails closed there.
+  if (classifyCodexEncryptedArtifactRejection(error)) return true;
+  // A ChatGPT plan that no longer includes the model refuses the compaction
+  // request exactly as it refuses an ordinary one (often with an empty 400).
+  // Failure settlement re-checks the plan and, when it proves the loss,
+  // excludes that account for the model and fails the same turn over; an
+  // unexplained rejection still fails there with typed copy.
+  if (classifyCodexEntitlementRejection(error)) return true;
   return agentRunFailurePayload(error).retryable === true;
 }
 
 export function classifyContextWindowOverflowError(
   error: unknown,
 ): { message: string; code?: string; detail?: string } | null {
+  if (isProviderSafetyRefusal(error)) return null;
   const fields = collectErrorStrings(error);
   const matched = fields.find(
     (value) =>
@@ -666,6 +1006,9 @@ export function collectErrorStrings(value: unknown, seen = new WeakSet<object>()
   }
   seen.add(value);
   const out: string[] = [];
+  // This diagnostic is already provider-owned and byte-bounded. Do not widen
+  // generic detail traversal to arbitrary application payloads.
+  if (value instanceof ResponsesStreamingTerminalError) out.push(value.detail);
   const record = value as Record<string, unknown>;
   for (const key of ["message", "code", "type", "name", "param"]) {
     const field = record[key];
@@ -728,9 +1071,12 @@ export function isExactStatuslessUpstreamConnectivityMessage(message: string): b
 }
 
 function providerSafetyRefusalDiagnostic(error: unknown): string | undefined {
+  if (error instanceof ResponsesStreamingTerminalError) {
+    return error.category === "safety" ? error.detail : undefined;
+  }
   return collectErrorStrings(error).find(
     (value) =>
-      /^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy)$/.test(
+      /^(?:content_policy_violation|content_filter|safety_violation|bio_policy|cyber_policy|misalignment_policy_violation)$/.test(
         value,
       ) || /\bthis request was blocked by our safety systems\b/i.test(value),
   );
@@ -741,6 +1087,9 @@ function isProviderSafetyRefusal(error: unknown): boolean {
 }
 
 export function isTransientProviderError(error: unknown): boolean {
+  if (error instanceof ResponsesStreamingTerminalError) {
+    return error.category === "unavailable";
+  }
   // A semantic refusal can arrive inside a 5xx transport envelope.
   if (isProviderSafetyRefusal(error)) return false;
   const status =
@@ -774,6 +1123,27 @@ export function isTransientProviderError(error: unknown): boolean {
   );
 }
 
+/** An explicit `usage_limit_reached` type or code string anywhere on the error chain. */
+function hasCodexUsageLimitType(error: unknown): boolean {
+  return collectErrorStrings(error).some((value) => value.includes(CODEX_USAGE_LIMIT_ERROR_TYPE));
+}
+
+/**
+ * Recognize an exhausted API-key provider quota (a daily or monthly allowance,
+ * a free-tier day cap, or an account out of credits) as distinct from an
+ * ordinary per-minute rate limit. Retrying within the bounded same-turn budget
+ * cannot succeed, so the turn fails promptly instead. Subscription transports
+ * own their quota semantics through credential rotation and durable capacity
+ * waits, so a Codex or SuperGrok transport error never classifies here.
+ */
+export function classifyProviderQuotaExhaustionError(
+  error: unknown,
+): ProviderQuotaExhaustion | null {
+  if (isCodexTransportError(error) || isXaiSubscriptionTransportError(error)) return null;
+  // The same reader the OpenAI SDK retry veto uses, so the two never disagree.
+  return classifyProviderQuotaError(error);
+}
+
 export type XaiCredentialFailure = {
   kind: "auth" | "forbidden" | "rate_limit";
   cooldownMs: number | null;
@@ -848,9 +1218,63 @@ function findPostgresDriverError(error: unknown): Record<string, unknown> | null
   return null;
 }
 
+function isRawDatabaseQueryError(error: unknown): boolean {
+  return (
+    error instanceof Error &&
+    "query" in error &&
+    typeof error.query === "string" &&
+    "params" in error &&
+    Array.isArray(error.params)
+  );
+}
+
+function anthropicRequestDiagnostic(error: unknown): AnthropicRequestError | undefined {
+  if (error instanceof AnthropicRequestError) return error;
+  return error instanceof Error && error.cause instanceof AnthropicRequestError
+    ? error.cause
+    : undefined;
+}
+
 export function agentRunFailurePayload(
   error: unknown,
   options: { isCodexTurn?: boolean } = {},
+): ReturnType<typeof baseAgentRunFailurePayload> {
+  const failure = baseAgentRunFailurePayload(error, options);
+  const diagnostic = materializationVerificationDiagnostic(error);
+  const anthropic = anthropicRequestDiagnostic(error);
+  if (anthropic) {
+    const authenticationRejected =
+      anthropic.status === 401 &&
+      (error === anthropic ||
+        (error instanceof Error && (error as Error & { status?: unknown }).status === 401));
+    return {
+      ...failure,
+      code:
+        failure.code ??
+        (authenticationRejected ? "anthropic_authentication_error" : anthropic.code),
+      retryable: failure.retryable ?? false,
+      ...(anthropic.detail ? { detail: anthropic.detail } : {}),
+      ...(anthropic.request_id ? { requestId: anthropic.request_id } : {}),
+    };
+  }
+  return diagnostic ? { ...failure, materializationDiagnostic: diagnostic } : failure;
+}
+
+/** Keep Anthropic provider text on terminal failures, never recovery events. */
+export function agentRunRecoveryFailurePayload(
+  error: unknown,
+  failure: ReturnType<typeof agentRunFailurePayload>,
+): ReturnType<typeof agentRunFailurePayload> {
+  if (!anthropicRequestDiagnostic(error)) return failure;
+  // Project a copy: retry exhaustion still needs the terminal diagnostic.
+  const recovery = { ...failure };
+  delete recovery.detail;
+  return recovery;
+}
+
+function baseAgentRunFailurePayload(
+  error: unknown,
+  options: { isCodexTurn?: boolean } = {},
 ): {
   error: string;
   code?: string;
@@ -870,12 +1294,30 @@ export function agentRunFailurePayload(
   database?: Record<string, string>;
   historyPersistenceStage?: MandatoryHistoryPersistenceStage;
   mcpTransportDiagnostic?: McpTransportRequestFailureDiagnostic;
+  materializationDiagnostic?: MaterializationVerificationDiagnostic;
+  quotaScope?: ProviderQuotaScope;
+  /** Closed Codex plan key on `codex_plan_entitlement` / `codex_request_rejected`. */
+  planType?: string | null;
+  /** Product model id a `codex_plan_entitlement` failure refers to. */
+  model?: string | null;
 } {
+  if (error instanceof SandboxMaterializationVerificationError) {
+    return {
+      error: error.message,
+      code: error.code,
+      retryable: false,
+      materializationDiagnostic: error.diagnostic,
+    };
+  }
   if (error instanceof RetainedAttachmentTransportLimitError) {
     return { error: error.message, code: "retained_attachment_transport_limit", retryable: false };
   }
   if (error instanceof MandatoryHistoryPersistenceError) {
-    const underlying = isSessionEventPersistenceError(error.cause)
+    const databaseFailure =
+      isSessionEventPersistenceError(error.cause) ||
+      findPostgresDriverError(error.cause) !== null ||
+      isRawDatabaseQueryError(error.cause);
+    const underlying = databaseFailure
       ? agentRunFailurePayload(error.cause, options)
       : {
           error: error.cause instanceof Error ? error.cause.message : String(error.cause),
@@ -885,6 +1327,32 @@ export function agentRunFailurePayload(
       historyPersistenceStage: error.stage,
     };
   }
+  // Raw ORM wrappers can contain the full SQL and its parameters. Classify a
+  // real driver before provider message heuristics; the original cause stays
+  // available to internal diagnostics. This payload grants no replay authority.
+  const postgresDriverError = findPostgresDriverError(error);
+  const rawOrmFailure = isRawDatabaseQueryError(error);
+  if ((postgresDriverError || rawOrmFailure) && !isSessionEventPersistenceError(error)) {
+    const database = safeDatabaseErrorFacts(postgresDriverError ?? error);
+    const sqlState = postgresDriverError ? nestedPostgresSqlState(postgresDriverError) : null;
+    return {
+      error: "Opengeni encountered a database error.",
+      code: databaseFailureCode(sqlState),
+      sqlState,
+      ...(Object.keys(database).length > 0 ? { database } : {}),
+    };
+  }
+  if (error instanceof ClaudeSubscriptionConnectionUnavailable) {
+    return { error: error.message, code: error.code, retryable: false };
+  }
+  if (error instanceof AnthropicProviderRejection) {
+    return {
+      error: error.message,
+      code: error.code === "content_policy_violation" ? "provider_safety_refusal" : error.code,
+      retryable: false,
+      ...(error.request_id ? { requestId: error.request_id } : {}),
+    };
+  }
   const safetyRefusalDiagnostic = providerSafetyRefusalDiagnostic(error);
   if (safetyRefusalDiagnostic !== undefined) {
     return {
@@ -895,6 +1363,41 @@ export function agentRunFailurePayload(
       detail: safetyRefusalDiagnostic,
     };
   }
+  if (error instanceof ResponsesStreamingTerminalError) {
+    const quota =
+      error.category === "unknown" || error.category === "rate_limit"
+        ? classifyProviderQuotaExhaustionError({
+            code: error.code,
+            error: { code: error.code, type: error.type, message: error.detail },
+            retryAfterSeconds: error.retryAfterSeconds,
+          })
+        : null;
+    if (quota) {
+      return {
+        error: providerQuotaExhaustedMessage(quota.scope),
+        code: PROVIDER_QUOTA_EXHAUSTED_CODE,
+        retryable: false,
+        quotaScope: quota.scope,
+        detail: error.detail,
+      };
+    }
+    return {
+      error:
+        error.category === "rate_limit"
+          ? "Model provider rate limit hit. Try again in a minute or lower the reasoning effort."
+          : error.category === "unavailable"
+            ? "The model provider is temporarily unavailable. The same turn will retry after a short delay."
+            : "The model provider rejected the response. Automatic retries stopped.",
+      code:
+        error.category === "rate_limit"
+          ? "provider_rate_limited"
+          : error.category === "unavailable"
+            ? "provider_unavailable"
+            : "provider_request_rejected",
+      retryable: error.category === "rate_limit" || error.category === "unavailable",
+      detail: error.detail,
+    };
+  }
   const message = error instanceof Error ? error.message : String(error);
   const status =
     typeof error === "object" && error !== null
@@ -933,10 +1436,29 @@ export function agentRunFailurePayload(
       retryable: true,
     };
   }
-  if (isModalTaskExecStartDnsResolutionError(error)) {
+  if (isProviderCommandObservationUnavailableError(error)) {
     return {
       error:
-        "The managed sandbox command transport was temporarily unreachable before the command started. The same turn will retry after a short delay.",
+        "A managed sandbox command cannot be observed. Its exact invocation and writer remain retained; setup is blocked without replay until the incomplete operation can be reconciled.",
+      code: "sandbox_command_observation_unavailable",
+      retryable: false,
+    };
+  }
+  if (isModalCommandStartOutcomeUnknownError(error)) {
+    return {
+      error:
+        "A managed sandbox command has an unknown outcome. Its original invocation remains fenced; setup is blocked without replay until the incomplete operation can be reconciled.",
+      code: "sandbox_command_start_outcome_unknown",
+      retryable: false,
+    };
+  }
+  if (
+    !isRoutingMutationOutcomeUnknownError(error) &&
+    isModalTaskExecStartPreDispatchUnavailableError(error)
+  ) {
+    return {
+      error:
+        "The managed sandbox command router was not ready before the command was sent. The same turn will retry after a short delay.",
       code: "sandbox_command_start_unavailable",
       retryable: true,
     };
@@ -1001,6 +1523,29 @@ export function agentRunFailurePayload(
       ...(Object.keys(details.database).length > 0 ? { database: details.database } : {}),
     };
   }
+  // Codex plan entitlement: the settlement path normally re-checks the plan
+  // and records a precise payload. These branches keep any other path from
+  // surfacing the SDK's raw "400 status code (no body)" text.
+  if (error instanceof CodexPlanEntitlementError) {
+    return error.payload;
+  }
+  const entitlementRejection = classifyCodexEntitlementRejection(error);
+  if (entitlementRejection) {
+    return entitlementRejection.evidence === "plan_entitlement"
+      ? codexPlanEntitlementFailurePayload({
+          accountLabel: null,
+          planType: null,
+          planChanged: false,
+          modelId: null,
+          rejection: entitlementRejection,
+        })
+      : codexRequestRejectedFailurePayload({
+          accountLabel: null,
+          planType: null,
+          rejection: entitlementRejection,
+          planChecked: false,
+        });
+  }
   // A ChatGPT/Codex usage cap is a HARD limit, not transient backpressure: it
   // must NOT be reported as a generic, retryable rate-limit (which would loop a
   // goal against a capped backend). Surface a precise, actionable message with
@@ -1011,8 +1556,11 @@ export function agentRunFailurePayload(
   // `usage_limit_reached` shape must still outrank generic 429 retryability.
   // Credential quarantine/failover remains separately provenance-gated by
   // `isCodexTransportError`; this branch only chooses the truthful user payload.
+  // The looser "429 ... usage limit" wording counts only on a Codex transport
+  // error: an API-key provider's 429 that says "usage limit" is provider quota
+  // evidence, not a ChatGPT/Codex subscription cap.
   const usageLimit = classifyCodexUsageLimitError(error);
-  if (usageLimit) {
+  if (usageLimit && (isCodexTransportError(error) || hasCodexUsageLimitType(error))) {
     return codexUsageLimitFailurePayload(usageLimit, message);
   }
   const codexTimeout = classifyCodexResponseTimeoutError(error, {
@@ -1064,10 +1612,33 @@ export function agentRunFailurePayload(
       retryable: true,
     };
   }
+  // An exhausted quota also arrives as HTTP 429 (or 402), but no retry within
+  // the finite same-turn budget can succeed. Fail the turn promptly with a
+  // distinct code so the client can offer another model; ordinary short rate
+  // limits fall through to the retryable branch below.
+  if (status === 402 && code === "anthropic_billing_error") {
+    return {
+      error: "Claude could not bill this request. Check the account's billing and payment details.",
+      code: "provider_billing_error",
+      retryable: false,
+      ...(message ? { detail: message } : {}),
+    };
+  }
+  const quota = classifyProviderQuotaExhaustionError(error);
+  if (quota) {
+    return {
+      error: providerQuotaExhaustedMessage(quota.scope),
+      code: PROVIDER_QUOTA_EXHAUSTED_CODE,
+      retryable: false,
+      quotaScope: quota.scope,
+      ...(message ? { detail: message } : {}),
+    };
+  }
   if (
     status === 429 ||
-    code === "rate_limit_exceeded" ||
-    /(?:too many requests|rate.?limit|\b429\b)/i.test(message)
+    ((status === undefined || !Number.isFinite(status)) &&
+      (code === "rate_limit_exceeded" ||
+        /(?:too many requests|rate.?limit|\b429\b)/i.test(message)))
   ) {
     return {
       error: "Model provider rate limit hit. Try again in a minute or lower the reasoning effort.",
@@ -1084,30 +1655,22 @@ export function agentRunFailurePayload(
     if (isExactStatuslessUpstreamConnectivityMessage(message)) {
       return {
         error:
-          "OpenGeni could not reach an upstream service. The same turn will retry after a short delay.",
+          "Opengeni could not reach an upstream service. The same turn will retry after a short delay.",
         code: "upstream_connectivity_unavailable",
         retryable: true,
       };
     }
     return { error: message, code: "provider_unavailable", retryable: true };
   }
-  const postgresDriverError = findPostgresDriverError(error);
-  if (postgresDriverError) {
-    const database = safeDatabaseErrorFacts(postgresDriverError);
-    const sqlState = nestedPostgresSqlState(postgresDriverError);
-    if (sqlState !== null || Object.keys(database).length > 0) {
-      return {
-        error: message,
-        sqlState,
-        ...(Object.keys(database).length > 0 ? { database } : {}),
-      };
-    }
-  }
   return { error: message };
 }
 
 export type CodexCredentialFailure = {
-  kind: "auth" | "forbidden" | "rate_limit" | "quota";
+  /**
+   * `plan_entitlement` is produced only after a plan re-check proves the
+   * serving account's current plan does not include the requested model.
+   */
+  kind: "auth" | "forbidden" | "rate_limit" | "quota" | "plan_entitlement";
   cooldownSeconds: number | null;
 };
 
@@ -1128,7 +1691,11 @@ export function codexCredentialCooldownUntil(
   > | null,
   now: Date,
 ): Date | null {
-  if (failure.kind === "auth" || failure.kind === "forbidden") {
+  if (
+    failure.kind === "auth" ||
+    failure.kind === "forbidden" ||
+    failure.kind === "plan_entitlement"
+  ) {
     return null;
   }
   const providerReset =
@@ -1188,6 +1755,12 @@ export function classifyCodexCredentialFailure(error: unknown): CodexCredentialF
   if (!isCodexTransportError(error)) {
     return null;
   }
+  // Plan/model entitlement evidence (an explicit plan refusal, or an empty
+  // HTTP 400) is not account health or quota. The worker re-checks the plan
+  // first; only a proven entitlement loss walks the pool, as `plan_entitlement`.
+  if (classifyCodexEntitlementRejection(error)) {
+    return null;
+  }
   const usageLimit = classifyCodexUsageLimitError(error);
   if (usageLimit) {
     return { kind: "quota", cooldownSeconds: usageLimit.resetsInSeconds };
@@ -1292,3 +1865,29 @@ export function codexUsageLimitFailurePayload(
 // open indefinitely for a goal-bearing session; cap the continuation hold so the
 // goal re-evaluates at most this far out (it will re-pause if still capped).
 export const CODEX_USAGE_LIMIT_MAX_RESUME_MS = 60 * 60_000; // 1h
+
+/** Only typed provider backpressure or a verified reconnect requirement can rotate Claude. */
+export function classifyClaudeCredentialFailure(
+  error: unknown,
+): (XaiCredentialFailure & { requestId?: string }) | null {
+  if (isProviderSafetyRefusal(error)) return null;
+  let current: unknown = error;
+  for (let depth = 0; depth < 6 && current instanceof Error; depth++) {
+    if (current instanceof ClaudeSubscriptionReconnectRequired)
+      return { kind: "auth", cooldownMs: null };
+    if (current instanceof AnthropicRequestError && current.status === 401)
+      return {
+        kind: "auth",
+        cooldownMs: null,
+        ...(current.request_id ? { requestId: current.request_id } : {}),
+      };
+    if (current instanceof AnthropicRequestError && current.status === 429)
+      return {
+        kind: "rate_limit",
+        cooldownMs: providerRetryAfterMs(current) ?? PROVIDER_BACKPRESSURE_DELAY_MS,
+        ...(current.request_id ? { requestId: current.request_id } : {}),
+      };
+    current = current.cause;
+  }
+  return null;
+}