@opengeni/worker-bundle 1.4.2 → 1.4.5-canary.37458084490001

package/dist/activities/agent-turn/errors.d.ts

@@ -4,9 +4,12 @@ import { ApplicationFailure } from "@temporalio/activity";
 import type { CodexAccountStatus } from "@opengeni/db";
 import type { EscapedMcpTimeoutRecoveryDetail } from "../types.js";
 import { type MandatoryHistoryPersistenceStage } from "./quiescence.js";
+import { type ProviderCondition } from "./provider-recovery-copy.js";
+export { PROVIDER_OVERLOAD_RECOVERY_CODE, MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES, PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS, } from "./provider-recovery-policy.js";
 export declare const PROVIDER_BACKPRESSURE_DELAY_MS = 60000;
 export declare const PROVIDER_CONNECTIVITY_BACKOFF_MS: readonly [2000, 5000, 15000, 30000, 60000];
 export declare const MAX_AUTOMATIC_PROVIDER_RECOVERIES: 5;
+export declare function providerRecoveryLimit(failureCode: unknown): number;
 /**
  * Minimum wait per rate-limited recovery. Providers such as Azure OpenAI often
  * answer a per-minute token limit with a `retry-after` of about a second, which
@@ -23,17 +26,44 @@ export declare class PostCompactionContinuationEmptyError extends Error {
 export type ProviderRecoveryResult = {
     status: "recovering";
     continueDelayMs: number;
+    maxProviderRecoveryCount?: number;
 } | {
     status: "exhausted";
     providerRecoveryCount: number;
     maxProviderRecoveryCount: number;
+    providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
 };
 export declare function providerRecoveryResult(input: {
     failureCode: string | undefined;
     attemptNumber: number;
     retryAfterMs?: number | null;
     jitterSample?: number;
+    /** Durable first-failure clock, never a new clock on each replacement attempt. */
+    recoveryStartedAt?: number | undefined;
+    now?: number;
 }): ProviderRecoveryResult;
+/** A timer/queue delay cannot authorize a provider call after the recovery window. */
+export declare class ProviderOverloadRecoveryExpiredError extends Error {
+    readonly failure: {
+        error: string;
+        code: string;
+        providerCondition: "overloaded";
+    } & {
+        retryable: false;
+        recoveryExhausted: true;
+        providerRecoveryCount: number;
+        maxProviderRecoveryCount: number;
+        lastRetryableError: string;
+        providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
+    };
+    constructor(count: number, reason: "deadline" | "retry_limit" | "invalid_clock");
+}
+export declare function assertProviderOverloadRecoveryActive(input: {
+    failureCode: unknown;
+    providerRecoveryCount: number;
+    recoveryStartedAt?: number | undefined;
+    now?: number;
+}): void;
 export declare function providerRecoveryExhaustedFailure<T extends Record<string, unknown> & {
     error: string;
 }>(failure: T, recovery: Extract<ProviderRecoveryResult, {
@@ -44,6 +74,23 @@ export declare function providerRecoveryExhaustedFailure<T extends Record<string
     providerRecoveryCount: number;
     maxProviderRecoveryCount: number;
     lastRetryableError: string;
+    providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
+};
+/**
+ * Name the exact accepted model route on model-provider recovery evidence so a
+ * person learns which model is affected. Display labels only: the route's
+ * credentials, base URL and upstream ids never enter the event.
+ */
+export declare function withModelRoutePresentation<T extends {
+    code?: string | undefined;
+}>(failure: T, route: {
+    model: string;
+    modelLabel: string;
+    providerLabel: string;
+} | undefined): T & {
+    model?: string;
+    modelLabel?: string;
+    providerLabel?: string;
 };
 export declare function providerRecoveryCountFromMetadata(metadata: Record<string, unknown>): number;
 export declare function providerRecoveryCountAfterModelRequestPhase(currentCount: number, phase: string): number;
@@ -100,6 +147,7 @@ export declare function postClaimDatabaseRecoveryFailure(input: {
     providerRecovery?: {
         failureCode: string;
         providerRecoveryCount: number;
+        continueDelayMs?: number;
     };
 }): ApplicationFailure | null;
 /**
@@ -246,6 +294,12 @@ export declare function collectErrorStrings(value: unknown, seen?: WeakSet<objec
  */
 export declare const STATUSLESS_UPSTREAM_CONNECTIVITY_MESSAGE = "unable to connect. is the computer able to access the url?";
 export declare function isExactStatuslessUpstreamConnectivityMessage(message: string): boolean;
+/**
+ * A fetch-layer timeout: the runtime's DOMException `TimeoutError` (message
+ * "The operation timed out."), possibly wrapped by SDK/runner `cause` chains.
+ * Explicit cancellation (`AbortError`) is deliberately not matched.
+ */
+export declare function isTransportTimeoutError(error: unknown): boolean;
 export declare function isTransientProviderError(error: unknown): boolean;
 /**
  * Recognize an exhausted API-key provider quota (a daily or monthly allowance,
@@ -270,6 +324,16 @@ export declare function classifyXaiCredentialFailure(error: unknown): XaiCredent
 export declare function agentRunFailurePayload(error: unknown, options?: {
     isCodexTurn?: boolean;
 }): ReturnType<typeof baseAgentRunFailurePayload>;
+/** HTTP 529, Anthropic `overloaded_error`, or explicit provider "overloaded" wording. */
+export declare function isProviderOverloadError(error: unknown, failure?: {
+    error?: string;
+    detail?: string;
+}): boolean;
+/** Retry authority must not depend on the looser presentation-only overload label. */
+export declare function providerRecoveryCode(error: unknown, failure: {
+    code?: string;
+    retryable?: boolean;
+}): string | undefined;
 /** Keep Anthropic provider text on terminal failures, never recovery events. */
 export declare function agentRunRecoveryFailurePayload(error: unknown, failure: ReturnType<typeof agentRunFailurePayload>): ReturnType<typeof agentRunFailurePayload>;
 declare function baseAgentRunFailurePayload(error: unknown, options?: {
@@ -297,8 +361,18 @@ declare function baseAgentRunFailurePayload(error: unknown, options?: {
     quotaScope?: ProviderQuotaScope;
     /** Closed Codex plan key on `codex_plan_entitlement` / `codex_request_rejected`. */
     planType?: string | null;
-    /** Product model id a `codex_plan_entitlement` failure refers to. */
+    /** Product model id a `codex_plan_entitlement` or model-provider recovery failure refers to. */
     model?: string | null;
+    /** Display label of the turn's model on model-provider recovery evidence. */
+    modelLabel?: string;
+    /** Display label of the serving provider on model-provider recovery evidence. */
+    providerLabel?: string;
+    /** Closed transient provider condition on `provider_unavailable` / `provider_rate_limited`. */
+    providerCondition?: ProviderCondition;
+    recoveryExhausted?: boolean;
+    providerRecoveryCount?: number;
+    maxProviderRecoveryCount?: number;
+    providerRecoveryExhaustedReason?: "deadline" | "retry_limit" | "invalid_clock";
 };
 export type CodexCredentialFailure = {
     /**
@@ -346,4 +420,3 @@ export declare const CODEX_USAGE_LIMIT_MAX_RESUME_MS: number;
 export declare function classifyClaudeCredentialFailure(error: unknown): (XaiCredentialFailure & {
     requestId?: string;
 }) | null;
-export {};

package/dist/activities/agent-turn/governance-model.d.ts

@@ -1,4 +1,4 @@
-import { materializeRigVersionForAttempt } from "@opengeni/db";
+import { materializeRigVersionForAttempt, freezeAgentLearningPolicy } from "@opengeni/db";
 import { renderWorkspaceGovernanceContext, type OpenGeniRuntime } from "@opengeni/runtime";
 import { type Settings } from "@opengeni/config";
 import { createModelHistoryAttachmentProjector } from "../run-input.js";
@@ -9,6 +9,7 @@ import { lazyToolTransportForTurn, openAiHostedImageProviderBindingForTurn, mode
 import type { ClaimTurnOk } from "./claim.js";
 import type { EventingState, WorkspaceRefState } from "./turn-context.js";
 export type GovernanceModelDeps = {
+    learningPolicy: Awaited<ReturnType<typeof freezeAgentLearningPolicy>>;
     input: RunAgentTurnInput;
     db: ActivityServices["db"];
     observability: ActivityServices["observability"];

package/dist/activities/agent-turn/model-usage.d.ts

@@ -227,7 +227,29 @@ export declare function recordModelUsageAndDebitCredits(settings: Settings, db:
     sourceKey: string;
     latencyMode?: LatencyMode;
     observability?: ActivityServices["observability"];
+    /** Configured provider id for metric labels; never billing authority. */
+    metricProvider?: string;
 }): Promise<ModelUsageBillingRecord | null>;
+/**
+ * The bounded `model` metric label: the deployment catalog product id for a
+ * catalog model, or `custom` for a workspace-owned model (workspace gateway or
+ * a customer-key provider) and for anything the deployment catalog does not
+ * list. Metrics only; never billing or routing authority.
+ */
+export declare function modelMetricProductId(settings: Settings, provider: string | undefined, model: string): string;
+/**
+ * Per-model usage and estimated provider cost for one authoritative response.
+ * Call only after the durable usage event was accepted as current (the same
+ * fence as the Insights fact). Best-effort: never throws.
+ */
+export declare function recordAuthoritativeModelUsageMetrics(input: {
+    observability: ActivityServices["observability"];
+    settings: Settings;
+    provider: string;
+    model: string;
+    externallyBilled: boolean;
+    billing: ModelUsageBillingRecord;
+}): void;
 /** Soft-fail Insights fact write — never throws into the billing/emit path. */
 export declare function recordAuthoritativeModelCallFact(input: {
     db: ActivityServices["db"];

package/dist/activities/agent-turn/provider-recovery-copy.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Person-facing copy for finite same-turn provider recovery. Presentation only:
+ * retry authority, pacing and the recovery budget stay in `errors.ts`.
+ *
+ * `@opengeni/react` (`lib/provider-recovery.ts`) composes the same sentences
+ * from the structured event fields, so live retry status and historical
+ * failures read identically in every surface. Keep the two in step.
+ */
+/** Closed provider condition recorded on model-provider recovery payloads. */
+export type ProviderCondition = "overloaded" | "unavailable" | "unresponsive" | "rate_limited";
+/** Failure codes whose transient condition belongs to the turn's model route. */
+export declare const MODEL_PROVIDER_RECOVERY_CODES: ReadonlySet<string>;
+export type ProviderRecoveryCopyInput = {
+    code?: string | null | undefined;
+    providerCondition?: ProviderCondition | null | undefined;
+    modelLabel?: string | null | undefined;
+    providerLabel?: string | null | undefined;
+};
+/** One present-tense clause naming what is unavailable, without a trailing period. */
+export declare function providerRecoverySubject(input: ProviderRecoveryCopyInput): string;
+/** Terminal copy once the finite automatic recovery budget is spent. */
+export declare function providerRecoveryExhaustedMessage(input: ProviderRecoveryCopyInput & {
+    providerRecoveryCount: number;
+}): string;

package/dist/activities/agent-turn/provider-recovery-policy.d.ts

@@ -0,0 +1,6 @@
+/** Closed policy facts shared with the workflow's content-free recovery wire. */
+export declare const PROVIDER_OVERLOAD_RECOVERY_CODE = "provider_overloaded";
+export declare const MAX_AUTOMATIC_PROVIDER_OVERLOAD_RECOVERIES = 15;
+export declare const PROVIDER_OVERLOAD_RECOVERY_WINDOW_MS: number;
+export declare function validProviderOverloadRecoveryDelay(code: unknown, delay: unknown): boolean;
+export declare function readProviderRecoveryStartedAt(metadata: Record<string, unknown>): number | undefined;

package/dist/activities/agent-turn/turn-context.d.ts

@@ -43,11 +43,20 @@ export type AttemptIdentityState = {
     triggerEventId: string | undefined;
     executionGeneration: number;
     providerRecoveryCount: number;
+    /** Durable recovery policy reason; never inferred from display wording. */
+    providerRecoveryPolicyCode?: string | undefined;
+    providerRecoveryStartedAt?: number | undefined;
     providerRecoveryObservation?: import("./provider-recovery-metrics.js").ProviderRecoveryObservation | undefined;
     modelMetricRoute?: {
         provider: string;
         model: string;
     };
+    /** Public display labels of the accepted model route, for recovery copy only. */
+    modelRoutePresentation?: {
+        model: string;
+        modelLabel: string;
+        providerLabel: string;
+    };
     claudeAuthRecovery?: {
         credentialId: string;
         credentialVersion: number;

package/dist/activities/capabilities.d.ts

@@ -24,4 +24,6 @@ export declare function withCodexProvider(settings: Settings): Settings;
 export declare function withXaiSubscriptionProvider(settings: Settings): Settings;
 export declare function settingsWithWorkspaceGatewayCredential(db: Database, accountId: string, workspaceId: string, settings: Settings, retainedProductModelId?: string | null): Promise<Settings>;
 export declare function settingsWithWorkspaceOpenRouterCredential(db: Database, accountId: string, workspaceId: string, settings: Settings, retainedProductModelId?: string | null): Promise<Settings>;
+/** Worker-only: overlay the workspace Opper catalog and, when active, its decrypted key. */
+export declare function settingsWithWorkspaceOpperCredential(db: Database, accountId: string, workspaceId: string, settings: Settings, retainedProductModelId?: string | null): Promise<Settings>;
 export declare function settingsWithOrganizationProviderCredentials(db: Database, accountId: string, workspaceId: string, settings: Settings, retainedProductModelId?: string | null): Promise<Settings>;

package/dist/activities/sandbox-lease.d.ts

@@ -209,6 +209,11 @@ export declare function probeRetainedProcessAtProvider(settings: ActivityService
 export declare function classifyRetainedProcessPollResult(result: unknown, providerSessionId: number, source?: object): RetainedProcessProbeResult;
 export declare function reconcileModalProviderCreates(db: ActivityServices["db"], settings: ActivityServices["settings"], observability: ActivityServices["observability"], discover?: typeof findModalProviderCreateReceipt): Promise<void>;
 export declare function modalOrphanTerminationStillEligible(latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>, candidate: ModalOrphanSweepTermination): boolean;
+/** Live leases still WARM on an instance the complete listing did not show.
+ * A draining lease legitimately loses its box mid-drain (capture, terminate,
+ * then cold), and warming has not published a box yet, so only a warm lease
+ * whose exact box is gone is a zombie. */
+export declare function countWarmModalLeasesMissingInstance(latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>, missingInstanceIds: string[]): number;
 /** Whether no live lease owns the candidate box by exact instance or by its
  * attribution tags. Unlike termination eligibility this ignores the pending
  * create postponement: a postponed orphan is still running without a lease. */

package/dist/activities/types.d.ts

@@ -226,6 +226,7 @@ export type FailSessionAttemptResult = {
     action: "failed";
 } | {
     action: "recovering";
+    continueDelayMs?: number;
 } | {
     action: "unclaimed";
 } | {
@@ -254,6 +255,8 @@ export type PostClaimDatabaseRecoveryDetail = {
     providerRecoveryCount?: number;
     /** Safe classified provider cause paired with providerRecoveryCount. */
     providerFailureCode?: string;
+    /** Preserve a confirmed-overload Retry-After through checkpoint DB outages. */
+    providerRecoveryContinueDelayMs?: number;
     /** Persist the no-replay setup marker, not a new provider/setup attempt. */
     sandboxSetupOutcomeUnknown?: true;
     /** Preserve the exhausted budget for a command proven not dispatched. */

package/dist/{activities-control-AKJZHDTY.js → activities-control-KBOSUF5Y.js}

@@ -8,6 +8,7 @@ import {
 import {
   INCIDENT_TELEMETRY_AUTHORITY_FENCE_LINEAGE_KEY,
   MAX_AUTOMATIC_PROVIDER_RECOVERIES,
+  PROVIDER_OVERLOAD_RECOVERY_CODE,
   assertReusableSessionRevivable,
   captureConnectedCommandOutput,
   createCodexCapacityActivities,
@@ -23,6 +24,7 @@ import {
   notifyParentOfChildIdle,
   persistWorkspaceArchiveCandidate,
   providerRecoveryCountFromMetadata,
+  providerRecoveryLimit,
   publishDurableSessionEvents,
   putVersion1TarArchiveOrInline,
   reconcileAutomaticSessionTitleFanout,
@@ -31,10 +33,11 @@ import {
   resolveIncidentTelemetryResponderMetadata,
   rigProviderImageSourceImage,
   scheduledUserMessagePayload,
+  validProviderOverloadRecoveryDelay,
   warnDrainSnapshotFailure,
   withFirstPartyTools,
   workflowIdForSession
-} from "./chunk-WVZQJBGD.js";
+} from "./chunk-WX5PX2C6.js";
 import {
   inspectOpenSandboxKubernetesInventory,
   recordActiveUserGauges,
@@ -64,7 +67,7 @@ import {
   recordVerifiedSignupTrialSwitchGauge,
   recordWorkerDeathRecoveryMetrics,
   runtimeMetricsHooksForObservability
-} from "./chunk-DCQ7XTXW.js";
+} from "./chunk-CMIUQ2D3.js";
 import "./chunk-PZ5AY32C.js";
 
 // src/activities/knowledge-indexing.ts
@@ -712,7 +715,8 @@ function createBrowserDeadlineCheckpointActivities(services, connectController =
 import {
   resolveTurnExecutionPolicyV1,
   WORKSPACE_GATEWAY_MODEL_ID_PREFIX,
-  WORKSPACE_OPENROUTER_MODEL_ID_PREFIX
+  WORKSPACE_OPENROUTER_MODEL_ID_PREFIX,
+  WORKSPACE_OPPER_MODEL_ID_PREFIX
 } from "@opengeni/config";
 import {
   applySessionAgentConfigWriteThrough,
@@ -800,7 +804,7 @@ function createAutomationActivities(services, options = {}) {
           return { action: "failed", reason: "credential_restriction_violation" };
         }
         const requestedModel = template.model ?? deploymentCatalogSettings.openaiModel;
-        const catalogSettings = requestedModel.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || requestedModel.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) ? (await resolveWorkspaceCatalogSettings(service.db, catalogSourceSettings, {
+        const catalogSettings = requestedModel.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || requestedModel.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) || requestedModel.startsWith(WORKSPACE_OPPER_MODEL_ID_PREFIX) ? (await resolveWorkspaceCatalogSettings(service.db, catalogSourceSettings, {
           accountId: input.accountId,
           workspaceId: input.workspaceId,
           retainedProductModelId: requestedModel
@@ -950,7 +954,7 @@ function createAutomationActivities(services, options = {}) {
           firstPartyMcpPermissions: template.firstPartyMcpPermissions,
           firstPartyMcpTools: agentWriteThrough.firstPartyMcpTools,
           ...agentWriteThrough.config ? { agentConfig: agentWriteThrough.config } : {},
-          retainWorkspaceCustomModel: model.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || model.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX),
+          retainWorkspaceCustomModel: model.startsWith(WORKSPACE_GATEWAY_MODEL_ID_PREFIX) || model.startsWith(WORKSPACE_OPENROUTER_MODEL_ID_PREFIX) || model.startsWith(WORKSPACE_OPPER_MODEL_ID_PREFIX),
           createIdempotencyKey: `automation-run:${run.id}`,
           subjectId: accepted.serviceSubjectId
         });
@@ -4767,6 +4771,11 @@ function modalOrphanTerminationStillEligible(latest, candidate) {
   }
   return modalOrphanCandidateStillUnowned(latest, candidate);
 }
+function countWarmModalLeasesMissingInstance(latest, missingInstanceIds) {
+  return missingInstanceIds.filter(
+    (instanceId) => latest.some((lease) => lease.liveness === "warm" && lease.instanceId === instanceId)
+  ).length;
+}
 function modalOrphanCandidateStillUnowned(latest, candidate) {
   if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
     return false;
@@ -4811,9 +4820,10 @@ async function sweepModalOrphansForConfiguredBackend(settings, db, observability
         unleased: result.inventory.unterminated.filter(
           (candidate) => modalOrphanCandidateStillUnowned(latest, candidate)
         ).length,
-        liveLeaseInstancesMissing: result.inventory.missingLiveLeaseInstanceIds.filter(
-          (instanceId) => latest.some((lease) => lease.instanceId === instanceId)
-        ).length
+        liveLeaseInstancesMissing: countWarmModalLeasesMissingInstance(
+          latest,
+          result.inventory.missingLiveLeaseInstanceIds
+        )
       });
       recordSandboxInventoryProjectionSuccess(observability, "modal_provider");
     } catch (error) {
@@ -8166,6 +8176,7 @@ import {
 } from "@opengeni/db";
 import { publishDurableSessionEvents as publishDurableSessionEvents3 } from "@opengeni/events";
 import { CancelledFailure } from "@temporalio/activity";
+import { interactionInterventionMetricObserver } from "@opengeni/observability";
 var WORKER_DEATH_MAX_REDISPATCHES = 3;
 function createSessionStateActivities(services, overrides = {}) {
   const settleSessionAttemptInterruptionsFn = overrides.settleSessionAttemptInterruptions ?? settleSessionAttemptInterruptions;
@@ -8280,13 +8291,14 @@ function createSessionStateActivities(services, overrides = {}) {
     const postClaimIdentityMatches = Boolean(
       postClaimRecovery && turn.id === postClaimRecovery.turnId && turn.triggerEventId === postClaimRecovery.triggerEventId && turn.executionGeneration === postClaimRecovery.executionGeneration
     );
-    if (postClaimRecovery?.sandboxSetupRecoveryExhausted && !postClaimIdentityMatches) {
+    if (postClaimRecovery && !postClaimIdentityMatches) {
       return { action: "stale" };
     }
     const providerRecoveryCount = postClaimIdentityMatches ? postClaimRecovery?.providerRecoveryCount : void 0;
     const providerFailureCode = postClaimIdentityMatches ? postClaimRecovery?.providerFailureCode : void 0;
     const hasProviderRecoveryCount = providerRecoveryCount !== void 0;
     const hasProviderFailureCode = providerFailureCode !== void 0;
+    const providerRecoveryContinueDelayMs = postClaimIdentityMatches ? postClaimRecovery?.providerRecoveryContinueDelayMs : void 0;
     const setupOutcomeUnknown = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupOutcomeUnknown === true;
     const setupRecoveryExhausted = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupRecoveryExhausted === true;
     if (setupOutcomeUnknown && setupRecoveryExhausted || (setupOutcomeUnknown || setupRecoveryExhausted) && (hasProviderRecoveryCount || hasProviderFailureCode) || setupRecoveryExhausted && turn.metadata?.providerRecoveryCount !== SANDBOX_SETUP_RECOVERY_LIMIT) {
@@ -8295,7 +8307,10 @@ function createSessionStateActivities(services, overrides = {}) {
     if (hasProviderRecoveryCount !== hasProviderFailureCode) {
       return { action: "stale" };
     }
-    if (providerRecoveryCount !== void 0 && (!Number.isSafeInteger(providerRecoveryCount) || providerRecoveryCount <= 0 || providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata ?? {}) + 1 || providerRecoveryCount > MAX_AUTOMATIC_PROVIDER_RECOVERIES || typeof providerFailureCode !== "string" || !/^[a-z][a-z0-9_]{0,63}$/.test(providerFailureCode))) {
+    if (!validProviderOverloadRecoveryDelay(providerFailureCode, providerRecoveryContinueDelayMs)) {
+      return { action: "stale" };
+    }
+    if (providerRecoveryCount !== void 0 && (!Number.isSafeInteger(providerRecoveryCount) || providerRecoveryCount <= 0 || providerRecoveryCount !== providerRecoveryCountFromMetadata(turn.metadata ?? {}) + 1 || providerRecoveryCount > providerRecoveryLimit(providerFailureCode) || typeof providerFailureCode !== "string" || !/^[a-z][a-z0-9_]{0,63}$/.test(providerFailureCode))) {
       return { action: "stale" };
     }
     const recoveredClaimCode = postClaimIdentityMatches ? postClaimRecovery.code : input.preClaimFailure?.disposition === "retryable" && input.preClaimFailure.code !== "claim_invariant" ? input.preClaimFailure.code : input.preClaimFailureDisposition === "retryable" ? "legacy_retryable_preclaim_database_failure" : null;
@@ -8310,7 +8325,7 @@ function createSessionStateActivities(services, overrides = {}) {
         ...setupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {},
         ...providerRecoveryCount !== void 0 ? { providerRecoveryCount } : {},
         detail: {
-          code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode ?? recoveredClaimCode,
+          code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode === PROVIDER_OVERLOAD_RECOVERY_CODE ? "provider_unavailable" : providerFailureCode ?? recoveredClaimCode,
           retryable: !setupOutcomeUnknown && !setupRecoveryExhausted,
           ...setupOutcomeUnknown ? { setupOutcome: "unknown", replay: "blocked" } : {},
           ...setupRecoveryExhausted ? {
@@ -8321,7 +8336,12 @@ function createSessionStateActivities(services, overrides = {}) {
           } : {},
           ...providerRecoveryCount !== void 0 ? {
             databaseFailureCode: recoveredClaimCode,
-            providerRecoveryCount
+            providerRecoveryCount,
+            ...providerFailureCode === PROVIDER_OVERLOAD_RECOVERY_CODE ? {
+              providerCondition: "overloaded",
+              maxProviderRecoveryCount: providerRecoveryLimit(providerFailureCode),
+              continueDelayMs: providerRecoveryContinueDelayMs
+            } : {}
           } : {},
           recoverySource: "workflow_activity_failure"
         },
@@ -8335,7 +8355,10 @@ function createSessionStateActivities(services, overrides = {}) {
         countQueuedTurnsFn,
         recordTurnsQueuedGaugeFn
       );
-      return { action: "recovering" };
+      return {
+        action: "recovering",
+        ...providerRecoveryContinueDelayMs !== void 0 ? { continueDelayMs: providerRecoveryContinueDelayMs } : {}
+      };
     }
     const trigger = await getSessionEventFn(db, input.workspaceId, turn.triggerEventId);
     const result = await applySessionTurnSettlementFn(db, input.workspaceId, {
@@ -8653,8 +8676,15 @@ function createSessionStateActivities(services, overrides = {}) {
     };
   }
   async function expireSessionInteractionIntervention(input) {
-    const { db, bus } = await services();
+    const { db, bus, observability } = await services();
     const result = await expireSessionInteractionInterventionFn(db, input);
+    if (result.action === "expired" && result.intervention) {
+      interactionInterventionMetricObserver(observability)({
+        kind: result.intervention.kind,
+        outcome: "expired",
+        waitMs: Date.now() - result.intervention.createdAt.getTime()
+      });
+    }
     if (result.events.length > 0) {
       await publishDurableSessionEventsFn(bus, input.workspaceId, input.sessionId, result.events);
     }
@@ -8853,4 +8883,4 @@ export {
   createControlActivities,
   createControlActivitiesFromServices
 };
-//# sourceMappingURL=activities-control-AKJZHDTY.js.map
+//# sourceMappingURL=activities-control-KBOSUF5Y.js.map