@opengeni/worker-bundle 1.0.2 → 1.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengeni/worker-bundle",
-  "version": "1.0.2",
+  "version": "1.2.0",
   "description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
   "license": "Apache-2.0",
   "repository": {
@@ -54,26 +54,26 @@
   },
   "dependencies": {
     "@llamaindex/liteparse": "2.14.2",
-    "@opengeni/agent-proto": "1.0.2",
-    "@opengeni/capabilities": "1.0.2",
-    "@opengeni/codemode": "1.0.2",
-    "@opengeni/codex": "1.0.2",
-    "@opengeni/config": "1.0.2",
-    "@opengeni/contracts": "1.0.2",
-    "@opengeni/core": "1.0.2",
-    "@opengeni/db": "1.0.2",
-    "@opengeni/documents": "1.0.2",
-    "@opengeni/events": "1.0.2",
-    "@opengeni/github": "1.0.2",
-    "@opengeni/interaction": "1.0.2",
-    "@opengeni/jev": "1.0.2",
-    "@opengeni/network": "1.0.2",
-    "@opengeni/observability": "1.0.2",
-    "@opengeni/runtime": "1.0.2",
-    "@opengeni/sdk": "1.0.2",
-    "@opengeni/storage": "1.0.2",
-    "@opengeni/tool-gateway": "1.0.2",
-    "@opengeni/xai-subscription": "1.0.2",
+    "@opengeni/agent-proto": "1.2.0",
+    "@opengeni/capabilities": "1.2.0",
+    "@opengeni/codemode": "1.2.0",
+    "@opengeni/codex": "1.2.0",
+    "@opengeni/config": "1.2.0",
+    "@opengeni/contracts": "1.2.0",
+    "@opengeni/core": "1.2.0",
+    "@opengeni/db": "1.2.0",
+    "@opengeni/documents": "1.2.0",
+    "@opengeni/events": "1.2.0",
+    "@opengeni/github": "1.2.0",
+    "@opengeni/interaction": "1.2.0",
+    "@opengeni/jev": "1.2.0",
+    "@opengeni/network": "1.2.0",
+    "@opengeni/observability": "1.2.0",
+    "@opengeni/runtime": "1.2.0",
+    "@opengeni/sdk": "1.2.0",
+    "@opengeni/storage": "1.2.0",
+    "@opengeni/tool-gateway": "1.2.0",
+    "@opengeni/xai-subscription": "1.2.0",
     "@temporalio/activity": "^1.17.0",
     "@temporalio/client": "^1.17.0",
     "@temporalio/worker": "^1.17.0",

package/src/activities/agent-run-admission.ts

@@ -1,98 +1 @@
-import { configuredStaticUsageLimits, type Settings } from "@opengeni/config";
-import { modelFundingForAdmission } from "@opengeni/core";
-import {
-  checkWorkspaceAllowance,
-  getBillingBalance,
-  isCodexBilledTurn,
-  sumUsageQuantity,
-} from "@opengeni/db";
-import type { ControlActivityServices } from "./types";
-
-export type AgentRunAdmissionDenial =
-  | "insufficient_credits"
-  | "allowance_exhausted"
-  | "monthly_model_cost_limit"
-  | "monthly_agent_run_limit";
-
-/** One worker-side admission boundary for service-authored agent runs. */
-export async function agentRunAdmissionDenial(
-  services: Pick<ControlActivityServices, "db" | "entitlements"> & { settings: Settings },
-  input: {
-    accountId: string;
-    workspaceId: string;
-    model: string;
-    requestedAgentRuns: number;
-    /** The accepted work's causal human, not the scheduler/service caller. */
-    initiatingHumanSubjectId?: string | null;
-  },
-): Promise<AgentRunAdmissionDenial | null> {
-  const codexBilled = await isCodexBilledTurn({
-    db: services.db,
-    settings: services.settings,
-    workspaceId: input.workspaceId,
-    model: input.model,
-  });
-  const externallyBilled = modelFundingForAdmission(
-    services.settings,
-    input.model,
-    codexBilled,
-  ).fundedWithoutCredits;
-  if (
-    !externallyBilled &&
-    (services.settings.billingMode === "stripe" || services.settings.usageLimitsMode === "managed")
-  ) {
-    if (services.entitlements) {
-      const decision = await services.entitlements.admitRun({
-        accountId: input.accountId,
-        workspaceId: input.workspaceId,
-        action: "agent_run:create",
-        quantity: input.requestedAgentRuns,
-      });
-      if (!decision.allowed) return "insufficient_credits";
-    } else {
-      const balance = await getBillingBalance(services.db, input.accountId);
-      if (balance.balanceMicros <= 0) return "insufficient_credits";
-    }
-  }
-  if (!externallyBilled) {
-    const refusal = await checkWorkspaceAllowance(services.db, {
-      accountId: input.accountId,
-      workspaceId: input.workspaceId,
-      subjectId: input.initiatingHumanSubjectId ?? null,
-    });
-    if (refusal) return refusal.code;
-  }
-  if (
-    services.settings.usageLimitsMode !== "static" &&
-    services.settings.usageLimitsMode !== "managed"
-  ) {
-    return null;
-  }
-  const limits = configuredStaticUsageLimits(services.settings);
-  if (!externallyBilled && limits.maxMonthlyCostMicrosPerAccount) {
-    const used = await sumUsageQuantity(services.db, {
-      accountId: input.accountId,
-      eventType: "model.cost",
-      since: startOfUtcMonth(),
-    });
-    if (used >= limits.maxMonthlyCostMicrosPerAccount) {
-      return "monthly_model_cost_limit";
-    }
-  }
-  if (limits.maxMonthlyAgentRunsPerWorkspace) {
-    const used = await sumUsageQuantity(services.db, {
-      workspaceId: input.workspaceId,
-      eventType: "agent_run.created",
-      since: startOfUtcMonth(),
-    });
-    if (used + input.requestedAgentRuns > limits.maxMonthlyAgentRunsPerWorkspace) {
-      return "monthly_agent_run_limit";
-    }
-  }
-  return null;
-}
-
-function startOfUtcMonth(): Date {
-  const now = new Date();
-  return new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
-}
+export { agentRunAdmissionDenial, type AgentRunAdmissionDenial } from "@opengeni/core";

package/src/activities/agent-turn/admission.ts

@@ -1,4 +1,4 @@
-import { checkWorkspaceAllowance, getBillingBalance, sumUsageQuantity } from "@opengeni/db";
+import { checkWorkspaceAllowance, getSpendableCreditBalance, sumUsageQuantity } from "@opengeni/db";
 import {
   configuredStaticUsageLimits,
   resolveTurnExecutionPolicyV1,
@@ -274,10 +274,11 @@ export async function ensureRunAllowedBetweenModelCalls(input: {
   chargesOpenGeniCredits: boolean;
   countsTowardTokenCap: boolean;
   initiatingHumanSubjectId: string | null;
+  modelId?: string;
   serializedRunState?: () => string | null;
-}): Promise<void> {
+}): Promise<number | undefined> {
   try {
-    await ensureRunAllowed(
+    return await ensureRunAllowed(
       input.settings,
       input.db,
       input.accountId,
@@ -287,6 +288,7 @@ export async function ensureRunAllowedBetweenModelCalls(input: {
       input.chargesOpenGeniCredits,
       input.countsTowardTokenCap,
       input.initiatingHumanSubjectId,
+      input.modelId,
     );
   } catch (limitError) {
     let serializedRunState: string | null = null;
@@ -317,7 +319,9 @@ export async function ensureRunAllowed(
   chargesOpenGeniCredits = !isExternallyBilledTurn,
   countsTowardTokenCap = !isExternallyBilledTurn,
   initiatingHumanSubjectId: string | null = null,
-): Promise<void> {
+  modelId?: string,
+): Promise<number | undefined> {
+  let creditPolicyRevision: number | undefined;
   // Upstream settlement and workspace-facing cost are independent. External
   // metering skips the token cap; free/subscription/workspace cost skips the
   // OpenGeni credit gate. The agent-run COUNT cap below is a volume/fairness
@@ -352,7 +356,8 @@ export async function ensureRunAllowed(
     chargesOpenGeniCredits &&
     (settings.billingMode === "stripe" || settings.usageLimitsMode === "managed")
   ) {
-    const balance = await getBillingBalance(db, accountId);
+    const balance = await getSpendableCreditBalance(db, accountId, modelId);
+    creditPolicyRevision = balance.creditPolicyRevision;
     if (balance.balanceMicros <= 0) {
       throw new Error("insufficient Opengeni credits");
     }
@@ -393,4 +398,5 @@ export async function ensureRunAllowed(
       }
     }
   }
+  return creditPolicyRevision;
 }

package/src/activities/agent-turn/claim.ts

@@ -60,6 +60,7 @@ import { createTurnCredentialLeases } from "./credential-leases";
 import { createTurnMediaArtifacts } from "./media-artifacts";
 import { readTurnExecutionPolicyV1 } from "@opengeni/contracts";
 import { turnCredentialRestriction } from "./credential-restriction";
+import { readProviderRecoveryObservation } from "./provider-recovery-metrics";
 
 import {
   credentialSubjectIdForTurnInitiator,
@@ -237,6 +238,7 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
   attempt.dispatchId = dispatchId;
   attempt.executionGeneration = turn.executionGeneration;
   attempt.providerRecoveryCount = providerRecoveryCountFromMetadata(turn.metadata);
+  attempt.providerRecoveryObservation = readProviderRecoveryObservation(turn.metadata ?? {});
   const authRecovery = turn.metadata?.claudeAuthRecovery;
   attempt.claudeAuthRecovery =
     authRecovery &&
@@ -391,6 +393,11 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
       latencyMode: turn.latencyMode,
     },
   );
+  // The durable same-turn recovery lane owns provider retries. Hidden SDK
+  // retries multiply that budget and keep the UI looking active during backoff.
+  // Apply before configuring/resolving clients so main, compaction and title
+  // requests all share this policy; standalone runtime consumers keep theirs.
+  capabilitySettings = { ...capabilitySettings, openaiMaxRetries: 0 };
   runtime.configure(capabilitySettings);
   const verifiedExecutionPolicy = assertTurnExecutionPolicyMatchesConfigV1(
     capabilitySettings,
@@ -402,6 +409,10 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
     },
   );
   const turnExecutionPolicy = verifiedExecutionPolicy.policy;
+  attempt.modelMetricRoute = {
+    provider: turnExecutionPolicy.providerId,
+    model: turnExecutionPolicy.productModelId,
+  };
   assertSessionAllowsProductModel(session, turnExecutionPolicy.productModelId);
   const billingIdentity = turnExecutionPolicyBillingIdentity(turnExecutionPolicy);
   billingState.isExternallyBilledTurn = billingIdentity.externallyBilled;
@@ -462,6 +473,7 @@ export async function claimTurnAttempt(deps: ClaimTurnDeps): Promise<ClaimTurnOu
         billingState.chargesOpenGeniCredits,
         billingState.countsTowardTokenCap,
         turn.initiatingHumanSubjectId,
+        turnExecutionPolicy.productModelId,
       ),
       cancellationSignal,
       undefined,

package/src/activities/agent-turn/compaction-prep.ts

@@ -1,3 +1,4 @@
+import { ensureRunAllowedBetweenModelCalls } from "./admission";
 import { hasPendingSteerAfterContextCompaction, isSessionCompactionRequested } from "@opengeni/db";
 import { publishDurableSessionEvents } from "@opengeni/events";
 import {
@@ -46,6 +47,7 @@ import {
   processCompactionModelUsageEvent,
 } from "./model-usage";
 import { waitForTurnOperation } from "./sandbox-provision";
+import { recordProviderRecoveryOutcome } from "./provider-recovery-metrics";
 
 import type { ClaimTurnOk } from "./claim";
 import type { GovernanceModelOk } from "./governance-model";
@@ -66,6 +68,7 @@ export type CompactionPrepDeps = {
   input: RunAgentTurnInput;
   settings: Settings;
   db: ActivityServices["db"];
+  entitlements?: ActivityServices["entitlements"];
   bus: ActivityServices["bus"];
   observability: ActivityServices["observability"];
   cancellationSignal: AbortSignal | undefined;
@@ -236,9 +239,11 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
 
   const promptCacheKey = acceptsPromptCacheKeyForTurn(resolvedModel) ? input.sessionId : undefined;
   const compactionUsageState = createCompactionModelUsageEventState(claimedModelUsageSourceKeys);
+  let compactionCreditPolicyRevision: number | undefined;
   const recordCompactionUsage = async (usage: ModelResponseUsage) => {
     await processCompactionModelUsageEvent({
       usage,
+      creditPolicyRevision: compactionCreditPolicyRevision,
       state: compactionUsageState,
       dispatchId: modelUsageDispatchId,
       settings,
@@ -266,7 +271,7 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
     });
   };
   const compactionSummarizerFor = (systemInstructions?: string): CompactionSummarizer => {
-    const summarize: CompactionSummarizer = resolvedModel
+    const summarizeModel: CompactionSummarizer = resolvedModel
       ? (s: Settings, m: Array<Record<string, unknown>>) =>
           withProviderRequestContext(() =>
             summarizeContextForCompaction(s, m, {
@@ -293,6 +298,21 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
             ...(systemInstructions ? { systemInstructions } : {}),
             ...(promptCacheKey ? { promptCacheKey } : {}),
           });
+    const summarize: CompactionSummarizer = async (s, m) => {
+      compactionCreditPolicyRevision = await ensureRunAllowedBetweenModelCalls({
+        settings: s,
+        db,
+        accountId: input.accountId,
+        workspaceId: input.workspaceId,
+        modelId: resolvedModel?.configured.id ?? turn.model,
+        isExternallyBilledTurn: billingState.isExternallyBilledTurn,
+        chargesOpenGeniCredits: billingState.chargesOpenGeniCredits,
+        countsTowardTokenCap: billingState.countsTowardTokenCap,
+        initiatingHumanSubjectId: turn.initiatingHumanSubjectId,
+        entitlements: deps.entitlements,
+      });
+      return await summarizeModel(s, m);
+    };
     summarize.estimatePrefixTokens = () => {
       if (resolvedModel?.provider.api === "chat") {
         return estimateSerializedValueTokens(systemInstructions ?? "");
@@ -330,6 +350,20 @@ export async function prepareCompaction(deps: CompactionPrepDeps): Promise<Compa
     await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, events);
   };
   const publishCompactionOutcomeEvents = async (events: SessionEvent[]) => {
+    if (events.some((event) => event.type === "session.context.compacted")) {
+      // The summary and recovery reset already committed under the attempt
+      // fence. Skipped compaction does not prove successful model progress.
+      attempt.providerRecoveryCount = 0;
+      if (attempt.providerRecoveryObservation) {
+        recordProviderRecoveryOutcome(observability, {
+          route: attempt.modelMetricRoute,
+          cause: attempt.providerRecoveryObservation.cause,
+          outcome: "recovered",
+          elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt,
+        });
+        attempt.providerRecoveryObservation = undefined;
+      }
+    }
     // `compaction.started` was already fanout via publishCompactionLiveEvents.
     await publishDurableSessionEvents(
       bus,

package/src/activities/agent-turn/errors.ts

@@ -11,6 +11,7 @@ import {
   safeDatabaseErrorFacts,
   isRetryableDatabaseTransportFailure,
   isSessionEventPersistenceError,
+  DatabaseTransactionError,
   SandboxLeaseTransitionError,
 } from "@opengeni/db";
 import {
@@ -100,6 +101,11 @@ export const MAX_AUTOMATIC_PROVIDER_RECOVERIES = PROVIDER_CONNECTIVITY_BACKOFF_M
  * alone would spend every automatic recovery before the window resets.
  */
 export const PROVIDER_RATE_LIMIT_BACKOFF_MS = [10_000, 20_000, 40_000, 60_000, 120_000] as const;
+/** Positive-only spread: never shorten the provider's minimum delay. */
+export function providerRecoveryJitterMs(delayMs: number, sample: number): number {
+  const bounded = Number.isFinite(sample) ? Math.max(0, Math.min(sample, 1)) : 0;
+  return Math.floor(Math.min(5_000, delayMs * 0.2) * bounded);
+}
 export const POST_COMPACTION_CONTINUATION_EMPTY_CODE = "post_compaction_continuation_empty";
 
 export class PostCompactionContinuationEmptyError extends Error {
@@ -126,6 +132,7 @@ export function providerRecoveryResult(input: {
   failureCode: string | undefined;
   attemptNumber: number;
   retryAfterMs?: number | null;
+  jitterSample?: number;
 }): ProviderRecoveryResult {
   if (input.attemptNumber > MAX_AUTOMATIC_PROVIDER_RECOVERIES) {
     return {
@@ -171,7 +178,13 @@ export function providerRecoveryResult(input: {
         : PROVIDER_BACKPRESSURE_DELAY_MS;
   return {
     status: "recovering",
-    continueDelayMs,
+    continueDelayMs:
+      continueDelayMs +
+      (input.failureCode === "provider_rate_limited" ||
+      input.failureCode === "provider_unavailable" ||
+      input.failureCode === "upstream_connectivity_unavailable"
+        ? providerRecoveryJitterMs(continueDelayMs, input.jitterSample ?? 0)
+        : 0),
   };
 }
 
@@ -407,6 +420,12 @@ function retryableDatabaseFailureCode(
     const boundaries = new Set<object>();
     const ownDatabaseNodes = new Set<object>();
     const codes = new Set<PostClaimDatabaseRecoveryDetail["code"]>();
+    const ownChildren = (node: object): object[] =>
+      node instanceof DatabaseTransactionError
+        ? // Callback failures retained beside a failed rollback supply vetoes,
+          // never driver provenance for a provider's connection-looking error.
+          [...graph.get(node)!].filter((child) => child === node.cause)
+        : [...graph.get(node)!];
     // Ask the canonical transport predicate about ONLY this node's facts. Its
     // recursive search must not pair a DB sibling with an unrelated provider.
     for (const node of graph.keys()) {
@@ -429,7 +448,11 @@ function retryableDatabaseFailureCode(
           : isDatabaseConnectionSqlState(sqlState)
       )
         transports.add(node);
-      if (node instanceof DrizzleQueryError || isSessionEventPersistenceError(node)) {
+      if (
+        node instanceof DrizzleQueryError ||
+        node instanceof DatabaseTransactionError ||
+        isSessionEventPersistenceError(node)
+      ) {
         // Only actual errors raised at our ORM/typed persistence boundary own
         // their driver subtree. A PostgresError name, SDK wrapper or provider
         // socket by itself is never own-client provenance for a running turn.
@@ -437,7 +460,7 @@ function retryableDatabaseFailureCode(
         for (const source of queue) {
           if (ownDatabaseNodes.has(source)) continue;
           ownDatabaseNodes.add(source);
-          queue.push(...graph.get(source)!);
+          queue.push(...ownChildren(source));
         }
       }
     }
@@ -448,7 +471,7 @@ function retryableDatabaseFailureCode(
         if (seen.has(node)) continue;
         seen.add(node);
         if (transports.has(node)) return true;
-        queue.push(...graph.get(node)!);
+        queue.push(...ownChildren(node));
       }
       return false;
     };
@@ -457,7 +480,7 @@ function retryableDatabaseFailureCode(
     // order; a deeper transport/reset cannot override it either.
     for (const node of graph.keys()) {
       const record = node as Record<string, unknown>;
-      if (node instanceof DrizzleQueryError) {
+      if (node instanceof DrizzleQueryError || node instanceof DatabaseTransactionError) {
         boundaries.add(node);
         if (hasOwnTransport(node)) codes.add("db_failure");
       }
@@ -501,6 +524,12 @@ function retryableDatabaseFailureCode(
 }
 
 const RUNNING_TURN_DATABASE_TRANSPORT_CODES = new Set([
+  // postgres.js reports these for a physically lost connection, including
+  // transaction cleanup after its socket has already closed. Own-client
+  // provenance and the unknown-outcome veto remain mandatory above.
+  "CONNECTION_CLOSED",
+  "CONNECTION_DESTROYED",
+  "CONNECTION_ENDED",
   "ECONNREFUSED",
   "ECONNRESET",
   "CONNECT_TIMEOUT",
@@ -1086,19 +1115,28 @@ function isProviderSafetyRefusal(error: unknown): boolean {
   return providerSafetyRefusalDiagnostic(error) !== undefined;
 }
 
+/** Preserve the closest real HTTP status through SDK Error.cause wrappers. */
+function providerHttpStatus(error: unknown): number | undefined {
+  let current = error;
+  const seen = new Set<unknown>();
+  for (let depth = 0; depth < 6 && current && typeof current === "object"; depth += 1) {
+    if (seen.has(current)) break;
+    seen.add(current);
+    const value = current as { status?: unknown; statusCode?: unknown; cause?: unknown };
+    const status = Number(value.status ?? value.statusCode);
+    if (Number.isInteger(status) && status >= 100 && status < 600) return status;
+    current = value.cause;
+  }
+  return undefined;
+}
+
 export function isTransientProviderError(error: unknown): boolean {
   if (error instanceof ResponsesStreamingTerminalError) {
     return error.category === "unavailable";
   }
   // A semantic refusal can arrive inside a 5xx transport envelope.
   if (isProviderSafetyRefusal(error)) return false;
-  const status =
-    typeof error === "object" && error !== null
-      ? Number(
-          (error as { status?: unknown; statusCode?: unknown }).status ??
-            (error as { statusCode?: unknown }).statusCode,
-        )
-      : undefined;
+  const status = providerHttpStatus(error);
   // A real HTTP status is AUTHORITATIVE: a 5xx is transient, and ANY other status
   // (4xx validation/auth/404, plus the 429 the earlier branches already handled) is
   // a request fault that must NOT auto-retry — even if its body happens to read like
@@ -1399,13 +1437,7 @@ function baseAgentRunFailurePayload(
     };
   }
   const message = error instanceof Error ? error.message : String(error);
-  const status =
-    typeof error === "object" && error !== null
-      ? Number(
-          (error as { status?: unknown; statusCode?: unknown }).status ??
-            (error as { statusCode?: unknown }).statusCode,
-        )
-      : undefined;
+  const status = providerHttpStatus(error);
   const code =
     typeof error === "object" && error !== null && "code" in error
       ? String((error as { code?: unknown }).code)

package/src/activities/agent-turn/failure-settlement.ts

@@ -100,6 +100,7 @@ import type {
 } from "./turn-context";
 import type { CodexCredentialPolicySnapshotV1 } from "@opengeni/contracts";
 import { armAndReconcileCodexCapacityWait } from "../codex-capacity";
+import { providerRecoveryCause, recordProviderRecoveryOutcome } from "./provider-recovery-metrics";
 
 export type TurnFailureDeps = {
   error: unknown;
@@ -276,12 +277,20 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
   try {
     return await settleTurnFailureInAttempt(deps);
   } catch (error) {
+    if (
+      deps.control.activityStatus === "recovering" &&
+      error instanceof ApplicationFailure &&
+      error.type === "OpenGeniPostClaimDatabaseRecovery"
+    )
+      throw error;
     // Connectivity can disappear while settling an unrelated run error too.
     // Do not overwrite a possibly committed settlement; the control lane
     // re-reads exact ownership and becomes a stale no-op if it already closed.
     if (deps.attempt.turnId && deps.attempt.triggerEventId) {
       const recovery = postClaimDatabaseRecoveryFailure({
-        error,
+        // A failed rollback/terminal write must not erase no-replay evidence
+        // from the failure we were settling (notably unknown tool effects).
+        error: new AggregateError([error, deps.error], "Turn failure settlement failed"),
         turnId: deps.attempt.turnId,
         triggerEventId: deps.attempt.triggerEventId,
         executionGeneration: deps.attempt.executionGeneration,
@@ -1995,6 +2004,7 @@ async function settleTurnFailureInAttempt(deps: TurnFailureDeps): Promise<RunAge
       failureCode: failure.code,
       attemptNumber: nextProviderRecoveryCount,
       retryAfterMs: providerRetryAfterMs(error),
+      jitterSample: Math.random(),
     });
     const setupRecoveryExhausted =
       earlyCommandStartUnavailable &&
@@ -2064,9 +2074,29 @@ async function settleTurnFailureInAttempt(deps: TurnFailureDeps): Promise<RunAge
         control.turnMetricOutcome = "recovering";
         control.activityStatus = "recovering";
         control.activityError = error;
+        const recoveryCause = providerRecoveryCause(failure.code);
+        if (recoveryCause) {
+          recordProviderRecoveryOutcome(observability, {
+            route: attempt.modelMetricRoute,
+            cause: recoveryCause,
+            outcome: "scheduled",
+            delayMs: recoveryResult.continueDelayMs,
+          });
+        }
         return claimedResult(recoveryResult);
       }
       failure = providerRecoveryExhaustedFailure(failure, recoveryResult);
+      const recoveryCause = providerRecoveryCause(failure.code);
+      if (recoveryCause) {
+        recordProviderRecoveryOutcome(observability, {
+          route: attempt.modelMetricRoute,
+          cause: recoveryCause,
+          outcome: "exhausted",
+          ...(attempt.providerRecoveryObservation
+            ? { elapsedMs: Date.now() - attempt.providerRecoveryObservation.startedAt }
+            : {}),
+        });
+      }
       if (earlyRecoverableSetup) {
         // Setup has no eventing sink yet. Carry only the fixed, safe diagnostic
         // through Temporal into exact-attempt workflow failure settlement.

package/src/activities/agent-turn/finalization-monitor.ts

@@ -1,4 +1,4 @@
-import type { Observability } from "@opengeni/observability";
+import { turnExecutionTelemetryKey, type Observability } from "@opengeni/observability";
 import type { TurnHeartbeatDetails } from "../../op-journal";
 import { armTurnQuiescenceWatchdog } from "./quiescence";
 
@@ -42,10 +42,21 @@ export function startTurnFinalizationMonitor(input: {
   details: TurnHeartbeatDetails;
   heartbeat: (details: TurnHeartbeatDetails) => void;
   requestWorkerDrain: () => void;
+  execution?: { workspaceId: string; sessionId: string; attemptId: string };
   timeoutMs?: number;
   slowAfterMs?: number;
 }) {
   const { observability, details } = input;
+  const correlation: { correlationId?: string } = {};
+  diagnose(() => {
+    if (input.execution) {
+      correlation.correlationId = turnExecutionTelemetryKey(
+        input.execution.workspaceId,
+        input.execution.sessionId,
+        input.execution.attemptId,
+      );
+    }
+  });
   if (!initialized.has(observability)) {
     for (const stage of TURN_FINALIZATION_STAGES) {
       diagnose(() => observability.incrementGauge({ ...INFLIGHT, labels: { stage }, amount: 0 }));
@@ -88,6 +99,7 @@ export function startTurnFinalizationMonitor(input: {
               surface: "turn_finalization",
               outcome: "containment",
               reason: next,
+              ...correlation,
             });
           } catch {
             // Diagnostic failure must never disable physical containment.
@@ -104,6 +116,7 @@ export function startTurnFinalizationMonitor(input: {
               surface: "turn_finalization",
               outcome: "slow",
               reason: next,
+              ...correlation,
             });
           }),
         input.slowAfterMs ?? 30_000,

package/src/activities/agent-turn/finalization.ts

@@ -128,6 +128,11 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
       }
     },
     requestWorkerDrain: deps.requestWorkerDrain,
+    execution: {
+      workspaceId: deps.input.workspaceId,
+      sessionId: deps.input.sessionId,
+      attemptId: deps.input.attemptId,
+    },
   });
   try {
     monitor.enter("tool_writers");