@opengeni/worker-bundle 2.3.1 → 2.3.2-canary.37111190284001

package/dist/{activities-turn-KU25J3VQ.js → activities-turn-E3EA7FPS.js}

@@ -4,6 +4,7 @@ import {
   PostCompactionContinuationEmptyError,
   SandboxDeadlineRotationError,
   agentRunFailurePayload,
+  agentRunRecoveryFailurePayload,
   armAndReconcileCodexCapacityWait,
   armTurnQuiescenceWatchdog,
   assertAgentStreamNotCancelled,
@@ -31,6 +32,7 @@ import {
   compactionFailureReasonFromError,
   compactionFailureTurnEventPayload,
   computeIdleDelayMs,
+  createClaudeUsageObserver,
   createFreshSandboxReadinessReplacementBudget,
   createKnowledgeSourceSyncActivities,
   createModelHistoryAttachmentProjector,
@@ -95,7 +97,7 @@ import {
   withFirstPartyTools,
   wrapLazyTurnBoxWithRouting,
   wrapTurnBoxWithRouting
-} from "./chunk-3K3ULWEE.js";
+} from "./chunk-PA7Z55P4.js";
 import {
   StreamTimingMetrics,
   makeMachineOpObserver,
@@ -127,7 +129,7 @@ import {
   runtimeMetricsHooksForObservability,
   sandboxAutomaticRecoveryOutcome,
   turnLifecycleMetricsFor
-} from "./chunk-YG27NEMX.js";
+} from "./chunk-YOI3Z3GF.js";
 import {
   __export
 } from "./chunk-PZ5AY32C.js";
@@ -183,82 +185,6 @@ import {
 import { codexUpstreamModelSlugs } from "@opengeni/config";
 import { parseModelProvidersJson } from "@opengeni/config";
 import { withClaudeUsageObserver } from "@opengeni/runtime";
-
-// src/activities/agent-turn/claude-usage-observer.ts
-import {
-  emptyClaudeUsage,
-  mergeClaudeUsage,
-  parseClaudeUsageHeaders
-} from "@opengeni/config";
-async function createClaudeUsageObserver(providers, latest, readCredential) {
-  const bindings = await Promise.all(
-    providers.filter(
-      (provider) => provider.kind === "claude-subscription-workspace" || provider.kind === "claude-subscription-organization"
-    ).map(async (provider) => {
-      const scope = provider.kind === "claude-subscription-workspace" ? "workspace" : "organization";
-      const binding = provider.anthropic?.credentialBinding;
-      if (binding && provider.apiKey)
-        return [
-          provider.id,
-          {
-            scope,
-            token: provider.apiKey,
-            expectedConnectionId: binding.connectionId,
-            expectedCredentialVersion: binding.credentialVersion
-          }
-        ];
-      const credential = await readCredential(scope).catch(() => null);
-      if (!credential || credential.token !== provider.apiKey) return null;
-      return [
-        provider.id,
-        {
-          scope,
-          token: credential.token,
-          expectedConnectionId: credential.connectionId,
-          expectedCredentialVersion: credential.credentialVersion
-        }
-      ];
-    })
-  );
-  const captured = new Map(bindings.filter((binding) => binding !== null));
-  const observe2 = (providerId, response) => {
-    const binding = captured.get(providerId);
-    if (!binding) return;
-    const { scope, ...identity } = binding;
-    const previous = latest.get(scope);
-    let observation = parseClaudeUsageHeaders(response.headers);
-    if (observation && previous?.observation) {
-      const merged = mergeClaudeUsage(
-        mergeClaudeUsage(emptyClaudeUsage(binding.expectedCredentialVersion), previous.observation),
-        observation
-      );
-      observation = {
-        windows: merged.windows,
-        observedAt: merged.observedAt,
-        source: merged.source
-      };
-    }
-    if (observation || response.status === 401)
-      latest.set(scope, {
-        ...identity,
-        ...previous?.observation && !observation ? { observation: previous.observation } : {},
-        ...observation ? { observation } : {},
-        ...response.status === 401 ? { refresh: { status: "reconnect", checkedAt: (/* @__PURE__ */ new Date()).toISOString() } } : {}
-      });
-  };
-  return Object.assign(observe2, {
-    binding(providerId) {
-      return captured.get(providerId);
-    },
-    renew(providerId, credential) {
-      const binding = captured.get(providerId);
-      if (binding && binding.expectedConnectionId === credential.connectionId && binding.expectedCredentialVersion === credential.credentialVersion)
-        captured.set(providerId, { ...binding, token: credential.token });
-    }
-  });
-}
-
-// src/activities/agent-turn/run.ts
 import {
   xaiSubscriptionRequestStorage
 } from "@opengeni/xai-subscription";
@@ -1261,161 +1187,72 @@ import {
   heartbeatXaiCredentialLeaseUntil
 } from "@opengeni/db";
 
-// src/activities/agent-turn/codex.ts
-import {
-  listCodexAccountStatuses,
-  fetchCodexUsageForAccount
-} from "@opengeni/db";
-import { createHash as createHash2 } from "crypto";
-function codexWorkspaceMetricKey(workspaceId) {
-  return createHash2("sha256").update(workspaceId).digest("hex").slice(0, 12);
-}
-function acceptsPromptCacheKeyForTurn(resolvedModel) {
-  if (!resolvedModel) {
-    return true;
-  }
-  return resolvedModel.provider.builtin === true || resolvedModel.provider.kind === "codex-subscription" || // Native Claude consumes this internal key as session identity; it never
-  // sends OpenAI's prompt_cache_key field to the Messages endpoint.
-  resolvedModel.provider.api === "anthropic-messages";
-}
-async function refreshCappedCodexUsageRows(db, settings, workspaceId, accounts, capacitySignals, acceptedTurnId) {
-  const now = /* @__PURE__ */ new Date();
-  const stale = accounts.filter(
-    (account) => account.status === "active" && codexAccountNeedsLiveCapacityRefresh(account, now)
-  );
-  if (stale.length === 0) {
-    return accounts;
-  }
-  await refreshCodexUsageAndRepairCapacityWaiters(
-    stale.map(
-      (account) => () => fetchCodexUsageForAccount(db, settings, workspaceId, account.id, void 0, acceptedTurnId)
-    ),
-    () => signalPendingCodexCapacityWakeTargets({ db, ...capacitySignals }, workspaceId)
-  );
-  return listCodexAccountStatuses(db, workspaceId, acceptedTurnId).catch(() => accounts);
-}
-function codexCredentialLeaseDeadlineExpired(confirmedUntilMs, nowMs = performance.now()) {
-  return confirmedUntilMs === null || !Number.isFinite(confirmedUntilMs) || confirmedUntilMs <= nowMs;
-}
-
-// src/activities/agent-turn/credential-leases.ts
-var CodexCredentialLeaseLostError = class extends Error {
-  constructor(reason) {
-    super("Codex credential lease is not usable for provider dispatch");
-    this.reason = reason;
-    this.name = "CodexCredentialLeaseLostError";
-  }
-  code = "codex_credential_lease_lost";
-};
-var CodexTurnLease = class {
-  constructor(deps) {
-    this.deps = deps;
+// src/activities/agent-turn/subscription-lease.ts
+var SubscriptionTurnLease = class {
+  constructor(leaseDeps) {
+    this.leaseDeps = leaseDeps;
+    if (!Number.isFinite(leaseDeps.ttlMs) || leaseDeps.ttlMs <= 0)
+      throw new Error("Subscription lease TTL must be positive");
+    this.now = leaseDeps.now ?? (() => performance.now());
   }
   held = false;
   lost = false;
   lossReason = null;
   holderId = null;
   generation = null;
-  // Monotonic worker deadline, not a comparison between the Postgres and
-  // worker wall clocks. It is advanced only after a database renewal confirms,
-  // from the request START + TTL; slow queries therefore shorten (never extend)
-  // the conservative ownership window.
   confirmedUntilMs = null;
   heartbeatInFlight = false;
   heartbeatTimer;
+  now;
+  expired = (deadline) => deadline === null || !Number.isFinite(deadline) || this.now() >= deadline;
   markLost = (reason) => {
     if (this.lost) return;
     this.lost = true;
     this.lossReason = reason;
-    this.deps.observability.incrementCounter({
-      name: "opengeni_codex_lease_renewals_total",
-      help: "Codex lease renewal checkpoints by outcome and reason.",
-      labels: { workspace_key: this.deps.codexWorkspaceKey, outcome: "lost", reason }
-    });
-    this.deps.observability.warn("Codex credential lease was lost during an active turn", {
-      workspaceId: this.deps.workspaceId,
-      turnId: this.deps.getTurnId(),
-      reason
-    });
+    this.leaseDeps.onLost(reason);
   };
-  /** Fail closed at the last worker-confirmed lease deadline before dispatch. */
   assertUsable = () => {
-    if (this.lost) {
-      throw new CodexCredentialLeaseLostError(this.lossReason ?? "not_found");
-    }
-    if (!this.held || this.holderId === null || this.generation === null || codexCredentialLeaseDeadlineExpired(this.confirmedUntilMs)) {
+    if (this.lost) throw this.leaseDeps.lostError(this.lossReason ?? "not_found");
+    if (!this.held || this.holderId === null || this.generation === null || this.expired(this.confirmedUntilMs)) {
       this.markLost("deadline");
-      throw new CodexCredentialLeaseLostError("deadline");
+      throw this.leaseDeps.lostError("deadline");
     }
   };
-  renew = async (reason) => {
-    const turnId = this.deps.getTurnId();
-    if (!turnId || !this.held || !this.holderId || this.generation === null || this.lost) {
-      return;
-    }
-    if (codexCredentialLeaseDeadlineExpired(this.confirmedUntilMs)) {
+  renew = async (reason = "timer") => {
+    const turnId = this.leaseDeps.getTurnId();
+    if (!turnId || !this.held || !this.holderId || this.generation === null || this.lost) return;
+    if (this.expired(this.confirmedUntilMs)) {
       this.markLost("deadline");
       return;
     }
     if (this.heartbeatInFlight) return;
     this.heartbeatInFlight = true;
-    const priorConfirmedUntilMs = this.confirmedUntilMs;
-    const renewalStartedAtMs = performance.now();
+    const priorDeadline = this.confirmedUntilMs;
+    const startedAt = this.now();
+    const identity = { turnId, holderId: this.holderId, generation: this.generation };
     try {
-      const renewedUntil = await heartbeatCodexCredentialLeaseUntil(
-        this.deps.db,
-        this.deps.accountId,
-        this.deps.workspaceId,
-        turnId,
-        this.holderId,
-        this.generation,
-        CODEX_CREDENTIAL_LEASE_TTL_MS
-      );
-      if (!renewedUntil) {
-        this.markLost("not_found");
-      } else if (codexCredentialLeaseDeadlineExpired(priorConfirmedUntilMs) || codexCredentialLeaseDeadlineExpired(renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS)) {
+      const renewedUntil = await this.leaseDeps.heartbeat(identity);
+      if (!this.held || this.lost || this.leaseDeps.getTurnId() !== identity.turnId || this.holderId !== identity.holderId || this.generation !== identity.generation)
+        return;
+      if (!renewedUntil) this.markLost("not_found");
+      else if (this.expired(priorDeadline) || this.expired(startedAt + this.leaseDeps.ttlMs))
         this.markLost("deadline");
-      } else {
-        this.confirmedUntilMs = renewalStartedAtMs + CODEX_CREDENTIAL_LEASE_TTL_MS;
-        this.deps.observability.incrementCounter({
-          name: "opengeni_codex_lease_renewals_total",
-          help: "Codex lease renewal checkpoints by outcome and reason.",
-          labels: {
-            workspace_key: this.deps.codexWorkspaceKey,
-            outcome: "completed",
-            reason
-          }
-        });
+      else {
+        this.confirmedUntilMs = startedAt + this.leaseDeps.ttlMs;
+        this.leaseDeps.onRenewed?.(reason);
       }
     } catch (error51) {
-      if (codexCredentialLeaseDeadlineExpired(this.confirmedUntilMs)) {
-        this.markLost("deadline");
+      if (!this.held || this.lost || this.leaseDeps.getTurnId() !== identity.turnId || this.holderId !== identity.holderId || this.generation !== identity.generation)
         return;
-      }
-      this.deps.observability.warn("Codex credential lease heartbeat failed", {
-        workspaceId: this.deps.workspaceId,
-        turnId,
-        reason,
-        ...safeErrorDiagnostic(error51)
-      });
-      this.deps.observability.incrementCounter({
-        name: "opengeni_codex_lease_renewals_total",
-        help: "Codex lease renewal checkpoints by outcome and reason.",
-        labels: {
-          workspace_key: this.deps.codexWorkspaceKey,
-          outcome: "error",
-          reason
-        }
-      });
+      if (this.expired(this.confirmedUntilMs)) this.markLost("deadline");
+      else this.leaseDeps.onError(error51, reason);
     } finally {
       this.heartbeatInFlight = false;
     }
   };
   startHeartbeat = () => {
-    if (!this.deps.getTurnId() || this.heartbeatTimer) return;
-    this.heartbeatTimer = setInterval(() => {
-      void this.renew("timer");
-    }, 6e4);
+    if (!this.leaseDeps.getTurnId() || this.heartbeatTimer) return;
+    this.heartbeatTimer = setInterval(() => void this.renew("timer"), 6e4);
     this.heartbeatTimer.unref?.();
   };
   stopHeartbeat = () => {
@@ -1424,79 +1261,94 @@ var CodexTurnLease = class {
     this.heartbeatTimer = void 0;
   };
 };
-var XaiTurnLease = class {
+
+// src/activities/agent-turn/credential-leases.ts
+var CodexCredentialLeaseLostError = class extends Error {
+  constructor(reason) {
+    super("Codex credential lease is not usable for provider dispatch");
+    this.reason = reason;
+    this.name = "CodexCredentialLeaseLostError";
+  }
+  code = "codex_credential_lease_lost";
+};
+var CodexTurnLease = class extends SubscriptionTurnLease {
   constructor(deps) {
-    this.deps = deps;
+    super({
+      ttlMs: CODEX_CREDENTIAL_LEASE_TTL_MS,
+      getTurnId: deps.getTurnId,
+      heartbeat: ({ turnId, holderId, generation }) => heartbeatCodexCredentialLeaseUntil(
+        deps.db,
+        deps.accountId,
+        deps.workspaceId,
+        turnId,
+        holderId,
+        generation,
+        CODEX_CREDENTIAL_LEASE_TTL_MS
+      ),
+      lostError: (reason) => new CodexCredentialLeaseLostError(reason),
+      onLost: (reason) => {
+        deps.observability.incrementCounter({
+          name: "opengeni_codex_lease_renewals_total",
+          help: "Codex lease renewal checkpoints by outcome and reason.",
+          labels: { workspace_key: deps.codexWorkspaceKey, outcome: "lost", reason }
+        });
+        deps.observability.warn("Codex credential lease was lost during an active turn", {
+          workspaceId: deps.workspaceId,
+          turnId: deps.getTurnId(),
+          reason
+        });
+      },
+      onRenewed: (reason) => deps.observability.incrementCounter({
+        name: "opengeni_codex_lease_renewals_total",
+        help: "Codex lease renewal checkpoints by outcome and reason.",
+        labels: { workspace_key: deps.codexWorkspaceKey, outcome: "completed", reason }
+      }),
+      onError: (error51, reason) => {
+        deps.observability.warn("Codex credential lease heartbeat failed", {
+          workspaceId: deps.workspaceId,
+          turnId: deps.getTurnId(),
+          reason,
+          ...safeErrorDiagnostic(error51)
+        });
+        deps.observability.incrementCounter({
+          name: "opengeni_codex_lease_renewals_total",
+          help: "Codex lease renewal checkpoints by outcome and reason.",
+          labels: { workspace_key: deps.codexWorkspaceKey, outcome: "error", reason }
+        });
+      }
+    });
   }
-  held = false;
-  lost = false;
+};
+var XaiTurnLease = class extends SubscriptionTurnLease {
   subjectId = null;
-  holderId = null;
-  generation = null;
-  confirmedUntilMs = null;
-  heartbeatInFlight = false;
-  heartbeatTimer;
-  markLost = (reason) => {
-    if (this.lost) return;
-    this.lost = true;
-    this.deps.observability.warn("xAI credential lease was lost during an active turn", {
-      workspaceId: this.deps.workspaceId,
-      turnId: this.deps.getTurnId(),
-      reason
-    });
-  };
-  renew = async () => {
-    const turnId = this.deps.getTurnId();
-    if (!turnId || !this.held || !this.subjectId || !this.holderId || this.generation === null || this.lost) {
-      return;
-    }
-    if (codexCredentialLeaseDeadlineExpired(this.confirmedUntilMs)) {
-      this.markLost("deadline");
-      return;
-    }
-    if (this.heartbeatInFlight) return;
-    this.heartbeatInFlight = true;
-    const renewalStartedAtMs = performance.now();
-    try {
-      const renewedUntil = await heartbeatXaiCredentialLeaseUntil(this.deps.db, {
-        workspaceId: this.deps.workspaceId,
+  constructor(deps) {
+    super({
+      ttlMs: XAI_CREDENTIAL_LEASE_TTL_MS,
+      getTurnId: deps.getTurnId,
+      heartbeat: ({ turnId, holderId, generation }) => this.subjectId ? heartbeatXaiCredentialLeaseUntil(deps.db, {
+        workspaceId: deps.workspaceId,
         subjectId: this.subjectId,
         turnId,
-        holderId: this.holderId,
-        generation: this.generation,
+        holderId,
+        generation,
         leaseTtlMs: XAI_CREDENTIAL_LEASE_TTL_MS
-      });
-      if (!renewedUntil) {
-        this.markLost("not_found");
-      } else {
-        this.confirmedUntilMs = renewalStartedAtMs + XAI_CREDENTIAL_LEASE_TTL_MS;
-      }
-    } catch (error51) {
-      if (codexCredentialLeaseDeadlineExpired(this.confirmedUntilMs)) {
-        this.markLost("deadline");
-        return;
-      }
-      this.deps.observability.warn("xAI credential lease heartbeat failed", {
-        workspaceId: this.deps.workspaceId,
-        turnId,
+      }) : Promise.resolve(null),
+      lostError: (reason) => Object.assign(new Error("SuperGrok credential lease is not usable for provider dispatch"), {
+        code: "xai_credential_lease_lost",
+        reason
+      }),
+      onLost: (reason) => deps.observability.warn("xAI credential lease was lost during an active turn", {
+        workspaceId: deps.workspaceId,
+        turnId: deps.getTurnId(),
+        reason
+      }),
+      onError: (error51) => deps.observability.warn("xAI credential lease heartbeat failed", {
+        workspaceId: deps.workspaceId,
+        turnId: deps.getTurnId(),
         ...safeErrorDiagnostic(error51)
-      });
-    } finally {
-      this.heartbeatInFlight = false;
-    }
-  };
-  startHeartbeat = () => {
-    if (!this.deps.getTurnId() || this.heartbeatTimer) return;
-    this.heartbeatTimer = setInterval(() => {
-      void this.renew();
-    }, 6e4);
-    this.heartbeatTimer.unref?.();
-  };
-  stopHeartbeat = () => {
-    if (!this.heartbeatTimer) return;
-    clearInterval(this.heartbeatTimer);
-    this.heartbeatTimer = void 0;
-  };
+      })
+    });
+  }
 };
 function createTurnCredentialLeases(deps) {
   const codex = new CodexTurnLease(deps);
@@ -1539,7 +1391,7 @@ import {
   settleGeneratedImageArtifactReady
 } from "@opengeni/db";
 import { retryWhileMissing as retryWhileMissing2 } from "@opengeni/storage";
-import { createHash as createHash4 } from "crypto";
+import { createHash as createHash3 } from "crypto";
 
 // src/activities/retained-screenshots.ts
 import {
@@ -1563,7 +1415,7 @@ import {
   settleRetainedScreenshotArtifactReady
 } from "@opengeni/db";
 import { retryWhileMissing } from "@opengeni/storage";
-import { createHash as createHash3 } from "crypto";
+import { createHash as createHash2 } from "crypto";
 var PNG_SIGNATURE = Uint8Array.of(137, 80, 78, 71, 13, 10, 26, 10);
 var PNG_ALLOWED_CRITICAL_CHUNKS = /* @__PURE__ */ new Set(["IHDR", "PLTE", "IDAT", "IEND"]);
 var PNG_MAX_CHUNKS = 16384;
@@ -1847,7 +1699,7 @@ function validateComputerScreenshot(input, limits = {}) {
     bytes: input.bytes,
     mediaType: "image/png",
     sizeBytes: input.bytes.byteLength,
-    sha256: createHash3("sha256").update(input.bytes).digest("hex"),
+    sha256: createHash2("sha256").update(input.bytes).digest("hex"),
     width,
     height
   };
@@ -1903,11 +1755,11 @@ function validateRetainableSessionImage(input, limits = {}) {
     bytes: input.bytes,
     ...identity,
     sizeBytes: input.bytes.byteLength,
-    sha256: createHash3("sha256").update(input.bytes).digest("hex")
+    sha256: createHash2("sha256").update(input.bytes).digest("hex")
   };
 }
 function retainedScreenshotIdentity(input) {
-  const settlementKey = createHash3("sha256").update(
+  const settlementKey = createHash2("sha256").update(
     input.kind === "browser_screenshot" ? "opengeni:browser-screenshot:v1\0" : "opengeni:computer-screenshot:v1\0"
   ).update(input.sessionId).update("\0").update(input.turnId).update("\0").update(input.attemptId).update("\0").update(input.toolCallId).update("\0").update(input.toolOutputId).digest("hex");
   const uuidBytes = Uint8Array.from(Buffer.from(settlementKey.slice(0, 32), "hex"));
@@ -2621,7 +2473,7 @@ function validateGeneratedImage(input) {
   }
 }
 function generatedImageIdentity(input) {
-  const hash2 = createHash4("sha256").update("opengeni:generated-image:v2\0").update(input.workspaceId).update("\0").update(input.sourceStrategy).update("\0").update(input.providerId).update("\0").update(input.providerBindingHash).update("\0");
+  const hash2 = createHash3("sha256").update("opengeni:generated-image:v2\0").update(input.workspaceId).update("\0").update(input.sourceStrategy).update("\0").update(input.providerId).update("\0").update(input.providerBindingHash).update("\0");
   if (input.sourceStrategy === "native_hosted") {
     hash2.update(input.providerItemId);
   } else {
@@ -3378,7 +3230,7 @@ function completedToolCallFromSdkEvent(event) {
 }
 
 // src/activities/agent-turn/history-prefix.ts
-import { createHash as createHash5 } from "crypto";
+import { createHash as createHash4 } from "crypto";
 import { normalizeProtocolJsonValue as normalizeProtocolJsonValue2, sanitizeHistoryItemsForModel as sanitizeHistoryItemsForModel2 } from "@opengeni/runtime";
 var HistoryPrefixGuard = class {
   persisted = [];
@@ -3404,7 +3256,7 @@ var HistoryPrefixGuard = class {
   }
   keys(items, truncationTokens) {
     return sanitizeHistoryItemsForModel2(items, truncationTokens).map(
-      (item) => createHash5("sha256").update(JSON.stringify(sorted(normalizeProtocolJsonValue2(item)))).digest("hex")
+      (item) => createHash4("sha256").update(JSON.stringify(sorted(normalizeProtocolJsonValue2(item)))).digest("hex")
     );
   }
 };
@@ -3611,6 +3463,40 @@ function safeObserveHistogram(observability, input) {
   }
 }
 
+// src/activities/agent-turn/codex.ts
+import {
+  listCodexAccountStatuses,
+  fetchCodexUsageForAccount
+} from "@opengeni/db";
+import { createHash as createHash5 } from "crypto";
+function codexWorkspaceMetricKey(workspaceId) {
+  return createHash5("sha256").update(workspaceId).digest("hex").slice(0, 12);
+}
+function acceptsPromptCacheKeyForTurn(resolvedModel) {
+  if (!resolvedModel) {
+    return true;
+  }
+  return resolvedModel.provider.builtin === true || resolvedModel.provider.kind === "codex-subscription" || // Native Claude consumes this internal key as session identity; it never
+  // sends OpenAI's prompt_cache_key field to the Messages endpoint.
+  resolvedModel.provider.api === "anthropic-messages";
+}
+async function refreshCappedCodexUsageRows(db, settings, workspaceId, accounts, capacitySignals, acceptedTurnId) {
+  const now = /* @__PURE__ */ new Date();
+  const stale = accounts.filter(
+    (account) => account.status === "active" && codexAccountNeedsLiveCapacityRefresh(account, now)
+  );
+  if (stale.length === 0) {
+    return accounts;
+  }
+  await refreshCodexUsageAndRepairCapacityWaiters(
+    stale.map(
+      (account) => () => fetchCodexUsageForAccount(db, settings, workspaceId, account.id, void 0, acceptedTurnId)
+    ),
+    () => signalPendingCodexCapacityWakeTargets({ db, ...capacitySignals }, workspaceId)
+  );
+  return listCodexAccountStatuses(db, workspaceId, acceptedTurnId).catch(() => accounts);
+}
+
 // src/activities/agent-turn/turn-context.ts
 function createTurnContext(input) {
   return {
@@ -3809,7 +3695,7 @@ function startTurnFinalizationMonitor(input) {
           } catch {
           }
         },
-        terminateWorker: input.terminateWorker
+        terminateWorker: input.requestWorkerDrain
       });
       slowTimer = setTimeout(
         () => diagnose(() => {
@@ -4671,6 +4557,7 @@ async function finalizeTurnAttempt(deps) {
   deps.eventing.heartbeatTimer = startActivityHeartbeat(deps.activityContext, details);
   const monitor = startTurnFinalizationMonitor({
     observability: deps.observability,
+    ...deps.turnFinalizationTimeoutMs === void 0 ? {} : { timeoutMs: deps.turnFinalizationTimeoutMs },
     details,
     heartbeat: (value) => {
       try {
@@ -4678,7 +4565,7 @@ async function finalizeTurnAttempt(deps) {
       } catch {
       }
     },
-    terminateWorker: () => process.exit(1)
+    requestWorkerDrain: deps.requestWorkerDrain
   });
   try {
     monitor.enter("tool_writers");
@@ -4912,12 +4799,12 @@ async function finalizeTurnAttemptSteps(deps, monitor) {
       );
     }
     monitor.enter("provider_leases");
-    for (const [scope, snapshot] of providerTurn.latestClaudeUsage) {
+    for (const snapshot of providerTurn.latestClaudeUsage.values()) {
       await waitForTurnFinalizerStep(
         recordClaudeSubscriptionUsage(
           db,
           settings,
-          { accountId: input.accountId, workspaceId: input.workspaceId, scope },
+          { accountId: input.accountId, workspaceId: input.workspaceId, scope: snapshot.scope },
           snapshot
         ).catch(() => null),
         finalizerSignal
@@ -5271,13 +5158,15 @@ import {
   settleCodexCredentialFailover,
   readLease,
   SandboxLeaseSupersededError,
-  isSessionEventPersistenceError as isSessionEventPersistenceError2
+  isSessionEventPersistenceError as isSessionEventPersistenceError2,
+  SANDBOX_SETUP_RECOVERY_LIMIT
 } from "@opengeni/db";
 import { publishDurableSessionEvents as publishDurableSessionEvents2 } from "@opengeni/events";
 import {
   maxTurnsExceededRunState,
   isModalTaskExecStartPreDispatchUnavailableError,
-  isModalCommandStartOutcomeUnknownError
+  isModalCommandStartOutcomeUnknownError,
+  isProviderCommandObservationUnavailableError
 } from "@opengeni/runtime";
 import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
 import { TurnExecutionPolicyDefinitionMismatchError } from "@opengeni/config";
@@ -6141,7 +6030,8 @@ async function settleTurnFailure(deps) {
     }
   }
   const recoveryTurnId = attempt.turnId;
-  if (isModalCommandStartOutcomeUnknownError(error51) && recoveryTurnId && attempt.triggerEventId && attempt.executionGeneration > 0) {
+  const observationUnavailable = isProviderCommandObservationUnavailableError(error51);
+  if ((isModalCommandStartOutcomeUnknownError(error51) || observationUnavailable) && recoveryTurnId && attempt.triggerEventId && attempt.executionGeneration > 0) {
     let recovery;
     try {
       if (eventing.turnStartedPublished) {
@@ -6153,10 +6043,10 @@ async function settleTurnFailure(deps) {
         turnId: recoveryTurnId,
         triggerEventId: attempt.triggerEventId,
         attemptId: input.attemptId,
-        reason: "sandbox_command_start_outcome_unknown",
+        reason: observationUnavailable ? "sandbox_command_observation_unavailable" : "sandbox_command_start_outcome_unknown",
         sandboxSetupOutcomeUnknown: true,
         detail: {
-          code: "sandbox_command_start_outcome_unknown",
+          code: observationUnavailable ? "sandbox_command_observation_unavailable" : "sandbox_command_start_outcome_unknown",
           retryable: false,
           setupOutcome: "unknown",
           replay: "blocked",
@@ -7218,7 +7108,39 @@ async function settleTurnFailure(deps) {
       attemptNumber: nextProviderRecoveryCount,
       retryAfterMs: providerRetryAfterMs(error51)
     });
+    const setupRecoveryExhausted = earlyCommandStartUnavailable && recoveryResult.status === "exhausted" && attempt.providerRecoveryCount === SANDBOX_SETUP_RECOVERY_LIMIT;
     try {
+      if (setupRecoveryExhausted) {
+        const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
+          sessionId: input.sessionId,
+          turnId: attempt.turnId,
+          triggerEventId: attempt.triggerEventId,
+          attemptId: input.attemptId,
+          reason: "sandbox_command_start_recovery_exhausted",
+          sandboxSetupRecoveryExhausted: true,
+          detail: {
+            code: failure2.code,
+            error: "Automatic sandbox setup recovery exhausted after five retries; the accepted turn remains parked without starting a command.",
+            retryable: false,
+            setupOutcome: "not_started",
+            replay: "blocked",
+            recoveryExhausted: true,
+            providerRecoveryCount: SANDBOX_SETUP_RECOVERY_LIMIT
+          }
+        });
+        if (recovery.action === "stale") {
+          acknowledgeLostAttemptOwnership();
+          control.activityStatus = "cancelled";
+          control.turnMetricOutcome = "cancelled";
+          return claimedResult({ status: "cancelled" });
+        }
+        acknowledgeRecoveryQuiescence();
+        await publishDurableSessionEvents2(bus, input.workspaceId, input.sessionId, recovery.events);
+        control.turnMetricOutcome = "recovering";
+        control.activityStatus = "recovering";
+        control.activityError = error51;
+        return claimedResult({ status: "recovering" });
+      }
       if (recoveryResult.status === "recovering") {
         if (!earlyRecoverableSetup) {
           await flushRuntimeBatcher();
@@ -7232,7 +7154,7 @@ async function settleTurnFailure(deps) {
           reason: failure2.code ?? "provider_unavailable",
           providerRecoveryCount: nextProviderRecoveryCount,
           detail: {
-            ...failure2,
+            ...agentRunRecoveryFailurePayload(error51, failure2),
             continueDelayMs: recoveryResult.continueDelayMs,
             providerRecoveryCount: nextProviderRecoveryCount
           }
@@ -7288,6 +7210,12 @@ async function settleTurnFailure(deps) {
           failureCode: failure2.code ?? "provider_unavailable",
           providerRecoveryCount: nextProviderRecoveryCount
         }
+      }) : setupRecoveryExhausted ? postClaimDatabaseRecoveryFailure({
+        error: recoveryError,
+        turnId: attempt.turnId,
+        triggerEventId: attempt.triggerEventId,
+        executionGeneration: attempt.executionGeneration,
+        sandboxSetupRecoveryExhausted: true
       }) : null;
       if (postClaimRecovery) {
         control.activityStatus = "recovering";
@@ -7899,6 +7827,13 @@ async function resultItemForOpenSuffixMember(input) {
   const name = toolNameFromCallItem(input.row.callItem);
   const rejected = () => {
     const payload2 = input.trigger.payload;
+    if (input.row.interruptionKind === "approval" && payload2.decision === "reject" && !(typeof payload2.message === "string" && payload2.message.trim().length > 0)) {
+      return memberResult({
+        callId: input.row.callId,
+        name,
+        output: "Tool approval was rejected. This proposed tool call was not executed."
+      });
+    }
     const message = typeof payload2.message === "string" && payload2.message.trim().length > 0 ? payload2.message : "Tool approval was rejected.";
     const interrupted = interruptedToolCallResult({
       callType: input.row.callType,
@@ -10079,6 +10014,10 @@ async function runTurnStreamAttempt(deps) {
   }
 }
 
+// src/activities/agent-turn/claim.ts
+import { withDirectModelProviders } from "@opengeni/config";
+import { loadDirectModelProviderConnection } from "@opengeni/db";
+
 // src/activities/agent-turn/recovery-warning.ts
 import { getSandboxRecoveryDiscontinuity } from "@opengeni/db";
 var FILESYSTEM_DISCONTINUITY_PROTOCOL = 3;
@@ -10511,6 +10450,15 @@ async function claimTurnAttempt(deps) {
     workspaceProviderSettings,
     claimedPolicy.kind === "valid" ? claimedPolicy.policy.productModelId : turn.model
   );
+  const selectedDirectConnection = await loadDirectModelProviderConnection(
+    db,
+    capabilitySettings,
+    input.workspaceId,
+    claimedPolicy.kind === "valid" ? claimedPolicy.policy.productModelId : turn.model ?? ""
+  );
+  if (selectedDirectConnection) {
+    capabilitySettings = withDirectModelProviders(capabilitySettings, [selectedDirectConnection]);
+  }
   const codexAppsCredentialId = capabilitySettings.codexConnectedAppsEnabled ? await resolveCodexAppsCredentialIdForRun(db, input.workspaceId) : null;
   const candidatePolicy = claimedPolicy.kind === "valid" ? claimedPolicy.policy : resolveTurnExecutionPolicyV12(capabilitySettings, legacyTurnExecutionPolicyInput(turn));
   const credentialRestriction = claimedPolicy.kind === "absent" ? turn.initiatorContext?.credentialRestriction === "developer_setup" ? "developer_setup" : turnCredentialRestriction(candidatePolicy, session.metadata) : void 0;
@@ -13001,6 +12949,7 @@ import { createProviderCommandRetainer } from "@opengeni/db/retained-provider-co
 import {
   RoutingMutationOutcomeUnknownError,
   ProviderCommandStartOutcomeUnknownError,
+  ProviderCommandObservationUnavailableError,
   isModalCommandStartOutcomeUnknownError as isModalCommandStartOutcomeUnknownError2,
   withModalCommandStartSignal,
   resolveModalCheckpointProviderBindingForSession,
@@ -13028,7 +12977,7 @@ function internalMutationUnknownCommand(error51, instanceId) {
     if (!value || typeof value !== "object" || seen.has(value)) continue;
     seen.add(value);
     try {
-      if (value instanceof ProviderCommandStartOutcomeUnknownError) {
+      if (value instanceof ProviderCommandStartOutcomeUnknownError || value instanceof ProviderCommandObservationUnavailableError) {
         unknown2 = true;
         let remaining = 64;
         const copyCommandData = (candidate, descriptorDepth = 0) => {
@@ -14161,7 +14110,8 @@ async function establishTurnSandbox(deps) {
               observability,
               freshSandboxReadinessReplacementBudget,
               onSandboxLost: publishSandboxLost,
-              objectStorage
+              objectStorage,
+              bus
             },
             {
               accountId: input.accountId,
@@ -14236,7 +14186,8 @@ async function establishTurnSandbox(deps) {
                 observability,
                 freshSandboxReadinessReplacementBudget,
                 onSandboxLost: publishSandboxLost,
-                objectStorage
+                objectStorage,
+                bus
               },
               {
                 accountId: input.accountId,
@@ -34291,6 +34242,7 @@ function configuredBundledSkillNames(context) {
     { name: "document-parsing", include: () => true },
     { name: "opengeni-skills", include: () => true },
     { name: "opengeni-projects", include: () => true },
+    { name: "opengeni-schedules", include: () => tools.has("scheduled_tasks_create") },
     { name: "opengeni-documents", include: artifacts },
     { name: "opengeni-spreadsheets", include: artifacts },
     { name: "opengeni-presentations", include: artifacts },
@@ -34308,6 +34260,7 @@ function loadConfiguredBundledSkills(context) {
   const names = new Set(configuredBundledSkillNames(context));
   const artifacts = loadNativeToolSkillArtifacts({
     projects: names.has("opengeni-projects"),
+    schedules: names.has("opengeni-schedules"),
     editableArtifacts: [
       "opengeni-documents",
       "opengeni-spreadsheets",
@@ -37388,6 +37341,8 @@ function createRunAgentTurnActivity(services) {
       bus,
       runtime,
       summarizeContextForCompaction,
+      requestWorkerDrain,
+      turnFinalizationTimeoutMs,
       objectStorage,
       observability,
       wakeSessionWorkflow,
@@ -38011,28 +37966,28 @@ function createRunAgentTurnActivity(services) {
             })
           );
           const withClaudeUsage = (fn) => withClaudeUsageObserver(claudeUsageObserver, fn, async (providerId, headers) => {
-            const binding = claudeUsageObserver.binding(providerId);
-            if (!binding) return headers;
-            const credential = await resolveClaudeSubscriptionCredential(
-              db,
-              settings,
-              {
-                accountId: input.accountId,
-                workspaceId: input.workspaceId,
-                scope: binding.scope
-              },
-              {
-                expectedConnectionId: binding.expectedConnectionId,
-                expectedCredentialVersion: binding.expectedCredentialVersion
+            return claudeUsageObserver.prepareRequestWithObserver(
+              providerId,
+              headers,
+              async (binding) => {
+                const credential = await resolveClaudeSubscriptionCredential(
+                  db,
+                  settings,
+                  {
+                    accountId: input.accountId,
+                    workspaceId: input.workspaceId,
+                    scope: binding.scope
+                  },
+                  {
+                    expectedConnectionId: binding.expectedConnectionId,
+                    expectedCredentialVersion: binding.expectedCredentialVersion
+                  }
+                );
+                if (credential && "reconnectRequired" in credential)
+                  throw new ClaudeSubscriptionReconnectRequired();
+                return credential;
               }
             );
-            if (!credential) return headers;
-            if ("reconnectRequired" in credential)
-              throw new ClaudeSubscriptionReconnectRequired();
-            claudeUsageObserver.renew(providerId, credential);
-            headers.set("authorization", `Bearer ${credential.token}`);
-            headers.delete("x-api-key");
-            return headers;
           });
           const withCodex = (fn) => codexContext ? codexRequestStorage.run(codexContext, fn) : fn();
           const withProviderRequestContext = (fn) => withClaudeUsage(
@@ -38842,6 +38797,8 @@ These are publication receipts for the human's earlier answer. Do not call the r
       });
     } finally {
       await finalizeTurnAttempt({
+        turnFinalizationTimeoutMs,
+        requestWorkerDrain,
         input,
         settings,
         db,
@@ -39982,12 +39939,17 @@ function createTurnActivityServices(dependencies) {
   let servicesPromise = null;
   return async () => {
     servicesPromise ??= (async () => {
+      if (!dependencies.requestWorkerDrain) {
+        throw new Error("Turn activities require a host requestWorkerDrain lifecycle edge");
+      }
       const services = await shared();
       return {
         ...services,
         runtime: dependencies.runtime ?? createProductionAgentRuntime({
           metrics: runtimeMetricsHooksForObservability(services.observability)
         }),
+        requestWorkerDrain: dependencies.requestWorkerDrain,
+        turnFinalizationTimeoutMs: dependencies.turnFinalizationTimeoutMs,
         summarizeContextForCompaction: dependencies.summarizeContextForCompaction ?? summarizeForCompaction2
       };
     })();
@@ -40007,4 +39969,4 @@ export {
   createTurnActivities,
   createTurnActivitiesFromServices
 };
-//# sourceMappingURL=activities-turn-KU25J3VQ.js.map
+//# sourceMappingURL=activities-turn-E3EA7FPS.js.map