@opengeni/worker-bundle 2.3.1 → 2.3.2-canary.37090756480001

package/src/activities/agent-turn/errors.ts

@@ -1,4 +1,5 @@
 import { SandboxCapabilitiesChangedError } from "./provider-dispatch-barrier";
+import { ClaudeSubscriptionConnectionUnavailable } from "./claude-usage-observer";
 import {
   ActiveSessionHistoryLimitExceededError,
   ApprovalRunStateLimitExceededError,
@@ -12,6 +13,7 @@ import {
 } from "@opengeni/db";
 import {
   ActiveBackendUnresolvableError,
+  AnthropicProviderRejection,
   CompactionProviderResponseError,
   EmptyCompactionSummaryError,
   compactionProviderRejection,
@@ -21,6 +23,7 @@ import {
   isMcpTransportConnectivityError,
   isModalTaskExecStartPreDispatchUnavailableError,
   isModalCommandStartOutcomeUnknownError,
+  isProviderCommandObservationUnavailableError,
   isRoutingMutationOutcomeUnknownError,
   RoutingWorkspaceRootChangedError,
   ResponsesStreamingTerminalError,
@@ -34,6 +37,7 @@ import {
   providerQuotaExhaustedMessage,
   SelfhostedWorkspaceRootChangedError,
   UNKNOWN_MODEL_FINISH_REASON_CODE,
+  AnthropicRequestError,
 } from "@opengeni/runtime";
 import {
   mcpTransportRequestFailureDiagnostic,
@@ -396,6 +400,7 @@ export function postClaimDatabaseRecoveryFailure(input: {
   triggerEventId: string;
   executionGeneration: number;
   sandboxSetupOutcomeUnknown?: true;
+  sandboxSetupRecoveryExhausted?: true;
   providerRecovery?: {
     failureCode: string;
     providerRecoveryCount: number;
@@ -403,7 +408,13 @@ export function postClaimDatabaseRecoveryFailure(input: {
 }): ApplicationFailure | null {
   const code = retryableDatabaseFailureCode(input.error);
   if (!code || input.executionGeneration < 1) return null;
-  if (input.sandboxSetupOutcomeUnknown && input.providerRecovery) return null;
+  if (
+    (input.sandboxSetupOutcomeUnknown && input.sandboxSetupRecoveryExhausted) ||
+    ((input.sandboxSetupOutcomeUnknown || input.sandboxSetupRecoveryExhausted) &&
+      input.providerRecovery)
+  ) {
+    return null;
+  }
   if (
     input.providerRecovery &&
     (!Number.isSafeInteger(input.providerRecovery.providerRecoveryCount) ||
@@ -419,6 +430,7 @@ export function postClaimDatabaseRecoveryFailure(input: {
     executionGeneration: input.executionGeneration,
     code,
     ...(input.sandboxSetupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {}),
+    ...(input.sandboxSetupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {}),
     ...(input.providerRecovery
       ? {
           providerFailureCode: input.providerRecovery.failureCode,
@@ -1032,15 +1044,44 @@ function isRawDatabaseQueryError(error: unknown): boolean {
   );
 }
 
+function anthropicRequestDiagnostic(error: unknown): AnthropicRequestError | undefined {
+  if (error instanceof AnthropicRequestError) return error;
+  return error instanceof Error && error.cause instanceof AnthropicRequestError
+    ? error.cause
+    : undefined;
+}
+
 export function agentRunFailurePayload(
   error: unknown,
   options: { isCodexTurn?: boolean } = {},
 ): ReturnType<typeof baseAgentRunFailurePayload> {
   const failure = baseAgentRunFailurePayload(error, options);
   const diagnostic = materializationVerificationDiagnostic(error);
+  const anthropic = anthropicRequestDiagnostic(error);
+  if (anthropic) {
+    return {
+      ...failure,
+      code: failure.code ?? anthropic.code,
+      retryable: failure.retryable ?? false,
+      ...(anthropic.detail ? { detail: anthropic.detail } : {}),
+      ...(anthropic.request_id ? { requestId: anthropic.request_id } : {}),
+    };
+  }
   return diagnostic ? { ...failure, materializationDiagnostic: diagnostic } : failure;
 }
 
+/** Keep Anthropic provider text on terminal failures, never recovery events. */
+export function agentRunRecoveryFailurePayload(
+  error: unknown,
+  failure: ReturnType<typeof agentRunFailurePayload>,
+): ReturnType<typeof agentRunFailurePayload> {
+  if (!anthropicRequestDiagnostic(error)) return failure;
+  // Project a copy: retry exhaustion still needs the terminal diagnostic.
+  const recovery = { ...failure };
+  delete recovery.detail;
+  return recovery;
+}
+
 function baseAgentRunFailurePayload(
   error: unknown,
   options: { isCodexTurn?: boolean } = {},
@@ -1111,6 +1152,17 @@ function baseAgentRunFailurePayload(
       ...(Object.keys(database).length > 0 ? { database } : {}),
     };
   }
+  if (error instanceof ClaudeSubscriptionConnectionUnavailable) {
+    return { error: error.message, code: error.code, retryable: false };
+  }
+  if (error instanceof AnthropicProviderRejection) {
+    return {
+      error: error.message,
+      code: error.code === "content_policy_violation" ? "provider_safety_refusal" : error.code,
+      retryable: false,
+      ...(error.request_id ? { requestId: error.request_id } : {}),
+    };
+  }
   const safetyRefusalDiagnostic = providerSafetyRefusalDiagnostic(error);
   if (safetyRefusalDiagnostic !== undefined) {
     return {
@@ -1194,6 +1246,14 @@ function baseAgentRunFailurePayload(
       retryable: true,
     };
   }
+  if (isProviderCommandObservationUnavailableError(error)) {
+    return {
+      error:
+        "A managed sandbox command cannot be observed. Its exact invocation and writer remain retained; setup is blocked without replay until the incomplete operation can be reconciled.",
+      code: "sandbox_command_observation_unavailable",
+      retryable: false,
+    };
+  }
   if (isModalCommandStartOutcomeUnknownError(error)) {
     return {
       error:

package/src/activities/agent-turn/failure-settlement.ts

@@ -13,6 +13,7 @@ import {
   readLease,
   SandboxLeaseSupersededError,
   isSessionEventPersistenceError,
+  SANDBOX_SETUP_RECOVERY_LIMIT,
   type CodexLeaseAccountStatus,
 } from "@opengeni/db";
 import { publishDurableSessionEvents } from "@opengeni/events";
@@ -20,6 +21,7 @@ import {
   maxTurnsExceededRunState,
   isModalTaskExecStartPreDispatchUnavailableError,
   isModalCommandStartOutcomeUnknownError,
+  isProviderCommandObservationUnavailableError,
 } from "@opengeni/runtime";
 import { ApplicationFailure, CancelledFailure } from "@temporalio/activity";
 import {
@@ -68,6 +70,7 @@ import {
   safeErrorDiagnostic,
   classifyXaiCredentialFailure,
   agentRunFailurePayload,
+  agentRunRecoveryFailurePayload,
   codexCredentialCooldownUntil,
   classifyCodexCredentialFailure,
   codexUsageLimitFailurePayload,
@@ -343,8 +346,9 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
   // cannot reconstruct setup on a replacement attempt. The exact command and
   // writer remain retained by sandbox-runtime; the logical turn is parked as
   // recovering with a durable no-replay marker, not failed or completed.
+  const observationUnavailable = isProviderCommandObservationUnavailableError(error);
   if (
-    isModalCommandStartOutcomeUnknownError(error) &&
+    (isModalCommandStartOutcomeUnknownError(error) || observationUnavailable) &&
     recoveryTurnId &&
     attempt.triggerEventId &&
     attempt.executionGeneration > 0
@@ -360,10 +364,14 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
         turnId: recoveryTurnId,
         triggerEventId: attempt.triggerEventId,
         attemptId: input.attemptId,
-        reason: "sandbox_command_start_outcome_unknown",
+        reason: observationUnavailable
+          ? "sandbox_command_observation_unavailable"
+          : "sandbox_command_start_outcome_unknown",
         sandboxSetupOutcomeUnknown: true,
         detail: {
-          code: "sandbox_command_start_outcome_unknown",
+          code: observationUnavailable
+            ? "sandbox_command_observation_unavailable"
+            : "sandbox_command_start_outcome_unknown",
           retryable: false,
           setupOutcome: "unknown",
           replay: "blocked",
@@ -1749,7 +1757,45 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
       attemptNumber: nextProviderRecoveryCount,
       retryAfterMs: providerRetryAfterMs(error),
     });
+    const setupRecoveryExhausted =
+      earlyCommandStartUnavailable &&
+      recoveryResult.status === "exhausted" &&
+      attempt.providerRecoveryCount === SANDBOX_SETUP_RECOVERY_LIMIT;
     try {
+      if (setupRecoveryExhausted) {
+        // This is positive pre-dispatch proof, not an ambiguous command. Park
+        // the SAME accepted turn without resetting or advancing its budget.
+        const recovery = await requestSessionTurnRecovery(db, input.workspaceId, {
+          sessionId: input.sessionId,
+          turnId: attempt.turnId,
+          triggerEventId: attempt.triggerEventId!,
+          attemptId: input.attemptId,
+          reason: "sandbox_command_start_recovery_exhausted",
+          sandboxSetupRecoveryExhausted: true,
+          detail: {
+            code: failure.code,
+            error:
+              "Automatic sandbox setup recovery exhausted after five retries; the accepted turn remains parked without starting a command.",
+            retryable: false,
+            setupOutcome: "not_started",
+            replay: "blocked",
+            recoveryExhausted: true,
+            providerRecoveryCount: SANDBOX_SETUP_RECOVERY_LIMIT,
+          },
+        });
+        if (recovery.action === "stale") {
+          acknowledgeLostAttemptOwnership();
+          control.activityStatus = "cancelled";
+          control.turnMetricOutcome = "cancelled";
+          return claimedResult({ status: "cancelled" });
+        }
+        acknowledgeRecoveryQuiescence();
+        await publishDurableSessionEvents(bus, input.workspaceId, input.sessionId, recovery.events);
+        control.turnMetricOutcome = "recovering";
+        control.activityStatus = "recovering";
+        control.activityError = error;
+        return claimedResult({ status: "recovering" });
+      }
       if (recoveryResult.status === "recovering") {
         if (!earlyRecoverableSetup) {
           await flushRuntimeBatcher();
@@ -1763,7 +1809,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
           reason: failure.code ?? "provider_unavailable",
           providerRecoveryCount: nextProviderRecoveryCount,
           detail: {
-            ...failure,
+            ...agentRunRecoveryFailurePayload(error, failure),
             continueDelayMs: recoveryResult.continueDelayMs,
             providerRecoveryCount: nextProviderRecoveryCount,
           },
@@ -1831,7 +1877,15 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
                 providerRecoveryCount: nextProviderRecoveryCount,
               },
             })
-          : null;
+          : setupRecoveryExhausted
+            ? postClaimDatabaseRecoveryFailure({
+                error: recoveryError,
+                turnId: attempt.turnId,
+                triggerEventId: attempt.triggerEventId!,
+                executionGeneration: attempt.executionGeneration,
+                sandboxSetupRecoveryExhausted: true,
+              })
+            : null;
       if (postClaimRecovery) {
         control.activityStatus = "recovering";
         control.turnMetricOutcome = "recovering";

package/src/activities/agent-turn/finalization-monitor.ts

@@ -41,7 +41,7 @@ export function startTurnFinalizationMonitor(input: {
   observability: Observability;
   details: TurnHeartbeatDetails;
   heartbeat: (details: TurnHeartbeatDetails) => void;
-  terminateWorker: () => void;
+  requestWorkerDrain: () => void;
   timeoutMs?: number;
   slowAfterMs?: number;
 }) {
@@ -93,10 +93,9 @@ export function startTurnFinalizationMonitor(input: {
             // Diagnostic failure must never disable physical containment.
           }
         },
-        terminateWorker: input.terminateWorker,
+        terminateWorker: input.requestWorkerDrain,
       });
-      // Record the precursor while the process remains scrapeable. A counter
-      // increment immediately before process.exit can never reach Prometheus.
+      // Record the precursor while the process remains scrapeable.
       slowTimer = setTimeout(
         () =>
           diagnose(() => {

package/src/activities/agent-turn/finalization.ts

@@ -74,6 +74,8 @@ export type TurnFinalizationDeps = {
   wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
   signalSessionAttemptQuiesced: ActivityServices["signalSessionAttemptQuiesced"];
   signalCodexCapacityWorkflow: ActivityServices["signalCodexCapacityWorkflow"];
+  requestWorkerDrain: ActivityServices["requestWorkerDrain"];
+  turnFinalizationTimeoutMs: ActivityServices["turnFinalizationTimeoutMs"];
   cancellationSignal: AbortSignal | undefined;
   sandboxResumeController: AbortController;
   activityContext: ReturnType<typeof currentActivityContext>;
@@ -113,6 +115,9 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
   deps.eventing.heartbeatTimer = startActivityHeartbeat(deps.activityContext, details);
   const monitor = startTurnFinalizationMonitor({
     observability: deps.observability,
+    ...(deps.turnFinalizationTimeoutMs === undefined
+      ? {}
+      : { timeoutMs: deps.turnFinalizationTimeoutMs }),
     details,
     heartbeat: (value) => {
       try {
@@ -121,7 +126,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
         // A closed Temporal transport is not proof of physical quiescence.
       }
     },
-    terminateWorker: () => process.exit(1),
+    requestWorkerDrain: deps.requestWorkerDrain,
   });
   try {
     monitor.enter("tool_writers");
@@ -411,12 +416,12 @@ async function finalizeTurnAttemptSteps(
     // best-effort (same discipline as today's usage write). Both writers skip
     // version/updatedAt, so neither can race the token-refresh CAS.
     monitor.enter("provider_leases");
-    for (const [scope, snapshot] of providerTurn.latestClaudeUsage) {
+    for (const snapshot of providerTurn.latestClaudeUsage.values()) {
       await waitForTurnFinalizerStep(
         recordClaudeSubscriptionUsage(
           db,
           settings,
-          { accountId: input.accountId, workspaceId: input.workspaceId, scope },
+          { accountId: input.accountId, workspaceId: input.workspaceId, scope: snapshot.scope },
           snapshot,
         ).catch(() => null),
         finalizerSignal,

package/src/activities/agent-turn/quiescence.ts

@@ -91,10 +91,9 @@ export const QUIESCENCE_PROOF_SIGNAL_MAX_RETRY_MS = 5_000;
 export const TURN_QUIESCENCE_WATCHDOG_MS = 5 * 60_000;
 export const TURN_FINALIZER_STEP_TIMEOUT_MS = 30_000;
 
-/** A fenced activity must never heartbeat forever while its physical writer
- * drain is stuck. The last-resort worker exit stops every in-process writer;
- * Kubernetes replaces the pod and the workflow's heartbeat-lease reconciler
- * then commits the exact quiescence receipt. */
+/** A fenced activity must never heartbeat forever while physical cleanup is
+ * stuck. The callback requests host-owned containment, which drains peer turns
+ * before applying its final exit policy. A timeout is never quiescence proof. */
 export function armTurnQuiescenceWatchdog(input: {
   enabled: boolean;
   timeoutMs?: number;

package/src/activities/agent-turn/run.ts

@@ -183,6 +183,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
       bus,
       runtime,
       summarizeContextForCompaction,
+      requestWorkerDrain,
+      turnFinalizationTimeoutMs,
       objectStorage,
       observability,
       wakeSessionWorkflow,
@@ -944,28 +946,28 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
           );
           const withClaudeUsage = <T>(fn: () => Promise<T>): Promise<T> =>
             withClaudeUsageObserver(claudeUsageObserver, fn, async (providerId, headers) => {
-              const binding = claudeUsageObserver.binding(providerId);
-              if (!binding) return headers;
-              const credential = await resolveClaudeSubscriptionCredential(
-                db,
-                settings,
-                {
-                  accountId: input.accountId,
-                  workspaceId: input.workspaceId,
-                  scope: binding.scope,
-                },
-                {
-                  expectedConnectionId: binding.expectedConnectionId,
-                  expectedCredentialVersion: binding.expectedCredentialVersion,
+              return claudeUsageObserver.prepareRequestWithObserver(
+                providerId,
+                headers,
+                async (binding) => {
+                  const credential = await resolveClaudeSubscriptionCredential(
+                    db,
+                    settings,
+                    {
+                      accountId: input.accountId,
+                      workspaceId: input.workspaceId,
+                      scope: binding.scope,
+                    },
+                    {
+                      expectedConnectionId: binding.expectedConnectionId,
+                      expectedCredentialVersion: binding.expectedCredentialVersion,
+                    },
+                  );
+                  if (credential && "reconnectRequired" in credential)
+                    throw new ClaudeSubscriptionReconnectRequired();
+                  return credential;
                 },
               );
-              if (!credential) return headers; // A replaced connection never lends its new token to this turn.
-              if ("reconnectRequired" in credential)
-                throw new ClaudeSubscriptionReconnectRequired();
-              claudeUsageObserver.renew(providerId, credential);
-              headers.set("authorization", `Bearer ${credential.token}`);
-              headers.delete("x-api-key");
-              return headers;
             });
           const withCodex = <T>(fn: () => Promise<T>): Promise<T> =>
             codexContext ? codexRequestStorage.run(codexContext, fn) : fn();
@@ -1844,6 +1846,8 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
       });
     } finally {
       await finalizeTurnAttempt({
+        turnFinalizationTimeoutMs,
+        requestWorkerDrain,
         input,
         settings,
         db,

package/src/activities/agent-turn/sandbox-establish.ts

@@ -656,6 +656,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
                 freshSandboxReadinessReplacementBudget,
                 onSandboxLost: publishSandboxLost,
                 objectStorage,
+                bus,
               },
               {
                 accountId: input.accountId,
@@ -735,6 +736,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
                 freshSandboxReadinessReplacementBudget,
                 onSandboxLost: publishSandboxLost,
                 objectStorage,
+                bus,
               },
               {
                 accountId: input.accountId,

package/src/activities/agent-turn/sandbox-runtime.ts

@@ -13,6 +13,7 @@ import { createProviderCommandRetainer } from "@opengeni/db/retained-provider-co
 import {
   RoutingMutationOutcomeUnknownError,
   ProviderCommandStartOutcomeUnknownError,
+  ProviderCommandObservationUnavailableError,
   isModalCommandStartOutcomeUnknownError,
   withModalCommandStartSignal,
   resolveModalCheckpointProviderBindingForSession,
@@ -84,7 +85,10 @@ function internalMutationUnknownCommand(
     if (!value || typeof value !== "object" || seen.has(value)) continue;
     seen.add(value);
     try {
-      if (value instanceof ProviderCommandStartOutcomeUnknownError) {
+      if (
+        value instanceof ProviderCommandStartOutcomeUnknownError ||
+        value instanceof ProviderCommandObservationUnavailableError
+      ) {
         unknown = true;
         // Zod reads ordinary properties. First make a bounded own-data-only
         // copy so neither validation nor equality executes descriptor getters.

package/src/activities/agent-turn/skill-selection.ts

@@ -20,6 +20,7 @@ export function configuredBundledSkillNames(context: BundledSkillConfiguration):
     { name: "document-parsing", include: () => true },
     { name: "opengeni-skills", include: () => true },
     { name: "opengeni-projects", include: () => true },
+    { name: "opengeni-schedules", include: () => tools.has("scheduled_tasks_create") },
     { name: "opengeni-documents", include: artifacts },
     { name: "opengeni-spreadsheets", include: artifacts },
     { name: "opengeni-presentations", include: artifacts },
@@ -43,6 +44,7 @@ export function loadConfiguredBundledSkills(context: BundledSkillConfiguration)
   const names = new Set(configuredBundledSkillNames(context));
   const artifacts = loadNativeToolSkillArtifacts({
     projects: names.has("opengeni-projects"),
+    schedules: names.has("opengeni-schedules"),
     editableArtifacts: [
       "opengeni-documents",
       "opengeni-spreadsheets",

package/src/activities/agent-turn/subscription-lease.ts

@@ -0,0 +1,115 @@
+export type LeaseRenewReason = "timer" | "runtime_event" | "model_usage";
+export type LeaseLossReason = "deadline" | "not_found";
+
+type LeaseIdentity = { turnId: string; holderId: string; generation: number };
+export type SubscriptionLeaseDeps = {
+  ttlMs: number;
+  getTurnId: () => string | undefined;
+  heartbeat: (identity: LeaseIdentity) => Promise<Date | null>;
+  onLost: (reason: LeaseLossReason) => void;
+  onRenewed?: (reason: LeaseRenewReason) => void;
+  onError: (error: unknown, reason: LeaseRenewReason) => void;
+  lostError: (reason: LeaseLossReason) => Error;
+  now?: () => number;
+};
+
+/** Cross-provider lease lifecycle. Only on-time DB confirmation extends ownership. */
+export class SubscriptionTurnLease {
+  held = false;
+  lost = false;
+  lossReason: LeaseLossReason | null = null;
+  holderId: string | null = null;
+  generation: number | null = null;
+  confirmedUntilMs: number | null = null;
+  private heartbeatInFlight = false;
+  private heartbeatTimer: ReturnType<typeof setInterval> | undefined;
+  private readonly now: () => number;
+
+  constructor(private readonly leaseDeps: SubscriptionLeaseDeps) {
+    if (!Number.isFinite(leaseDeps.ttlMs) || leaseDeps.ttlMs <= 0)
+      throw new Error("Subscription lease TTL must be positive");
+    this.now = leaseDeps.now ?? (() => performance.now());
+  }
+
+  private expired = (deadline: number | null): boolean =>
+    deadline === null || !Number.isFinite(deadline) || this.now() >= deadline;
+
+  markLost = (reason: LeaseLossReason): void => {
+    if (this.lost) return;
+    this.lost = true;
+    this.lossReason = reason;
+    this.leaseDeps.onLost(reason);
+  };
+
+  assertUsable = (): void => {
+    if (this.lost) throw this.leaseDeps.lostError(this.lossReason ?? "not_found");
+    if (
+      !this.held ||
+      this.holderId === null ||
+      this.generation === null ||
+      this.expired(this.confirmedUntilMs)
+    ) {
+      this.markLost("deadline");
+      throw this.leaseDeps.lostError("deadline");
+    }
+  };
+
+  renew = async (reason: LeaseRenewReason = "timer"): Promise<void> => {
+    const turnId = this.leaseDeps.getTurnId();
+    if (!turnId || !this.held || !this.holderId || this.generation === null || this.lost) return;
+    if (this.expired(this.confirmedUntilMs)) {
+      this.markLost("deadline");
+      return;
+    }
+    if (this.heartbeatInFlight) return;
+    this.heartbeatInFlight = true;
+    const priorDeadline = this.confirmedUntilMs;
+    const startedAt = this.now();
+    const identity = { turnId, holderId: this.holderId, generation: this.generation };
+    try {
+      const renewedUntil = await this.leaseDeps.heartbeat(identity);
+      // A late confirmation is not proof of uninterrupted ownership, even if
+      // Postgres extended the row. A changed holder must also discard its reply.
+      if (
+        !this.held ||
+        this.lost ||
+        this.leaseDeps.getTurnId() !== identity.turnId ||
+        this.holderId !== identity.holderId ||
+        this.generation !== identity.generation
+      )
+        return;
+      if (!renewedUntil) this.markLost("not_found");
+      else if (this.expired(priorDeadline) || this.expired(startedAt + this.leaseDeps.ttlMs))
+        this.markLost("deadline");
+      else {
+        this.confirmedUntilMs = startedAt + this.leaseDeps.ttlMs;
+        this.leaseDeps.onRenewed?.(reason);
+      }
+    } catch (error) {
+      if (
+        !this.held ||
+        this.lost ||
+        this.leaseDeps.getTurnId() !== identity.turnId ||
+        this.holderId !== identity.holderId ||
+        this.generation !== identity.generation
+      )
+        return;
+      if (this.expired(this.confirmedUntilMs)) this.markLost("deadline");
+      else this.leaseDeps.onError(error, reason);
+    } finally {
+      this.heartbeatInFlight = false;
+    }
+  };
+
+  startHeartbeat = (): void => {
+    if (!this.leaseDeps.getTurnId() || this.heartbeatTimer) return;
+    this.heartbeatTimer = setInterval(() => void this.renew("timer"), 60_000);
+    this.heartbeatTimer.unref?.();
+  };
+
+  stopHeartbeat = (): void => {
+    if (!this.heartbeatTimer) return;
+    clearInterval(this.heartbeatTimer);
+    this.heartbeatTimer = undefined;
+  };
+}

package/src/activities/agent-turn/turn-context.ts

@@ -191,10 +191,7 @@ export type ProviderTurnState = {
   // scraped. Lives on the turn context so the finalizer sees it; the sink is
   // wired into codexContext.onUsageHeaders by the orchestrator.
   latestCodexUsage: CodexUsageHeaderSnapshot | null;
-  latestClaudeUsage: Map<
-    "workspace" | "organization",
-    import("./claude-usage-observer").CapturedClaudeUsage
-  >;
+  latestClaudeUsage: Map<string, import("./claude-usage-observer").CapturedClaudeUsage>;
   lastCodexRequestOpaqueArtifacts: readonly string[];
 };
 

package/src/activities/codex-rotation.ts

@@ -1,3 +1,4 @@
+import { subscriptionAccountShardIndex } from "@opengeni/config";
 // Multi-account P3 — the PURE rotation ranker. Zero I/O: no provider calls, no
 // decrypts, no db. It consumes the already-loaded, metadata-only account list
 // (cached usage columns + the exhausted_until cooldown column) and returns the
@@ -223,17 +224,6 @@ export function isCodexAccountEligible(acct: CodexRotationAccount, now: Date): b
   return isCodexCredentialEligible(acct, now);
 }
 
-/** Deterministic 32-bit FNV-1a over a UTF-16 code-unit stream. Pure, allocation-free. */
-function fnv1a32(input: string): number {
-  let hash = 0x811c9dc5;
-  for (let i = 0; i < input.length; i++) {
-    hash ^= input.charCodeAt(i);
-    // 32-bit FNV prime multiply via Math.imul; `>>> 0` keeps it unsigned.
-    hash = Math.imul(hash, 0x01000193);
-  }
-  return hash >>> 0;
-}
-
 /**
  * Session-sharded HOME account (AM-6): the deterministic account a session runs on
  * under the "sharded" strategy — `stableAccountList[ hash(sessionId) % N ]` over the
@@ -263,7 +253,7 @@ export function shardCredentialForSession(args: {
   if (eligibles.length === 0) {
     return null;
   }
-  const index = fnv1a32(sessionId) % eligibles.length;
+  const index = subscriptionAccountShardIndex(sessionId, eligibles.length);
   return eligibles[index]!.id;
 }