@opengeni/worker-bundle 1.0.1 → 2.0.0

package/src/activities/agent-turn/errors.ts

@@ -11,10 +11,16 @@ import {
   ActiveBackendUnresolvableError,
   CompactionProviderResponseError,
   EmptyCompactionSummaryError,
+  compactionProviderRejection,
+  describeCompactionProviderRejection,
+  type CompactionProviderRejection,
   isMcpRequestTimeoutError,
   isMcpTransportConnectivityError,
   isModalTaskExecStartDnsResolutionError,
   RoutingWorkspaceRootChangedError,
+  SandboxMaterializationVerificationError,
+  materializationVerificationDiagnostic,
+  type MaterializationVerificationDiagnostic,
   SelfhostedWorkspaceRootChangedError,
   UNKNOWN_MODEL_FINISH_REASON_CODE,
 } from "@opengeni/runtime";
@@ -28,6 +34,7 @@ import { RetainedAttachmentTransportLimitError } from "../run-input";
 import type { CodexAccountStatus } from "@opengeni/db";
 import {
   CodexReloginRequired,
+  classifyCodexEncryptedArtifactRejection,
   classifyCodexResponseTimeoutError,
   classifyCodexUsageLimitError,
   isCodexTransportError,
@@ -269,22 +276,38 @@ export function escapedMcpTimeoutRecoveryFailure(input: {
  * Convert the atomic claim transaction's failure into a small, stable
  * Temporal wire contract. The original error remains in activity diagnostics,
  * but SQL text, parameters, and arbitrary invariant messages never enter
- * workflow history. Every database failure is safe to re-read after backoff
- * because the claim transaction contains no model/tool effects. Database
- * constraints and authorization guards can also be repaired by a deployment or
- * authority refresh, so they must preserve pending work. Only a non-database
- * claim invariant is terminal.
+ * workflow history. Operational failures retry; other persistence rejections
+ * retain accepted work behind a durable explicit-recheck fence. Repeating the
+ * same rejected transaction without changed conditions is not recovery.
  */
 export function preClaimAdmissionFailure(error: unknown): ApplicationFailure {
   const persistenceFailure = isSessionEventPersistenceError(error) ? error : null;
   const retryableCode = retryableDatabaseFailureCode(error);
-  // The workflow retries after a durable re-read and bounded delay. A
-  // SessionEventPersistenceError proves the failure happened inside the atomic
-  // claim boundary, before provider or tool work; retaining it is safer than
-  // destroying accepted machine input on an application SQLSTATE.
+  const rejectedClaim =
+    persistenceFailure?.details.stage === "session_attempts.claim" &&
+    persistenceFailure.details.retryOutcome === "not_retryable" &&
+    !retryableCode;
   const detail: PreClaimFailureDetail = {
-    disposition: persistenceFailure || retryableCode ? "retryable" : "permanent",
+    disposition: rejectedClaim
+      ? "blocked"
+      : persistenceFailure || retryableCode
+        ? "retryable"
+        : "permanent",
     code: retryableCode ?? persistenceFailure?.details.code ?? "claim_invariant",
+    ...(persistenceFailure && rejectedClaim
+      ? {
+          sqlState: persistenceFailure.details.sqlState,
+          reason:
+            persistenceFailure.details.stage === "session_attempts.claim" &&
+            persistenceFailure.details.sqlState === "OG001"
+              ? ("initiator_membership_required" as const)
+              : persistenceFailure.details.stage === "session_attempts.claim" &&
+                  persistenceFailure.details.sqlState === "OG002"
+                ? ("personal_resource_grant_required" as const)
+                : ("database_claim_rejected" as const),
+          retryPolicy: "explicit_recheck" as const,
+        }
+      : {}),
   };
   return ApplicationFailure.create({
     message: PRE_CLAIM_FAILURE_MESSAGE,
@@ -574,7 +597,21 @@ export function safeErrorForTelemetry(error: unknown): Error {
   return safe;
 }
 
+/**
+ * Guidance appended to a definitive provider rejection. Repeating the exact
+ * request cannot succeed, so the generic "send another message to retry"
+ * advice is wrong here: a new prompt re-sends the same rejected history.
+ */
+export const COMPACTION_PROVIDER_REJECTION_GUIDANCE =
+  "The provider refused this exact request, so repeating it fails the same way until the conversation changes. If a new message fails again, start a new session.";
+
 export function compactionFailureReasonFromError(error: unknown): string {
+  const rejection = compactionProviderRejection(error);
+  if (rejection) {
+    return compactionFailureReason(
+      `the model provider rejected the compaction request (${describeCompactionProviderRejection(rejection)}). Active history was preserved. ${COMPACTION_PROVIDER_REJECTION_GUIDANCE}`,
+    );
+  }
   if (
     error instanceof CompactionProviderResponseError ||
     error instanceof EmptyCompactionSummaryError
@@ -585,6 +622,34 @@ export function compactionFailureReasonFromError(error: unknown): string {
   return compactionFailureReason(`unexpected ${errorName}`);
 }
 
+/**
+ * Exact `turn.failed` payload for a terminal compaction failure. A definitive
+ * provider rejection additionally carries its closed identifier record so the
+ * timeline, API consumers, and operators can name the rejected field without
+ * parsing the message.
+ */
+export function compactionFailureTurnEventPayload(
+  error: unknown,
+  overrides: { error?: string } = {},
+): {
+  error: string;
+  code: "context_compaction_failed";
+  retryable: false;
+  recovery: "user_message";
+  compacted: false;
+  providerRejection?: CompactionProviderRejection;
+} {
+  const rejection = compactionProviderRejection(error);
+  return {
+    error: overrides.error ?? compactionFailureReasonFromError(error),
+    code: "context_compaction_failed",
+    retryable: false,
+    recovery: "user_message",
+    compacted: false,
+    ...(rejection ? { providerRejection: rejection } : {}),
+  };
+}
+
 export function isCompactionSummaryFailure(error: unknown): boolean {
   return (
     error instanceof CompactionProviderResponseError || error instanceof EmptyCompactionSummaryError
@@ -594,6 +659,11 @@ export function isCompactionSummaryFailure(error: unknown): boolean {
 export function shouldRecoverCompactionProviderFailure(error: unknown): boolean {
   if (!(error instanceof CompactionProviderResponseError)) return false;
   if (isCodexTransportError(error) && classifyCodexUsageLimitError(error)) return true;
+  // Codex may reject an opaque artifact it minted itself on the compaction
+  // request exactly as it can on an ordinary request. Failure settlement
+  // invalidates only the exact participating artifacts and recovers the same
+  // logical turn; when nothing can be invalidated it fails closed there.
+  if (classifyCodexEncryptedArtifactRejection(error)) return true;
   return agentRunFailurePayload(error).retryable === true;
 }
 
@@ -851,6 +921,15 @@ function findPostgresDriverError(error: unknown): Record<string, unknown> | null
 export function agentRunFailurePayload(
   error: unknown,
   options: { isCodexTurn?: boolean } = {},
+): ReturnType<typeof baseAgentRunFailurePayload> {
+  const failure = baseAgentRunFailurePayload(error, options);
+  const diagnostic = materializationVerificationDiagnostic(error);
+  return diagnostic ? { ...failure, materializationDiagnostic: diagnostic } : failure;
+}
+
+function baseAgentRunFailurePayload(
+  error: unknown,
+  options: { isCodexTurn?: boolean } = {},
 ): {
   error: string;
   code?: string;
@@ -870,7 +949,16 @@ export function agentRunFailurePayload(
   database?: Record<string, string>;
   historyPersistenceStage?: MandatoryHistoryPersistenceStage;
   mcpTransportDiagnostic?: McpTransportRequestFailureDiagnostic;
+  materializationDiagnostic?: MaterializationVerificationDiagnostic;
 } {
+  if (error instanceof SandboxMaterializationVerificationError) {
+    return {
+      error: error.message,
+      code: error.code,
+      retryable: false,
+      materializationDiagnostic: error.diagnostic,
+    };
+  }
   if (error instanceof RetainedAttachmentTransportLimitError) {
     return { error: error.message, code: "retained_attachment_transport_limit", retryable: false };
   }

package/src/activities/agent-turn/failure-settlement.ts

@@ -225,6 +225,16 @@ function acceptedCodexPolicySnapshot(
 }
 
 export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgentTurnResult> {
+  if (deps.settings.environment === "local" && deps.error instanceof Error) {
+    // Keep local startup failures diagnosable without logging error messages,
+    // absolute host paths, prompts, credentials, or provider response bodies.
+    const locations = (deps.error.stack ?? "")
+      .split("\n")
+      .slice(1)
+      .flatMap((line) => line.match(/(?:apps|packages)\/[A-Za-z0-9_./-]+:\d+:\d+/g) ?? [])
+      .slice(0, 8);
+    console.error(JSON.stringify({ message: "Local turn failure source locations", locations }));
+  }
   const {
     error,
     input,
@@ -249,6 +259,36 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
     acknowledgeLostAttemptOwnership,
     acknowledgeRecoveryQuiescence,
   } = deps;
+  // Capture before any recovery/checkpoint DB operation can fail again.
+  if (isSessionEventPersistenceError(error)) {
+    try {
+      const diagnosticId = observability.recordFailureDiagnostic({
+        code: error.details.code,
+        stage:
+          error.details.stage === "session_events.append_generic"
+            ? "session_events.append_generic"
+            : error.details.stage === "session_events.append_for_turn_attempt"
+              ? "session_events.append_for_turn_attempt"
+              : error.details.stage === "session_attempts.claim"
+                ? "session_attempts.claim"
+                : "failure_settlement",
+        retryDecision: error.details.retryOutcome,
+        error,
+        sessionId: input.sessionId,
+        ...(attempt.turnId ? { turnId: attempt.turnId } : {}),
+        attemptId: input.attemptId,
+        attempts: error.details.attempts,
+        eventTypes: error.details.eventTypes,
+        sqlState: error.details.sqlState,
+        ...(error.details.database.constraint
+          ? { constraint: error.details.database.constraint }
+          : {}),
+      });
+      observability.error("session event persistence failed", { correlationId: diagnosticId });
+    } catch {
+      // Failure settlement must not depend on telemetry availability.
+    }
+  }
   // Graceful worker shutdown (deploy / rollout restart): checkpoint the
   // same current inference for a new fenced attempt instead of failing the
   // session. Conversation truth is already persisted per model response;
@@ -724,7 +764,9 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
         labels: { workspace_key: codexWorkspaceKey, outcome: "completed" },
       });
       const now = new Date();
-      const before = await listCodexAccountStatuses(db, input.workspaceId).catch(() => []);
+      const before = await listCodexAccountStatuses(db, input.workspaceId, attempt.turnId).catch(
+        () => [],
+      );
       const servingCached = before.find(
         (account) => account.id === providerTurn.effectiveCodexCredentialId,
       );
@@ -892,7 +934,7 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
       }
       let accounts: Awaited<ReturnType<typeof listCodexAccountStatuses>>;
       try {
-        accounts = await listCodexAccountStatuses(db, input.workspaceId);
+        accounts = await listCodexAccountStatuses(db, input.workspaceId, attempt.turnId);
       } catch (metadataError) {
         // Current account health/cooldown metadata is still required after
         // quarantine. Operational database failures re-enter the existing
@@ -1126,6 +1168,11 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
           { onArmed: () => (leases.codex.held = false) },
         );
         control.activityError = error;
+        if (evaluated.action === "stopped") {
+          control.activityStatus = evaluated.sessionStatus === "queued" ? "idle" : "failed";
+          control.turnMetricOutcome = "failed";
+          return claimedResult({ status: control.activityStatus });
+        }
         if (evaluated.action === "resumed") {
           control.activityStatus = "recovering";
           control.turnMetricOutcome = "recovering";
@@ -1455,32 +1502,6 @@ export async function settleTurnFailure(deps: TurnFailureDeps): Promise<RunAgent
   let failure = agentRunFailurePayload(error, {
     isCodexTurn: billingState.isCodexTurn,
   }) as ReturnType<typeof agentRunFailurePayload>;
-  if (isSessionEventPersistenceError(error)) {
-    // Preserve the exact source message in the internal runtime diagnostic;
-    // SQLSTATE/catalog facts remain separate classification attributes.
-    observability.error("session event persistence failed", {
-      accountId: input.accountId,
-      workspaceId: input.workspaceId,
-      sessionId: input.sessionId,
-      turnId: attempt.turnId,
-      attemptId: input.attemptId,
-      code: error.details.code,
-      sqlState: error.details.sqlState ?? "unknown",
-      stage: error.details.stage,
-      eventTypes: error.details.eventTypes.join(","),
-      correlationId: error.details.correlationId,
-      attempts: error.details.attempts,
-      retryOutcome: error.details.retryOutcome,
-      dbSeverity: error.details.database.severity,
-      dbSchema: error.details.database.schema,
-      dbTable: error.details.database.table,
-      dbColumn: error.details.database.column,
-      dbDataType: error.details.database.dataType,
-      dbConstraint: error.details.database.constraint,
-      dbRoutine: error.details.database.routine,
-      error: error.message,
-    });
-  }
   if (failure.retryable && eventing.publish && attempt.turnId && eventing.turnStartedPublished) {
     const nextProviderRecoveryCount = attempt.providerRecoveryCount + 1;
     const recoveryResult = providerRecoveryResult({

package/src/activities/agent-turn/finalization-monitor.ts

@@ -0,0 +1,121 @@
+import type { Observability } from "@opengeni/observability";
+import type { TurnHeartbeatDetails } from "../../op-journal";
+import { armTurnQuiescenceWatchdog } from "./quiescence";
+
+export const TURN_FINALIZATION_STAGES = [
+  "tool_writers",
+  "credential_renewals",
+  "credential_cleanup",
+  "quiescence_receipt",
+  "event_flush",
+  "provider_leases",
+  "workspace_capture",
+  "tool_close",
+  "sandbox_provisioning",
+  "sandbox_rotation",
+  "workspace_snapshot",
+  "sandbox_release",
+] as const;
+export type TurnFinalizationStage = (typeof TURN_FINALIZATION_STAGES)[number];
+
+const INFLIGHT = {
+  name: "opengeni_turn_finalization_inflight",
+  help: "Physical turn finalizers currently waiting in each cleanup stage.",
+};
+const SLOW_STAGES = {
+  name: "opengeni_turn_finalization_slow_total",
+  help: "Turn cleanup stages observed to exceed thirty seconds.",
+};
+const initialized = new WeakSet<Observability>();
+function diagnose(operation: () => void): void {
+  try {
+    operation();
+  } catch {
+    // Diagnostics cannot own physical cleanup or containment.
+  }
+}
+
+/** Starts only after agent execution ends. A cleanup-stage deadline never caps
+ * agent work and never detaches a physical writer to manufacture quiescence. */
+export function startTurnFinalizationMonitor(input: {
+  observability: Observability;
+  details: TurnHeartbeatDetails;
+  heartbeat: (details: TurnHeartbeatDetails) => void;
+  terminateWorker: () => void;
+  timeoutMs?: number;
+  slowAfterMs?: number;
+}) {
+  const { observability, details } = input;
+  if (!initialized.has(observability)) {
+    for (const stage of TURN_FINALIZATION_STAGES) {
+      diagnose(() => observability.incrementGauge({ ...INFLIGHT, labels: { stage }, amount: 0 }));
+      diagnose(() =>
+        observability.incrementCounter({ ...SLOW_STAGES, labels: { stage }, amount: 0 }),
+      );
+    }
+    initialized.add(observability);
+  }
+  let stage: TurnFinalizationStage | null = null;
+  let disarm = () => {};
+  let stopped = false;
+  let slowTimer: ReturnType<typeof setTimeout> | undefined;
+  const leave = () => {
+    disarm();
+    clearTimeout(slowTimer);
+    if (stage) {
+      const previous = stage;
+      diagnose(() =>
+        observability.incrementGauge({ ...INFLIGHT, labels: { stage: previous }, amount: -1 }),
+      );
+    }
+    stage = null;
+  };
+  return {
+    enter(next: TurnFinalizationStage) {
+      if (stopped || stage === next) return;
+      leave();
+      stage = next;
+      details.phase = "finalizing";
+      details.finalizationStage = next;
+      details.finalizationStageStartedAt = new Date().toISOString();
+      diagnose(() => observability.incrementGauge({ ...INFLIGHT, labels: { stage: next } }));
+      disarm = armTurnQuiescenceWatchdog({
+        enabled: true,
+        ...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }),
+        onTimeout: () => {
+          try {
+            observability.error("turn finalization stalled; containing worker", {
+              surface: "turn_finalization",
+              outcome: "containment",
+              reason: next,
+            });
+          } catch {
+            // Diagnostic failure must never disable physical containment.
+          }
+        },
+        terminateWorker: input.terminateWorker,
+      });
+      // Record the precursor while the process remains scrapeable. A counter
+      // increment immediately before process.exit can never reach Prometheus.
+      slowTimer = setTimeout(
+        () =>
+          diagnose(() => {
+            observability.incrementCounter({ ...SLOW_STAGES, labels: { stage: next } });
+            observability.warn("turn finalization is taking longer than expected", {
+              surface: "turn_finalization",
+              outcome: "slow",
+              reason: next,
+            });
+          }),
+        input.slowAfterMs ?? 30_000,
+      );
+      slowTimer.unref?.();
+      diagnose(() => input.heartbeat(details));
+    },
+    stop() {
+      if (stopped) return;
+      stopped = true;
+      leave();
+    },
+  };
+}

package/src/activities/agent-turn/finalization.ts

@@ -1,7 +1,7 @@
 import {
   commitSessionAttemptQuiescence,
   listPendingSessionTurns,
-  recordCodexAccountUsageWithWakeTargets,
+  recordCodexAccountUsageForFinalization,
   releaseCodexCredentialLease,
   releaseXaiCredentialLease,
   updateXaiQuotaMetadata,
@@ -13,7 +13,8 @@ import { clearRunCredentialsForAttempt } from "@opengeni/runtime";
 import { fetchXaiSubscriptionQuota } from "@opengeni/xai-subscription";
 import type { Settings } from "@opengeni/config";
 import { signalCodexCapacityWakeTargets } from "../codex-capacity";
-import type { currentActivityContext } from "../streaming";
+import { startActivityHeartbeat, type currentActivityContext } from "../streaming";
+import { startTurnFinalizationMonitor } from "./finalization-monitor";
 import type { CodemodeTokenRenewalController } from "../codemode-token-renewal";
 import type { RunCredentialRenewalController } from "../run-credential-renewal";
 import type {
@@ -32,6 +33,7 @@ import {
   makeMachineOpObserver,
   recordSessionEventAppendPhase,
   turnLifecycleMetricsFor,
+  runtimeMetricsHooksForObservability,
 } from "../../observability-metrics";
 import {
   maybePersistWarmWorkspaceSnapshot,
@@ -43,7 +45,6 @@ import { safeErrorDiagnostic, safeErrorForTelemetry } from "./errors";
 import {
   assertPhysicalToolQuiescenceForCancellation,
   assertSessionAttemptQuiescenceRecoveryDurable,
-  armTurnQuiescenceWatchdog,
   clearAttemptCredentialsWithSettledFence,
   drainAttemptOwnedSandboxWriters,
   persistOrSignalSessionAttemptQuiescence,
@@ -99,6 +100,42 @@ export type TurnFinalizationDeps = {
 };
 
 export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<void> {
+  const details = deps.eventing.heartbeatDetails ?? {
+    sessionId: deps.input.sessionId,
+    turnId: deps.attempt.turnId,
+    opAcks: {},
+  };
+  deps.eventing.heartbeatDetails = details;
+  // Keep one heartbeat owner through all physical cleanup, with the same op
+  // acknowledgements used during execution. It must no longer say "running".
+  if (deps.eventing.heartbeatTimer) clearInterval(deps.eventing.heartbeatTimer);
+  deps.eventing.heartbeatTimer = startActivityHeartbeat(deps.activityContext, details);
+  const monitor = startTurnFinalizationMonitor({
+    observability: deps.observability,
+    details,
+    heartbeat: (value) => {
+      try {
+        deps.activityContext?.heartbeat({ ...value, at: new Date().toISOString() });
+      } catch {
+        // A closed Temporal transport is not proof of physical quiescence.
+      }
+    },
+    terminateWorker: () => process.exit(1),
+  });
+  try {
+    monitor.enter("tool_writers");
+    await finalizeTurnAttemptSteps(deps, monitor);
+  } finally {
+    monitor.stop();
+    if (deps.eventing.heartbeatTimer) clearInterval(deps.eventing.heartbeatTimer);
+    deps.eventing.heartbeatTimer = undefined;
+  }
+}
+
+async function finalizeTurnAttemptSteps(
+  deps: TurnFinalizationDeps,
+  monitor: ReturnType<typeof startTurnFinalizationMonitor>,
+): Promise<void> {
   const {
     input,
     settings,
@@ -142,17 +179,6 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
   let finalizationError: unknown;
   let physicalToolQuiescenceConfirmed = !control.acknowledgeQuiescence;
   let quiescenceReceiptOrProofDurable = !control.acknowledgeQuiescence;
-  const disarmQuiescenceWatchdog = armTurnQuiescenceWatchdog({
-    enabled: control.acknowledgeQuiescence,
-    onTimeout: () => {
-      observability.error("turn quiescence drain exceeded hard containment deadline", {
-        "opengeni.session_id": input.sessionId,
-        "opengeni.turn_id": attempt.turnId ?? "",
-        "opengeni.attempt_id": input.attemptId,
-      });
-    },
-    terminateWorker: () => process.exit(1),
-  });
   const finalizerSignal = turnFinalizerCancellationSignal(
     cancellationSignal,
     control.activityStatus,
@@ -200,6 +226,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
       // and durably settled before workspace capture. Only receipt
       // publication remains conditional on acknowledgeQuiescence.
       toolCancellationFence,
+      onStage: monitor.enter,
       cancellationReason: cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
       gitCredentialRenewals: gitRenewalsToStop,
       codemodeTokenRenewal: codemodeRenewalToStop,
@@ -211,6 +238,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
     // attempt's generation before publishing physical quiescence. A failure
     // deliberately keeps the receipt closed.
     if (credentialSessionToClear) {
+      monitor.enter("credential_cleanup");
       const clearAttemptCredentials = async (): Promise<void> =>
         await clearRunCredentialsForAttempt(credentialSessionToClear, {
           sessionId: input.sessionId,
@@ -258,6 +286,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
       // and before lease, cache, recording, or provider cleanup. Its
       // transaction also enqueues the exact workflow wake that will admit
       // the replacement; Temporal activity terminalization does neither.
+      monitor.enter("quiescence_receipt");
       const proof: SessionAttemptQuiescenceProof = {
         accountId: input.accountId,
         workspaceId: input.workspaceId,
@@ -304,6 +333,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
         signalProof: signalSessionAttemptQuiesced,
         heartbeat: (deliveryAttempt, retryMs) => {
           activityContext?.heartbeat({
+            ...eventing.heartbeatDetails,
             phase: "quiescence-proof-delivery",
             sessionId: input.sessionId,
             attemptId: input.attemptId,
@@ -336,7 +366,6 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
         },
       });
       quiescenceReceiptOrProofDurable = true;
-      disarmQuiescenceWatchdog();
       if (recoveryMode === "signal") {
         observability.info("agent turn quiescence proof handed to workflow recovery", {
           "opengeni.session_id": input.sessionId,
@@ -351,6 +380,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
     // the turn. Sync observer → buffer → single awaited append here (no unawaited
     // DB write inside the activity). Scoped to this turn; skipped if no turnId
     // (the op ran under a turn, so on the normal path turnId is set).
+    monitor.enter("event_flush");
     const machineOpEvents = machineOpObserver.drainEvents();
     if (machineOpEvents.length > 0 && attempt.turnId && attempt.executionGeneration > 0) {
       await waitForTurnFinalizerStep(
@@ -377,17 +407,34 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
     // Multi-account P4: flush the serving account's free per-turn caches ONCE,
     // best-effort (same discipline as today's usage write). Both writers skip
     // version/updatedAt, so neither can race the token-refresh CAS.
+    monitor.enter("provider_leases");
     if (providerTurn.effectiveCodexCredentialId) {
       // Part A: the latest scraped usage-header snapshot → the P2 usage cache. A
       // full both-windows snapshot (parseCodexUsageHeaders gates on both), so this
       // is byte-identical to the /wham/usage write — no partial-window clobber.
-      if (providerTurn.latestCodexUsage) {
+      if (
+        providerTurn.latestCodexUsage &&
+        attempt.turnId &&
+        leases.codex.held &&
+        leases.codex.holderId &&
+        leases.codex.generation !== null &&
+        providerTurn.effectiveCodexCredentialVersion !== null
+      ) {
         const usageMutation = await waitForTurnFinalizerStep(
-          recordCodexAccountUsageWithWakeTargets(
+          recordCodexAccountUsageForFinalization(
             db,
             input.workspaceId,
             providerTurn.effectiveCodexCredentialId,
             providerTurn.latestCodexUsage,
+            {
+              turnId: attempt.turnId,
+              sessionId: input.sessionId,
+              attemptId: input.attemptId,
+              executionGeneration: attempt.executionGeneration,
+              holderId: leases.codex.holderId,
+              generation: leases.codex.generation,
+              credentialVersion: providerTurn.effectiveCodexCredentialVersion,
+            },
           ).catch(() => null),
           finalizerSignal,
         );
@@ -502,6 +549,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
     ) {
       // Review reads yield to queued work. Recovery snapshots have their own
       // physical-settlement requirement and are drained later in finalization.
+      monitor.enter("workspace_capture");
       sandboxState.turnEndCaptureInProgress = true;
       const captureSandbox = sandboxState.resolvedSandbox;
       const captureSession = sandboxState.setupBoxSession as ChannelASession;
@@ -549,6 +597,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
         },
       });
     }
+    monitor.enter("tool_close");
     eventing.toolPreparationClosing = true;
     if (eventing.toolPreparationReady) {
       await waitForTurnFinalizerStep(
@@ -569,9 +618,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
         finalizerSignal,
       );
     }
-    if (eventing.heartbeatTimer) {
-      clearInterval(eventing.heartbeatTimer);
-    }
+    monitor.enter("sandbox_provisioning");
     if (sandboxState.turnSandboxProvisioner?.hasStarted()) {
       await waitForTurnFinalizerStep(
         sandboxState.turnSandboxProvisioner.waitForSettled(30_000),
@@ -599,12 +646,14 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
     if (sandboxState.leaseHeartbeatTimer) {
       stopLeaseHeartbeat();
     }
+    monitor.enter("sandbox_rotation");
     if (sandboxState.rotationPreemptionInFlight) {
       await sandboxState.rotationPreemptionInFlight.catch(() => undefined);
     }
     // No heartbeat can register another capture after this point. Join the
     // actual provider continuation before either starting a turn-end capture
     // or entering the proof-bearing holder release.
+    monitor.enter("workspace_snapshot");
     await drainInFlightWarmSnapshot();
     if (sandboxState.resolvedSandbox) {
       // TURN-END mid-session snapshot (sandbox-file-persistence): fold the
@@ -622,7 +671,13 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
         settledTurnId
       ) {
         const snapshot = maybePersistWarmWorkspaceSnapshot(
-          { db, settings, objectStorage },
+          {
+            db,
+            settings,
+            objectStorage,
+            sandboxMetrics: runtimeMetricsHooksForObservability(observability),
+            observability,
+          },
           {
             accountId: input.accountId,
             workspaceId: input.workspaceId,
@@ -667,10 +722,13 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
     // Temporal cancellation: the eager listener may already have dropped
     // the holder, and this idempotent proof-bearing pass still must run.
     stopLeaseHeartbeat();
+    monitor.enter("sandbox_rotation");
     if (sandboxState.rotationPreemptionInFlight) {
       await sandboxState.rotationPreemptionInFlight.catch(() => undefined);
     }
+    monitor.enter("workspace_snapshot");
     await drainInFlightWarmSnapshot();
+    monitor.enter("sandbox_release");
     const sandboxReleaseTargets = new Set(sandboxState.lateSandboxesAwaitingWriterDrain);
     sandboxState.lateSandboxesAwaitingWriterDrain.clear();
     if (sandboxState.resolvedSandbox) {