@opengeni/worker-bundle 1.0.0-canary.1 → 1.0.0

package/src/activities/agent-turn/final-reply.ts

@@ -0,0 +1,46 @@
+/** One durable, same-logical-turn handoff, not a retry of model/tool effects. */
+export function finalReplyNudge(turnId: string) {
+  return {
+    type: "message" as const,
+    role: "developer" as const,
+    content: [
+      {
+        type: "input_text" as const,
+        text: `[Runtime final-reply handoff for turn ${turnId}]\nYour final user-facing answer was empty. Reply to the user now with the requested deliverable or a concise outcome and its retained artifact link. Goal evidence is a short ledger proof, not the answer, and goal_complete does not deliver a chat reply. Use the completed work and tool results already in history; do not repeat completed tools or restart the task. If this is a late child result after goal completion, integrate material new findings and deliver any still-missing handoff. Do not change the completed goal or invent a result.`,
+      },
+    ],
+  };
+}
+
+export function hasFinalReplyNudge(history: readonly unknown[], turnId: string): boolean {
+  const text = finalReplyNudge(turnId).content[0]!.text;
+  return history.some((item) => {
+    const message = item as { role?: unknown; content?: unknown } | null;
+    return (
+      message?.role === "developer" &&
+      Array.isArray(message.content) &&
+      message.content.some(
+        (part: { type?: unknown; text?: unknown }) =>
+          part?.type === "input_text" && part.text === text,
+      )
+    );
+  });
+}
+
+export function needsFinalReply(input: {
+  output: unknown;
+  inputWaitYielded: boolean;
+  interrupted: boolean;
+  maintenance: boolean;
+  toolsExecuted: boolean;
+  completedGoal: boolean;
+}): boolean {
+  return (
+    typeof input.output === "string" &&
+    input.output.trim().length === 0 &&
+    !input.inputWaitYielded &&
+    !input.interrupted &&
+    !input.maintenance &&
+    (input.toolsExecuted || input.completedGoal)
+  );
+}

package/src/activities/agent-turn/finalization-monitor.ts

@@ -0,0 +1,120 @@
+import type { Observability } from "@opengeni/observability";
+import type { TurnHeartbeatDetails } from "../../op-journal";
+import { armTurnQuiescenceWatchdog } from "./quiescence";
+
+export const TURN_FINALIZATION_STAGES = [
+  "tool_writers",
+  "credential_renewals",
+  "credential_cleanup",
+  "quiescence_receipt",
+  "event_flush",
+  "provider_leases",
+  "workspace_capture",
+  "tool_close",
+  "sandbox_provisioning",
+  "sandbox_rotation",
+  "workspace_snapshot",
+  "sandbox_release",
+] as const;
+export type TurnFinalizationStage = (typeof TURN_FINALIZATION_STAGES)[number];
+
+const INFLIGHT = {
+  name: "opengeni_turn_finalization_inflight",
+  help: "Physical turn finalizers currently waiting in each cleanup stage.",
+};
+const SLOW_STAGES = {
+  name: "opengeni_turn_finalization_slow_total",
+  help: "Turn cleanup stages observed to exceed thirty seconds.",
+};
+const initialized = new WeakSet<Observability>();
+function diagnose(operation: () => void): void {
+  try {
+    operation();
+  } catch {
+    // Diagnostics cannot own physical cleanup or containment.
+  }
+}
+
+/** Starts only after agent execution ends. A cleanup-stage deadline never caps
+ * agent work and never detaches a physical writer to manufacture quiescence. */
+export function startTurnFinalizationMonitor(input: {
+  observability: Observability;
+  details: TurnHeartbeatDetails;
+  heartbeat: (details: TurnHeartbeatDetails) => void;
+  requestWorkerDrain: () => void;
+  timeoutMs?: number;
+  slowAfterMs?: number;
+}) {
+  const { observability, details } = input;
+  if (!initialized.has(observability)) {
+    for (const stage of TURN_FINALIZATION_STAGES) {
+      diagnose(() => observability.incrementGauge({ ...INFLIGHT, labels: { stage }, amount: 0 }));
+      diagnose(() =>
+        observability.incrementCounter({ ...SLOW_STAGES, labels: { stage }, amount: 0 }),
+      );
+    }
+    initialized.add(observability);
+  }
+  let stage: TurnFinalizationStage | null = null;
+  let disarm = () => {};
+  let stopped = false;
+  let slowTimer: ReturnType<typeof setTimeout> | undefined;
+  const leave = () => {
+    disarm();
+    clearTimeout(slowTimer);
+    if (stage) {
+      const previous = stage;
+      diagnose(() =>
+        observability.incrementGauge({ ...INFLIGHT, labels: { stage: previous }, amount: -1 }),
+      );
+    }
+    stage = null;
+  };
+  return {
+    enter(next: TurnFinalizationStage) {
+      if (stopped || stage === next) return;
+      leave();
+      stage = next;
+      details.phase = "finalizing";
+      details.finalizationStage = next;
+      details.finalizationStageStartedAt = new Date().toISOString();
+      diagnose(() => observability.incrementGauge({ ...INFLIGHT, labels: { stage: next } }));
+      disarm = armTurnQuiescenceWatchdog({
+        enabled: true,
+        ...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }),
+        onTimeout: () => {
+          try {
+            observability.error("turn finalization stalled; containing worker", {
+              surface: "turn_finalization",
+              outcome: "containment",
+              reason: next,
+            });
+          } catch {
+            // Diagnostic failure must never disable physical containment.
+          }
+        },
+        terminateWorker: input.requestWorkerDrain,
+      });
+      // Record the precursor while the process remains scrapeable.
+      slowTimer = setTimeout(
+        () =>
+          diagnose(() => {
+            observability.incrementCounter({ ...SLOW_STAGES, labels: { stage: next } });
+            observability.warn("turn finalization is taking longer than expected", {
+              surface: "turn_finalization",
+              outcome: "slow",
+              reason: next,
+            });
+          }),
+        input.slowAfterMs ?? 30_000,
+      );
+      slowTimer.unref?.();
+      diagnose(() => input.heartbeat(details));
+    },
+    stop() {
+      if (stopped) return;
+      stopped = true;
+      leave();
+    },
+  };
+}

package/src/activities/agent-turn/finalization.ts

@@ -1,7 +1,9 @@
 import {
   commitSessionAttemptQuiescence,
   listPendingSessionTurns,
-  recordCodexAccountUsageWithWakeTargets,
+  recordCodexAccountUsageForFinalization,
+  recordClaudeAccountUsage,
+  releaseClaudeCredentialLease,
   releaseCodexCredentialLease,
   releaseXaiCredentialLease,
   updateXaiQuotaMetadata,
@@ -11,9 +13,10 @@ import { appendAndPublishTurnEventsFenced, publishDurableSessionEvents } from "@
 import { sandboxLeaseTelemetryKey } from "@opengeni/observability";
 import { clearRunCredentialsForAttempt } from "@opengeni/runtime";
 import { fetchXaiSubscriptionQuota } from "@opengeni/xai-subscription";
-import type { Settings } from "@opengeni/config";
+import { environmentsEncryptionKeyBytes, type Settings } from "@opengeni/config";
 import { signalCodexCapacityWakeTargets } from "../codex-capacity";
-import type { currentActivityContext } from "../streaming";
+import { startActivityHeartbeat, type currentActivityContext } from "../streaming";
+import { startTurnFinalizationMonitor } from "./finalization-monitor";
 import type { CodemodeTokenRenewalController } from "../codemode-token-renewal";
 import type { RunCredentialRenewalController } from "../run-credential-renewal";
 import type {
@@ -32,6 +35,7 @@ import {
   makeMachineOpObserver,
   recordSessionEventAppendPhase,
   turnLifecycleMetricsFor,
+  runtimeMetricsHooksForObservability,
 } from "../../observability-metrics";
 import {
   maybePersistWarmWorkspaceSnapshot,
@@ -43,7 +47,6 @@ import { safeErrorDiagnostic, safeErrorForTelemetry } from "./errors";
 import {
   assertPhysicalToolQuiescenceForCancellation,
   assertSessionAttemptQuiescenceRecoveryDurable,
-  armTurnQuiescenceWatchdog,
   clearAttemptCredentialsWithSettledFence,
   drainAttemptOwnedSandboxWriters,
   persistOrSignalSessionAttemptQuiescence,
@@ -72,6 +75,8 @@ export type TurnFinalizationDeps = {
   wakeSessionWorkflow: ActivityServices["wakeSessionWorkflow"];
   signalSessionAttemptQuiesced: ActivityServices["signalSessionAttemptQuiesced"];
   signalCodexCapacityWorkflow: ActivityServices["signalCodexCapacityWorkflow"];
+  requestWorkerDrain: ActivityServices["requestWorkerDrain"];
+  turnFinalizationTimeoutMs: ActivityServices["turnFinalizationTimeoutMs"];
   cancellationSignal: AbortSignal | undefined;
   sandboxResumeController: AbortController;
   activityContext: ReturnType<typeof currentActivityContext>;
@@ -99,6 +104,45 @@ export type TurnFinalizationDeps = {
 };
 
 export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<void> {
+  const details = deps.eventing.heartbeatDetails ?? {
+    sessionId: deps.input.sessionId,
+    turnId: deps.attempt.turnId,
+    opAcks: {},
+  };
+  deps.eventing.heartbeatDetails = details;
+  // Keep one heartbeat owner through all physical cleanup, with the same op
+  // acknowledgements used during execution. It must no longer say "running".
+  if (deps.eventing.heartbeatTimer) clearInterval(deps.eventing.heartbeatTimer);
+  deps.eventing.heartbeatTimer = startActivityHeartbeat(deps.activityContext, details);
+  const monitor = startTurnFinalizationMonitor({
+    observability: deps.observability,
+    ...(deps.turnFinalizationTimeoutMs === undefined
+      ? {}
+      : { timeoutMs: deps.turnFinalizationTimeoutMs }),
+    details,
+    heartbeat: (value) => {
+      try {
+        deps.activityContext?.heartbeat({ ...value, at: new Date().toISOString() });
+      } catch {
+        // A closed Temporal transport is not proof of physical quiescence.
+      }
+    },
+    requestWorkerDrain: deps.requestWorkerDrain,
+  });
+  try {
+    monitor.enter("tool_writers");
+    await finalizeTurnAttemptSteps(deps, monitor);
+  } finally {
+    monitor.stop();
+    if (deps.eventing.heartbeatTimer) clearInterval(deps.eventing.heartbeatTimer);
+    deps.eventing.heartbeatTimer = undefined;
+  }
+}
+
+async function finalizeTurnAttemptSteps(
+  deps: TurnFinalizationDeps,
+  monitor: ReturnType<typeof startTurnFinalizationMonitor>,
+): Promise<void> {
   const {
     input,
     settings,
@@ -142,17 +186,6 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
   let finalizationError: unknown;
   let physicalToolQuiescenceConfirmed = !control.acknowledgeQuiescence;
   let quiescenceReceiptOrProofDurable = !control.acknowledgeQuiescence;
-  const disarmQuiescenceWatchdog = armTurnQuiescenceWatchdog({
-    enabled: control.acknowledgeQuiescence,
-    onTimeout: () => {
-      observability.error("turn quiescence drain exceeded hard containment deadline", {
-        "opengeni.session_id": input.sessionId,
-        "opengeni.turn_id": attempt.turnId ?? "",
-        "opengeni.attempt_id": input.attemptId,
-      });
-    },
-    terminateWorker: () => process.exit(1),
-  });
   const finalizerSignal = turnFinalizerCancellationSignal(
     cancellationSignal,
     control.activityStatus,
@@ -187,6 +220,8 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
       renewals.codemodeTokenRenewal as CodemodeTokenRenewalController | null;
     renewals.codemodeTokenRenewal = null;
     renewals.runCredentialRenewalClosed = true;
+    renewals.runMcpCredentials?.close();
+    delete renewals.runMcpCredentials;
     const runRenewalToStop = renewals.runCredentialRenewal as RunCredentialRenewalController | null;
     renewals.runCredentialRenewal = null;
 
@@ -200,6 +235,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
       // and durably settled before workspace capture. Only receipt
       // publication remains conditional on acknowledgeQuiescence.
       toolCancellationFence,
+      onStage: monitor.enter,
       cancellationReason: cancellationSignal?.reason ?? new Error("TURN_ATTEMPT_FINALIZED"),
       gitCredentialRenewals: gitRenewalsToStop,
       codemodeTokenRenewal: codemodeRenewalToStop,
@@ -211,6 +247,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
     // attempt's generation before publishing physical quiescence. A failure
     // deliberately keeps the receipt closed.
     if (credentialSessionToClear) {
+      monitor.enter("credential_cleanup");
       const clearAttemptCredentials = async (): Promise<void> =>
         await clearRunCredentialsForAttempt(credentialSessionToClear, {
           sessionId: input.sessionId,
@@ -258,6 +295,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
       // and before lease, cache, recording, or provider cleanup. Its
       // transaction also enqueues the exact workflow wake that will admit
       // the replacement; Temporal activity terminalization does neither.
+      monitor.enter("quiescence_receipt");
       const proof: SessionAttemptQuiescenceProof = {
         accountId: input.accountId,
         workspaceId: input.workspaceId,
@@ -304,6 +342,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
         signalProof: signalSessionAttemptQuiesced,
         heartbeat: (deliveryAttempt, retryMs) => {
           activityContext?.heartbeat({
+            ...eventing.heartbeatDetails,
             phase: "quiescence-proof-delivery",
             sessionId: input.sessionId,
             attemptId: input.attemptId,
@@ -336,7 +375,6 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
         },
       });
       quiescenceReceiptOrProofDurable = true;
-      disarmQuiescenceWatchdog();
       if (recoveryMode === "signal") {
         observability.info("agent turn quiescence proof handed to workflow recovery", {
           "opengeni.session_id": input.sessionId,
@@ -351,6 +389,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
     // the turn. Sync observer → buffer → single awaited append here (no unawaited
     // DB write inside the activity). Scoped to this turn; skipped if no turnId
     // (the op ran under a turn, so on the normal path turnId is set).
+    monitor.enter("event_flush");
     const machineOpEvents = machineOpObserver.drainEvents();
     if (machineOpEvents.length > 0 && attempt.turnId && attempt.executionGeneration > 0) {
       await waitForTurnFinalizerStep(
@@ -377,17 +416,65 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
     // Multi-account P4: flush the serving account's free per-turn caches ONCE,
     // best-effort (same discipline as today's usage write). Both writers skip
     // version/updatedAt, so neither can race the token-refresh CAS.
+    monitor.enter("provider_leases");
+    const claudeEncryptionKey = environmentsEncryptionKeyBytes(settings);
+    if (claudeEncryptionKey && providerTurn.claudeAuthoritySnapshot && leases.claude.subjectId) {
+      for (const snapshot of providerTurn.latestClaudeUsage.values()) {
+        if (
+          (!snapshot.observation && !snapshot.refresh) ||
+          snapshot.expectedConnectionId !== providerTurn.effectiveClaudeCredentialId ||
+          snapshot.expectedCredentialVersion !== providerTurn.effectiveClaudeCredentialVersion
+        )
+          continue;
+        await waitForTurnFinalizerStep(
+          recordClaudeAccountUsage(
+            db,
+            {
+              accountId: input.accountId,
+              workspaceId: input.workspaceId,
+              subjectId: leases.claude.subjectId,
+              credentialId: snapshot.expectedConnectionId,
+              authoritySnapshot: providerTurn.claudeAuthoritySnapshot,
+            },
+            {
+              encryptionKey: claudeEncryptionKey,
+              token: snapshot.token,
+              expectedCredentialVersion: snapshot.expectedCredentialVersion,
+              ...(snapshot.observation ? { observation: snapshot.observation } : {}),
+              ...(snapshot.refresh ? { refresh: snapshot.refresh } : {}),
+            },
+          ).catch(() => null),
+          finalizerSignal,
+        );
+      }
+    }
     if (providerTurn.effectiveCodexCredentialId) {
       // Part A: the latest scraped usage-header snapshot → the P2 usage cache. A
-      // full both-windows snapshot (parseCodexUsageHeaders gates on both), so this
-      // is byte-identical to the /wham/usage write — no partial-window clobber.
-      if (providerTurn.latestCodexUsage) {
+      // full duration-identified snapshot (parseCodexUsageHeaders gates on both),
+      // so untyped response headers cannot mislabel weekly-only quota.
+      if (
+        providerTurn.latestCodexUsage &&
+        attempt.turnId &&
+        leases.codex.held &&
+        leases.codex.holderId &&
+        leases.codex.generation !== null &&
+        providerTurn.effectiveCodexCredentialVersion !== null
+      ) {
         const usageMutation = await waitForTurnFinalizerStep(
-          recordCodexAccountUsageWithWakeTargets(
+          recordCodexAccountUsageForFinalization(
             db,
             input.workspaceId,
             providerTurn.effectiveCodexCredentialId,
             providerTurn.latestCodexUsage,
+            {
+              turnId: attempt.turnId,
+              sessionId: input.sessionId,
+              attemptId: input.attemptId,
+              executionGeneration: attempt.executionGeneration,
+              holderId: leases.codex.holderId,
+              generation: leases.codex.generation,
+              credentialVersion: providerTurn.effectiveCodexCredentialVersion,
+            },
           ).catch(() => null),
           finalizerSignal,
         );
@@ -450,25 +537,30 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
         );
       }
     }
-    leases.xai.stopHeartbeat();
-    if (
-      leases.xai.held &&
-      attempt.turnId &&
-      leases.xai.subjectId &&
-      leases.xai.holderId &&
-      leases.xai.generation !== null
-    ) {
-      await waitForTurnFinalizerStep(
-        releaseXaiCredentialLease(db, {
-          workspaceId: input.workspaceId,
-          subjectId: leases.xai.subjectId,
-          turnId: attempt.turnId,
-          holderId: leases.xai.holderId,
-          generation: leases.xai.generation,
-        }).catch(() => undefined),
-        finalizerSignal,
-      );
-      leases.xai.held = false;
+    for (const [lease, release] of [
+      [leases.xai, releaseXaiCredentialLease],
+      [leases.claude, releaseClaudeCredentialLease],
+    ] as const) {
+      lease.stopHeartbeat();
+      if (
+        lease.held &&
+        attempt.turnId &&
+        lease.subjectId &&
+        lease.holderId &&
+        lease.generation !== null
+      ) {
+        await waitForTurnFinalizerStep(
+          release(db, {
+            workspaceId: input.workspaceId,
+            subjectId: lease.subjectId,
+            turnId: attempt.turnId,
+            holderId: lease.holderId,
+            generation: lease.generation,
+          }).catch(() => undefined),
+          finalizerSignal,
+        );
+        lease.held = false;
+      }
     }
     // Workbench v2 turn-end workspace capture — runs FIRST in
     // the turn-end finally, while the box is MAXIMALLY ALIVE. The agent's last
@@ -502,6 +594,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
     ) {
       // Review reads yield to queued work. Recovery snapshots have their own
       // physical-settlement requirement and are drained later in finalization.
+      monitor.enter("workspace_capture");
       sandboxState.turnEndCaptureInProgress = true;
       const captureSandbox = sandboxState.resolvedSandbox;
       const captureSession = sandboxState.setupBoxSession as ChannelASession;
@@ -549,6 +642,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
         },
       });
     }
+    monitor.enter("tool_close");
     eventing.toolPreparationClosing = true;
     if (eventing.toolPreparationReady) {
       await waitForTurnFinalizerStep(
@@ -569,9 +663,7 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
         finalizerSignal,
       );
     }
-    if (eventing.heartbeatTimer) {
-      clearInterval(eventing.heartbeatTimer);
-    }
+    monitor.enter("sandbox_provisioning");
     if (sandboxState.turnSandboxProvisioner?.hasStarted()) {
       await waitForTurnFinalizerStep(
         sandboxState.turnSandboxProvisioner.waitForSettled(30_000),
@@ -599,12 +691,14 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
     if (sandboxState.leaseHeartbeatTimer) {
       stopLeaseHeartbeat();
     }
+    monitor.enter("sandbox_rotation");
     if (sandboxState.rotationPreemptionInFlight) {
       await sandboxState.rotationPreemptionInFlight.catch(() => undefined);
     }
     // No heartbeat can register another capture after this point. Join the
     // actual provider continuation before either starting a turn-end capture
     // or entering the proof-bearing holder release.
+    monitor.enter("workspace_snapshot");
     await drainInFlightWarmSnapshot();
     if (sandboxState.resolvedSandbox) {
       // TURN-END mid-session snapshot (sandbox-file-persistence): fold the
@@ -622,7 +716,13 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
         settledTurnId
       ) {
         const snapshot = maybePersistWarmWorkspaceSnapshot(
-          { db, settings, objectStorage },
+          {
+            db,
+            settings,
+            objectStorage,
+            sandboxMetrics: runtimeMetricsHooksForObservability(observability),
+            observability,
+          },
           {
             accountId: input.accountId,
             workspaceId: input.workspaceId,
@@ -667,10 +767,13 @@ export async function finalizeTurnAttempt(deps: TurnFinalizationDeps): Promise<v
     // Temporal cancellation: the eager listener may already have dropped
     // the holder, and this idempotent proof-bearing pass still must run.
     stopLeaseHeartbeat();
+    monitor.enter("sandbox_rotation");
     if (sandboxState.rotationPreemptionInFlight) {
       await sandboxState.rotationPreemptionInFlight.catch(() => undefined);
     }
+    monitor.enter("workspace_snapshot");
     await drainInFlightWarmSnapshot();
+    monitor.enter("sandbox_release");
     const sandboxReleaseTargets = new Set(sandboxState.lateSandboxesAwaitingWriterDrain);
     sandboxState.lateSandboxesAwaitingWriterDrain.clear();
     if (sandboxState.resolvedSandbox) {

package/src/activities/agent-turn/github-repository-bindings.ts

@@ -229,6 +229,8 @@ export type ApplyTurnGitHubRepositoryBindingsInput = {
   workspaceId: string;
   sessionId: string;
   activeSandboxBackend: Settings["sandboxBackend"] | undefined;
+  /** Product-owned credentials take precedence over inferred platform bindings. */
+  hasCredentialProvider?: boolean;
   claimedTurnResources: readonly ResourceRef[];
   claimedRuntimeResources: readonly ResourceRef[];
   /** Publishes one attempt-fenced session event batch. */
@@ -258,7 +260,7 @@ export async function applyTurnGitHubRepositoryBindings(
     turnResources: [...input.claimedTurnResources],
     runtimeResources: [...input.claimedRuntimeResources],
   };
-  if (input.activeSandboxBackend === "selfhosted") {
+  if (input.activeSandboxBackend === "selfhosted" || input.hasCredentialProvider) {
     return passthrough;
   }
   let resolved: TurnGitHubRepositoryBindings;