@opengeni/worker-bundle 1.4.0 → 1.4.4

package/dist/observability-metrics.d.ts

@@ -176,7 +176,7 @@ export declare function startContextCompactionPendingMonitor(input: {
 };
 export declare function recordSandboxLeaseGauges(observability: Observability, counts: Partial<Record<SandboxLeaseLiveness, number>>): void;
 export declare function recordOpenSandboxKubernetesInventoryGauges(observability: Observability, inventory: OpenSandboxKubernetesInventory): void;
-export declare const SANDBOX_INVENTORY_PROJECTION_DOMAINS: readonly ["leases", "checkpoint_artifacts", "recovery_observations", "rotation_backlog", "retained_processes", "expired_drains", "opensandbox_kubernetes"];
+export declare const SANDBOX_INVENTORY_PROJECTION_DOMAINS: readonly ["leases", "checkpoint_artifacts", "recovery_observations", "rotation_backlog", "retained_processes", "expired_drains", "opensandbox_kubernetes", "modal_provider", "interaction_idle"];
 export type SandboxInventoryProjectionDomain = (typeof SANDBOX_INVENTORY_PROJECTION_DOMAINS)[number];
 export declare function recordSandboxInventoryProjectionSuccess(observability: Observability, domain: SandboxInventoryProjectionDomain, timestampSeconds?: number): void;
 export declare function recordSandboxInventoryProjectionFailure(observability: Observability, domain: SandboxInventoryProjectionDomain): void;
@@ -218,6 +218,22 @@ export declare function recordManagedAuthNewSignupsSwitchGauge(observability: Ob
  * grants happen only while this gauge and the runtime switch gauge are both 1.
  */
 export declare function recordVerifiedSignupTrialDeploymentFlagGauge(observability: Observability, enabled: boolean): void;
+/**
+ * Provider-side Modal inventory reconciled against live leases by the orphan
+ * sweep: `running` = every running box in the app, `unleased` = running boxes
+ * no live lease protects (orphans, counted before termination), and
+ * `lease_missing_instance` = warm leases whose exact provider instance is no
+ * longer running (zombies).
+ */
+export declare function recordModalSandboxInventoryGauges(observability: Observability, inventory: {
+    running: number;
+    unleased: number;
+    liveLeaseInstancesMissing: number;
+}): void;
+/** Warm leases held only by Browser/Computer interaction holders, by time since
+ * the newest interaction activity. Such boxes never drain on their own until the
+ * interaction session ends or the provider deadline. */
+export declare function recordInteractionOnlyLeaseGauges(observability: Observability, counts: Record<string, number>): void;
 export declare function recordSandboxOrphansTerminated(observability: Observability, count: number): void;
 export type SandboxCheckpointArtifactMetricState = "candidate" | "current" | "previous" | "delete_pending" | "deleting" | "delete_failed" | "deleted";
 export declare function recordSandboxCheckpointArtifactGauges(observability: Observability, counts: Record<SandboxCheckpointArtifactMetricState, number>): void;
@@ -284,7 +300,7 @@ export declare function recordModelRequestPhase(observability: Observability, in
     outcome?: "completed" | "failed" | "timed_out";
     durationSeconds: number;
 }): void;
-export type TurnStartupPhase = "services_initialization" | "claim_catalog_read" | "claim_atomic" | "claim_session_read" | "claim_capability_settings" | "learning_policy_freeze" | "attachment_authority_projection" | "claim_and_policy" | "turn_start_settlement" | "credential_selection" | "runtime_preparation" | "session_mcp_settings" | "sandbox_establish" | "file_resolution" | "tool_context_preparation" | "tool_preparation" | "tool_server_construction" | "tool_required_connect" | "tool_optional_connect" | "tool_attempt_catalog_build" | "tool_attempt_catalog_persist" | "tool_workspace_gateway_catalog_build" | "post_tool_preparation" | "agent_construction" | "post_agent_preparation" | "programmatic_operation_recovery" | "file_materialization" | "history_preparation" | "history_system_update_load" | "history_current_attachment_resolution" | "history_durable_history_load" | "history_sandbox_envelope_load" | "history_canonical_projection" | "history_provider_projection" | "history_attachment_ref_projection" | "history_screenshot_materialization" | "history_model_attachment_projection" | "history_runtime_input_assembly" | "history_artifact_candidate_scan" | "history_generated_image_materialization" | "history_position_load" | "owned_sandbox_setup" | "runtime_stream_initialization" | "model_request_preparation" | "model_sdk_serialization" | "model_prepare_sandbox_agent_preparation" | "model_prepare_sandbox_agent_manifest_inventory" | "model_prepare_sandbox_session_manifest_inventory" | "model_prepare_sandbox_manifest_apply" | "model_prepare_sandbox_entry_materialization" | "model_prepare_sandbox_running_check" | "model_prepare_sandbox_start" | "model_prepare_sandbox_client_create" | "model_prepare_sandbox_client_resume" | "model_prepare_sandbox_client_delete" | "model_prepare_sandbox_client_state_serialize" | "model_prepare_sandbox_client_reuse_check" | "model_prepare_sandbox_workspace_mutation_admission" | "model_prepare_sandbox_workspace_mutation_provider" | "model_prepare_sandbox_workspace_mutation_settlement" | "model_prepare_sandbox_first_routed_resolution_other" | "model_prepare_sandbox_first_routed_mutation_admission" | "model_prepare_sandbox_first_routed_provider_operation" | "model_prepare_sandbox_first_routed_mutation_settlement" | "model_prepare_sandbox_first_routed_other" | "model_prepare_sandbox_snapshot_wait" | "model_prepare_runner_before_first_sandbox_operation" | "model_prepare_sdk_after_first_sandbox_operation" | "model_prepare_runner_before_mcp_tools" | "model_prepare_mcp_tools_snapshot" | "model_prepare_mcp_tools_before_input_filter" | "model_prepare_mcp_tools_before_repository_skill_discovery" | "model_prepare_repository_skill_discovery" | "model_prepare_repository_skill_discovery_before_input_filter" | "model_prepare_input_filter_base" | "model_prepare_input_filter_genesis" | "model_prepare_input_filter_host" | "model_prepare_input_filter_tool_output" | "model_prepare_input_filter_modality" | "model_prepare_input_filter_context" | "model_prepare_responses_input_conversion" | "model_prepare_responses_request_build" | "model_credential_resolution" | "model_wire_normalization" | "model_request_audit" | "stream_bootstrap";
+export type TurnStartupPhase = "services_initialization" | "claim_catalog_read" | "claim_atomic" | "claim_session_read" | "claim_capability_settings" | "learning_policy_freeze" | "attachment_authority_projection" | "claim_and_policy" | "turn_start_settlement" | "credential_selection" | "runtime_preparation" | "session_mcp_settings" | "sandbox_establish" | "file_resolution" | "tool_context_preparation" | "tool_preparation" | "tool_server_construction" | "tool_required_connect" | "tool_optional_connect" | "tool_attempt_catalog_build" | "tool_attempt_catalog_persist" | "tool_workspace_gateway_catalog_build" | "post_tool_preparation" | "agent_construction" | "post_agent_preparation" | "file_materialization" | "history_preparation" | "history_system_update_load" | "history_current_attachment_resolution" | "history_durable_history_load" | "history_sandbox_envelope_load" | "history_canonical_projection" | "history_provider_projection" | "history_attachment_ref_projection" | "history_screenshot_materialization" | "history_model_attachment_projection" | "history_runtime_input_assembly" | "history_artifact_candidate_scan" | "history_generated_image_materialization" | "history_position_load" | "owned_sandbox_setup" | "runtime_stream_initialization" | "model_request_preparation" | "model_sdk_serialization" | "model_prepare_sandbox_agent_preparation" | "model_prepare_sandbox_agent_manifest_inventory" | "model_prepare_sandbox_session_manifest_inventory" | "model_prepare_sandbox_manifest_apply" | "model_prepare_sandbox_entry_materialization" | "model_prepare_sandbox_running_check" | "model_prepare_sandbox_start" | "model_prepare_sandbox_client_create" | "model_prepare_sandbox_client_resume" | "model_prepare_sandbox_client_delete" | "model_prepare_sandbox_client_state_serialize" | "model_prepare_sandbox_client_reuse_check" | "model_prepare_sandbox_workspace_mutation_admission" | "model_prepare_sandbox_workspace_mutation_provider" | "model_prepare_sandbox_workspace_mutation_settlement" | "model_prepare_sandbox_first_routed_resolution_other" | "model_prepare_sandbox_first_routed_mutation_admission" | "model_prepare_sandbox_first_routed_provider_operation" | "model_prepare_sandbox_first_routed_mutation_settlement" | "model_prepare_sandbox_first_routed_other" | "model_prepare_sandbox_snapshot_wait" | "model_prepare_runner_before_first_sandbox_operation" | "model_prepare_sdk_after_first_sandbox_operation" | "model_prepare_runner_before_mcp_tools" | "model_prepare_mcp_tools_snapshot" | "model_prepare_mcp_tools_before_input_filter" | "model_prepare_mcp_tools_before_repository_skill_discovery" | "model_prepare_repository_skill_discovery" | "model_prepare_repository_skill_discovery_before_input_filter" | "model_prepare_input_filter_base" | "model_prepare_input_filter_genesis" | "model_prepare_input_filter_host" | "model_prepare_input_filter_tool_output" | "model_prepare_input_filter_modality" | "model_prepare_input_filter_context" | "model_prepare_responses_input_conversion" | "model_prepare_responses_request_build" | "model_credential_resolution" | "model_wire_normalization" | "model_request_audit" | "stream_bootstrap";
 export type TurnStartupOutcome = "completed" | "failed";
 export type TurnStartupMilestone = "queue" | "provider_dispatch" | "first_byte";
 export type TurnStartupCache = "hit" | "miss" | "disabled" | "none";
@@ -375,22 +391,35 @@ export declare function recordTurnStartupMilestone(observability: Observability,
     durationSeconds: number;
 }): void;
 export type StreamDeltaClass = "message" | "reasoning";
+/** First streamed content of a provider response. `any` is the first reasoning
+ *  or answer delta; `text` is the first answer-text delta. They overlap — never
+ *  sum across `content`. */
+export type ProviderFirstContent = "any" | "text";
 /**
- * Per-turn stream-timing tracker fed every normalized runtime event in push order.
- * It emits two model-responsiveness SLIs from the worker's seat on the stream:
+ * Per-stream timing tracker fed every normalized runtime event in push order,
+ * plus two producer-side hooks. It separates OUR latency from the PROVIDER's:
  *
- *   - `opengeni_stream_ttft_seconds{provider}` — time from a model (re)start to its
- *     first streamed content delta. The anchor starts at construction (≈ runStream
- *     start, so the first observation is "how long until text appears") and re-arms
- *     on every non-content event (a tool call, a completed message, a usage frame),
- *     so a post-tool response measures the model's restart latency, NOT our own
- *     tool-execution time.
+ *   - `opengeni_stream_ttft_seconds{provider}` — user-perceived (re)start latency:
+ *     time from a stream start / structural boundary (tool call, tool result,
+ *     completed message, usage frame) to the first streamed content delta. It
+ *     mixes our between-call work with provider time; keep it for the absolute
+ *     dashboard view, not for provider alerting.
+ *   - `opengeni_model_request_pre_dispatch_seconds{provider}` — OUR per-request
+ *     latency: SDK model entry (the first admission check) to the literal
+ *     provider dispatch. Covers admission, the durable history checkpoint,
+ *     credit revalidation, the durable request audit and request build.
+ *   - `opengeni_model_provider_ttft_seconds{provider,content}` — the PROVIDER's
+ *     latency: literal dispatch to the first streamed `any` (reasoning or text)
+ *     delta and to the first answer `text` delta. Includes network, provider
+ *     queueing, prompt prefill and reasoning before the first streamed token.
  *   - `opengeni_stream_inter_delta_gap_seconds{provider,class}` — gap between
- *     consecutive content deltas of the SAME class. The run resets on any
- *     non-content event so a gap never spans a tool call or a model boundary — it
- *     measures only the choppiness of a live token stream.
+ *     consecutive content deltas of the SAME class, reset at every structural
+ *     event so a gap never spans a tool call or a model boundary.
  *
- * Purely observational and clock-injectable; it never touches the events it sees.
+ * Dispatch timing deliberately survives structural events: the consumer may
+ * still be persisting the previous response's tool results when the producer
+ * dispatches the next request, but the next request's first delta always
+ * follows its own dispatch. Purely observational and clock-injectable.
  */
 export declare class StreamTimingMetrics {
     private readonly observability;
@@ -399,11 +428,20 @@ export declare class StreamTimingMetrics {
     private ttftAnchor;
     private ttftArmed;
     private readonly lastDeltaAt;
+    private modelEntryAt;
+    private dispatchedAt;
+    private readonly providerFirstRecorded;
     constructor(observability: Observability, options: {
         provider: string;
         now?: () => number;
     });
+    /** Producer side: the SDK entered a model request (first admission check).
+     *  Admission can be re-entered for the same request; keep the earliest. */
+    onModelRequestEntry(): void;
+    /** Producer side: request bytes are about to leave this process. */
+    onProviderDispatch(): void;
     onEvent(type: SessionEventType): void;
+    private observeProviderFirst;
 }
 /** Flush shape of the streaming batcher: how many events coalesced into one flush
  *  (the coalescing win) and how long that flush took (append + publish). */

package/dist/sandbox-resume.d.ts

@@ -54,6 +54,9 @@ export type SandboxResumeServices = {
      * replacement. Production uses {@link freshSandboxReadinessReplacementDelayMs}.
      * It may be async so a test can observe the rolled-back lease in between. */
     freshSandboxReadinessReplacementDelayMs?: () => number | Promise<number>;
+    /** Test seam: runs immediately after the elected spawner published its box
+     * warm, before the final cancellation check. */
+    onSpawnedSandboxPublished?: () => void | Promise<void>;
     /**
      * The turn attempt's fresh-box readiness replacement budget. The lazy
      * provisioner may call resumeBoxForTurn again after a typed lease

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@opengeni/worker-bundle",
-  "version": "1.4.0",
+  "version": "1.4.4",
   "description": "OpenGeni worker entry and reusable embedded lifecycle, shipped with a release-coherent pre-bundled Temporal workflow artifact.",
   "license": "Apache-2.0",
   "repository": {
@@ -54,26 +54,26 @@
   },
   "dependencies": {
     "@llamaindex/liteparse": "2.14.2",
-    "@opengeni/agent-proto": "1.4.0",
-    "@opengeni/capabilities": "1.4.0",
-    "@opengeni/codemode": "1.4.0",
-    "@opengeni/codex": "1.4.0",
-    "@opengeni/config": "1.4.0",
-    "@opengeni/contracts": "1.4.0",
-    "@opengeni/core": "1.4.0",
-    "@opengeni/db": "1.4.0",
-    "@opengeni/documents": "1.4.0",
-    "@opengeni/events": "1.4.0",
-    "@opengeni/github": "1.4.0",
-    "@opengeni/interaction": "1.4.0",
-    "@opengeni/jev": "1.4.0",
-    "@opengeni/network": "1.4.0",
-    "@opengeni/observability": "1.4.0",
-    "@opengeni/runtime": "1.4.0",
-    "@opengeni/sdk": "1.4.0",
-    "@opengeni/storage": "1.4.0",
-    "@opengeni/tool-gateway": "1.4.0",
-    "@opengeni/xai-subscription": "1.4.0",
+    "@opengeni/agent-proto": "1.4.4",
+    "@opengeni/capabilities": "1.4.4",
+    "@opengeni/codemode": "1.4.4",
+    "@opengeni/codex": "1.4.4",
+    "@opengeni/config": "1.4.4",
+    "@opengeni/contracts": "1.4.4",
+    "@opengeni/core": "1.4.4",
+    "@opengeni/db": "1.4.4",
+    "@opengeni/documents": "1.4.4",
+    "@opengeni/events": "1.4.4",
+    "@opengeni/github": "1.4.4",
+    "@opengeni/interaction": "1.4.4",
+    "@opengeni/jev": "1.4.4",
+    "@opengeni/network": "1.4.4",
+    "@opengeni/observability": "1.4.4",
+    "@opengeni/runtime": "1.4.4",
+    "@opengeni/sdk": "1.4.4",
+    "@opengeni/storage": "1.4.4",
+    "@opengeni/tool-gateway": "1.4.4",
+    "@opengeni/xai-subscription": "1.4.4",
     "@temporalio/activity": "^1.17.0",
     "@temporalio/client": "^1.17.0",
     "@temporalio/worker": "^1.17.0",

package/src/activities/agent-turn/errors.ts

@@ -34,6 +34,7 @@ import {
   RoutingWorkspaceRootChangedError,
   RoutingBackendRecoveryRequiredError,
   ResponsesStreamingTerminalError,
+  classifyModelStreamIdleTimeoutError,
   SandboxMaterializationVerificationError,
   materializationVerificationDiagnostic,
   type MaterializationVerificationDiagnostic,
@@ -500,15 +501,20 @@ function retryableDatabaseFailureCode(
       const connectionOutage = requireDatabaseProvenance
         ? isRunningTurnDatabaseConnectionSqlState(sqlState)
         : isDatabaseConnectionSqlState(sqlState);
+      // PostgreSQL aborts a deadlock or serialization victim's whole
+      // transaction: unlike a lost connection, that write certainly did not
+      // commit, so it is at least as safe for exact-attempt recovery.
+      const rolledBack = isRetryablePersistenceSqlState(sqlState);
       const code = typed
         ? retryablePersistenceFailureCode(sqlState)
-        : connectionOutage || isRetryablePersistenceSqlState(sqlState)
+        : connectionOutage || rolledBack
           ? databaseFailureCode(sqlState)
           : null;
       if (
         !code ||
         (requireDatabaseProvenance &&
           !connectionOutage &&
+          !rolledBack &&
           !(sqlState === null && hasOwnTransport(node)))
       )
         return null;
@@ -603,12 +609,7 @@ export function postClaimDatabaseRecoveryFailure(input: {
   };
 }): ApplicationFailure | null {
   const code = retryableDatabaseFailureCode(input.error, input.requireDatabaseProvenance);
-  if (
-    !code ||
-    input.executionGeneration < 1 ||
-    (input.requireDatabaseProvenance && code !== "db_failure")
-  )
-    return null;
+  if (!code || input.executionGeneration < 1) return null;
   if (
     (input.sandboxSetupOutcomeUnknown && input.sandboxSetupRecoveryExhausted) ||
     ((input.sandboxSetupOutcomeUnknown || input.sandboxSetupRecoveryExhausted) &&
@@ -1178,6 +1179,26 @@ function providerHttpStatus(error: unknown): number | undefined {
   return undefined;
 }
 
+/**
+ * A fetch-layer timeout: the runtime's DOMException `TimeoutError` (message
+ * "The operation timed out."), possibly wrapped by SDK/runner `cause` chains.
+ * Explicit cancellation (`AbortError`) is deliberately not matched.
+ */
+export function isTransportTimeoutError(error: unknown): boolean {
+  let current: unknown = error;
+  const seen = new Set<unknown>();
+  for (let depth = 0; depth < 8 && current && typeof current === "object"; depth += 1) {
+    if (seen.has(current)) break;
+    seen.add(current);
+    const value = current as { name?: unknown; message?: unknown; cause?: unknown };
+    if (value.name === "TimeoutError" || value.message === "The operation timed out.") {
+      return true;
+    }
+    current = value.cause;
+  }
+  return false;
+}
+
 export function isTransientProviderError(error: unknown): boolean {
   if (error instanceof ResponsesStreamingTerminalError) {
     return error.category === "unavailable";
@@ -1715,6 +1736,39 @@ function baseAgentRunFailurePayload(
       ...(mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}),
     };
   }
+  // A generic model stream that went silent mid-response (typed idle bound) or
+  // a transport timeout ("The operation timed out." DOMException TimeoutError
+  // from the fetch layer) is a dead upstream connection, not a request fault:
+  // checkpoint durable truth and recover the same turn within the finite
+  // provider recovery budget. Codex/SuperGrok typed timeouts are classified
+  // above under their own transport policy.
+  const streamIdle = classifyModelStreamIdleTimeoutError(error);
+  if (streamIdle) {
+    const seconds =
+      streamIdle.idleTimeoutMs !== null ? Math.round(streamIdle.idleTimeoutMs / 1_000) : null;
+    const window = seconds !== null ? ` for ${seconds}s` : "";
+    return {
+      error:
+        streamIdle.kind === "progress"
+          ? `The model provider stream stalled: only keepalive traffic and no response progress${window}. The same turn will retry after a short delay.`
+          : `The model provider stopped sending response data${window}. The same turn will retry after a short delay.`,
+      code: "provider_unavailable",
+      retryable: true,
+      timeoutClass: streamIdle.kind === "progress" ? "progress_stream" : "idle_stream",
+      responseObserved: true,
+      detail: streamIdle.message,
+    };
+  }
+  if (status === undefined && isTransportTimeoutError(error)) {
+    return {
+      error:
+        "The model provider connection timed out. The same turn will retry after a short delay.",
+      code: "provider_unavailable",
+      retryable: true,
+      timeoutClass: "transport",
+      detail: message,
+    };
+  }
   if (code === UNKNOWN_MODEL_FINISH_REASON_CODE) {
     return {
       error:

package/src/activities/agent-turn/finalization.ts

@@ -43,6 +43,7 @@ import {
   type ResumedTurnSandbox,
 } from "../../sandbox-resume";
 import { createTurnCredentialLeases } from "./credential-leases";
+import { drainPhysicalSandboxResumes } from "./sandbox-provision";
 import { safeErrorDiagnostic, safeErrorForTelemetry } from "./errors";
 import {
   assertPhysicalToolQuiescenceForCancellation,
@@ -778,6 +779,21 @@ async function finalizeTurnAttemptSteps(
     }
     monitor.enter("workspace_snapshot");
     await drainInFlightWarmSnapshot();
+    // Join any provider establish still running behind a cancelled wrapper
+    // (finalization already aborted provisioning when no box resolved). Its
+    // own exact cleanup (roll an unpublished box back to cold, or keep a
+    // published box and drop its holder) must commit before this activity can
+    // let a shutting-down worker exit. A late success is routed through
+    // releaseLateSandbox before the release targets are collected below.
+    monitor.enter("sandbox_provisioning");
+    if (sandboxState.inFlightSandboxResumes.size > 0) {
+      const drained = await drainPhysicalSandboxResumes(sandboxState.inFlightSandboxResumes);
+      if (drained === "timed_out") {
+        console.error(
+          "in-flight sandbox establish did not settle before finalization; the lease reaper owns its warming row",
+        );
+      }
+    }
     monitor.enter("sandbox_release");
     const sandboxReleaseTargets = new Set(sandboxState.lateSandboxesAwaitingWriterDrain);
     sandboxState.lateSandboxesAwaitingWriterDrain.clear();

package/src/activities/agent-turn/run.ts

@@ -385,6 +385,14 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
       getModelRunSettings: () => eventing.modelRunSettings,
       getExecutionGeneration: () => attempt.executionGeneration,
     });
+    // Diagnostic only: a timing observer can never fence or fail a request.
+    const observeProviderDispatch = () => {
+      try {
+        eventing.providerDispatchObserver?.();
+      } catch {
+        // Metrics emission must never affect a model call.
+      }
+    };
     const checkpointBeforeProviderDispatch = async () => {
       await awaitModelCallAdmission();
       await checkpointHistoryBeforeProviderDispatch(historySink, {
@@ -682,6 +690,7 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
                     }, // latest wins; flushed once in finally
                     beforeProviderDispatch: () => {
                       leases.codex.assertUsable();
+                      observeProviderDispatch();
                     },
                     onRequestPreparationDiagnostic: (phase) => {
                       if (
@@ -959,6 +968,9 @@ export function createRunAgentTurnActivity(services: () => Promise<ActivityServi
                       },
                     },
                   ]);
+                  // SuperGrok places bytes on the wire immediately after this
+                  // awaited audit returns.
+                  if (event.phase === "started") observeProviderDispatch();
                   attempt.providerRecoveryCount = providerRecoveryCountAfterModelRequestPhase(
                     attempt.providerRecoveryCount,
                     event.phase,

package/src/activities/agent-turn/sandbox-establish.ts

@@ -66,6 +66,7 @@ import {
   classifySandboxLogicalProvisionFailure,
   isLazySandboxProvisionRetryable,
   createTurnSandboxProvisioner,
+  trackPhysicalSandboxResume,
 } from "./sandbox-provision";
 import {
   resolveActiveSandboxBackend,
@@ -645,40 +646,43 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
           const lazyHolderId = sandboxState.sandboxHolderId;
           const lazyGroupId = sandboxState.sandboxGroupId;
           sandboxState.resumeManagedGroupBox = () =>
-            resumeBoxForTurn(
-              {
-                db,
-                settings: runSettings,
-                logicalFallbackSettings: logicalSandboxSettings,
-                cancellationSignal: sandboxResumeSignal,
-                sandboxMetrics: runtimeMetricsHooksForObservability(observability),
-                observability,
-                freshSandboxReadinessReplacementBudget,
-                onSandboxLost: publishSandboxLost,
-                objectStorage,
-                bus,
-              },
-              {
-                accountId: input.accountId,
-                workspaceId: input.workspaceId,
-                sandboxGroupId: lazyGroupId,
-                sessionId: input.sessionId,
-                backend: groupBoxBackend,
-                os: groupBoxOs,
-                environment: sandboxEnvironment,
-                ...(groupBoxImage
-                  ? {
-                      image: groupBoxImage,
-                      imagePolicy: groupBoxImagePolicy,
-                    }
-                  : {}),
-                // The lazy acquire must enforce the same frozen rig authority
-                // as the eager acquire; otherwise a warm box for another rig
-                // can bypass the shared-state conflict/rotation fence.
-                ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
-              },
-              "turn",
-              lazyHolderId,
+            trackPhysicalSandboxResume(
+              sandboxState.inFlightSandboxResumes,
+              resumeBoxForTurn(
+                {
+                  db,
+                  settings: runSettings,
+                  logicalFallbackSettings: logicalSandboxSettings,
+                  cancellationSignal: sandboxResumeSignal,
+                  sandboxMetrics: runtimeMetricsHooksForObservability(observability),
+                  observability,
+                  freshSandboxReadinessReplacementBudget,
+                  onSandboxLost: publishSandboxLost,
+                  objectStorage,
+                  bus,
+                },
+                {
+                  accountId: input.accountId,
+                  workspaceId: input.workspaceId,
+                  sandboxGroupId: lazyGroupId,
+                  sessionId: input.sessionId,
+                  backend: groupBoxBackend,
+                  os: groupBoxOs,
+                  environment: sandboxEnvironment,
+                  ...(groupBoxImage
+                    ? {
+                        image: groupBoxImage,
+                        imagePolicy: groupBoxImagePolicy,
+                      }
+                    : {}),
+                  // The lazy acquire must enforce the same frozen rig authority
+                  // as the eager acquire; otherwise a warm box for another rig
+                  // can bypass the shared-state conflict/rotation fence.
+                  ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
+                },
+                "turn",
+                lazyHolderId,
+              ),
             );
           if (
             shouldPrefetchManagedSandbox({
@@ -725,52 +729,55 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
         );
         try {
           sandboxState.resolvedSandbox = await waitForTurnOperation(
-            resumeBoxForTurn(
-              {
-                db,
-                settings: runSettings,
-                logicalFallbackSettings: logicalSandboxSettings,
-                cancellationSignal: sandboxResumeSignal,
-                sandboxMetrics: runtimeMetricsHooksForObservability(observability),
-                observability,
-                freshSandboxReadinessReplacementBudget,
-                onSandboxLost: publishSandboxLost,
-                objectStorage,
-                bus,
-              },
-              {
-                accountId: input.accountId,
-                workspaceId: input.workspaceId,
-                sandboxGroupId: session.sandboxGroupId,
-                sessionId: input.sessionId,
-                // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
-                // is not machine-primary resumes a real cloud group box (the
-                // deployment default), never a "selfhosted" box (which would throw
-                // for lack of a bound agentId).
-                backend: groupBoxBackend,
-                os: groupBoxOs,
-                environment: sandboxEnvironment,
-                // Deployment/workspace pins select NEW creates only. The lease
-                // retains a live group's image through between-turn repins, even
-                // with other holders; only rotation/reaping elects a new image.
-                // Select the image for the actual group-box backend; a configured Modal image must
-                // never override a Docker run. The selfhosted branch
-                // (establishSelfhostedTurnSession) NEVER passes
-                // an image — B3 lives only on this managed-box branch.
-                ...(groupBoxImage
-                  ? {
-                      image: groupBoxImage,
-                      imagePolicy: groupBoxImagePolicy,
-                    }
-                  : {}),
-                // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
-                // conflicts on a live shared box set up under a different rig (solo
-                // durable rotation / N-holders SandboxRigConflictError). Omitted for a
-                // rig-less turn -> never stamped or enforced (shares exactly as today).
-                ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
-              },
-              "turn",
-              managedOwnership!.holderId,
+            trackPhysicalSandboxResume(
+              sandboxState.inFlightSandboxResumes,
+              resumeBoxForTurn(
+                {
+                  db,
+                  settings: runSettings,
+                  logicalFallbackSettings: logicalSandboxSettings,
+                  cancellationSignal: sandboxResumeSignal,
+                  sandboxMetrics: runtimeMetricsHooksForObservability(observability),
+                  observability,
+                  freshSandboxReadinessReplacementBudget,
+                  onSandboxLost: publishSandboxLost,
+                  objectStorage,
+                  bus,
+                },
+                {
+                  accountId: input.accountId,
+                  workspaceId: input.workspaceId,
+                  sandboxGroupId: session.sandboxGroupId,
+                  sessionId: input.sessionId,
+                  // groupBoxBackend, not turn.sandboxBackend: a machine-home turn that
+                  // is not machine-primary resumes a real cloud group box (the
+                  // deployment default), never a "selfhosted" box (which would throw
+                  // for lack of a bound agentId).
+                  backend: groupBoxBackend,
+                  os: groupBoxOs,
+                  environment: sandboxEnvironment,
+                  // Deployment/workspace pins select NEW creates only. The lease
+                  // retains a live group's image through between-turn repins, even
+                  // with other holders; only rotation/reaping elects a new image.
+                  // Select the image for the actual group-box backend; a configured Modal image must
+                  // never override a Docker run. The selfhosted branch
+                  // (establishSelfhostedTurnSession) NEVER passes
+                  // an image — B3 lives only on this managed-box branch.
+                  ...(groupBoxImage
+                    ? {
+                        image: groupBoxImage,
+                        imagePolicy: groupBoxImagePolicy,
+                      }
+                    : {}),
+                  // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
+                  // conflicts on a live shared box set up under a different rig (solo
+                  // durable rotation / N-holders SandboxRigConflictError). Omitted for a
+                  // rig-less turn -> never stamped or enforced (shares exactly as today).
+                  ...(rigVersion ? { rigVersionId: rigVersion.id } : {}),
+                },
+                "turn",
+                managedOwnership!.holderId,
+              ),
             ),
             sandboxResumeSignal,
             releaseLateSandbox,

package/src/activities/agent-turn/sandbox-provision.ts

@@ -153,6 +153,57 @@ export async function waitForTurnOperation<T>(
   }
 }
 
+/**
+ * Upper bound for joining a cancelled attempt's in-flight provider establish.
+ * Worker shutdown cancels activities 5s after SIGTERM and force-stops 100s
+ * after it; this leaves room for the remaining finalization steps.
+ */
+export const PHYSICAL_SANDBOX_RESUME_DRAIN_TIMEOUT_MS = 60_000;
+
+/** Register the PHYSICAL lease acquire/provider establish promise, not the
+ * cancellable wrapper awaiting it. */
+export function trackPhysicalSandboxResume<T>(
+  inFlight: Set<Promise<unknown>>,
+  operation: Promise<T>,
+): Promise<T> {
+  inFlight.add(operation);
+  const settle = (): void => {
+    inFlight.delete(operation);
+  };
+  operation.then(settle, settle);
+  return operation;
+}
+
+/**
+ * Join every lease acquire/provider establish this attempt started, even after
+ * the turn-facing wrapper rejected on cancellation. An establish has no
+ * portable abort: after cancellation it still finishes its own exact cleanup
+ * (terminate an unpublished box and roll warming back to cold, or keep a
+ * published box and drop its holder). If the activity returned first, a
+ * shutting-down worker exited mid-cleanup and stranded a warming lease naming a
+ * terminated box; the replacement attempt then waited out the warming TTL and
+ * inherited a lost-workspace verdict. This wait is deliberately independent of
+ * the finalizer cancellation signal (cancellation is what started it) and is
+ * bounded so an uncooperative provider call cannot pin the activity.
+ */
+export async function drainPhysicalSandboxResumes(
+  inFlight: Set<Promise<unknown>>,
+  timeoutMs: number = PHYSICAL_SANDBOX_RESUME_DRAIN_TIMEOUT_MS,
+): Promise<"none" | "settled" | "timed_out"> {
+  if (inFlight.size === 0) return "none";
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    return await Promise.race([
+      Promise.allSettled([...inFlight]).then(() => "settled" as const),
+      new Promise<"timed_out">((resolve) => {
+        timer = setTimeout(() => resolve("timed_out"), Math.max(1, timeoutMs));
+      }),
+    ]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+
 export function sleep(ms: number): Promise<void> {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }