@opengeni/worker-bundle 1.4.0 → 1.4.2

package/src/activities/agent-turn/stream-attempt.ts

@@ -78,11 +78,8 @@ import {
   generatedImageFromSdkEvent,
   isCompletedGeneratedImageSdkEvent,
 } from "../generated-images";
-import {
-  programmaticApproval,
-  programmaticContinuationNote,
-  recoverProgrammaticOperationsBeforeFirstRequest,
-} from "../programmatic-approvals";
+import { programmaticApproval, programmaticContinuationNote } from "../programmatic-approvals";
+import { CodemodeAttemptDispatcher } from "../codemode-dispatcher";
 import { ToolResultSpill } from "./tool-result-spill";
 import { ownedTurnSandboxForAgent } from "./turn-sandbox-access";
 import { createTurnCredentialLeases } from "./credential-leases";
@@ -665,6 +662,16 @@ export async function runTurnStreamAttempt(
     const providerPublishesNativeRequestEvents =
       resolvedModel?.provider.kind === "codex-subscription" ||
       resolvedModel?.provider.kind === "xai-subscription";
+    // Bounded provider label for the streaming SLIs — the resolved registry
+    // provider id (or the built-in OpenAI/Azure provider), never a raw
+    // user-supplied model string. Created before the stream starts so the
+    // first request's model entry and provider dispatch are observed.
+    const streamTiming = new StreamTimingMetrics(observability, {
+      provider: streamProvider,
+    });
+    // Native Codex/SuperGrok transports report their literal dispatch through
+    // the shared turn context; generic transports use the wire-dispatch clock.
+    eventing.providerDispatchObserver = () => streamTiming.onProviderDispatch();
     let fallbackProviderRequestStartedAt: number | null = null;
     let fallbackProviderRequestLifecycleStartedAt: number | null = null;
     const recordFallbackProviderDispatchAtWire = async (): Promise<void> => {
@@ -857,7 +864,10 @@ export async function runTurnStreamAttempt(
         }
         attempt.modelRequestStarted = true;
         return await runtime.runStream(agent, runInput!, eventing.modelRunSettings, {
-          beforeModelRequest: modelCallAdmission.beforeModelRequest,
+          beforeModelRequest: () => {
+            streamTiming.onModelRequestEntry();
+            return modelCallAdmission.beforeModelRequest();
+          },
           onModelResponse: (event) => {
             responseCreditPolicyRevisions.add(creditPolicyRevision);
             return modelCallAdmission.onModelResponse(event);
@@ -969,6 +979,7 @@ export async function runTurnStreamAttempt(
             ? {
                 onModelTransportStarted: recordFallbackProviderDispatchAtWire,
                 onModelTransportDispatched: (clock) => {
+                  streamTiming.onProviderDispatch();
                   eventing.initialModelWireDispatch.record(
                     { provider: streamProvider, dispatchId },
                     clock,
@@ -1010,12 +1021,6 @@ export async function runTurnStreamAttempt(
       modelCallAdmission.close();
       throw error;
     }
-    // Bounded provider label for the streaming SLIs — the resolved registry
-    // provider id (or the built-in OpenAI/Azure provider), never a raw
-    // user-supplied model string.
-    const streamTiming = new StreamTimingMetrics(observability, {
-      provider: streamProvider,
-    });
     eventing.batcher = createRuntimeBatcher(
       async (events) => {
         await eventing.publish!(events);
@@ -1964,44 +1969,26 @@ export async function runTurnStreamAttempt(
 
   // Recover committed waits/results before any model dispatch, including crash replacement.
   // A new attempt uses its own signed caller; origin provenance never changes.
-  // Lazy MCP preparation is never awaited unless stored work must resume.
-  // Measured as its own startup phase so a first-request barrier cannot hide.
-  const programmaticRecoveryStartedAt = performance.now();
-  let programmaticRecoveryOutcome: "completed" | "failed" = "failed";
-  const codemodeOperations = await recoverProgrammaticOperationsBeforeFirstRequest({
-    approvalDecisionId:
-      trigger.type === "user.approvalDecision"
-        ? (trigger.payload as { approvalId: string }).approvalId
-        : undefined,
-    hasUnfinishedOperations: async () =>
-      (
-        await listTurnCodemodeApprovals(db, {
-          accountId: input.accountId,
-          workspaceId: input.workspaceId,
-          sessionId: input.sessionId,
-          turnId: activeTurnId,
-        })
-      ).length > 0,
-    toolPreparationReady: eventing.toolPreparationReady,
-    resumeApproved: (decisionOperationId) =>
-      eventing.codemodeDispatcher?.resumeApproved(
-        `sandbox:${input.attemptId}`,
-        decisionOperationId,
-      ),
-  })
-    .then((operations) => {
-      programmaticRecoveryOutcome = "completed";
-      return operations;
-    })
-    .finally(() => {
-      recordTurnStartupPhase(observability, {
-        phase: "programmatic_operation_recovery",
-        provider: turnExecutionPolicy.providerId,
-        backend: activeSandboxBackend ?? groupBoxBackend,
-        outcome: programmaticRecoveryOutcome,
-        durationSeconds: (performance.now() - programmaticRecoveryStartedAt) / 1_000,
-      });
-    });
+  // Join-on-use: lazy MCP preparation is awaited only when stored work must resume.
+  const codemodeOperations = await CodemodeAttemptDispatcher.resumeApproved(
+    db,
+    {
+      accountId: input.accountId,
+      workspaceId: input.workspaceId,
+      sessionId: input.sessionId,
+      turnId: activeTurnId,
+      attemptId: input.attemptId,
+      executionGeneration: attempt.executionGeneration,
+    },
+    `sandbox:${input.attemptId}`,
+    trigger.type === "user.approvalDecision"
+      ? (trigger.payload as { approvalId: string }).approvalId
+      : undefined,
+    async () => {
+      await eventing.toolPreparationReady;
+      return eventing.codemodeDispatcher;
+    },
+  );
   codemodeContinuationNote = programmaticContinuationNote(codemodeOperations);
   programmaticApprovalAcknowledged =
     trigger.type === "user.approvalDecision" &&

package/src/activities/agent-turn/turn-context.ts

@@ -103,6 +103,9 @@ export type SandboxRuntimeState = {
   }>;
   turnSandboxProvisioner: TurnSandboxProvisioner<ResumedTurnSandbox> | null;
   resumeManagedGroupBox: (() => Promise<ResumedTurnSandbox>) | null;
+  /** Physical resumeBoxForTurn promises still running for this attempt; the
+   * finalizer joins them so a cancelled establish completes its own cleanup. */
+  inFlightSandboxResumes: Set<Promise<unknown>>;
   prefetchedManagedBox: Promise<ResumedTurnSandbox> | null;
   prefetchedManagedBoxResult: ResumedTurnSandbox | null;
   setupBoxSession: unknown;
@@ -153,6 +156,9 @@ export type EventingState = {
   firstModelRequestPreparationRecorded: boolean;
   firstModelRequestCheckpointAt: number | null;
   initialModelWireDispatch: InitialModelWireDispatchClock;
+  /** Diagnostic only: the current stream's timing hook for a native transport's
+   *  literal provider dispatch. Never joins, fences or fails the request. */
+  providerDispatchObserver: (() => void) | null;
   companyBrainContextContributions: readonly ModelContextContributionSummary[] | null;
   /** Skill ids in this turn's frozen, model-visible Skill index; telemetry only. */
   modelVisibleSkillIds: ReadonlySet<string> | null;
@@ -259,6 +265,7 @@ export function createTurnContext(input: {
       firstModelPreparationNestedSandboxPhases: [],
       turnSandboxProvisioner: null,
       resumeManagedGroupBox: null,
+      inFlightSandboxResumes: new Set(),
       prefetchedManagedBox: null,
       prefetchedManagedBoxResult: null,
       setupBoxSession: null,
@@ -301,6 +308,7 @@ export function createTurnContext(input: {
       firstModelRequestPreparationRecorded: false,
       firstModelRequestCheckpointAt: null,
       initialModelWireDispatch: new InitialModelWireDispatchClock(),
+      providerDispatchObserver: null,
       companyBrainContextContributions: null,
       modelVisibleSkillIds: null,
     },

package/src/activities/agent-turn/web-search.ts

@@ -41,7 +41,10 @@ import { hostedWebSearchForTurn } from "./tool-policy";
  * call this with the same accepted model and settings, so they agree.
  */
 export function turnWebSearchPlan(
-  resolvedModel: { configured: { hostedWebSearch: boolean }; provider: { kind?: string } } | null,
+  resolvedModel: {
+    configured: { hostedWebSearch: boolean };
+    provider: { kind?: string };
+  } | null,
   settings: WebSearchSettings,
 ) {
   return webSearchToolPlan(settings, {
@@ -92,6 +95,48 @@ function providerFailure(error: WebSearchProviderError): string {
   return `${error.message}${error.retryable ? ". You may retry shortly." : "."}`;
 }
 
+/** Closed outcome set for `opengeni_web_search_calls_total`. */
+export type WebSearchCallOutcome =
+  | "ok"
+  | "provider_error"
+  | "provider_retryable"
+  | "billing_refused"
+  | "error";
+
+type WebSearchObservability = Pick<Observability, "warn" | "incrementCounter" | "observeHistogram">;
+
+const WEB_SEARCH_DURATION_BUCKETS = [0.25, 0.5, 1, 2, 4, 8, 15, 30];
+
+/**
+ * One provider call: a counter by fixed outcome plus its latency. Labels are
+ * the operation, the configured provider id, and the outcome only, never the
+ * query, URL, or session.
+ */
+function recordWebSearchCall(
+  observability: WebSearchObservability,
+  operation: "search" | "fetch",
+  provider: string,
+  outcome: WebSearchCallOutcome,
+  startedAt: number,
+): void {
+  observability.incrementCounter({
+    name: "opengeni_web_search_calls_total",
+    help: "Provider web search/fetch calls made by the worker, by operation, provider and outcome.",
+    labels: { operation, provider, outcome },
+  });
+  observability.observeHistogram({
+    name: "opengeni_web_search_call_duration_seconds",
+    help: "Provider web search/fetch call duration in seconds.",
+    buckets: WEB_SEARCH_DURATION_BUCKETS,
+    value: (performance.now() - startedAt) / 1000,
+    labels: { operation, provider },
+  });
+}
+
+function providerFailureOutcome(error: WebSearchProviderError): WebSearchCallOutcome {
+  return error.retryable ? "provider_retryable" : "provider_error";
+}
+
 /**
  * The `web_search` and `web_fetch` attempt tools for one turn. The caller
  * decides which names the turn gets ({@link webSearchToolPlan}); this only
@@ -102,7 +147,7 @@ export function webSearchToolDefinitions(input: {
   tools: readonly WebSearchProviderToolName[];
   scope: WebSearchCallScope;
   billing: WebSearchBilling;
-  observability: Pick<Observability, "warn">;
+  observability: WebSearchObservability;
   fetch?: typeof fetch;
 }): AttemptToolDefinition[] {
   const config: WebSearchProviderConfig | null = webSearchProviderConfig(input.settings);
@@ -127,7 +172,10 @@ export function webSearchToolDefinitions(input: {
   const definitions: AttemptToolDefinition[] = [];
 
   if (input.tools.includes(WEB_SEARCH_TOOL_NAME)) {
-    const provider = createWebSearchProvider({ ...adapterInput, endpoint: config.search });
+    const provider = createWebSearchProvider({
+      ...adapterInput,
+      endpoint: config.search,
+    });
     const pricing = webSearchCallPricing(config, "search");
     definitions.push({
       identity: { serverId: "opengeni", toolName: WEB_SEARCH_TOOL_NAME },
@@ -151,6 +199,15 @@ export function webSearchToolDefinitions(input: {
           request = parseWebSearchArguments(args);
           await input.billing.admit(input.scope, pricing.providerMicros);
         } catch (error) {
+          if (error instanceof WebSearchBillingRefusedError) {
+            recordWebSearchCall(
+              input.observability,
+              "search",
+              config.search.provider,
+              "billing_refused",
+              performance.now(),
+            );
+          }
           if (
             error instanceof WebToolArgumentError ||
             error instanceof WebSearchBillingRefusedError
@@ -158,8 +215,18 @@ export function webSearchToolDefinitions(input: {
             return textResult(error.message, true);
           throw error;
         }
+        const startedAt = performance.now();
         try {
-          const response = await provider.search(request, { signal: context.signal });
+          const response = await provider.search(request, {
+            signal: context.signal,
+          });
+          recordWebSearchCall(
+            input.observability,
+            "search",
+            config.search.provider,
+            "ok",
+            startedAt,
+          );
           await settle(
             settledCost(
               context.operationId,
@@ -173,8 +240,29 @@ export function webSearchToolDefinitions(input: {
         } catch (error) {
           if (context.signal?.aborted) throw error;
           if (error instanceof WebSearchProviderError) {
+            recordWebSearchCall(
+              input.observability,
+              "search",
+              config.search.provider,
+              providerFailureOutcome(error),
+              startedAt,
+            );
+            input.observability.warn("web search provider call failed", {
+              operation: "search",
+              provider: config.search.provider,
+              status: error.status ?? undefined,
+              retryable: error.retryable,
+              error: error.message,
+            });
             return textResult(`Web search failed: ${providerFailure(error)}`, true);
           }
+          recordWebSearchCall(
+            input.observability,
+            "search",
+            config.search.provider,
+            "error",
+            startedAt,
+          );
           throw error;
         }
       },
@@ -183,7 +271,10 @@ export function webSearchToolDefinitions(input: {
 
   if (input.tools.includes(WEB_FETCH_TOOL_NAME) && config.fetch) {
     const fetchEndpoint = config.fetch;
-    const provider = createWebFetchProvider({ ...adapterInput, endpoint: fetchEndpoint });
+    const provider = createWebFetchProvider({
+      ...adapterInput,
+      endpoint: fetchEndpoint,
+    });
     const pricing = webSearchCallPricing(config, "fetch");
     const pages = new Map<string, WebPage>();
     definitions.push({
@@ -215,11 +306,28 @@ export function webSearchToolDefinitions(input: {
         try {
           await input.billing.admit(input.scope, pricing.providerMicros);
         } catch (error) {
-          if (error instanceof WebSearchBillingRefusedError) return textResult(error.message, true);
+          if (error instanceof WebSearchBillingRefusedError) {
+            recordWebSearchCall(
+              input.observability,
+              "fetch",
+              fetchEndpoint.provider,
+              "billing_refused",
+              performance.now(),
+            );
+            return textResult(error.message, true);
+          }
           throw error;
         }
+        const startedAt = performance.now();
         try {
           const page = await provider.fetch({ url: request.url }, { signal: context.signal });
+          recordWebSearchCall(
+            input.observability,
+            "fetch",
+            fetchEndpoint.provider,
+            "ok",
+            startedAt,
+          );
           await settle(
             settledCost(
               context.operationId,
@@ -241,8 +349,29 @@ export function webSearchToolDefinitions(input: {
         } catch (error) {
           if (context.signal?.aborted) throw error;
           if (error instanceof WebSearchProviderError) {
+            recordWebSearchCall(
+              input.observability,
+              "fetch",
+              fetchEndpoint.provider,
+              providerFailureOutcome(error),
+              startedAt,
+            );
+            input.observability.warn("web search provider call failed", {
+              operation: "fetch",
+              provider: fetchEndpoint.provider,
+              status: error.status ?? undefined,
+              retryable: error.retryable,
+              error: error.message,
+            });
             return textResult(`Web fetch failed: ${providerFailure(error)}`, true);
           }
+          recordWebSearchCall(
+            input.observability,
+            "fetch",
+            fetchEndpoint.provider,
+            "error",
+            startedAt,
+          );
           throw error;
         }
       },

package/src/activities/codemode-dispatcher.ts

@@ -61,6 +61,12 @@ export type CodemodeDispatcherTimings = {
   claimHeartbeatMs?: number;
 };
 
+const RESUMABLE_STATES: readonly CodemodeOperation["state"][] = [
+  "waiting_for_approval",
+  "queued",
+  "running",
+];
+
 /**
  * Owns the only Codemode execution edge for one exact attempt. NATS carries a
  * wake-up only; the durable row is claimed before any side effect and the call
@@ -190,11 +196,20 @@ export class CodemodeAttemptDispatcher {
     );
   }
 
-  async resumeApproved(
+  /**
+   * Join-on-use recovery before the first model request. The journal read needs
+   * no tool environment, so a turn with nothing unfinished returns at once and
+   * never waits on lazy MCP preparation. Only a stored waiting/queued/running
+   * operation awaits the prepared dispatcher to adopt and execute it.
+   */
+  static async resumeApproved(
+    db: Database,
+    scope: CodemodeDispatcherScope,
     callerSubjectId: string,
-    decisionOperationId?: string,
+    decisionOperationId: string | undefined,
+    prepared: () => Promise<CodemodeAttemptDispatcher | null>,
   ): Promise<CodemodeOperation[]> {
-    const operations = await listTurnCodemodeApprovals(this.db, this.scope);
+    const operations = await listTurnCodemodeApprovals(db, scope);
     if (
       decisionOperationId &&
       // Model-tool approval IDs are opaque strings. Only programmatic UUIDs
@@ -202,15 +217,38 @@ export class CodemodeAttemptDispatcher {
       CodemodeOperation.shape.operationId.safeParse(decisionOperationId).success &&
       !operations.some((operation) => operation.operationId === decisionOperationId)
     ) {
-      const decided = await readTurnCodemodeOperation(this.db, {
-        ...this.scope,
+      const decided = await readTurnCodemodeOperation(db, {
+        ...scope,
         callerSubjectId,
         operationId: decisionOperationId,
       });
       if (decided) operations.push(decided);
     }
+    if (!operations.some((operation) => RESUMABLE_STATES.includes(operation.state))) {
+      return operations;
+    }
+    return (await (await prepared())?.resumeOperations(operations, callerSubjectId)) ?? [];
+  }
+
+  async resumeApproved(
+    callerSubjectId: string,
+    decisionOperationId?: string,
+  ): Promise<CodemodeOperation[]> {
+    return await CodemodeAttemptDispatcher.resumeApproved(
+      this.db,
+      this.scope,
+      callerSubjectId,
+      decisionOperationId,
+      async () => this,
+    );
+  }
+
+  private async resumeOperations(
+    operations: CodemodeOperation[],
+    callerSubjectId: string,
+  ): Promise<CodemodeOperation[]> {
     for (const operation of operations) {
-      if (!["waiting_for_approval", "queued", "running"].includes(operation.state)) continue;
+      if (!RESUMABLE_STATES.includes(operation.state)) continue;
       let effectDigest: string | null = null;
       try {
         effectDigest = this.environment.effectDigest(operation.identity);
@@ -380,7 +418,17 @@ export class CodemodeAttemptDispatcher {
     try {
       if (operation.durableApproval) {
         if (!this.approvalWait) throw new Error("Durable approval yield is unavailable");
-        releaseWait = this.approvalWait.beginWait();
+        // The wait gate is sealed while a model request is in flight or the
+        // stream is settling. A call arriving then cannot durably wait, but it
+        // must not fail outright: prepare it like a non-waiting client, so an
+        // allowed call still executes and an Ask settles approval_required.
+        try {
+          releaseWait = this.approvalWait.beginWait();
+        } catch {
+          releaseWait = undefined;
+        }
+      }
+      if (releaseWait) {
         if (
           !(await bindCodemodePreparation(this.db, {
             ...this.scope,
@@ -407,10 +455,11 @@ export class CodemodeAttemptDispatcher {
         },
         {
           signal,
-          ...(operation.durableApproval ? { transportMeta: { durableApproval: true } } : {}),
+          ...(releaseWait ? { transportMeta: { durableApproval: true } } : {}),
         },
       );
       if (preparedCall.waitingForApproval) {
+        if (!releaseWait) throw new AttemptToolApprovalRequiredError();
         const waiting = await waitForCodemodeApproval(this.db, {
           ...this.scope,
           operationId: operation.operationId,

package/src/activities/programmatic-approvals.ts

@@ -24,29 +24,3 @@ export function programmaticContinuationNote(
     "Approval resumes the stored operation, not the earlier program's JavaScript stack. Read results with the current Codemode client's status/resume handle. Do not resubmit arguments for completed, running, waiting, or uncertain operations.",
   ].join("\n");
 }
-
-/**
- * Recover committed programmatic approval waits/results before the first model
- * request without turning lazy MCP preparation into a first-request barrier.
- *
- * Only eager servers may gate the first request. Resuming a stored operation
- * needs the prepared attempt tool environment, so this waits for full tool
- * preparation only when there is something to resume: an approval decision
- * trigger, or a cheap durable read showing unfinished operations for this
- * logical turn (which covers every recovery that has stored work). Every other
- * turn sends its first request without waiting on any lazy server.
- */
-export async function recoverProgrammaticOperationsBeforeFirstRequest(input: {
-  approvalDecisionId: string | undefined;
-  hasUnfinishedOperations: () => Promise<boolean>;
-  toolPreparationReady: Promise<void> | null;
-  resumeApproved: (
-    decisionOperationId: string | undefined,
-  ) => Promise<CodemodeOperation[]> | undefined;
-}): Promise<CodemodeOperation[]> {
-  if (input.approvalDecisionId === undefined && !(await input.hasUnfinishedOperations())) {
-    return [];
-  }
-  await input.toolPreparationReady;
-  return (await input.resumeApproved(input.approvalDecisionId)) ?? [];
-}

package/src/activities/sandbox-lease.ts

@@ -53,6 +53,7 @@ import {
   recordRecoveredModalProviderCreate,
   markWarmBillingStopCutoff,
   listMeterableWarmLeases,
+  countInteractionOnlyWarmLeasesByIdle,
   listSandboxViewerForceDrainWorkspaceIds,
   markSandboxCheckpointArtifactDeletePending,
   persistDrainSnapshot,
@@ -174,6 +175,8 @@ import {
   type RetainedProcessReconciliationOutcome,
   type SandboxInventoryProjectionDomain,
   recordSandboxLeaseGauges,
+  recordModalSandboxInventoryGauges,
+  recordInteractionOnlyLeaseGauges,
   recordSandboxOrphansTerminated,
   recordSandboxCommandContainment,
   recordSandboxProviderMissingBeforeCapture,
@@ -2688,6 +2691,17 @@ async function refreshQueueLeaseAndCreditGauges(
         );
       },
     ),
+    refreshSandboxInventoryGauge(
+      observability,
+      "interaction_idle",
+      "interaction-only-lease",
+      async () => {
+        recordInteractionOnlyLeaseGauges(
+          observability,
+          await countInteractionOnlyWarmLeasesByIdle(db, await listMeterableWarmLeases(db)),
+        );
+      },
+    ),
     refreshSandboxInventoryGauge(observability, "expired_drains", "expired-draining", async () => {
       recordExpiredDrainingSandboxLeaseGauges(
         observability,
@@ -2876,6 +2890,16 @@ export function modalOrphanTerminationStillEligible(
   if (latest.some((lease) => lease.liveness === "warming" && lease.instanceId === null)) {
     return false;
   }
+  return modalOrphanCandidateStillUnowned(latest, candidate);
+}
+
+/** Whether no live lease owns the candidate box by exact instance or by its
+ * attribution tags. Unlike termination eligibility this ignores the pending
+ * create postponement: a postponed orphan is still running without a lease. */
+export function modalOrphanCandidateStillUnowned(
+  latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,
+  candidate: ModalOrphanSweepTermination,
+): boolean {
   if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
     return false;
   }
@@ -2925,6 +2949,30 @@ async function sweepModalOrphansForConfiguredBackend(
       return modalOrphanTerminationStillEligible(latest, candidate);
     },
   });
+  // Only a complete listing is an inventory; a pass cut short by the
+  // termination budget leaves the previous projection to age out as stale.
+  // Re-read durable ownership after the listing so a lease that started or
+  // finished while the app was being listed is not reported as a leak.
+  if (result.inventory.complete) {
+    try {
+      const latest = await listLiveModalSandboxLeaseAttributions(db);
+      recordModalSandboxInventoryGauges(observability, {
+        running: result.inventory.running,
+        unleased: result.inventory.unterminated.filter((candidate) =>
+          modalOrphanCandidateStillUnowned(latest, candidate),
+        ).length,
+        liveLeaseInstancesMissing: result.inventory.missingLiveLeaseInstanceIds.filter(
+          (instanceId) => latest.some((lease) => lease.instanceId === instanceId),
+        ).length,
+      });
+      recordSandboxInventoryProjectionSuccess(observability, "modal_provider");
+    } catch (error) {
+      recordSandboxInventoryProjectionFailure(observability, "modal_provider");
+      observability.warn("sandbox reaper: Modal inventory ownership re-read failed", {
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
   for (const terminated of result.terminated) {
     observability.warn("sandbox reaper: terminated Modal orphan sandbox", {
       sandboxId: terminated.sandboxId,
@@ -3524,15 +3572,18 @@ async function terminateDrainableBox(
   // with draining->cold. Until this succeeds, arrivals remain fenced by that
   // exact claim; a timestamp or a failed provider call can never reopen a box
   // while termination may still be in flight.
-  const { wentCold, backgroundCommandEvents } = await confirmDrainCold(db, {
-    accountId,
-    workspaceId: row.workspaceId,
-    sandboxGroupId: row.sandboxGroupId,
-    expectedEpoch: row.leaseEpoch,
-    ...(captureClaim ? { expectedCaptureId: captureClaim.id } : {}),
-    providerMissingBeforeCapture: providerMissing,
-    idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
-  });
+  const { wentCold, unpublishedProviderLost, backgroundCommandEvents } = await confirmDrainCold(
+    db,
+    {
+      accountId,
+      workspaceId: row.workspaceId,
+      sandboxGroupId: row.sandboxGroupId,
+      expectedEpoch: row.leaseEpoch,
+      ...(captureClaim ? { expectedCaptureId: captureClaim.id } : {}),
+      providerMissingBeforeCapture: providerMissing,
+      idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
+    },
+  );
   // The command terminal events and agent inputs are already durable in the
   // cold commit; this is only best-effort live fanout.
   await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {
@@ -3545,7 +3596,9 @@ async function terminateDrainableBox(
     // Only the exact successful cold commit counts provider loss. A missing
     // probe, a stale capture, a failed commit, or a retried child is not another
     // observed loss. Keep this outside the best-effort session event writer.
-    if (providerMissing) {
+    // An unpublished warming replacement (its creator died before warm
+    // publication) held no workspace; its absence is not a capture loss.
+    if (providerMissing && !unpublishedProviderLost) {
       recordSandboxProviderMissingBeforeCapture(observability, backend);
     }
     if (lease.unobservableCommandDrainIds?.length) {