@opengeni/worker-bundle 2.3.1 → 2.3.2-canary.37090756480001

package/src/activities/model-call-fact-reconciler.ts

@@ -0,0 +1,112 @@
+import { randomUUID } from "node:crypto";
+import { listWorkspaceIdsAfter, reconcileModelCallFacts } from "@opengeni/db";
+import type { ControlActivityServices } from "./types";
+
+/** Recent charged calls are rechecked on every run; older gaps use the backfill script. */
+export const MODEL_CALL_FACT_RECONCILE_LOOKBACK_MS = 2 * 60 * 60 * 1_000;
+/** Skip the newest calls so a live fact write is not raced by its own repair. */
+export const MODEL_CALL_FACT_RECONCILE_SETTLE_MS = 10 * 60 * 1_000;
+export const MODEL_CALL_FACT_RECONCILE_WORKSPACE_LIMIT = 500;
+export const MODEL_CALL_FACT_RECONCILE_RUN_BUDGET_MS = 3 * 60 * 1_000;
+const WORKSPACE_PAGE_SIZE = 200;
+
+export type ReconcileRecentModelCallFactsResult = {
+  workspaces: number;
+  missing: number;
+  repaired: number;
+  unrepaired: number;
+  /** Workspaces with more missing facts than one bounded pass examines. */
+  truncatedWorkspaces: number;
+  failedWorkspaces: number;
+  budgetExhausted: boolean;
+};
+
+export type ModelCallFactReconcilerOptions = {
+  now?: () => number;
+  runBudgetMs?: number;
+  startAfterWorkspaceId?: () => string;
+  reconcile?: typeof reconcileModelCallFacts;
+  listWorkspaces?: typeof listWorkspaceIdsAfter;
+};
+
+/**
+ * Rebuild missing Insights call facts for recently charged model calls. Each
+ * workspace pass is bounded and idempotent, one failure never stops the rest,
+ * and each run starts at a random workspace so a run that exhausts its budget
+ * does not starve the same tail of workspaces every time.
+ */
+export function createModelCallFactReconcilerActivities(
+  services: () => Promise<ControlActivityServices>,
+  options: ModelCallFactReconcilerOptions = {},
+) {
+  const now = options.now ?? Date.now;
+  const runBudgetMs = options.runBudgetMs ?? MODEL_CALL_FACT_RECONCILE_RUN_BUDGET_MS;
+  const startAfterWorkspaceId = options.startAfterWorkspaceId ?? randomUUID;
+  const reconcile = options.reconcile ?? reconcileModelCallFacts;
+  const listWorkspaces = options.listWorkspaces ?? listWorkspaceIdsAfter;
+
+  async function reconcileRecentModelCallFacts(): Promise<ReconcileRecentModelCallFactsResult> {
+    const { db, observability } = await services();
+    const startedAt = now();
+    const until = new Date(startedAt - MODEL_CALL_FACT_RECONCILE_SETTLE_MS);
+    const since = new Date(until.getTime() - MODEL_CALL_FACT_RECONCILE_LOOKBACK_MS);
+    const result: ReconcileRecentModelCallFactsResult = {
+      workspaces: 0,
+      missing: 0,
+      repaired: 0,
+      unrepaired: 0,
+      truncatedWorkspaces: 0,
+      failedWorkspaces: 0,
+      budgetExhausted: false,
+    };
+    const start = startAfterWorkspaceId();
+    let cursor: string | null = start;
+    let wrapped = false;
+    while (true) {
+      const page = await listWorkspaces(db, { afterId: cursor, limit: WORKSPACE_PAGE_SIZE });
+      const ids = wrapped ? page.filter((id) => id <= start) : page;
+      for (const workspaceId of ids) {
+        if (now() - startedAt >= runBudgetMs) {
+          result.budgetExhausted = true;
+          break;
+        }
+        result.workspaces += 1;
+        try {
+          const outcome = await reconcile(db, {
+            workspaceId,
+            since,
+            until,
+            limit: MODEL_CALL_FACT_RECONCILE_WORKSPACE_LIMIT,
+          });
+          result.missing += outcome.missing;
+          result.repaired += outcome.repaired;
+          result.unrepaired += outcome.unrepaired;
+          if (outcome.truncated) result.truncatedWorkspaces += 1;
+        } catch (error) {
+          result.failedWorkspaces += 1;
+          observability.warn("model call fact reconciliation failed for a workspace", {
+            workspaceId,
+            errorName: error instanceof Error ? error.name : "unknown",
+          });
+        }
+      }
+      if (result.budgetExhausted) break;
+      if (wrapped && ids.length < page.length) break;
+      if (page.length < WORKSPACE_PAGE_SIZE) {
+        if (wrapped) break;
+        wrapped = true;
+        cursor = null;
+        continue;
+      }
+      cursor = page[page.length - 1]!;
+    }
+    if (result.failedWorkspaces > 0 || result.truncatedWorkspaces > 0 || result.budgetExhausted) {
+      observability.warn("model call fact reconciliation incomplete", { ...result });
+    } else if (result.missing > 0) {
+      observability.info("model call fact reconciliation", { ...result });
+    }
+    return result;
+  }
+
+  return { reconcileRecentModelCallFacts };
+}

package/src/activities/sandbox-lease.ts

@@ -14,6 +14,7 @@
 // timer, viewer activity, owner task queue, or provider-specific lifecycle path
 // in the normal drain state machine.
 import { warnDrainSnapshotFailure } from "../sandbox-snapshot-diagnostics";
+import { publishDurableSessionEvents } from "../session-event-fanout";
 import { warnRetainedProcessProofFailure } from "../retained-process-diagnostics";
 import { retainedProcessDeadlineRetryMs } from "../retained-process-retry";
 
@@ -68,6 +69,7 @@ import {
   workspaceArchiveCaptureDeadlineElapsed,
   retainedProcessReconciliationProof,
   retainedProcessSettlementIdentity,
+  recoverManagedSessionBackgroundCommand,
   settleClaimedConnectedMachineBackgroundCommand,
   settleSandboxCheckpointArtifactGc,
   rlsContextForWorkspace,
@@ -169,6 +171,7 @@ import {
   type SandboxInventoryProjectionDomain,
   recordSandboxLeaseGauges,
   recordSandboxOrphansTerminated,
+  recordSandboxCommandContainment,
   recordSandboxProviderMissingBeforeCapture,
   recordSandboxRecoveryObservationGauges,
   recordSandboxRotationBacklogGauges,
@@ -562,10 +565,15 @@ export function createSandboxLeaseActivities(
     try {
       const { db, settings, observability } = await services();
       const timing = sandboxDrainTiming(settings);
-      const onUnobservableCommandDrainError = (error: unknown) => {
-        observability.warn("sandbox reaper: unobservable command drain inspection failed", {
-          error: error instanceof Error ? error.message : String(error),
-        });
+      const commandContainment = {
+        idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
+        onCommandContainment: (outcome: Parameters<typeof recordSandboxCommandContainment>[1]) =>
+          recordSandboxCommandContainment(observability, outcome),
+        onCommandContainmentError: (error: unknown) => {
+          observability.warn("sandbox reaper: retained command containment inspection failed", {
+            error: error instanceof Error ? error.message : String(error),
+          });
+        },
       };
       if (!settings.sandboxOwnershipEnabled) {
         // Turns skip leases when the flag is off, but Computer/Browser attach
@@ -573,7 +581,7 @@ export function createSandboxLeaseActivities(
         // behind rotation_in_progress forever. Inventory and drain those rows;
         // do not request NEW deadline rotations or meter warm time.
         const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
-          onUnobservableCommandDrainError,
+          ...commandContainment,
           viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
           turnHolderTtlMs: settings.sandboxLeaseTtlMs,
           interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
@@ -610,7 +618,7 @@ export function createSandboxLeaseActivities(
       // Billing, reconciliation, provider-orphan cleanup, artifact GC, and gauges
       // run only after every drainable box has its own durable child.
       const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
-        onUnobservableCommandDrainError,
+        ...commandContainment,
         viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
         // Dead-worker turn holders: a live holder is touched every 10s from the
         // moment it is registered (resumeBoxForTurn's holder-liveness loop covers
@@ -654,7 +662,7 @@ export function createSandboxLeaseActivities(
       instanceId: input.target.instanceId,
     });
     try {
-      const { db, settings, observability, objectStorage } = await services();
+      const { db, settings, observability, objectStorage, bus } = await services();
       assertSandboxDrainInputTiming(input);
       const drainSettings =
         settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs
@@ -678,6 +686,7 @@ export function createSandboxLeaseActivities(
           probeDrainableProvider,
           captureAttempt,
           objectStorage,
+          bus,
         );
         return { status: drainedCold ? "terminated" : "skipped" };
       } catch (error) {
@@ -1477,6 +1486,32 @@ async function reconcileTerminalRetainedProcesses(
       sessionId: process.sessionId,
       processId: process.id,
     };
+    // Recover a yield whose observation failed before normal receipt adoption.
+    // Do this before provider I/O: an unavailable observer must not fence every
+    // subsequent turn. Physical retention and unknown outcome stay unchanged.
+    if (
+      process.ownerActorKind === "turn" &&
+      process.providerBackend === "modal" &&
+      !proof &&
+      !claim.ownerState.startsWith("background_")
+    ) {
+      try {
+        const command = await recoverManagedSessionBackgroundCommand(db, {
+          ...processScope,
+          expected,
+          reconciliationClaimId: claim.claimId,
+        });
+        if (command) {
+          claim.ownerState = `background_${command.state}`;
+          recordRetainedProcessReconciliation(observability, "background_recovered");
+        }
+      } catch (error) {
+        observability.warn("sandbox reaper: retained-command background recovery failed", {
+          processId: process.id,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    }
     const storedCommandPersistence = retainedProviderCommandPersistence(
       db,
       processScope,
@@ -2858,6 +2893,7 @@ async function terminateDrainableBox(
   probeDrainableProvider: DrainableProviderProbeFn,
   attempt: SandboxDrainCaptureAttempt,
   objectStorage: ObjectStorage | null,
+  bus: ActivityServices["bus"] | null = null,
 ): Promise<boolean> {
   // Resolve the account for the RLS-scoped confirmDrainCold (the global sweep
   // returns no account_id; the workspace->account map is the bootstrap read).
@@ -3318,13 +3354,22 @@ async function terminateDrainableBox(
   // with draining->cold. Until this succeeds, arrivals remain fenced by that
   // exact claim; a timestamp or a failed provider call can never reopen a box
   // while termination may still be in flight.
-  const { wentCold } = await confirmDrainCold(db, {
+  const { wentCold, backgroundCommandEvents } = await confirmDrainCold(db, {
     accountId,
     workspaceId: row.workspaceId,
     sandboxGroupId: row.sandboxGroupId,
     expectedEpoch: row.leaseEpoch,
     ...(captureClaim ? { expectedCaptureId: captureClaim.id } : {}),
     providerMissingBeforeCapture: providerMissing,
+    idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
+  });
+  // The command terminal events and agent inputs are already durable in the
+  // cold commit; this is only best-effort live fanout.
+  await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {
+    observability.warn("sandbox reaper: contained command event fanout failed", {
+      sandboxGroupId: row.sandboxGroupId,
+      error: error instanceof Error ? error.message : String(error),
+    });
   });
   if (wentCold) {
     // Only the exact successful cold commit counts provider loss. A missing
@@ -3333,6 +3378,12 @@ async function terminateDrainableBox(
     if (providerMissing) {
       recordSandboxProviderMissingBeforeCapture(observability, backend);
     }
+    if (lease.unobservableCommandDrainIds?.length) {
+      recordSandboxCommandContainment(
+        observability,
+        providerMissing ? "provider_missing" : "contained",
+      );
+    }
     // Durable termination record (sandbox-file-persistence observability): who
     // ended this box and whether its /workspace was captured first, appended to
     // every session sharing the group's box. Best-effort: attribution must

package/src/activities/session-state.ts

@@ -19,6 +19,7 @@ import {
   markSessionAttemptQuiesced,
   requireSession,
   settleSessionIdleWithParentOutbox,
+  SANDBOX_SETUP_RECOVERY_LIMIT,
 } from "@opengeni/db";
 import { publishDurableSessionEvents } from "@opengeni/events";
 import { CancelledFailure } from "@temporalio/activity";
@@ -239,6 +240,9 @@ export function createSessionStateActivities(
       turn.triggerEventId === postClaimRecovery.triggerEventId &&
       turn.executionGeneration === postClaimRecovery.executionGeneration,
     );
+    if (postClaimRecovery?.sandboxSetupRecoveryExhausted && !postClaimIdentityMatches) {
+      return { action: "stale" };
+    }
     const providerRecoveryCount = postClaimIdentityMatches
       ? postClaimRecovery?.providerRecoveryCount
       : undefined;
@@ -249,7 +253,15 @@ export function createSessionStateActivities(
     const hasProviderFailureCode = providerFailureCode !== undefined;
     const setupOutcomeUnknown =
       postClaimIdentityMatches && postClaimRecovery?.sandboxSetupOutcomeUnknown === true;
-    if (setupOutcomeUnknown && (hasProviderRecoveryCount || hasProviderFailureCode)) {
+    const setupRecoveryExhausted =
+      postClaimIdentityMatches && postClaimRecovery?.sandboxSetupRecoveryExhausted === true;
+    if (
+      (setupOutcomeUnknown && setupRecoveryExhausted) ||
+      ((setupOutcomeUnknown || setupRecoveryExhausted) &&
+        (hasProviderRecoveryCount || hasProviderFailureCode)) ||
+      (setupRecoveryExhausted &&
+        turn.metadata?.providerRecoveryCount !== SANDBOX_SETUP_RECOVERY_LIMIT)
+    ) {
       return { action: "stale" };
     }
     if (hasProviderRecoveryCount !== hasProviderFailureCode) {
@@ -282,15 +294,28 @@ export function createSessionStateActivities(
         attemptId: input.attemptId,
         reason: setupOutcomeUnknown
           ? "sandbox_command_start_outcome_unknown"
-          : (providerFailureCode ?? "claimed_attempt_database_failure"),
+          : setupRecoveryExhausted
+            ? "sandbox_command_start_recovery_exhausted"
+            : (providerFailureCode ?? "claimed_attempt_database_failure"),
         ...(setupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {}),
+        ...(setupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {}),
         ...(providerRecoveryCount !== undefined ? { providerRecoveryCount } : {}),
         detail: {
           code: setupOutcomeUnknown
             ? "sandbox_command_start_outcome_unknown"
-            : (providerFailureCode ?? recoveredClaimCode),
-          retryable: !setupOutcomeUnknown,
+            : setupRecoveryExhausted
+              ? "sandbox_command_start_recovery_exhausted"
+              : (providerFailureCode ?? recoveredClaimCode),
+          retryable: !setupOutcomeUnknown && !setupRecoveryExhausted,
           ...(setupOutcomeUnknown ? { setupOutcome: "unknown", replay: "blocked" } : {}),
+          ...(setupRecoveryExhausted
+            ? {
+                setupOutcome: "not_started",
+                replay: "blocked",
+                recoveryExhausted: true,
+                providerRecoveryCount: SANDBOX_SETUP_RECOVERY_LIMIT,
+              }
+            : {}),
           ...(providerRecoveryCount !== undefined
             ? {
                 databaseFailureCode: recoveredClaimCode,

package/src/activities/types.ts

@@ -144,6 +144,11 @@ export type ControlActivityServices = SharedActivityServices;
 
 /** Turn workers own the model loop and never construct document parsers. */
 export type TurnActivityServices = SharedActivityServices & {
+  /** Host-owned containment: stop polling and checkpoint peer turns before exit.
+   * Bare activity embedders must provide their own worker lifecycle edge. */
+  requestWorkerDrain: () => void;
+  /** Deterministic test clock for the cleanup-only monitor. */
+  turnFinalizationTimeoutMs?: number | undefined;
   runtime: OpenGeniRuntime;
   /** Provider-free test/profiling seam; production injects the real runtime summarizer. */
   summarizeContextForCompaction: typeof import("@opengeni/runtime").summarizeForCompaction;
@@ -304,6 +309,8 @@ export type PostClaimDatabaseRecoveryDetail = {
   providerFailureCode?: string;
   /** Persist the no-replay setup marker, not a new provider/setup attempt. */
   sandboxSetupOutcomeUnknown?: true;
+  /** Preserve the exhausted budget for a command proven not dispatched. */
+  sandboxSetupRecoveryExhausted?: true;
 };
 
 export const POST_CLAIM_DATABASE_RECOVERY_FAILURE_TYPE = "OpenGeniPostClaimDatabaseRecovery";

package/src/activities-control.ts

@@ -7,6 +7,7 @@ import { createAutomationActivities } from "./activities/automations";
 import { createDocumentActivities } from "./activities/documents";
 import { createFileUploadReaperActivities } from "./activities/file-upload-reaper";
 import { createGoalActivities } from "./activities/goals";
+import { createModelCallFactReconcilerActivities } from "./activities/model-call-fact-reconciler";
 import { createKnowledgeSourceSyncActivities } from "./activities/knowledge-source-sync";
 import { createRetainedScreenshotMaintenanceActivities } from "./activities/retained-screenshot-reaper";
 import { createRigVerificationActivities } from "./activities/rig-verification";
@@ -42,6 +43,7 @@ export function createControlActivitiesFromServices(
     ...createBrowserStateArtifactMaintenanceActivities(services),
     ...createBrowserDeadlineCheckpointActivities(services),
     ...createFileUploadReaperActivities(services),
+    ...createModelCallFactReconcilerActivities(services),
     ...createRetainedScreenshotMaintenanceActivities(services),
     ...createWorkflowWakeActivities(services),
     ...createSandboxLeaseActivities(services),

package/src/activities-turn.ts

@@ -12,6 +12,9 @@ function createTurnActivityServices(
   let servicesPromise: Promise<TurnActivityServices> | null = null;
   return async () => {
     servicesPromise ??= (async () => {
+      if (!dependencies.requestWorkerDrain) {
+        throw new Error("Turn activities require a host requestWorkerDrain lifecycle edge");
+      }
       const services = await shared();
       return {
         ...services,
@@ -20,6 +23,8 @@ function createTurnActivityServices(
           createProductionAgentRuntime({
             metrics: runtimeMetricsHooksForObservability(services.observability),
           }),
+        requestWorkerDrain: dependencies.requestWorkerDrain,
+        turnFinalizationTimeoutMs: dependencies.turnFinalizationTimeoutMs,
         summarizeContextForCompaction:
           dependencies.summarizeContextForCompaction ?? summarizeForCompaction,
       };

package/src/activities.ts

@@ -69,6 +69,12 @@ export function createActivityTestHarness(dependencies: ActivityDependencies = {
           createProductionAgentRuntime({
             metrics: runtimeMetricsHooksForObservability(common.observability),
           }),
+        requestWorkerDrain:
+          dependencies.requestWorkerDrain ??
+          (() => {
+            throw new Error("Test host must provide requestWorkerDrain for stalled cleanup");
+          }),
+        turnFinalizationTimeoutMs: dependencies.turnFinalizationTimeoutMs,
         summarizeContextForCompaction:
           dependencies.summarizeContextForCompaction ?? summarizeForCompaction,
         documentServices: dependencies.documentServices ?? createDocumentServices(common.settings),
@@ -128,6 +134,7 @@ export const recoverVideoGenerationWorkflows =
   defaultControlActivities.recoverVideoGenerationWorkflows;
 export const maintainRetainedScreenshots = defaultControlActivities.maintainRetainedScreenshots;
 export const maintainBrowserStateArtifacts = defaultControlActivities.maintainBrowserStateArtifacts;
+export const reconcileRecentModelCallFacts = defaultControlActivities.reconcileRecentModelCallFacts;
 export const maintainSiteAuthConnections = defaultControlActivities.maintainSiteAuthConnections;
 export const dispatchSessionWorkflowWakes = defaultControlActivities.dispatchSessionWorkflowWakes;
 export const verifyRigChange = defaultControlActivities.verifyRigChange;

package/src/http.ts

@@ -120,6 +120,7 @@ export async function runReadinessChecks(
 export function dbReadyCheck(
   db: Database,
   posture?: RuntimeDatabasePostureOptions,
+  afterDatabaseReady?: () => Promise<void>,
 ): () => Promise<void> {
   let inFlight: Promise<void> | undefined;
   return () => {
@@ -130,9 +131,10 @@ export function dbReadyCheck(
     inFlight = (async () => {
       if (posture) {
         await assertRuntimeDatabasePosture(db, posture);
-        return;
+      } else {
+        await db.execute(dbSql`select 1`);
       }
-      await db.execute(dbSql`select 1`);
+      await afterDatabaseReady?.();
     })().finally(() => {
       inFlight = undefined;
     });

package/src/index.ts

@@ -85,6 +85,7 @@ import {
   combineWorkerRunTargets,
   constructWithOwnedConnection,
   createWorkerServiceLifecycle,
+  createWorkerCleanupContainment,
   type WorkerServiceLifecycle,
   type WorkerRunTarget,
 } from "./worker-service-lifecycle";
@@ -271,10 +272,20 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
         },
       ),
     async (connection) => {
+      let turnWorker: Worker | undefined;
+      let cleanupDrainRequested = false;
       const activityDependencies = {
         ...options.activityDependencies,
         settings,
         observability,
+        requestWorkerDrain:
+          options.activityDependencies?.requestWorkerDrain ??
+          (() => {
+            if (cleanupDrainRequested) return;
+            if (!turnWorker) throw new Error("turn worker lifecycle is not initialized");
+            turnWorker.shutdown();
+            cleanupDrainRequested = true;
+          }),
       };
       const activities =
         options.activities ??
@@ -323,6 +334,7 @@ export async function createOpenGeniWorker(options: WorkerOptions): Promise<{
           : turnConcurrency!.options),
       });
       if (options.role !== "control") {
+        turnWorker = worker;
         turnConcurrency?.admission?.finalizeStartupBaseline();
         return { worker, connection };
       }
@@ -822,6 +834,9 @@ export async function registerSessionWorkflowWakeDispatcherSchedule(
 
 export type OpenGeniWorkerServiceOptions = Omit<WorkerOptions, "activityDependencies"> & {
   activityDependencies: ActivityDependencies & { db: Database; bus: EventBus };
+  /** Host-owned final containment after a stalled cleanup has drained peers.
+   * The standalone process exits; embedded hosts own their termination policy. */
+  terminateWorker?: () => void;
   /**
    * Exact catalog posture required by the standalone runtime. Embedded hosts
    * may omit this when they own an equivalent database isolation contract.
@@ -878,6 +893,7 @@ export async function createOpenGeniWorkerService(
   const onRetry = (event: Parameters<typeof logStartupDependencyRetry>[1]) =>
     logStartupDependencyRetry(observability, event);
   let lifecycle: WorkerServiceLifecycle | undefined;
+  let cleanupContainment: ReturnType<typeof createWorkerCleanupContainment> | undefined;
   let signaler: Awaited<ReturnType<typeof createWorkerWorkflowSignaler>> | undefined;
   let workerBundle: Awaited<ReturnType<typeof createOpenGeniWorker>> | undefined;
   let turnCapacityMonitor: ReturnType<typeof startTurnCapacityMonitor> | undefined;
@@ -957,6 +973,10 @@ export async function createOpenGeniWorkerService(
         signalCodexCapacityWorkflow,
         startSandboxReaperWorkflow,
         startVideoGenerationWorkflow,
+        requestWorkerDrain: () => {
+          if (!cleanupContainment) throw new Error("worker cleanup containment is not initialized");
+          cleanupContainment.request();
+        },
       },
     });
 
@@ -1018,18 +1038,15 @@ export async function createOpenGeniWorkerService(
     }
 
     if (options.http !== false) {
-      const databaseReady = dbReadyCheck(
-        options.http?.readinessDb ?? options.activityDependencies.db,
-        options.databasePosture,
-      );
+      const readinessDb = options.http?.readinessDb ?? options.activityDependencies.db;
+      const databaseReady = dbReadyCheck(readinessDb, options.databasePosture, async () => {
+        await resolveCatalogSettings(readinessDb, settings);
+      });
       httpServer = startWorkerHttpServer({
         settings,
         observability,
         checks: {
-          db: async () => {
-            await databaseReady();
-            await resolveCatalogSettings(options.activityDependencies.db, settings);
-          },
+          db: databaseReady,
           nats: natsReadyCheck(options.activityDependencies.bus),
           temporal: temporalReadyCheck(workerBundle.connection),
         },
@@ -1056,9 +1073,21 @@ export async function createOpenGeniWorkerService(
     throw new Error("OpenGeni worker service initialization did not complete");
   }
 
+  cleanupContainment = createWorkerCleanupContainment({
+    drain: () => lifecycle?.drain("stalled turn finalization") ?? false,
+    ...(options.terminateWorker ? { terminate: options.terminateWorker } : {}),
+    observability,
+  });
+  const activeCleanupContainment = cleanupContainment;
   lifecycle = createWorkerServiceLifecycle({
     role: options.role,
-    worker: activeWorkerBundle.worker,
+    worker: {
+      shutdown: () => activeWorkerBundle.worker.shutdown(),
+      run: async () => {
+        await activeWorkerBundle.worker.run();
+        activeCleanupContainment.finished();
+      },
+    },
     observability,
     closeOwnedResources: async () => {
       memoryPressureGuard?.close();
@@ -1210,6 +1239,7 @@ export async function startWorker() {
     );
     await runOpenGeniWorker({
       role,
+      terminateWorker: () => process.exit(1),
       settings,
       databasePosture,
       http: { readinessDb: readinessDbClient.db },

package/src/observability-metrics.ts

@@ -1285,6 +1285,32 @@ export function recordSandboxDeadlineRotationsRequested(
   });
 }
 
+/** Fixed outcomes of legacy retained-command containment: per-candidate
+ * enrollment inspection, then the enrolled drain's exact cold commit. */
+export const SANDBOX_COMMAND_CONTAINMENT_OUTCOMES = [
+  "idle_enrolled",
+  "deadline_enrolled",
+  "resumed_enrolled",
+  "not_eligible",
+  "inspection_failed",
+  "contained",
+  "provider_missing",
+] as const;
+
+export type SandboxCommandContainmentOutcome =
+  (typeof SANDBOX_COMMAND_CONTAINMENT_OUTCOMES)[number];
+
+export function recordSandboxCommandContainment(
+  observability: Observability,
+  outcome: SandboxCommandContainmentOutcome,
+): void {
+  observability.incrementCounter({
+    name: "opengeni_sandbox_command_containment_total",
+    help: "Legacy retained-command containment inspections and enrolled drain commits by fixed outcome.",
+    labels: { outcome },
+  });
+}
+
 /** Only call after the exact draining->cold commit reports wentCold. The
  * backend is validated against the closed contract so provider IDs and other
  * per-sandbox values can never become metric labels. */
@@ -1471,6 +1497,7 @@ export function recordExpiredDrainingSandboxLeaseGauges(
 
 export const RETAINED_PROCESS_RECONCILIATION_OUTCOMES = [
   "claim_failed",
+  "background_recovered",
   "proof_exited",
   "proof_lost",
   "proof_checkpoint_failed",

package/src/sandbox-resume.ts

@@ -20,6 +20,8 @@
 //
 // Liveness between turns is the lease refcount; there is no keepalive loop.
 
+import type { EventBus } from "@opengeni/events";
+import { publishDurableSessionEvents } from "./session-event-fanout";
 import {
   effectiveSandboxLifecycle,
   sandboxArchiveCaptureTimeoutMs,
@@ -171,6 +173,8 @@ export type SandboxResumeServices = {
    * call. Absent, the call gets its own single replacement.
    */
   freshSandboxReadinessReplacementBudget?: FreshSandboxReadinessReplacementBudget;
+  /** Live fanout for background commands a provider loss settled. */
+  bus?: EventBus | null;
   /** Called only by the observer that wins the exact warm->cold loss CAS. */
   onSandboxLost?: (input: {
     sandboxGroupId: string;
@@ -2155,6 +2159,11 @@ async function resumeBoxForTurnOnce(
         expectedBackend: ids.backend,
       });
       if (marked.status === "marked") {
+        await publishDurableSessionEvents(
+          services.bus,
+          ids.workspaceId,
+          marked.backgroundCommandEvents,
+        );
         await services.onSandboxLost?.({
           sandboxGroupId: ids.sandboxGroupId,
           instanceId: live.instanceId,