@opengeni/worker-bundle 1.0.0-canary.0 → 1.0.0

package/src/activities/sandbox-lease.ts

@@ -13,14 +13,19 @@
 // harness and executes admitted drains concurrently. There is no per-session
 // timer, viewer activity, owner task queue, or provider-specific lifecycle path
 // in the normal drain state machine.
+import { warnDrainSnapshotFailure } from "../sandbox-snapshot-diagnostics";
+import { publishDurableSessionEvents } from "../session-event-fanout";
+import { warnRetainedProcessProofFailure } from "../retained-process-diagnostics";
+import { retainedProcessDeadlineRetryMs } from "../retained-process-retry";
 
 import { createHash, randomUUID } from "node:crypto";
-import { retainedProviderCommandPersistence } from "@opengeni/db/retained-provider-commands";
+import { requestRetainedProcessDeadlineCancellation } from "@opengeni/db/retained-provider-commands";
 import type { ProviderCommandPersistence, ProviderCommandSession } from "@opengeni/runtime";
 import { Context } from "@temporalio/activity";
 import { OpLostReason, OpState, type OpStatus } from "@opengeni/agent-proto";
 import {
   accrueWarmSeconds,
+  retainedProviderCommandPersistence,
   adoptLegacyModalCheckpointArtifact,
   confirmDrainCold,
   appendSessionEventToSandboxGroup,
@@ -36,10 +41,17 @@ import {
   countSandboxLeasesByLiveness,
   deferRetainedProcessReconciliation,
   forceDrainOverLimitViewerOnlyBoxes,
-  getBillingBalance,
   listCreditBalancesByAccount,
+  countActiveUsers,
+  readCreditGrantTotals,
+  readManagedAuthNewSignupsSwitch,
+  readVerifiedSignupTrialSwitch,
   listLegacyModalCheckpointSlots,
   listLiveModalSandboxLeaseAttributions,
+  listPendingModalProviderCreates,
+  claimModalProviderCreateRecovery,
+  recordRecoveredModalProviderCreate,
+  markWarmBillingStopCutoff,
   listMeterableWarmLeases,
   listSandboxViewerForceDrainWorkspaceIds,
   markSandboxCheckpointArtifactDeletePending,
@@ -49,13 +61,16 @@ import {
   recordRetainedProcessReconciliationProof,
   replaceWorkspaceArchiveCaptureAfterProof,
   readLease,
+  readWorkspaceArchiveCapturePreflight,
   reapExpiredSessionListSnapshots,
   reapStaleLeaseHoldersGlobal,
   requestDueSandboxRotationsGlobal,
   readSandboxRotationBacklog,
+  readRecentSandboxRecoveryObservations,
   workspaceArchiveCaptureDeadlineElapsed,
   retainedProcessReconciliationProof,
   retainedProcessSettlementIdentity,
+  recoverManagedSessionBackgroundCommand,
   settleClaimedConnectedMachineBackgroundCommand,
   settleSandboxCheckpointArtifactGc,
   rlsContextForWorkspace,
@@ -67,9 +82,12 @@ import {
   type LeaseSnapshot,
 } from "@opengeni/db";
 import {
+  claimConnectedCommandOutputReleases,
   claimConnectedMachineSessionBackgroundCommands,
   deferConnectedMachineBackgroundCommandReconciliation,
   recordConnectedMachineBackgroundCommandProof,
+  recordConnectedCommandOutputConsumption,
+  settleConnectedCommandOutputReleaseClaim,
   type ConnectedMachineBackgroundCommandClaim,
   type ConnectedMachineBackgroundCommandProof,
 } from "@opengeni/db/session-background-commands";
@@ -107,6 +125,7 @@ import {
   providerWorkspaceCapturePolicy,
   resolveModalCheckpointProviderBindingForLiveSandbox,
   resolveModalCheckpointProviderBinding,
+  findModalProviderCreateReceipt,
   resolveModalCheckpointProviderBindingForSession,
   querySelfhostedOp,
   resumeExactSandboxSession,
@@ -139,7 +158,9 @@ import { assertSandboxDrainInputTiming, sandboxDrainTiming } from "../sandbox-re
 import type { ControlActivityServices as ActivityServices } from "./types";
 import { reconcilePendingParentSystemUpdates } from "./parent-wake";
 import {
+  recordActiveUserGauges,
   recordCreditBalanceGauges,
+  recordCreditGrantGauges,
   recordCreditMicros,
   recordExpiredDrainingSandboxLeaseGauges,
   recordOpenSandboxKubernetesInventoryGauges,
@@ -154,8 +175,14 @@ import {
   type SandboxInventoryProjectionDomain,
   recordSandboxLeaseGauges,
   recordSandboxOrphansTerminated,
+  recordSandboxCommandContainment,
+  recordSandboxProviderMissingBeforeCapture,
+  recordSandboxRecoveryObservationGauges,
   recordSandboxRotationBacklogGauges,
   recordTurnsQueuedGauge,
+  recordVerifiedSignupTrialDeploymentFlagGauge,
+  recordManagedAuthNewSignupsSwitchGauge,
+  recordVerifiedSignupTrialSwitchGauge,
   runtimeMetricsHooksForObservability,
 } from "../observability-metrics";
 import {
@@ -182,8 +209,7 @@ export type ReapSandboxLeasesResult = {
   skipped: number;
   /** Warm viewer-only leases that accrued warm-seconds this tick (P2.1). */
   metered: number;
-  /** Viewer-only boxes force-drained because their workspace is over a limit
-   *  (0 balance / over the warm cap) — turn-held boxes are never drained (P2.1). */
+  /** Holderless boxes accelerated to draining by the optional warm cap. */
   forceDrained: number;
   /** Provider-side Modal orphan sandboxes terminated by the defensive sweep. */
   modalOrphansTerminated: number;
@@ -251,6 +277,8 @@ export type TerminateBoxFn = (
   capturePolicy?: ProviderWorkspaceCapturePolicy | null,
   diskBackedArchives?: boolean,
   releaseFailedCapture?: () => Promise<void>,
+  workspaceId?: string,
+  beforeProviderStop?: () => Promise<void>,
 ) => Promise<boolean | ProviderTerminationOutcome>;
 
 export type SweepModalOrphansFn = (
@@ -270,6 +298,8 @@ export type SandboxLeaseActivityOptions = {
   /** Override the provider-side Modal orphan sweep (tests spy this; defaults to
    *  Modal list+tag comparison when Modal is configured). */
   sweepModalOrphans?: SweepModalOrphansFn;
+  /** Only provider discovery is injected; durable recovery remains real. */
+  findModalProviderCreateReceipt?: typeof findModalProviderCreateReceipt;
   /** Override the read-only bounded OpenSandbox Kubernetes projection. */
   inspectOpenSandboxKubernetesInventory?: InspectOpenSandboxKubernetesInventoryFn;
   /** Override only the read-only provider process probe. The canonical DB
@@ -303,6 +333,8 @@ export type RetainedProcessProbeResult =
         | "provider_binding_missing"
         | "provider_binding_mismatch"
         | "process_observation_unavailable";
+      /** A running command whose last read still returned a large backlog. */
+      outputBacklog?: boolean;
     };
 
 export type RetainedProcessProbeFn = (
@@ -317,8 +349,12 @@ export type RetainedProcessProbeFn = (
     streamFidelity?: "separate" | "merged",
   ) => Promise<void>,
   providerPersistence?: ProviderCommandPersistence,
+  drainBudget?: RetainedProcessDrainBudget,
 ) => Promise<RetainedProcessProbeResult>;
 
+/** Shared wall-clock bound on output draining within one reconciliation sweep. */
+export type RetainedProcessDrainBudget = { until: number };
+
 export type HistoricalModalSandboxLifecycleProbeFn = typeof inspectModalSandboxLifecycle;
 
 export type DrainableProviderProbeFn = (
@@ -329,6 +365,16 @@ export type DrainableProviderProbeFn = (
 export const RETAINED_PROCESS_RECONCILIATION_LIMIT = 20;
 export const RETAINED_PROCESS_RECONCILIATION_CLAIM_TTL_MS = 5 * 60_000;
 export const RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS = 5_000;
+/** Bounds on reading a command's output backlog: per claim, and across one
+ * reconciliation sweep so the reaper activity stays well inside its timeout.
+ * Each read returns up to one provider page per stream. */
+export const RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS = 8;
+export const RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS = 10_000;
+export const RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS = 60_000;
+/** A live process whose read returned at least this much output left a backlog
+ * behind it. A server or watcher that only trickles output stays on ordinary
+ * backoff; a process the provider reports exited is always read to EOF. */
+export const RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES = 256 * 1024;
 export const RETAINED_PROCESS_BINDING_QUARANTINE_AFTER_ATTEMPTS = 5;
 export const RETAINED_PROCESS_BINDING_QUARANTINE_RETRY_MS = 24 * 60 * 60_000;
 export const CONNECTED_COMMAND_RECONCILIATION_LIMIT = 20;
@@ -477,7 +523,14 @@ export function createSandboxLeaseActivities(
   options: SandboxLeaseActivityOptions = {},
 ) {
   const terminateBox: TerminateBoxFn =
-    options.terminateBox ??
+    (options.terminateBox
+      ? async (...args: Parameters<TerminateBoxFn>) => {
+          // The injected test seam represents the stop itself; the real
+          // implementation invokes this only after capture has succeeded.
+          await args[10]?.();
+          return options.terminateBox!(...args);
+        }
+      : null) ??
     (async (
       settings,
       lease,
@@ -488,6 +541,8 @@ export function createSandboxLeaseActivities(
       capturePolicy,
       diskBackedArchives,
       releaseFailedCapture,
+      workspaceId,
+      beforeProviderStop,
     ) =>
       await terminateProviderBox(
         settings,
@@ -501,6 +556,8 @@ export function createSandboxLeaseActivities(
         capturePolicy,
         diskBackedArchives,
         releaseFailedCapture,
+        workspaceId,
+        beforeProviderStop,
       ));
   const sweepModalOrphans: SweepModalOrphansFn =
     options.sweepModalOrphans ?? sweepModalOrphansForConfiguredBackend;
@@ -513,10 +570,15 @@ export function createSandboxLeaseActivities(
     try {
       const { db, settings, observability } = await services();
       const timing = sandboxDrainTiming(settings);
-      const onUnobservableCommandDrainError = (error: unknown) => {
-        observability.warn("sandbox reaper: unobservable command drain inspection failed", {
-          error: error instanceof Error ? error.message : String(error),
-        });
+      const commandContainment = {
+        idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
+        onCommandContainment: (outcome: Parameters<typeof recordSandboxCommandContainment>[1]) =>
+          recordSandboxCommandContainment(observability, outcome),
+        onCommandContainmentError: (error: unknown) => {
+          observability.warn("sandbox reaper: retained command containment inspection failed", {
+            error: error instanceof Error ? error.message : String(error),
+          });
+        },
       };
       if (!settings.sandboxOwnershipEnabled) {
         // Turns skip leases when the flag is off, but Computer/Browser attach
@@ -524,7 +586,7 @@ export function createSandboxLeaseActivities(
         // behind rotation_in_progress forever. Inventory and drain those rows;
         // do not request NEW deadline rotations or meter warm time.
         const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
-          onUnobservableCommandDrainError,
+          ...commandContainment,
           viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
           turnHolderTtlMs: settings.sandboxLeaseTtlMs,
           interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
@@ -561,7 +623,7 @@ export function createSandboxLeaseActivities(
       // Billing, reconciliation, provider-orphan cleanup, artifact GC, and gauges
       // run only after every drainable box has its own durable child.
       const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
-        onUnobservableCommandDrainError,
+        ...commandContainment,
         viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
         // Dead-worker turn holders: a live holder is touched every 10s from the
         // moment it is registered (resumeBoxForTurn's holder-liveness loop covers
@@ -605,7 +667,7 @@ export function createSandboxLeaseActivities(
       instanceId: input.target.instanceId,
     });
     try {
-      const { db, settings, observability, objectStorage } = await services();
+      const { db, settings, observability, objectStorage, bus } = await services();
       assertSandboxDrainInputTiming(input);
       const drainSettings =
         settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs
@@ -629,6 +691,7 @@ export function createSandboxLeaseActivities(
           probeDrainableProvider,
           captureAttempt,
           objectStorage,
+          bus,
         );
         return { status: drainedCold ? "terminated" : "skipped" };
       } catch (error) {
@@ -703,6 +766,23 @@ export function createSandboxLeaseActivities(
       // immutable launch locators must reconcile even when managed ownership is
       // disabled, otherwise an exact runner exit/loss proof can remain stranded.
       await reconcileConnectedMachineBackgroundCommands(db, settings, observability, service.bus);
+      await reconcileConnectedCommandOutputReleases(db, settings, observability, service.bus);
+
+      // Disabling new lease ownership cannot strand a previously dispatched
+      // operation. This only attributes a positive receipt; normal draining
+      // already runs in both ownership modes.
+      try {
+        await reconcileModalProviderCreates(
+          db,
+          settings,
+          observability,
+          options.findModalProviderCreateReceipt,
+        );
+      } catch (error) {
+        observability.warn("sandbox reaper: Modal create recovery inventory failed", {
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
 
       if (!settings.sandboxOwnershipEnabled) {
         // Inventory remains useful while provider ownership is intentionally
@@ -728,8 +808,7 @@ export function createSandboxLeaseActivities(
       metered = meterResult.accrued;
       const forceDrainWorkspaceIds = new Set<string>();
       if (
-        settings.billingMode === "stripe" ||
-        settings.usageLimitsMode === "managed" ||
+        settings.sandboxWarmBillingMode === "credits" ||
         settings.sandboxMaxWarmSecondsPerWorkspace > 0
       ) {
         for (const workspaceId of meterResult.workspaceIds) {
@@ -903,6 +982,9 @@ type ConnectedCommandReconciliationOutcome =
   | "proof_checkpoint_failed"
   | "settled_exited"
   | "settled_lost"
+  | "output_unavailable"
+  | "output_ack_published"
+  | "output_released"
   | "settlement_failed"
   | "defer_failed";
 
@@ -1103,6 +1185,7 @@ export async function reconcileConnectedMachineBackgroundCommands(
       );
     await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
       let proof = claim.proof;
+      let outputReplay: Awaited<ReturnType<OpStreamExecClient["readExisting"]>> | null = null;
       const connectionKey = JSON.stringify([
         claim.controlWorkspaceId,
         claim.enrollmentId,
@@ -1160,18 +1243,14 @@ export async function reconcileConnectedMachineBackgroundCommands(
               retryClock: defaultSelfhostedRetryClock,
               journal: { attachGeneration: () => String(Date.now()), persistSettled: () => {} },
             });
-            const replay = await client.readExisting(
+            outputReplay = await replayConnectedCommandOutput(
+              client,
               claim.opId,
-              proof ? 5_000 : 250,
+              Boolean(proof),
               async (frames) => {
                 await captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames);
               },
             );
-            if (proof && replay.status !== "completed" && !replay.terminal) {
-              throw new Error(
-                "Connected command terminal output replay has not reached its exit frontier",
-              );
-            }
           }
           if (!proof) {
             await deferConnectedCommandClaim(
@@ -1226,6 +1305,18 @@ export async function reconcileConnectedMachineBackgroundCommands(
         if (!settlement.settled) {
           throw new Error("Connected command settlement lost its exact claim or proof");
         }
+        const receipt =
+          outputReplay?.outputReceipt ??
+          (outputReplay?.status === "running" ? outputReplay.terminal?.outputReceipt : undefined);
+        const terminal =
+          outputReplay?.status === "completed" ? outputReplay : outputReplay?.terminal;
+        if (proof.outcome === "exited" && receipt && terminal) {
+          await recordConnectedCommandOutputConsumption(db, {
+            ...claim,
+            receipt,
+            exitCode: terminal.outcome.response.exitCode,
+          });
+        }
         if (settlement.events.length > 0) {
           try {
             await bus.publish(claim.workspaceId, claim.sessionId, settlement.events);
@@ -1250,6 +1341,168 @@ export async function reconcileConnectedMachineBackgroundCommands(
   } while (Date.now() < deadline);
 }
 
+export async function replayConnectedCommandOutput(
+  client: Pick<OpStreamExecClient, "readExisting">,
+  opId: string,
+  terminalKnown: boolean,
+  capture: Parameters<OpStreamExecClient["readExisting"]>[2],
+): Promise<Awaited<ReturnType<OpStreamExecClient["readExisting"]>>> {
+  // A completed operation can need several bounded reads to replay its retained
+  // frames. Keep the reader's integrity checkpoint until the exit is verified;
+  // replacing it after each partial read would restart forever at frame one.
+  let capturedThrough = 0n;
+  while (true) {
+    const before = capturedThrough;
+    const replay = await client.readExisting(opId, terminalKnown ? 5_000 : 250, capture);
+    if (!terminalKnown || replay.status === "completed" || replay.terminal) return replay;
+    // Quiet jobs can retain many heartbeat frames (or an incomplete UTF-8
+    // chunk). These are real replay progress even when capture receives no
+    // stdout/stderr. Use the reader's verified, contiguous protocol frontier.
+    capturedThrough = BigInt(replay.replaySequence);
+    // This is a finite terminal drain, not a poller for a running command.
+    // Failed persistence, transport/integrity errors, and no-progress reads
+    // return control to normal reconciliation without licensing settlement.
+    if (capturedThrough === before) {
+      throw new Error("Connected command terminal output replay has not reached its exit frontier");
+    }
+  }
+}
+
+/** Drain the durable terminal-output obligation independently of the launching
+ * attempt or model completion notice. Existing terminal rows receive no inferred
+ * receipt: they must replay and capture all retained bytes before any final ACK. */
+export async function reconcileConnectedCommandOutputReleases(
+  db: ActivityServices["db"],
+  settings: ActivityServices["settings"],
+  observability: ActivityServices["observability"],
+  bus: ActivityServices["bus"],
+  controlRpcOverride?: ControlRpc,
+): Promise<void> {
+  const dueBefore = new Date();
+  const deadline = Date.now() + settings.sandboxLeaseReaperPeriodMs;
+  const controlRpc =
+    controlRpcOverride ?? new NatsControlRpc(async () => bus.getRequestConnection());
+  do {
+    let claims: Awaited<ReturnType<typeof claimConnectedCommandOutputReleases>>;
+    try {
+      claims = await claimConnectedCommandOutputReleases(db, {
+        claimId: crypto.randomUUID(),
+        dueBefore,
+        limit: CONNECTED_COMMAND_RECONCILIATION_LIMIT,
+        claimTtlMs: CONNECTED_COMMAND_RECONCILIATION_CLAIM_TTL_MS,
+      });
+    } catch (error) {
+      recordConnectedCommandReconciliation(observability, "claim_failed");
+      observability.warn("sandbox reaper: connected-command output claim failed", {
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return;
+    }
+    await forEachWithConcurrency(claims, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (claim) => {
+      try {
+        if (!bus.getOpStreamConnection)
+          throw new Error("Connected command output transport unavailable");
+        const rpcSubject = subjectFor(
+          claim.controlWorkspaceId,
+          claim.enrollmentId,
+          claim.connectionInstanceId,
+        );
+        const controlTimeoutMs = Math.min(
+          settings.sandboxSelfhostedControlTimeoutMs,
+          RETAINED_PROCESS_PROVIDER_PROBE_TIMEOUT_MS,
+        );
+        const client = new OpStreamExecClient({
+          workspaceId: claim.controlWorkspaceId,
+          agentId: claim.enrollmentId,
+          connectionInstanceId: claim.connectionInstanceId,
+          epoch: 0,
+          controlRpc,
+          rpcSubject,
+          transport: new NatsOpStreamTransport(async () => bus.getOpStreamConnection?.() ?? null),
+          controlTimeoutMs,
+          retryClock: defaultSelfhostedRetryClock,
+          journal: {
+            attachGeneration: () => String(Date.now()),
+            persistSettled: () => {
+              throw new Error("Background output release requires its database receipt");
+            },
+          },
+        });
+        let receipt = claim.receipt;
+        if (!receipt) {
+          const status = await querySelfhostedOp({
+            controlRpc,
+            rpcSubject,
+            opId: claim.opId,
+            controlTimeoutMs,
+          });
+          if (status.state === OpState.OP_STATE_LOST) {
+            if (
+              !(await settleConnectedCommandOutputReleaseClaim(db, {
+                claim,
+                outcome: "unavailable",
+                retryAfterMs: 0,
+              }))
+            )
+              throw new Error("Connected command output loss no longer owns its exact claim");
+            recordConnectedCommandReconciliation(observability, "output_unavailable");
+            return;
+          }
+          if (status.state !== OpState.OP_STATE_COMPLETE)
+            throw new Error("Terminal command output returned a live operation");
+          const replay = await replayConnectedCommandOutput(
+            client,
+            claim.opId,
+            true,
+            async (frames) =>
+              captureConnectedCommandOutput(db, claim, bus)(claim.commandId, frames),
+          );
+          const terminal = replay.status === "completed" ? replay : replay.terminal;
+          receipt =
+            replay.outputReceipt ??
+            (replay.status === "running" ? replay.terminal?.outputReceipt : undefined) ??
+            null;
+          if (!terminal || !receipt)
+            throw new Error("Connected command output replay has no complete custody receipt");
+          await recordConnectedCommandOutputConsumption(db, {
+            ...claim,
+            receipt,
+            exitCode: terminal.outcome.response.exitCode,
+          });
+        }
+        const outcome = await client.releaseCapturedOutput(claim.opId, receipt);
+        if (
+          !(await settleConnectedCommandOutputReleaseClaim(db, {
+            claim,
+            outcome,
+            retryAfterMs: outcome === "published" ? settings.sandboxLeaseReaperPeriodMs : 0,
+          }))
+        )
+          throw new Error("Connected command output release no longer owns its exact claim");
+        recordConnectedCommandReconciliation(
+          observability,
+          outcome === "published" ? "output_ack_published" : "output_released",
+        );
+      } catch (error) {
+        await settleConnectedCommandOutputReleaseClaim(db, {
+          claim,
+          outcome: "retry",
+          retryAfterMs: Math.min(
+            5 * 60_000,
+            Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) *
+              2 ** Math.min(4, claim.reconcileAttempts - 1),
+          ),
+        }).catch(() => false);
+        observability.warn("sandbox reaper: connected-command output release deferred", {
+          commandId: claim.commandId,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    });
+    if (claims.length < CONNECTED_COMMAND_RECONCILIATION_LIMIT) break;
+  } while (Date.now() < deadline);
+}
+
 export async function probeConnectedMachineBackgroundCommand(
   claim: ConnectedMachineBackgroundCommandClaim,
   controlRpc: ControlRpc,
@@ -1377,10 +1630,80 @@ async function reconcileTerminalRetainedProcesses(
     return;
   }
 
+  const drainBudget: RetainedProcessDrainBudget = {
+    until: Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_SWEEP_BUDGET_MS,
+  };
   for (const claim of claims) {
     let process = claim.process;
     const expected = retainedProcessSettlementIdentity(process);
     let proof = retainedProcessReconciliationProof(process);
+    const processScope = {
+      accountId: process.accountId,
+      workspaceId: process.workspaceId,
+      sessionId: process.sessionId,
+      processId: process.id,
+    };
+    // Recover a yield whose observation failed before normal receipt adoption.
+    // Do this before provider I/O: an unavailable observer must not fence every
+    // subsequent turn. Physical retention and unknown outcome stay unchanged.
+    if (
+      process.ownerActorKind === "turn" &&
+      process.providerBackend === "modal" &&
+      !proof &&
+      !claim.ownerState.startsWith("background_")
+    ) {
+      try {
+        const command = await recoverManagedSessionBackgroundCommand(db, {
+          ...processScope,
+          expected,
+          reconciliationClaimId: claim.claimId,
+        });
+        if (command) {
+          claim.ownerState = `background_${command.state}`;
+          recordRetainedProcessReconciliation(observability, "background_recovered");
+        }
+      } catch (error) {
+        observability.warn("sandbox reaper: retained-command background recovery failed", {
+          processId: process.id,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    }
+    const storedCommandPersistence = retainedProviderCommandPersistence(
+      db,
+      processScope,
+      bus ? (events) => bus.publish(process.workspaceId, process.sessionId, events) : undefined,
+    );
+    const commandPersistence = {
+      ...storedCommandPersistence,
+      requestCancellation: async (reason: "provider_deadline" | "explicit_stop") => {
+        if (reason === "provider_deadline") {
+          await requestRetainedProcessDeadlineCancellation(db, processScope);
+          if (!(await storedCommandPersistence.cancellationRequested()))
+            throw new Error(
+              "Supervised deadline cancellation no longer owns its original rotating lease",
+            );
+        } else await storedCommandPersistence.requestCancellation(reason);
+      },
+    };
+    if (proof && process.providerBackend === "modal") {
+      try {
+        const command = await commandPersistence.load();
+        if (
+          command?.kind === "modal-router-v1" &&
+          command.supervision &&
+          (!(await commandPersistence.loadSupervisionReceipt()) ||
+            ![command.streams.stdout, command.streams.stderr].every(
+              (stream) => stream.eof && stream.exitCode === 0,
+            ))
+        )
+          proof = null;
+      } catch {
+        // An earlier terminal observation cannot make missing control/output
+        // proof unreachable. Reprobe the original binding; DB remains the gate.
+        proof = null;
+      }
+    }
     if (!proof) {
       let observation: RetainedProcessProbeResult | null = null;
 
@@ -1434,6 +1757,25 @@ async function reconcileTerminalRetainedProcesses(
         }
       }
 
+      if (
+        observation?.status === "deferred" &&
+        process.providerBackend === "modal" &&
+        !process.providerBindingKey &&
+        !process.deadlineCancellationRequestedAt
+      ) {
+        // An unbound historical command cannot be signalled safely. Still
+        // record deadline intent when its lease matches: a failed observer
+        // must not block capture until the provider expires.
+        try {
+          await requestRetainedProcessDeadlineCancellation(db, processScope);
+        } catch (error) {
+          observability.warn("sandbox reaper: legacy deadline cancellation intent failed", {
+            processId: process.id,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        }
+      }
+
       if (observation === null) {
         const lease = await readLease(db, process.workspaceId, process.sandboxGroupId).catch(
           () => null,
@@ -1490,11 +1832,39 @@ async function reconcileTerminalRetainedProcesses(
           }
         } else {
           try {
-            observation = await probe(
+            const command = await commandPersistence.load();
+            const supervised = command?.kind === "modal-router-v1" && Boolean(command.supervision);
+            const supervisionMetric = (outcome: string) =>
+              observability.incrementCounter({
+                name: "opengeni_command_supervision_total",
+                help: "Native retained command supervision reconciliation outcomes.",
+                labels: { outcome },
+              });
+            if (
+              supervised &&
+              (lease!.rotationReason === "provider_deadline" ||
+                claim.ownerState === "background_stopping")
+            ) {
+              await commandPersistence.requestCancellation(
+                lease!.rotationReason === "provider_deadline"
+                  ? "provider_deadline"
+                  : "explicit_stop",
+              );
+              supervisionMetric("cancellation_intent");
+            }
+            const legacyDeadlineStop =
+              lease!.rotationReason === "provider_deadline" &&
+              !supervised &&
+              !process.deadlineCancellationRequestedAt;
+            const probeOutcome = await probe(
               settings,
               lease!,
               process,
-              claim.ownerState === "background_stopping" ? "cancel" : "observe",
+              (claim.ownerState === "background_stopping" &&
+                !process.deadlineCancellationRequestedAt) ||
+                legacyDeadlineStop
+                ? "cancel"
+                : "observe",
               async (result, chunkId, stream, streamFidelity) => {
                 if (
                   typeof result !== "string" ||
@@ -1516,13 +1886,35 @@ async function reconcileTerminalRetainedProcesses(
                     .publish(process.workspaceId, process.sessionId, events)
                     .catch(() => undefined);
               },
-              retainedProviderCommandPersistence(db, {
-                accountId: process.accountId,
-                workspaceId: process.workspaceId,
-                sessionId: process.sessionId,
-                processId: process.id,
-              }),
+              commandPersistence,
+              drainBudget,
+            ).then(
+              (value) => ({ ok: true as const, value }),
+              (error: unknown) => ({ ok: false as const, error }),
             );
+            // Legacy PTY input reserves Ctrl-C before the cancellation fence.
+            // Non-PTY has no command signal in the Modal SDK: probe it without
+            // injecting Ctrl-C as data, then fence and save after the grace.
+            if (
+              legacyDeadlineStop &&
+              !(await requestRetainedProcessDeadlineCancellation(db, processScope))
+            )
+              throw new Error("Deadline cancellation no longer owns its rotating lease");
+            if (!probeOutcome.ok) throw probeOutcome.error;
+            observation = probeOutcome.value;
+            if (supervised) {
+              supervisionMetric(
+                (await commandPersistence.loadSupervisionReceipt())
+                  ? "proof_retained"
+                  : "proof_missing",
+              );
+              if (observation.status === "deferred")
+                supervisionMetric(
+                  observation.reason === "provider_running"
+                    ? "checkpoint_blocked"
+                    : "provider_failure",
+                );
+            }
           } catch (error) {
             observability.warn("sandbox reaper: retained-process provider probe failed", {
               processId: process.id,
@@ -1564,6 +1956,7 @@ async function reconcileTerminalRetainedProcesses(
           expected,
           claim.claimId,
           observation.reason,
+          observation.outputBacklog === true,
         );
         continue;
       }
@@ -1581,10 +1974,7 @@ async function reconcileTerminalRetainedProcesses(
         recordRetainedProcessReconciliation(observability, `proof_${proof.outcome}`);
       } catch (error) {
         recordRetainedProcessReconciliation(observability, "proof_checkpoint_failed");
-        observability.warn("sandbox reaper: retained-process proof checkpoint failed", {
-          processId: process.id,
-          error: error instanceof Error ? error.message : String(error),
-        });
+        warnRetainedProcessProofFailure(observability, error, process);
         continue;
       }
     }
@@ -1647,9 +2037,17 @@ async function deferRetainedProcessClaim(
   expected: ReturnType<typeof retainedProcessSettlementIdentity>,
   claimId: string,
   outcome: RetainedProcessDeferralOutcome,
+  outputBacklog = false,
 ): Promise<void> {
-  const deferral = retainedProcessReconciliationDeferral(settings, process, outcome);
+  const deferral = retainedProcessReconciliationDeferral(settings, process, outcome, outputBacklog);
   try {
+    // A healthy long-lived command can already be on a five-minute backoff
+    // when rotation becomes due. Wake it at the lead boundary, then at the
+    // reaper cadence, so observation does not consume the capture window.
+    const lease =
+      process.providerBackend === "modal" && process.routeTargetId === null
+        ? await readLease(db, process.workspaceId, process.sandboxGroupId)
+        : null;
     await deferRetainedProcessReconciliation(db, {
       accountId: process.accountId,
       workspaceId: process.workspaceId,
@@ -1658,7 +2056,7 @@ async function deferRetainedProcessClaim(
       expected,
       claimId,
       outcome: deferral.durableOutcome,
-      retryAfterMs: deferral.retryAfterMs,
+      retryAfterMs: retainedProcessDeadlineRetryMs(process, lease, settings, deferral.retryAfterMs),
     });
     recordRetainedProcessReconciliation(observability, deferral.metricOutcome);
   } catch (error) {
@@ -1678,6 +2076,7 @@ export function retainedProcessReconciliationDeferral(
   settings: Pick<ActivityServices["settings"], "sandboxLeaseReaperPeriodMs">,
   process: Pick<SandboxRetainedProcess, "reconcileAttempts">,
   outcome: RetainedProcessDeferralOutcome,
+  outputBacklog = false,
 ): {
   durableOutcome: string;
   metricOutcome: RetainedProcessReconciliationOutcome;
@@ -1702,11 +2101,14 @@ export function retainedProcessReconciliationDeferral(
       quarantined: true,
     };
   }
-  const exponent = Math.min(4, Math.max(0, process.reconcileAttempts - 1));
-  const retryAfterMs = Math.min(
-    5 * 60_000,
-    Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000) * 2 ** exponent,
-  );
+  const baseRetryMs = Math.max(settings.sandboxLeaseReaperPeriodMs, 30_000);
+  // A backlog still being drained is progress, not an unhealthy provider:
+  // come back at the reaper cadence, without backoff.
+  const exponent =
+    outcome === "provider_running" && outputBacklog
+      ? 0
+      : Math.min(4, Math.max(0, process.reconcileAttempts - 1));
+  const retryAfterMs = Math.min(5 * 60_000, baseRetryMs * 2 ** exponent);
   return {
     durableOutcome: outcome,
     metricOutcome: outcome,
@@ -1733,6 +2135,77 @@ type RetainedProcessProbeSession = ProviderCommandSession & {
 
 const RETAINED_PROCESS_PROBE_TIMEOUT = Symbol("retained-process-provider-probe-timeout");
 
+/** A command's exit is reported only after both output streams reach EOF, so a
+ * finished command with a large backlog looks running until it is read. While a
+ * backlog remains, keep reading within the per-claim and per-sweep budgets, so
+ * it settles in this claim instead of one page per backoff step. Every page,
+ * including `first`, is captured here. */
+export async function drainRetainedCommandBacklog(
+  session: RetainedProcessProbeSession,
+  providerSessionId: number,
+  first: unknown,
+  capturePage: (value: unknown) => Promise<void>,
+  budget?: RetainedProcessDrainBudget,
+): Promise<RetainedProcessProbeResult> {
+  // Measure before capture: capturing a byte-offset page consumes its receipt.
+  let backlog = retainedBacklogRemains(session, first);
+  await capturePage(first);
+  let observation = classifyRetainedProcessPollResult(first, providerSessionId, session);
+  if (typeof session.writeStdin !== "function") return observation;
+  const until = Math.min(
+    Date.now() + RETAINED_PROCESS_OUTPUT_DRAIN_BUDGET_MS,
+    budget?.until ?? Number.POSITIVE_INFINITY,
+  );
+  for (
+    let reads = 0;
+    backlog &&
+    observation.status === "deferred" &&
+    observation.reason === "provider_running" &&
+    reads < RETAINED_PROCESS_OUTPUT_DRAIN_MAX_READS &&
+    Date.now() < until;
+    reads++
+  ) {
+    let next: unknown;
+    try {
+      next = await withRetainedProcessProbeTimeout(
+        session.writeStdin({
+          sessionId: providerSessionId,
+          chars: "",
+          yieldTimeMs: 1_000,
+          maxOutputTokens: 2_000,
+        }),
+      );
+    } catch {
+      // The observation already made stays authoritative; the next sweep
+      // classifies any provider failure through the ordinary path.
+      break;
+    }
+    backlog = retainedBacklogRemains(session, next);
+    await capturePage(next);
+    observation = classifyRetainedProcessPollResult(next, providerSessionId, session);
+  }
+  return backlog && observation.status === "deferred" && observation.reason === "provider_running"
+    ? { ...observation, outputBacklog: true }
+    : observation;
+}
+
+/** Whether a byte-offset page leaves a backlog worth reading now: the provider
+ * reports the process exited but its output is not yet read to EOF, or a live
+ * process produced a large page. Must run before the page is captured, which
+ * consumes its receipt. Supervised commands are left to their proof path. */
+function retainedBacklogRemains(session: RetainedProcessProbeSession, result: unknown): boolean {
+  const page = session.getProviderCommandOutput?.(result);
+  if (!page?.expected || page.command.kind !== "modal-router-v1") return false;
+  if (page.command.supervision || page.exitCode !== null) return false;
+  if (page.providerExited) return true;
+  const { command, expected } = page;
+  return (["stdout", "stderr"] as const).some(
+    (stream) =>
+      command.streams[stream].byteOffset - expected.streams[stream].byteOffset >=
+      RETAINED_PROCESS_OUTPUT_BACKLOG_BYTES,
+  );
+}
+
 async function withRetainedProcessProbeTimeout<T>(promise: Promise<T>): Promise<T> {
   let timer: ReturnType<typeof setTimeout> | undefined;
   try {
@@ -1781,6 +2254,7 @@ export async function probeRetainedProcessAtProvider(
     streamFidelity?: "separate" | "merged",
   ) => Promise<void>,
   providerPersistence?: ProviderCommandPersistence,
+  drainBudget?: RetainedProcessDrainBudget,
 ): Promise<RetainedProcessProbeResult> {
   if (
     lease.id !== process.leaseId ||
@@ -1788,7 +2262,8 @@ export async function probeRetainedProcessAtProvider(
     lease.leaseEpoch !== process.leaseEpoch ||
     lease.backend !== process.providerBackend ||
     lease.instanceId !== process.providerInstanceId ||
-    lease.resumeBackendId !== process.providerBackend ||
+    !lease.resumeBackendId ||
+    sandboxBackendForSdkBackendId(lease.resumeBackendId) !== process.providerBackend ||
     process.routeTargetId !== null
   ) {
     return { status: "deferred", reason: "identity_mismatch" };
@@ -1800,10 +2275,20 @@ export async function probeRetainedProcessAtProvider(
   if (pending) {
     if (!captureOutput) throw new Error("Pending command output requires its persistence callback");
     await captureRetainedProbeOutput(process.id, pending.result, captureOutput);
-    return classifyRetainedProcessPollResult(pending.result, process.providerSessionId);
+    const pendingObservation = classifyRetainedProcessPollResult(
+      pending.result,
+      process.providerSessionId,
+    );
+    // Capture of a previous page must not consume the only deadline stop
+    // attempt. Terminal proof needs no further provider call.
+    if (mode !== "cancel" || pendingObservation.status === "proved") return pendingObservation;
   }
   const envelopeBackend = (lease.resumeState as { backendId?: unknown }).backendId;
-  if (envelopeBackend !== undefined && envelopeBackend !== process.providerBackend) {
+  if (
+    envelopeBackend !== undefined &&
+    (typeof envelopeBackend !== "string" ||
+      sandboxBackendForSdkBackendId(envelopeBackend) !== process.providerBackend)
+  ) {
     return { status: "deferred", reason: "identity_mismatch" };
   }
   if (sandboxProviderInstanceIdFromEnvelope(lease.resumeState) !== process.providerInstanceId) {
@@ -1837,7 +2322,11 @@ export async function probeRetainedProcessAtProvider(
   } catch {
     return { status: "deferred", reason: "provider_error" };
   }
-  if (!client || client.backendId !== process.providerBackend || !client.resume) {
+  if (
+    !client ||
+    sandboxBackendForSdkBackendId(client.backendId) !== process.providerBackend ||
+    !client.resume
+  ) {
     return { status: "deferred", reason: "backend_unsupported" };
   }
   const envelopeSessionState =
@@ -1889,6 +2378,7 @@ export async function probeRetainedProcessAtProvider(
   if (typeof session.writeStdin !== "function") {
     return { status: "deferred", reason: "backend_unsupported" };
   }
+  let legacyPty = false;
   if (process.providerBackend === "modal") {
     let liveBinding: Awaited<ReturnType<typeof resolveModalCheckpointProviderBindingForSession>>;
     try {
@@ -1920,6 +2410,7 @@ export async function probeRetainedProcessAtProvider(
     if (!command || !providerPersistence || !session.bindProviderCommand)
       return { status: "deferred", reason: "process_observation_unavailable" };
     session.bindProviderCommand(process.providerSessionId, command, providerPersistence);
+    legacyPty = command.pty === true;
   }
   const capturePage = async (value: unknown): Promise<void> => {
     if (!captureOutput) {
@@ -1928,6 +2419,11 @@ export async function probeRetainedProcessAtProvider(
       return;
     }
     const page = session.getProviderCommandOutput?.(value);
+    if (page?.command.kind === "modal-router-v1" && typeof value === "string") {
+      if (!session.captureCommandOutput || !(await session.captureCommandOutput(value)))
+        throw new Error("Byte-offset output requires atomic capture before settlement");
+      return;
+    }
     if (page) {
       for (const chunk of page.chunks)
         if (chunk.text)
@@ -1940,10 +2436,24 @@ export async function probeRetainedProcessAtProvider(
 
   let result: unknown;
   try {
+    const supervisedCancelled =
+      mode === "cancel" || lease.rotationReason === "provider_deadline"
+        ? ((await session.cancelSupervisedCommand?.(
+            process.providerSessionId,
+            lease.rotationReason === "provider_deadline" ? "provider_deadline" : "explicit_stop",
+          )) ?? false)
+        : false;
     result = await withRetainedProcessProbeTimeout(
       session.writeStdin({
         sessionId: process.providerSessionId,
-        chars: mode === "cancel" ? "\u0003" : "",
+        // Ctrl-C is a signal only through a PTY. For a non-PTY command it is
+        // ordinary stdin data and can corrupt the command's input.
+        chars:
+          mode === "cancel" &&
+          !supervisedCancelled &&
+          (process.providerBackend !== "modal" || legacyPty)
+            ? "\u0003"
+            : "",
         yieldTimeMs: 1_000,
         maxOutputTokens: 2_000,
       }),
@@ -1967,12 +2477,25 @@ export async function probeRetainedProcessAtProvider(
     }
     return { status: "deferred", reason: "provider_error" };
   }
-  await capturePage(result);
-  const observation = classifyRetainedProcessPollResult(result, process.providerSessionId, session);
+  // Cancellation keeps draining with empty reads too: a stopped command's exit
+  // is also reported only after its unread output reaches EOF.
+  const observation = await drainRetainedCommandBacklog(
+    session,
+    process.providerSessionId,
+    result,
+    capturePage,
+    drainBudget,
+  );
   if (
     observation.status === "deferred" &&
     observation.reason === "provider_running" &&
-    lease.rotationRequestedAt !== null
+    lease.rotationRequestedAt !== null &&
+    mode !== "cancel" &&
+    !process.cancellationRequestedAt &&
+    legacyPty &&
+    !(await providerPersistence
+      ?.load()
+      .then((command) => command?.kind === "modal-router-v1" && command.supervision))
   ) {
     // A background PTY cannot outlive the finite provider box. Interrupt only
     // this exact durable provider session after rotation admission is fenced;
@@ -2075,19 +2598,25 @@ async function accrueWarmTick(
   }
   await forEachWithConcurrency(leases, SANDBOX_MAINTENANCE_ITEM_CONCURRENCY, async (lease) => {
     try {
-      const rate = sandboxWarmRateMicrosPerSecond(settings, lease.backend);
+      const rate =
+        settings.sandboxWarmBillingMode === "usage_only"
+          ? 0
+          : sandboxWarmRateMicrosPerSecond(settings, lease.backend);
       const result = await accrueWarmSeconds(db, {
         accountId: lease.accountId,
         workspaceId: lease.workspaceId,
         sandboxGroupId: lease.sandboxGroupId,
         expectedEpoch: lease.leaseEpoch,
         warmRateMicrosPerSecond: rate,
+        billingMode: settings.sandboxWarmBillingMode,
         subjectId: lease.sandboxGroupId,
       });
       if (result.accrued) {
         accrued += 1;
       }
-      recordCreditMicros(observability, "usage", result.costMicros);
+      if (result.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
+        recordCreditMicros(observability, "usage", result.costMicros);
+      }
     } catch (error) {
       observability.warn("sandbox reaper: warm-seconds accrual failed for lease", {
         workspaceId: lease.workspaceId,
@@ -2129,6 +2658,17 @@ async function refreshQueueLeaseAndCreditGauges(
         );
       },
     ),
+    refreshSandboxInventoryGauge(
+      observability,
+      "recovery_observations",
+      "recovery-observations",
+      async () => {
+        recordSandboxRecoveryObservationGauges(
+          observability,
+          await readRecentSandboxRecoveryObservations(db),
+        );
+      },
+    ),
     refreshSandboxInventoryGauge(
       observability,
       "rotation_backlog",
@@ -2181,6 +2721,51 @@ async function refreshQueueLeaseAndCreditGauges(
         });
       }
     })(),
+    (async () => {
+      try {
+        recordActiveUserGauges(observability, await countActiveUsers(db));
+      } catch (error) {
+        observability.warn("sandbox reaper: active-user gauge refresh failed", {
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    })(),
+    (async () => {
+      try {
+        recordCreditGrantGauges(observability, await readCreditGrantTotals(db));
+      } catch (error) {
+        observability.warn("sandbox reaper: credit-grant gauge refresh failed", {
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    })(),
+    (async () => {
+      recordVerifiedSignupTrialDeploymentFlagGauge(
+        observability,
+        settings.verifiedSignupTrialCreditsEnabled,
+      );
+      try {
+        const trialSwitch = await readVerifiedSignupTrialSwitch(db);
+        recordVerifiedSignupTrialSwitchGauge(observability, trialSwitch?.grantsEnabled === true);
+      } catch (error) {
+        observability.warn("sandbox reaper: trial-credit switch gauge refresh failed", {
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    })(),
+    (async () => {
+      try {
+        const signupsSwitch = await readManagedAuthNewSignupsSwitch(db);
+        recordManagedAuthNewSignupsSwitchGauge(
+          observability,
+          signupsSwitch?.signupsEnabled !== false,
+        );
+      } catch (error) {
+        observability.warn("sandbox reaper: new-signups switch gauge refresh failed", {
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    })(),
   ]);
 }
 
@@ -2203,8 +2788,8 @@ async function refreshSandboxInventoryGauge(
 
 /**
  * The per-workspace warm-cap + force-drain pass (P2.1). For each workspace with a
- * live warm box, under the usage lock: if it is at 0 balance (when a billing /
- * managed mode is on) or over its warm cap, force-drain its VIEWER-ONLY boxes
+ * live warm box, under the usage lock: if it is over its optional warm cap,
+ * accelerate only holderless boxes to draining
  * (guarded turn_holders=0 — a paying turn is never killed). Returns the count of
  * viewer-only boxes force-drained. Per-workspace best-effort.
  */
@@ -2214,8 +2799,7 @@ async function forceDrainOverLimitWorkspaces(
   workspaceIds: Set<string>,
   observability: ActivityServices["observability"],
 ): Promise<number> {
-  const enforceBalance =
-    settings.billingMode === "stripe" || settings.usageLimitsMode === "managed";
+  const enforceBalance = false;
   const cap = settings.sandboxMaxWarmSecondsPerWorkspace;
   let forceDrained = 0;
   await forEachWithConcurrency(
@@ -2223,13 +2807,8 @@ async function forceDrainOverLimitWorkspaces(
     SANDBOX_MAINTENANCE_ITEM_CONCURRENCY,
     async (workspaceId) => {
       try {
-        const { accountId } = await rlsContextForWorkspace(db, workspaceId);
-        const balance = enforceBalance
-          ? await getBillingBalance(db, accountId)
-          : ({ balanceMicros: 1 } as { balanceMicros: number });
         const result = await forceDrainOverLimitViewerOnlyBoxes(db, {
           workspaceId,
-          balanceMicros: balance.balanceMicros,
           enforceBalance,
           maxWarmSecondsPerWorkspace: cap,
           idleGraceMs: settings.sandboxIdleGraceMs,
@@ -2253,10 +2832,50 @@ async function forceDrainOverLimitWorkspaces(
   return forceDrained;
 }
 
+export async function reconcileModalProviderCreates(
+  db: ActivityServices["db"],
+  settings: ActivityServices["settings"],
+  observability: ActivityServices["observability"],
+  discover: typeof findModalProviderCreateReceipt = findModalProviderCreateReceipt,
+): Promise<void> {
+  if (settings.sandboxBackend !== "modal" && !settings.modalTokenId && !settings.modalTokenSecret)
+    return;
+  const candidates = await listPendingModalProviderCreates(db);
+  await forEachWithConcurrency(candidates, 2, async (candidate) => {
+    try {
+      const { accountId } = await rlsContextForWorkspace(db, candidate.workspaceId);
+      const identity = { ...candidate, accountId };
+      const attempt = await claimModalProviderCreateRecovery(db, identity);
+      if (!attempt) return;
+      const instanceId = await discover(settings, attempt);
+      if (!instanceId) return; // Absence never settles an unknown dispatch.
+      if (await recordRecoveredModalProviderCreate(db, { ...identity, attempt, instanceId })) {
+        observability.info("sandbox reaper: recovered Modal create receipt", {
+          ...candidate,
+          operationId: attempt.operationId,
+          instanceId,
+        });
+      }
+    } catch (error) {
+      observability.warn("sandbox reaper: Modal create remains fenced pending discovery", {
+        ...candidate,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  });
+}
+
 export function modalOrphanTerminationStillEligible(
   latest: Awaited<ReturnType<typeof listLiveModalSandboxLeaseAttributions>>,
   candidate: ModalOrphanSweepTermination,
 ): boolean {
+  // The provider may have accepted a create whose reply has not arrived.
+  // Before attribution, neither its ID nor its separately-applied tags are
+  // available here. Postpone orphan deletion while any such owner exists;
+  // age/absence cannot distinguish its provider from an abandoned one.
+  if (latest.some((lease) => lease.liveness === "warming" && lease.instanceId === null)) {
+    return false;
+  }
   if (latest.some((lease) => lease.instanceId === candidate.sandboxId)) {
     return false;
   }
@@ -2444,6 +3063,7 @@ async function terminateDrainableBox(
   probeDrainableProvider: DrainableProviderProbeFn,
   attempt: SandboxDrainCaptureAttempt,
   objectStorage: ObjectStorage | null,
+  bus: ActivityServices["bus"] | null = null,
 ): Promise<boolean> {
   // Resolve the account for the RLS-scoped confirmDrainCold (the global sweep
   // returns no account_id; the workspace->account map is the bootstrap read).
@@ -2799,6 +3419,38 @@ async function terminateDrainableBox(
   // lease draining for a later sweep (NEVER terminate a box whose files we
   // could not capture). A persist CAS miss means the box was re-armed and left
   // running, so the cold commit is skipped.
+  // A published capture retry skips persistence. Revalidate legacy containment
+  // even on that path: stale enrollment is not native-supervisor exit proof.
+  // Typed provider absence is independently fenced by the canonical loss commit.
+  if (
+    !providerMissingBeforeCapture &&
+    backend === "modal" &&
+    lease.instanceId &&
+    !(await readWorkspaceArchiveCapturePreflight(db, {
+      accountId,
+      workspaceId: row.workspaceId,
+      sandboxGroupId: row.sandboxGroupId,
+      expectedEpoch: row.leaseEpoch,
+      expectedInstanceId: lease.instanceId,
+      liveness: "draining",
+    }))
+  ) {
+    return false;
+  }
+  // Stamp the charge horizon after verified capture and immediately before
+  // provider stop. Capture can fail and release its claim, allowing re-arm.
+  // A retry after stop success cannot charge post-termination wall time.
+  let stopCutoffMarked = false;
+  const beforeProviderStop = async (): Promise<void> => {
+    const cutoff = await markWarmBillingStopCutoff(db, {
+      accountId,
+      workspaceId: row.workspaceId,
+      sandboxGroupId: row.sandboxGroupId,
+      expectedEpoch: row.leaseEpoch,
+    });
+    if (!cutoff) throw new Error("sandbox stop cutoff was fenced before provider termination");
+    stopCutoffMarked = true;
+  };
   const termination: ProviderTerminationOutcome | boolean = providerMissingBeforeCapture
     ? { terminated: true, providerMissingBeforeCapture: true }
     : await terminateBox(
@@ -2821,12 +3473,50 @@ async function terminateDrainableBox(
             expectedInstanceId: lease.instanceId,
           });
         },
+        row.workspaceId,
+        beforeProviderStop,
       );
   const terminated = typeof termination === "boolean" ? termination : termination.terminated;
   if (!terminated) {
     return false;
   }
 
+  // Self-hosted/none and an unassigned lease have no provider stop callback;
+  // finish their usage at logical detach without touching customer compute.
+  // A managed instance that was stopped without its callback must fail closed.
+  const providerMissingDuringStop =
+    providerMissingBeforeCapture ||
+    (typeof termination !== "boolean" && termination.providerMissingBeforeCapture);
+  if (!stopCutoffMarked && !providerMissingDuringStop) {
+    if (managedProvider && lease.instanceId) {
+      throw new Error("managed sandbox stopped without a warm usage cutoff");
+    }
+    await beforeProviderStop();
+  }
+
+  // Stop was confirmed but the lease is not cold yet. This final epoch-fenced
+  // tick settles the interval since the last heartbeat/reaper sweep, including
+  // a box drained before its first periodic tick. If settlement fails, keep
+  // the draining receipt for retry instead of silently losing the charge.
+  const finalRate =
+    settings.sandboxWarmBillingMode === "usage_only"
+      ? 0
+      : sandboxWarmRateMicrosPerSecond(settings, backend);
+  if (!providerMissingDuringStop) {
+    const finalTick = await accrueWarmSeconds(db, {
+      accountId,
+      workspaceId: row.workspaceId,
+      sandboxGroupId: row.sandboxGroupId,
+      expectedEpoch: row.leaseEpoch,
+      warmRateMicrosPerSecond: finalRate,
+      billingMode: settings.sandboxWarmBillingMode,
+      finalDrain: true,
+    });
+    if (finalTick.costMicros > 0 && settings.sandboxWarmBillingMode === "credits") {
+      recordCreditMicros(observability, "usage", finalTick.costMicros);
+    }
+  }
+
   const providerMissing =
     providerMissingBeforeCapture ||
     (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
@@ -2834,15 +3524,36 @@ async function terminateDrainableBox(
   // with draining->cold. Until this succeeds, arrivals remain fenced by that
   // exact claim; a timestamp or a failed provider call can never reopen a box
   // while termination may still be in flight.
-  const { wentCold } = await confirmDrainCold(db, {
+  const { wentCold, backgroundCommandEvents } = await confirmDrainCold(db, {
     accountId,
     workspaceId: row.workspaceId,
     sandboxGroupId: row.sandboxGroupId,
     expectedEpoch: row.leaseEpoch,
     ...(captureClaim ? { expectedCaptureId: captureClaim.id } : {}),
     providerMissingBeforeCapture: providerMissing,
+    idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
+  });
+  // The command terminal events and agent inputs are already durable in the
+  // cold commit; this is only best-effort live fanout.
+  await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {
+    observability.warn("sandbox reaper: contained command event fanout failed", {
+      sandboxGroupId: row.sandboxGroupId,
+      error: error instanceof Error ? error.message : String(error),
+    });
   });
   if (wentCold) {
+    // Only the exact successful cold commit counts provider loss. A missing
+    // probe, a stale capture, a failed commit, or a retried child is not another
+    // observed loss. Keep this outside the best-effort session event writer.
+    if (providerMissing) {
+      recordSandboxProviderMissingBeforeCapture(observability, backend);
+    }
+    if (lease.unobservableCommandDrainIds?.length) {
+      recordSandboxCommandContainment(
+        observability,
+        providerMissing ? "provider_missing" : "contained",
+      );
+    }
     // Durable termination record (sandbox-file-persistence observability): who
     // ended this box and whether its /workspace was captured first, appended to
     // every session sharing the group's box. Best-effort: attribution must
@@ -2955,18 +3666,23 @@ export async function terminateProviderBox(
   claimedCapturePolicy?: ProviderWorkspaceCapturePolicy | null,
   diskBackedArchives = false,
   releaseFailedCapture?: () => Promise<void>,
+  workspaceId?: string,
+  beforeProviderStop?: () => Promise<void>,
 ): Promise<ProviderTerminationOutcome> {
   const durableBackendId = (lease.resumeBackendId ?? lease.backend) as string;
   const backend = sandboxBackendForSdkBackendId(durableBackendId) ?? durableBackendId;
   const logIdentity = providerDrainLogIdentity(lease, backend);
+  const warnCaptureFailure = (message: string, error: unknown): void =>
+    warnDrainSnapshotFailure(observability, message, error, {
+      sandboxGroupId: lease.sandboxGroupId,
+      leaseEpoch: lease.leaseEpoch,
+      ...(workspaceId ? { workspaceId } : {}),
+    });
   const releaseAfterCaptureFailure = async (): Promise<void> => {
     try {
       await releaseFailedCapture?.();
     } catch (error) {
-      observability.warn("sandbox reaper: failed capture claim release failed", {
-        ...logIdentity,
-        error: error instanceof Error ? error.message : String(error),
-      });
+      warnCaptureFailure("sandbox reaper: failed capture claim release failed", error);
     }
   };
   // 'none' / no backend -> nothing to terminate.
@@ -3016,6 +3732,11 @@ export async function terminateProviderBox(
   // fail closed instead of silently leaving a live provider behind while the
   // caller commits the lease cold.
   if (!lease.instanceId) {
+    if (lease.providerCreateAttempt?.instanceId === null) {
+      throw new Error(
+        "provider_create_outcome_unknown: cannot infer termination from missing instance ID",
+      );
+    }
     if (persistedInstanceId) {
       throw new Error(
         `sandbox backend ${backend} has persisted provider identity ${persistedInstanceId} but no authoritative lease instance; refusing teardown`,
@@ -3051,7 +3772,13 @@ export async function terminateProviderBox(
       return { terminated: false, providerMissingBeforeCapture: false };
     }
     try {
-      await terminateModalById(settings, lease.instanceId);
+      await beforeProviderStop?.();
+      await terminateModalById(
+        settings,
+        lease.instanceId,
+        undefined,
+        lease.providerCreateAttempt?.providerBindingKey,
+      );
     } catch (error) {
       if (!isProviderSandboxNotFoundError("modal", error)) {
         throw error;
@@ -3208,29 +3935,20 @@ export async function terminateProviderBox(
               },
             );
           } catch (error) {
-            observability.warn("sandbox reaper: late workspace capture publication failed", {
-              ...logIdentity,
-              error: error instanceof Error ? error.message : String(error),
-            });
+            warnCaptureFailure("sandbox reaper: late workspace capture publication failed", error);
           } finally {
             await disposeWorkspaceArchive(archive);
           }
         },
         observeLatePublicationFailure: (error) => {
-          observability.warn("sandbox reaper: late capture publication cleanup failed", {
-            ...logIdentity,
-            error: error instanceof Error ? error.message : String(error),
-          });
+          warnCaptureFailure("sandbox reaper: late capture publication cleanup failed", error);
         },
         observeLateFailure: (error) => {
           // Capture has now rejected and this timed-out path can never reach
           // provider teardown. Release only our unpublished claim so a waiting
           // turn can re-arm the intact live box; stale callbacks are DB-fenced.
           void releaseAfterCaptureFailure();
-          observability.warn("sandbox reaper: timed-out provider capture later failed", {
-            ...logIdentity,
-            error: error instanceof Error ? error.message : String(error),
-          });
+          warnCaptureFailure("sandbox reaper: timed-out provider capture later failed", error);
         },
       });
     }
@@ -3241,12 +3959,9 @@ export async function terminateProviderBox(
       });
       return { terminated: true, providerMissingBeforeCapture: true };
     }
-    observability.warn(
+    warnCaptureFailure(
       "sandbox reaper: persistWorkspace failed — leaving box draining (files NOT lost)",
-      {
-        ...logIdentity,
-        error: error instanceof Error ? error.message : String(error),
-      },
+      error,
     );
     // An unresolved timeout is not proof of capture failure: retain the fence
     // until its provider promise settles. A settled failure cannot enter the
@@ -3306,6 +4021,7 @@ export async function terminateProviderBox(
       throw new Error(`sandbox backend ${backend} returned no terminable session state`);
     }
     prepareProviderForTeardownAfterCapture(backend, session);
+    await beforeProviderStop?.();
     await terminateManagedSandboxSession(client, sessionState, session);
     return { terminated: true, providerMissingBeforeCapture: false };
   } catch (error) {