@opengeni/worker-bundle 1.0.1 → 2.0.0

package/src/observability-metrics.ts

@@ -16,6 +16,7 @@ import {
   type RuntimeMetricsHooks,
   type SelfhostedOpObservation,
   type SelfhostedOpObserver,
+  type ToolPreparationPhaseMeasurement,
 } from "@opengeni/runtime";
 
 export type TurnOutcome = "completed" | "failed" | "cancelled" | "recovering";
@@ -119,6 +120,10 @@ export function runtimeMetricsHooksForObservability(
 ): RuntimeMetricsHooks {
   return {
     onModelCall: ({ provider, outcome, durationSeconds }) => {
+      completedOperationSpan(observability, "worker.model.call", durationSeconds, {
+        provider,
+        outcome,
+      });
       observability.incrementCounter({
         name: "opengeni_model_calls_total",
         help: "Total model calls by provider and outcome.",
@@ -172,6 +177,20 @@ export function runtimeMetricsHooksForObservability(
         labels: { outcome, port: String(port) },
       });
     },
+    onWorkspaceCapture: ({ backend, outcome, durationSeconds }) => {
+      if (!Number.isFinite(durationSeconds) || durationSeconds < 0) return;
+      const safeBackend = SandboxBackend.safeParse(backend).success ? backend : "unknown";
+      completedOperationSpan(observability, "worker.workspace_capture", durationSeconds, {
+        backend: safeBackend,
+        outcome,
+      });
+      observability.observeHistogram({
+        name: "opengeni_workspace_capture_duration_seconds",
+        help: "Physical warm workspace capture and publication duration, including late settlement after caller timeout.",
+        labels: { backend: safeBackend, outcome },
+        value: durationSeconds,
+      });
+    },
     onWorkspaceArchiveObject: ({ outcome, backend }) => {
       observability.incrementCounter({
         name: "opengeni_workspace_archive_object_total",
@@ -180,6 +199,7 @@ export function runtimeMetricsHooksForObservability(
       });
     },
     onMcpToolCall: ({ outcome, durationSeconds }) => {
+      completedOperationSpan(observability, "worker.mcp.tool_call", durationSeconds, { outcome });
       observability.incrementCounter({
         name: "opengeni_mcp_tool_calls_total",
         help: "Total physical MCP tool calls by bounded structural outcome.",
@@ -1386,7 +1406,7 @@ export type TurnStartupPhase =
   | "history_generated_image_materialization"
   | "history_position_load"
   | "owned_sandbox_setup"
-  | "provider_dispatch"
+  | "runtime_stream_initialization"
   | "model_request_preparation"
   | "model_sdk_serialization"
   | "model_prepare_sandbox_agent_preparation"
@@ -1492,7 +1512,8 @@ export function turnStartupCountBucket(count: number | null): TurnStartupCountBu
 }
 
 /**
- * Measure the critical path from a durable turn start to provider dispatch.
+ * Measure startup operations, including nested and parallel work. These are
+ * not additive critical-path intervals; use milestones for elapsed latency.
  * Every label is a closed or configuration-derived enum; high-cardinality turn,
  * session, credential, connection, file, and model identifiers are forbidden.
  */
@@ -1506,8 +1527,19 @@ export function recordTurnStartupPhase(
     durationSeconds: number;
     count?: number | null;
     cache?: TurnStartupCache;
+    /** Trace-only lookup for the completed claim while its execution root is still open. */
+    executionCorrelationId?: string;
   },
 ): void {
+  completedOperationSpan(observability, `worker.prepare.${input.phase}`, input.durationSeconds, {
+    provider: input.provider,
+    backend: input.backend,
+    outcome: input.outcome,
+    ...(input.phase === "claim_and_policy" &&
+    /^turn_[0-9a-f]{32}$/.test(input.executionCorrelationId ?? "")
+      ? { correlationId: input.executionCorrelationId }
+      : {}),
+  });
   observability.observeHistogram({
     name: "opengeni_turn_startup_phase_duration_seconds",
     help: "Turn startup phase duration before the model response stream begins.",
@@ -1524,6 +1556,77 @@ export function recordTurnStartupPhase(
   });
 }
 
+/** Background MCP preparation must not inflate startup phase distributions. */
+export function recordToolPreparationPhase(
+  observability: Observability,
+  input: ToolPreparationPhaseMeasurement & { provider: string; backend: string },
+): void {
+  if (input.execution === "blocking") {
+    recordTurnStartupPhase(observability, {
+      ...input,
+      phase: `tool_${input.phase}`,
+    });
+    return;
+  }
+  completedOperationSpan(
+    observability,
+    `worker.tool_prepare.${input.phase}`,
+    input.durationSeconds,
+    {
+      provider: input.provider,
+      backend: input.backend,
+      outcome: input.outcome,
+    },
+  );
+  observability.observeHistogram({
+    name: "opengeni_tool_background_preparation_duration_seconds",
+    help: "Nonblocking MCP preparation operations; may overlap startup and later execution.",
+    buckets: TURN_STARTUP_PHASE_BUCKETS,
+    labels: {
+      phase: input.phase,
+      provider: input.provider,
+      backend: input.backend,
+      outcome: input.outcome,
+    },
+    value: Math.max(0, input.durationSeconds),
+  });
+}
+
+/** Completed measurements become siblings under the scoped physical attempt.
+ * They never establish ambient ancestry for work which has already finished. */
+function completedOperationSpan(
+  observability: Observability,
+  name: string,
+  durationSeconds: number,
+  attributes: {
+    outcome: string;
+    provider?: string;
+    backend?: string;
+    correlationId?: string | undefined;
+  },
+): void {
+  if (!Number.isFinite(durationSeconds) || durationSeconds < 0) return;
+  try {
+    const failed = [
+      "failed",
+      "error",
+      "provider_declared_error",
+      "auth_needed",
+      "outcome_uncertain",
+      "timeout",
+      "thrown_transport_error",
+      "thrown_protocol_error",
+    ].includes(attributes.outcome);
+    observability
+      .startSpan(name, attributes, { startTimeMs: Date.now() - durationSeconds * 1_000 })
+      .end({
+        ...(failed ? { error: true } : {}),
+      });
+  } catch {
+    // Observers cannot change the completed model/tool/phase outcome.
+  }
+}
+
 /**
  * Explain why a turn did or did not keep provider sandbox work off the
  * pre-model critical path. The reason is a closed enum: never attach session,

package/src/personal-github-git-credentials.ts

@@ -329,7 +329,6 @@ function exactPersonalGitHubDelegation(
     delegation.providerDomain !== "github.com" ||
     delegation.kind !== "oauth2" ||
     !delegation.originWorkspaceId ||
-    !delegation.userDelegation ||
     !delegation.personalGitHubRepositorySelection
   ) {
     throw new Error("personal GitHub Git delegation is unavailable");

package/src/pr-review-credentials.ts

@@ -5,7 +5,6 @@ import type {
   GitCredentialsRequest,
 } from "@opengeni/contracts";
 import { prReviewRegistrationIdFromCredentialBinding } from "@opengeni/core";
-import { createNativeRemoteMcpCredentialsPort } from "@opengeni/core/remote-mcp-credentials";
 import { decryptVariableSetValue, resolvePrReviewGitCredential, type Database } from "@opengeni/db";
 import {
   createGitHubAppInstallationTokenWithExpiry,
@@ -19,7 +18,6 @@ export function createStandaloneConnectionCredentialsPort(
   db: Database,
 ): ConnectionCredentialsPort {
   return {
-    ...createNativeRemoteMcpCredentialsPort(settings, db),
     gitCredentials: async (request) => await resolveStandaloneGitCredentials(settings, db, request),
   };
 }

package/src/retained-process-diagnostics.ts

@@ -0,0 +1,50 @@
+import {
+  databaseFailureCode,
+  nestedPostgresSqlState,
+  safeDatabaseErrorFacts,
+  SandboxWorkspaceMutationFencedError,
+  type SandboxRetainedProcess,
+} from "@opengeni/db";
+import type { Observability } from "@opengeni/observability";
+
+/** Protected cause locations and schema facts; public logs receive only the diagnostic key. */
+export function warnRetainedProcessProofFailure(
+  observability: Pick<Observability, "recordFailureDiagnostic" | "warn">,
+  error: unknown,
+  process: Pick<
+    SandboxRetainedProcess,
+    "id" | "sessionId" | "ownerAttemptId" | "ownerTurnId" | "reconcileAttempts"
+  >,
+): void {
+  let correlationId: string | undefined;
+  try {
+    const sqlState = nestedPostgresSqlState(error);
+    const constraint = safeDatabaseErrorFacts(error).constraint;
+    correlationId = observability.recordFailureDiagnostic({
+      code:
+        error instanceof SandboxWorkspaceMutationFencedError
+          ? "retained_process_fenced"
+          : sqlState
+            ? databaseFailureCode(sqlState)
+            : "retained_process_proof_failed",
+      stage: "sandbox_retained_processes.proof",
+      error,
+      processId: process.id,
+      sessionId: process.sessionId,
+      ...(process.ownerAttemptId ? { attemptId: process.ownerAttemptId } : {}),
+      ...(process.ownerTurnId ? { turnId: process.ownerTurnId } : {}),
+      attempts: process.reconcileAttempts,
+      sqlState,
+      ...(constraint ? { constraint } : {}),
+    });
+  } catch {
+    // A diagnostic/export failure must not change reconciliation or release its fence.
+  }
+  try {
+    observability.warn("sandbox reaper: retained-process proof checkpoint failed", {
+      ...(correlationId ? { correlationId } : {}),
+    });
+  } catch {
+    // Keep the original failure behavior even when the public sink fails.
+  }
+}

package/src/sandbox-resume.ts

@@ -27,6 +27,7 @@ import {
   type Settings,
 } from "@opengeni/config";
 import { randomUUID } from "node:crypto";
+import { retainsHolderForWarmCapture } from "./warm-capture-holder";
 import {
   acquireLease,
   adoptLegacyModalCheckpointArtifact,
@@ -86,7 +87,9 @@ import {
   WorkspaceArchiveStorageError,
   type ObjectStorage,
 } from "@opengeni/storage";
-import type { Observability } from "@opengeni/observability";
+import { sandboxLeaseTelemetryKey, type Observability } from "@opengeni/observability";
+import { safeSnapshotError } from "./sandbox-snapshot-diagnostics";
+export { safeSnapshotError } from "./sandbox-snapshot-diagnostics";
 import { parseWorkspaceArchiveObjectRef } from "@opengeni/contracts";
 import {
   persistWorkspaceArchiveCandidate,
@@ -125,9 +128,7 @@ export type SandboxResumeServices = {
   db: Database;
   settings: Settings;
   objectStorage?: ObjectStorage | null;
-  /** Exact settings before a verified rig provider image overlaid the logical
-   * pack/deployment image. Fresh-create NotFound fallback must preserve an
-   * ID-only logical base instead of selecting the provider default. */
+
   logicalFallbackSettings?: Settings;
   sandboxMetrics?: RuntimeMetricsHooks;
   /** Structured logger for lifecycle decisions (holder release reasons). Falls
@@ -369,54 +370,6 @@ class SnapshotTimeoutError extends Error {
   }
 }
 
-export function safeSnapshotError(error: unknown): {
-  errorClass: "SnapshotOperationError";
-  errorCode: "snapshot_operation_failed";
-  status?: number;
-  origin: "sandbox-resume";
-  causeName?: string;
-  integrityCode?: string;
-} {
-  const fields: {
-    errorClass: "SnapshotOperationError";
-    errorCode: "snapshot_operation_failed";
-    status?: number;
-    origin: "sandbox-resume";
-    causeName?: string;
-    integrityCode?: string;
-  } = {
-    errorClass: "SnapshotOperationError",
-    errorCode: "snapshot_operation_failed",
-    origin: "sandbox-resume",
-  };
-  try {
-    if (error && typeof error === "object") {
-      const candidate = error as {
-        name?: unknown;
-        code?: unknown;
-        status?: unknown;
-        statusCode?: unknown;
-      };
-      if (
-        typeof candidate.name === "string" &&
-        /^[A-Z][A-Za-z0-9]{2,62}Error$/u.test(candidate.name)
-      ) {
-        fields.causeName = candidate.name;
-      }
-      if (typeof candidate.code === "string" && /^[a-z0-9_]{1,64}$/u.test(candidate.code)) {
-        fields.integrityCode = candidate.code;
-      }
-      const rawStatus = candidate.status ?? candidate.statusCode;
-      const status = Number(rawStatus);
-      if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
-    }
-  } catch {
-    // Public diagnostics are best-effort and must never replace the exact
-    // internal snapshot failure.
-  }
-  return fields;
-}
-
 export async function waitForWarmSnapshot(
   snapshot: Promise<unknown>,
   timeoutMs: number,
@@ -787,6 +740,19 @@ async function persistWarmWorkspaceSnapshot(
   registerOwnedContinuation: (continuation: Promise<void>) => void,
 ): Promise<boolean> {
   const { db, settings } = services;
+  const warnSnapshotFailure = (message: string, error: unknown): void => {
+    const fields = {
+      ...safeSnapshotError(error),
+      sandboxLeaseKey: sandboxLeaseTelemetryKey(ids.workspaceId, ids.sandboxGroupId),
+      leaseEpoch,
+    };
+    try {
+      if (services.observability) services.observability.warn(message, fields);
+      else console.warn(message, fields);
+    } catch {
+      // Diagnostics cannot change provider settlement or capture-gate cleanup.
+    }
+  };
   const intervalMs = settings.sandboxSnapshotIntervalMs;
   if (intervalMs <= 0 && !force) {
     return false;
@@ -896,6 +862,9 @@ async function persistWarmWorkspaceSnapshot(
         : {}),
     });
     if (claimed.status !== "claimed") {
+      // A scheduled interval not being due is normal, including after a failed
+      // attempt. Do not turn every ten-second heartbeat into an error log.
+      if (claimed.status === "throttled") return false;
       console.error("mid-session workspace snapshot skipped (capture claim)", {
         sandboxGroupId: ids.sandboxGroupId,
         status: claimed.status,
@@ -951,6 +920,8 @@ async function persistWarmWorkspaceSnapshot(
     // turn signal resolves first. Its finally block is the only normal release
     // of the exact admission gate; a late callback cannot release a successor.
     const captureAndPublish = (async (): Promise<boolean> => {
+      const captureStarted = performance.now();
+      let captureOutcome: "completed" | "failed" = "failed";
       let archive: VerifiedWorkspaceArchivePayload | undefined;
       let candidate: { id: string } | null = null;
       let publicationAttempted = false;
@@ -1001,6 +972,7 @@ async function persistWarmWorkspaceSnapshot(
         if (!wrote && candidate) {
           await abandonCandidate(candidate.id, "snapshot_publication_fenced");
         }
+        if (wrote) captureOutcome = "completed";
         return wrote;
       } catch (error) {
         if (candidate && !publicationAttempted)
@@ -1018,20 +990,24 @@ async function persistWarmWorkspaceSnapshot(
           expectedEpoch: leaseEpoch,
           expectedInstanceId: instanceId,
         }).catch((error) => {
-          console.error(
-            "mid-session workspace capture gate release failed",
-            safeSnapshotError(error),
-          );
+          captureOutcome = "failed";
+          warnSnapshotFailure("mid-session workspace capture gate release failed", error);
         });
+        try {
+          services.sandboxMetrics?.onWorkspaceCapture?.({
+            backend: lease.backend,
+            outcome: captureOutcome,
+            durationSeconds: Math.max(0, performance.now() - captureStarted) / 1_000,
+          });
+        } catch {
+          // A completed capture and released gate cannot depend on telemetry.
+        }
       }
     })();
     const settled = captureAndPublish.then(
       (persisted) => ({ kind: "settled" as const, persisted }),
       (error) => {
-        console.error(
-          "mid-session workspace snapshot failed (turn unaffected)",
-          safeSnapshotError(error),
-        );
+        warnSnapshotFailure("mid-session workspace snapshot failed (turn unaffected)", error);
         return { kind: "settled" as const, persisted: false };
       },
     );
@@ -1063,12 +1039,10 @@ async function persistWarmWorkspaceSnapshot(
     // exact claim cleanup, so it cannot leak an unhandled promise or artifact.
     return outcome.kind === "settled" ? outcome.persisted : false;
   } catch (error) {
-    // Protection, not a dependency: a failed snapshot must never fail (or slow
-    // down retrying) the turn. The next heartbeat/turn-end tick retries.
-    console.error(
-      "mid-session workspace snapshot failed (turn unaffected)",
-      safeSnapshotError(error),
-    );
+    // A failed snapshot does not fail the turn. Physical captures can still
+    // fence commands until their exact gate settles; the caller timeout is not
+    // permission to release that gate. The next eligible tick can retry.
+    warnSnapshotFailure("mid-session workspace snapshot failed (turn unaffected)", error);
     return false;
   }
 }
@@ -1276,7 +1250,16 @@ export async function resumeBoxForTurn(
   // provider operation may still be mutating /workspace. Both loops therefore
   // converge on "holder alive -> never release"; finalization performs the
   // proof-bearing release.
-  const releaseDeadHolder = (reason: string): void => {
+  const releaseDeadHolder = async (reason: string): Promise<void> => {
+    // Logical closure deliberately fences heartbeat authority. It is not proof
+    // that the already-admitted capture has physically settled: the finalizer
+    // owns that wait and the original deadline bounds dead-worker recovery.
+    // Match the reaper's retention rule without touching holder/lease TTLs.
+    if (reason === "holder_gone" && kind === "turn" && holderLeaseHeartbeat) {
+      const expectedEpoch = holderLeaseHeartbeat.expectedEpoch;
+      const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
+      if (retainsHolderForWarmCapture(lease, expectedEpoch)) return;
+    }
     const message = "sandbox resume holder released: holder no longer live";
     const fields = {
       workspaceId: ids.workspaceId,
@@ -1301,13 +1284,13 @@ export async function resumeBoxForTurn(
         expectedEpoch: heartbeat.expectedEpoch,
         leaseTtlMs: heartbeat.leaseTtlMs,
       })
-        .then((status) => {
+        .then(async (status) => {
           if (heartbeat !== holderLeaseHeartbeat) return;
           if (!status.holderAlive) {
             // Reaped, released, or its exact attempt is no longer the active
             // writer. Stop this otherwise-unbounded provider operation and
             // idempotently drop any remaining holder state.
-            releaseDeadHolder(status.fence);
+            await releaseDeadHolder(status.fence);
             return;
           }
           if (status.fence === "epoch") {
@@ -1316,7 +1299,7 @@ export async function resumeBoxForTurn(
             // instance replaced, re-established by a later turn). The holder
             // row is stale authority for an instance that no longer exists;
             // drop it promptly instead of pinning a successor's lease.
-            releaseDeadHolder(status.fence);
+            await releaseDeadHolder(status.fence);
             return;
           }
           // The box is still serving this live holder: keep the provider TTL
@@ -1338,13 +1321,13 @@ export async function resumeBoxForTurn(
       kind,
       holderId,
     })
-      .then((touched) => {
+      .then(async (touched) => {
         if (heartbeat !== holderLeaseHeartbeat) return;
         // A canonical turn holder is rejected once its exact attempt is no
         // longer the active writer (touch is attempt-fenced only; it never
         // consults lease authority, so false here always means holder death).
         if (!touched) {
-          releaseDeadHolder("holder_gone");
+          await releaseDeadHolder("holder_gone");
           return;
         }
         void maybeRenewProviderExpiration().catch(() => undefined);
@@ -1422,7 +1405,8 @@ export async function resumeBoxForTurn(
       const continuityRecovery = acquired.lease.recovery.continuity;
       if (
         (acquired.lease.recovery.archive.status === "available" &&
-          acquired.lease.archiveComplete) ||
+          (acquired.lease.archiveComplete ||
+            acquired.lease.historicalRecoveryAuthorized === true)) ||
         (acquired.lease.recovery.archive.status === "none" &&
           workspaceArchiveFieldsFromEnvelope(archiveSource) !== null)
       ) {
@@ -1875,6 +1859,7 @@ export async function resumeBoxForTurn(
         sandboxGroupId: ids.sandboxGroupId,
         expectedEpoch: leaseEpoch,
         expectedInstanceId: live.instanceId,
+        expectedBackend: ids.backend,
       });
       if (marked.status === "marked") {
         await services.onSandboxLost?.({

package/src/sandbox-routing.ts

@@ -17,12 +17,13 @@
 import { sandboxLifecycleTransitionWaitMs, type Settings } from "@opengeni/config";
 import {
   createProviderCommandRetainer,
-  retainedProviderCommandPersistence,
+  supervisedCommandProtocolReady,
 } from "@opengeni/db/retained-provider-commands";
 import {
   adoptConnectedMachineSessionBackgroundCommand,
   adoptManagedSessionBackgroundCommand,
   advanceWorkspaceGenerationForRetainedProcess,
+  retainedProviderCommandPersistence,
   advanceWorkspaceGeneration,
   getRetainedProcess,
   retainWorkspaceMutationProcess,
@@ -72,6 +73,7 @@ import {
   type RoutingMutationSettlementResult,
   type RoutingSandboxFirstOperationObserver,
   type RoutingSandboxOperationObserver,
+  type RoutingSandboxCaptureWaitObserver,
   type RoutingSandboxWaitObservation,
   type RoutingRetainedProcess,
   type RoutingRetainedProcessTerminalProof,
@@ -112,6 +114,7 @@ export type RoutingWiringServices = {
   onOp?: SelfhostedOpObserver;
   /** Every physical routed provider call, across cloud and selfhosted homes. */
   onSandboxOperation?: RoutingSandboxOperationObserver;
+  onSandboxCaptureWait?: RoutingSandboxCaptureWaitObserver;
   /** The op-stream durable-resume journal (the Temporal adaptation from
    *  op-journal.ts): attach generation + settled-frontier persistence. Absent ⇒
    *  the runtime defaults (generation "1", no persistence) — tests / non-turn
@@ -124,11 +127,12 @@ export type RoutingWiringServices = {
     instanceId: string;
     leaseEpoch: number;
   }) => Promise<void>;
-  /** Called when a route repair replaces the worker's original home handle. */
+  /** Prepare and publish the worker's home handle. Rejection leaves the route
+   * unpublished; replacement preparation must settle before the first tool op. */
   onHomeSandboxRebound?: (input: {
     established: EstablishedSandboxSession;
     leaseEpoch: number;
-  }) => void;
+  }) => void | Promise<void>;
   /** Cancel bounded capture waits with the owning turn activity. */
   waitSignal?: AbortSignal;
 };
@@ -342,7 +346,7 @@ async function resolveCurrentHomeBackend(
   // A route epoch can advance without a provider replacement. Reuse the exact
   // established handle only when its identity still equals the durable one.
   if (lease.instanceId === established.instanceId) {
-    services.onHomeSandboxRebound?.({
+    await services.onHomeSandboxRebound?.({
       established,
       leaseEpoch: lease.leaseEpoch,
     });
@@ -393,6 +397,7 @@ async function resolveCurrentHomeBackend(
       sandboxGroupId: ids.sandboxGroupId,
       expectedEpoch: lease.leaseEpoch,
       expectedInstanceId: lease.instanceId,
+      expectedBackend: lease.backend,
       diagnostic: "provider_not_found_during_home_route_rebind",
     });
     if (marked.status === "marked") {
@@ -416,7 +421,7 @@ async function resolveCurrentHomeBackend(
   if (rebound.instanceId !== lease.instanceId || reboundBackend !== resumeBackend) {
     throw homeRouteRecoveryError(lease, lease.leaseEpoch);
   }
-  services.onHomeSandboxRebound?.({
+  await services.onHomeSandboxRebound?.({
     established: rebound,
     leaseEpoch: lease.leaseEpoch,
   });
@@ -747,7 +752,7 @@ function settleRetainedProcessForTurn(
     if (
       !durable ||
       durable.providerSessionId !== process.providerSessionId ||
-      durable.providerBackend !== backend.kind ||
+      durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) ||
       durable.providerInstanceId !== backend.providerInstanceId ||
       durable.leaseEpoch !== backend.leaseEpoch ||
       durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
@@ -795,7 +800,7 @@ function adoptRetainedProcessAsBackgroundCommandForTurn(
     if (
       !durable ||
       durable.providerSessionId !== process.providerSessionId ||
-      durable.providerBackend !== backend.kind ||
+      durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) ||
       durable.providerInstanceId !== backend.providerInstanceId ||
       durable.leaseEpoch !== backend.leaseEpoch ||
       durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
@@ -1026,16 +1031,25 @@ export function wrapTurnBoxWithRouting(
     },
     resolveActiveBackend: resolver,
     ...(services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {}),
+    ...(services.onSandboxCaptureWait ? { onCaptureWait: services.onSandboxCaptureWait } : {}),
     providerCommandHandle: admittedCommandHandle,
+    providerSupervisionReady: async () =>
+      settings.modalCommandSupervisionEnabled && (await supervisedCommandProtocolReady(db)),
     ...(ids.workspaceMutationFence
       ? {
           providerCommandPersistence: (process: RoutingRetainedProcess) =>
-            retainedProviderCommandPersistence(db, {
-              accountId: ids.workspaceMutationFence!.accountId,
-              workspaceId: ids.workspaceId,
-              sessionId: ids.sessionId,
-              processId: process.id,
-            }),
+            retainedProviderCommandPersistence(
+              db,
+              {
+                accountId: ids.workspaceMutationFence!.accountId,
+                workspaceId: ids.workspaceId,
+                sessionId: ids.sessionId,
+                processId: process.id,
+              },
+              services.bus
+                ? (events) => services.bus!.publish(ids.workspaceId, ids.sessionId, events)
+                : undefined,
+            ),
         }
       : {}),
     ...(beforeMutation ? { beforeMutation } : {}),
@@ -1069,6 +1083,7 @@ export function wrapTurnBoxWithRouting(
               sandboxGroupId: home.sandboxGroupId,
               expectedEpoch,
               expectedInstanceId,
+              expectedBackend: home.backend,
               diagnostic: "provider_not_found_during_routed_operation",
             });
             if (marked.status === "marked") {
@@ -1276,17 +1291,26 @@ export function wrapLazyTurnBoxWithRouting(
       return routedResolver(pointer);
     },
     ...(services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {}),
+    ...(services.onSandboxCaptureWait ? { onCaptureWait: services.onSandboxCaptureWait } : {}),
     ...(args.onFirstOperation ? { onFirstOperation: args.onFirstOperation } : {}),
     providerCommandHandle: admittedCommandHandle,
+    providerSupervisionReady: async () =>
+      settings.modalCommandSupervisionEnabled && (await supervisedCommandProtocolReady(db)),
     ...(ids.workspaceMutationFence
       ? {
           providerCommandPersistence: (process: RoutingRetainedProcess) =>
-            retainedProviderCommandPersistence(db, {
-              accountId: ids.workspaceMutationFence!.accountId,
-              workspaceId: ids.workspaceId,
-              sessionId: ids.sessionId,
-              processId: process.id,
-            }),
+            retainedProviderCommandPersistence(
+              db,
+              {
+                accountId: ids.workspaceMutationFence!.accountId,
+                workspaceId: ids.workspaceId,
+                sessionId: ids.sessionId,
+                processId: process.id,
+              },
+              services.bus
+                ? (events) => services.bus!.publish(ids.workspaceId, ids.sessionId, events)
+                : undefined,
+            ),
         }
       : {}),
     ...(beforeMutation ? { beforeMutation } : {}),
@@ -1321,6 +1345,7 @@ export function wrapLazyTurnBoxWithRouting(
               sandboxGroupId: home.sandboxGroupId,
               expectedEpoch: backend.leaseEpoch,
               expectedInstanceId: backend.providerInstanceId,
+              expectedBackend: home.backend,
               diagnostic: "provider_not_found_during_routed_operation",
             });
             if (marked.status === "marked") {