@opengeni/worker-bundle 1.0.0-canary.1 → 1.0.0

package/src/sandbox-routing.ts

@@ -1,3 +1,4 @@
+import { isDeepStrictEqual } from "node:util";
 // apps/worker/src/sandbox-routing.ts — wire the agent-loop-free routing proxy
 // (`@opengeni/runtime` RoutingSandboxSession + makeActiveBackendResolver) to the
 // real DB pointer + the live NATS control plane for the WORKER TURN path (M7).
@@ -17,12 +18,13 @@
 import { sandboxLifecycleTransitionWaitMs, type Settings } from "@opengeni/config";
 import {
   createProviderCommandRetainer,
-  retainedProviderCommandPersistence,
+  supervisedCommandProtocolReady,
 } from "@opengeni/db/retained-provider-commands";
 import {
   adoptConnectedMachineSessionBackgroundCommand,
   adoptManagedSessionBackgroundCommand,
   advanceWorkspaceGenerationForRetainedProcess,
+  retainedProviderCommandPersistence,
   advanceWorkspaceGeneration,
   getRetainedProcess,
   retainWorkspaceMutationProcess,
@@ -42,10 +44,14 @@ import {
   type EnrollmentRecord,
   type SandboxWorkspaceMutationAdmission,
 } from "@opengeni/db";
-import { observeSessionBackgroundCommandCompletion } from "@opengeni/db/session-background-commands";
+import {
+  observeSessionBackgroundCommandCompletion,
+  recordConnectedCommandOutputConsumption,
+} from "@opengeni/db/session-background-commands";
 import { appendSessionCommandOutput } from "@opengeni/db/session-command-output";
 import type { OpStreamOutputFrame } from "@opengeni/runtime/sandbox";
 import type { EventBus } from "@opengeni/events";
+import { publishDurableSessionEvents } from "./session-event-fanout";
 import {
   buildSelfhostedBackendSession,
   ActiveBackendUnresolvableError,
@@ -72,6 +78,7 @@ import {
   type RoutingMutationSettlementResult,
   type RoutingSandboxFirstOperationObserver,
   type RoutingSandboxOperationObserver,
+  type RoutingSandboxCaptureWaitObserver,
   type RoutingSandboxWaitObservation,
   type RoutingRetainedProcess,
   type RoutingRetainedProcessTerminalProof,
@@ -112,6 +119,7 @@ export type RoutingWiringServices = {
   onOp?: SelfhostedOpObserver;
   /** Every physical routed provider call, across cloud and selfhosted homes. */
   onSandboxOperation?: RoutingSandboxOperationObserver;
+  onSandboxCaptureWait?: RoutingSandboxCaptureWaitObserver;
   /** The op-stream durable-resume journal (the Temporal adaptation from
    *  op-journal.ts): attach generation + settled-frontier persistence. Absent ⇒
    *  the runtime defaults (generation "1", no persistence) — tests / non-turn
@@ -124,11 +132,12 @@ export type RoutingWiringServices = {
     instanceId: string;
     leaseEpoch: number;
   }) => Promise<void>;
-  /** Called when a route repair replaces the worker's original home handle. */
+  /** Prepare and publish the worker's home handle. Rejection leaves the route
+   * unpublished; replacement preparation must settle before the first tool op. */
   onHomeSandboxRebound?: (input: {
     established: EstablishedSandboxSession;
     leaseEpoch: number;
-  }) => void;
+  }) => void | Promise<void>;
   /** Cancel bounded capture waits with the owning turn activity. */
   waitSignal?: AbortSignal;
 };
@@ -342,7 +351,7 @@ async function resolveCurrentHomeBackend(
   // A route epoch can advance without a provider replacement. Reuse the exact
   // established handle only when its identity still equals the durable one.
   if (lease.instanceId === established.instanceId) {
-    services.onHomeSandboxRebound?.({
+    await services.onHomeSandboxRebound?.({
       established,
       leaseEpoch: lease.leaseEpoch,
     });
@@ -393,9 +402,15 @@ async function resolveCurrentHomeBackend(
       sandboxGroupId: ids.sandboxGroupId,
       expectedEpoch: lease.leaseEpoch,
       expectedInstanceId: lease.instanceId,
+      expectedBackend: lease.backend,
       diagnostic: "provider_not_found_during_home_route_rebind",
     });
     if (marked.status === "marked") {
+      await publishDurableSessionEvents(
+        services.bus,
+        ids.workspaceId,
+        marked.backgroundCommandEvents,
+      );
       await services.onHomeSandboxLost?.({
         sandboxGroupId: ids.sandboxGroupId,
         instanceId: lease.instanceId,
@@ -416,7 +431,7 @@ async function resolveCurrentHomeBackend(
   if (rebound.instanceId !== lease.instanceId || reboundBackend !== resumeBackend) {
     throw homeRouteRecoveryError(lease, lease.leaseEpoch);
   }
-  services.onHomeSandboxRebound?.({
+  await services.onHomeSandboxRebound?.({
     established: rebound,
     leaseEpoch: lease.leaseEpoch,
   });
@@ -491,7 +506,7 @@ function afterPersistableHomeMutation(
       op: string;
       backend: ResolvedActiveBackend;
       admission: unknown;
-      outcome: "resolved" | "rejected";
+      outcome: "resolved" | "rejected" | "outcome_unknown";
       result?: unknown;
       retainedProcess?: RoutingRetainedProcess;
     }) => Promise<void | RoutingMutationSettlementResult>)
@@ -518,7 +533,7 @@ function afterPersistableHomeMutation(
     ) {
       throw new Error("Persistable home mutation settlement lacked its bound admission");
     }
-    if (outcome === "resolved" && retainedProcess) {
+    if ((outcome === "resolved" || outcome === "outcome_unknown") && retainedProcess) {
       try {
         await retainWorkspaceProviderCommand(services.db, {
           accountId: fence.accountId,
@@ -577,6 +592,8 @@ function afterPersistableHomeMutation(
       }
       return;
     }
+    if (outcome === "outcome_unknown")
+      throw new Error("Outcome-unknown command settlement requires its exact retained invocation");
     await verifyWorkspaceMutationSettlement(services.db, {
       accountId: fence.accountId,
       workspaceId: ids.workspaceId,
@@ -747,7 +764,7 @@ function settleRetainedProcessForTurn(
     if (
       !durable ||
       durable.providerSessionId !== process.providerSessionId ||
-      durable.providerBackend !== backend.kind ||
+      durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) ||
       durable.providerInstanceId !== backend.providerInstanceId ||
       durable.leaseEpoch !== backend.leaseEpoch ||
       durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
@@ -778,15 +795,75 @@ function settleRetainedProcessForTurn(
   };
 }
 
+/** Cleanup can observe the independently settled row even when this worker's
+ * pending output receipt or original provider transport can no longer advance. */
+function isRetainedProcessSettledForTurn(services: RoutingWiringServices, ids: RoutingWiringIds) {
+  const fence = ids.workspaceMutationFence;
+  if (!fence) return undefined;
+  return async ({
+    backend,
+    process,
+  }: {
+    backend: ResolvedActiveBackend;
+    process: RoutingRetainedProcess;
+  }): Promise<boolean> => {
+    const durable = await getRetainedProcess(services.db, {
+      workspaceId: ids.workspaceId,
+      sessionId: ids.sessionId,
+      processId: process.id,
+    });
+    if (
+      !durable ||
+      durable.state === "active" ||
+      durable.providerSessionId !== process.providerSessionId ||
+      durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) ||
+      durable.providerInstanceId !== backend.providerInstanceId ||
+      durable.leaseEpoch !== backend.leaseEpoch ||
+      durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
+      durable.routeTargetId !== backend.sandboxId ||
+      durable.routeEpoch !== backend.activeEpoch
+    )
+      return false;
+    if (process.providerCommand) {
+      // The general process projection deliberately omits the provider locator.
+      // Load its immutable identity through the exact protected persistence seam.
+      const command = await retainedProviderCommandPersistence(services.db, {
+        accountId: fence.accountId,
+        workspaceId: ids.workspaceId,
+        sessionId: ids.sessionId,
+        processId: process.id,
+      }).load();
+      if (
+        !command ||
+        command.kind !== process.providerCommand.kind ||
+        command.execId !== process.providerCommand.execId ||
+        command.taskId !== process.providerCommand.taskId ||
+        command.sandboxId !== process.providerCommand.sandboxId ||
+        Boolean(command.pty) !== Boolean(process.providerCommand.pty) ||
+        (command.kind === "modal-router-v1" &&
+          process.providerCommand.kind === "modal-router-v1" &&
+          !isDeepStrictEqual(command.supervision, process.providerCommand.supervision))
+      )
+        return false;
+    }
+    // Terminal rows are written only through proof-gated physical settlement.
+    return true;
+  };
+}
+
 function adoptRetainedProcessAsBackgroundCommandForTurn(
   services: RoutingWiringServices,
   ids: RoutingWiringIds,
 ):
-  | ((input: { backend: ResolvedActiveBackend; process: RoutingRetainedProcess }) => Promise<void>)
+  | ((input: {
+      backend: ResolvedActiveBackend;
+      process: RoutingRetainedProcess;
+      command?: string | undefined;
+    }) => Promise<void>)
   | undefined {
   const fence = ids.workspaceMutationFence;
   if (!fence) return undefined;
-  return async ({ backend, process }) => {
+  return async ({ backend, process, command }) => {
     const durable = await getRetainedProcess(services.db, {
       workspaceId: ids.workspaceId,
       sessionId: ids.sessionId,
@@ -795,7 +872,7 @@ function adoptRetainedProcessAsBackgroundCommandForTurn(
     if (
       !durable ||
       durable.providerSessionId !== process.providerSessionId ||
-      durable.providerBackend !== backend.kind ||
+      durable.providerBackend !== (sandboxBackendForSdkBackendId(backend.kind) ?? backend.kind) ||
       durable.providerInstanceId !== backend.providerInstanceId ||
       durable.leaseEpoch !== backend.leaseEpoch ||
       durable.routeKind !== (backend.sandboxId === null ? "home" : "active") ||
@@ -813,7 +890,7 @@ function adoptRetainedProcessAsBackgroundCommandForTurn(
       attemptId: fence.attemptId,
       processId: process.id,
       expected: retainedProcessSettlementIdentity(durable),
-      command: "execCommand",
+      command: command ?? "Background command",
     });
   };
 }
@@ -856,6 +933,7 @@ export function wrapTurnBoxWithRouting(
   const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
   const afterProcessMutation = afterRetainedProcessMutation(services, ids);
   const settleProcess = settleRetainedProcessForTurn(services, ids);
+  const isProcessSettled = isRetainedProcessSettledForTurn(services, ids);
   const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
     services,
     ids,
@@ -1026,16 +1104,25 @@ export function wrapTurnBoxWithRouting(
     },
     resolveActiveBackend: resolver,
     ...(services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {}),
+    ...(services.onSandboxCaptureWait ? { onCaptureWait: services.onSandboxCaptureWait } : {}),
     providerCommandHandle: admittedCommandHandle,
+    providerSupervisionReady: async () =>
+      settings.modalCommandSupervisionEnabled && (await supervisedCommandProtocolReady(db)),
     ...(ids.workspaceMutationFence
       ? {
           providerCommandPersistence: (process: RoutingRetainedProcess) =>
-            retainedProviderCommandPersistence(db, {
-              accountId: ids.workspaceMutationFence!.accountId,
-              workspaceId: ids.workspaceId,
-              sessionId: ids.sessionId,
-              processId: process.id,
-            }),
+            retainedProviderCommandPersistence(
+              db,
+              {
+                accountId: ids.workspaceMutationFence!.accountId,
+                workspaceId: ids.workspaceId,
+                sessionId: ids.sessionId,
+                processId: process.id,
+              },
+              services.bus
+                ? (events) => services.bus!.publish(ids.workspaceId, ids.sessionId, events)
+                : undefined,
+            ),
         }
       : {}),
     ...(beforeMutation ? { beforeMutation } : {}),
@@ -1043,6 +1130,7 @@ export function wrapTurnBoxWithRouting(
     ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
     ...(afterProcessMutation ? { afterProcessMutation } : {}),
     ...(settleProcess ? { settleProcess } : {}),
+    ...(isProcessSettled ? { isProcessSettled } : {}),
     ...(ids.workspaceMutationFence
       ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids)! }
       : {}),
@@ -1069,9 +1157,15 @@ export function wrapTurnBoxWithRouting(
               sandboxGroupId: home.sandboxGroupId,
               expectedEpoch,
               expectedInstanceId,
+              expectedBackend: home.backend,
               diagnostic: "provider_not_found_during_routed_operation",
             });
             if (marked.status === "marked") {
+              await publishDurableSessionEvents(
+                services.bus,
+                ids.workspaceId,
+                marked.backgroundCommandEvents,
+              );
               await services.onHomeSandboxLost?.({
                 sandboxGroupId: home.sandboxGroupId,
                 instanceId: expectedInstanceId,
@@ -1128,6 +1222,7 @@ export function wrapLazyTurnBoxWithRouting(
   const beforeProcessMutation = beforeRetainedProcessMutation(services, ids);
   const afterProcessMutation = afterRetainedProcessMutation(services, ids);
   const settleProcess = settleRetainedProcessForTurn(services, ids);
+  const isProcessSettled = isRetainedProcessSettledForTurn(services, ids);
   const adoptProcessAsBackgroundCommand = adoptRetainedProcessAsBackgroundCommandForTurn(
     services,
     ids,
@@ -1276,17 +1371,26 @@ export function wrapLazyTurnBoxWithRouting(
       return routedResolver(pointer);
     },
     ...(services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {}),
+    ...(services.onSandboxCaptureWait ? { onCaptureWait: services.onSandboxCaptureWait } : {}),
     ...(args.onFirstOperation ? { onFirstOperation: args.onFirstOperation } : {}),
     providerCommandHandle: admittedCommandHandle,
+    providerSupervisionReady: async () =>
+      settings.modalCommandSupervisionEnabled && (await supervisedCommandProtocolReady(db)),
     ...(ids.workspaceMutationFence
       ? {
           providerCommandPersistence: (process: RoutingRetainedProcess) =>
-            retainedProviderCommandPersistence(db, {
-              accountId: ids.workspaceMutationFence!.accountId,
-              workspaceId: ids.workspaceId,
-              sessionId: ids.sessionId,
-              processId: process.id,
-            }),
+            retainedProviderCommandPersistence(
+              db,
+              {
+                accountId: ids.workspaceMutationFence!.accountId,
+                workspaceId: ids.workspaceId,
+                sessionId: ids.sessionId,
+                processId: process.id,
+              },
+              services.bus
+                ? (events) => services.bus!.publish(ids.workspaceId, ids.sessionId, events)
+                : undefined,
+            ),
         }
       : {}),
     ...(beforeMutation ? { beforeMutation } : {}),
@@ -1294,6 +1398,7 @@ export function wrapLazyTurnBoxWithRouting(
     ...(beforeProcessMutation ? { beforeProcessMutation } : {}),
     ...(afterProcessMutation ? { afterProcessMutation } : {}),
     ...(settleProcess ? { settleProcess } : {}),
+    ...(isProcessSettled ? { isProcessSettled } : {}),
     ...(ids.workspaceMutationFence
       ? { observeProcessTerminal: observeRetainedProcessTerminalForTurn(services, ids)! }
       : {}),
@@ -1321,9 +1426,15 @@ export function wrapLazyTurnBoxWithRouting(
               sandboxGroupId: home.sandboxGroupId,
               expectedEpoch: backend.leaseEpoch,
               expectedInstanceId: backend.providerInstanceId,
+              expectedBackend: home.backend,
               diagnostic: "provider_not_found_during_routed_operation",
             });
             if (marked.status === "marked") {
+              await publishDurableSessionEvents(
+                services.bus,
+                ids.workspaceId,
+                marked.backgroundCommandEvents,
+              );
               await services.onHomeSandboxLost?.({
                 sandboxGroupId: home.sandboxGroupId,
                 instanceId: backend.providerInstanceId,
@@ -1606,6 +1717,20 @@ export async function establishSelfhostedTurnSession(
         reason: command.reason,
         ...(command.failure ? { failure: command.failure } : {}),
       });
+      if (command.outputReceipt && command.outcome === "exited" && command.exitCode !== null) {
+        await recordConnectedCommandOutputConsumption(db, {
+          accountId: args.accountId,
+          workspaceId: args.workspaceId,
+          sessionId: args.sessionId,
+          commandId: command.commandId,
+          controlWorkspaceId: command.controlWorkspaceId,
+          enrollmentId: command.enrollmentId,
+          connectionInstanceId: command.connectionInstanceId,
+          opId: command.opId,
+          receipt: command.outputReceipt,
+          exitCode: command.exitCode,
+        });
+      }
       if (settlement && bus) {
         await bus
           .publish(args.workspaceId, args.sessionId, settlement.events)

package/src/sandbox-snapshot-diagnostics.ts

@@ -0,0 +1,112 @@
+import { sandboxLeaseTelemetryKey, type Observability } from "@opengeni/observability";
+
+type SafeSnapshotError = {
+  errorClass: "SnapshotOperationError";
+  errorCode: "snapshot_operation_failed";
+  status?: number;
+  origin: "sandbox-resume" | "worker-lifecycle";
+  causeName?: string;
+  integrityCode?: string;
+  providerErrorName?: string;
+  providerGrpcCode?: number;
+  providerHttpStatus?: number;
+  providerRetryable?: boolean;
+};
+
+export function safeSnapshotError(
+  error: unknown,
+  origin: SafeSnapshotError["origin"] = "sandbox-resume",
+): SafeSnapshotError {
+  const fields: SafeSnapshotError = {
+    errorClass: "SnapshotOperationError",
+    errorCode: "snapshot_operation_failed",
+    origin,
+  };
+  try {
+    if (error && typeof error === "object") {
+      const candidate = error as {
+        name?: unknown;
+        code?: unknown;
+        status?: unknown;
+        statusCode?: unknown;
+      };
+      if (
+        typeof candidate.name === "string" &&
+        /^[A-Z][A-Za-z0-9]{2,62}Error$/u.test(candidate.name)
+      ) {
+        fields.causeName = candidate.name;
+      }
+      if (typeof candidate.code === "string" && /^[a-z0-9_]{1,64}$/u.test(candidate.code)) {
+        fields.integrityCode = candidate.code;
+      }
+      const status = Number(candidate.status ?? candidate.statusCode);
+      if (Number.isInteger(status) && status >= 100 && status <= 599) fields.status = status;
+    }
+  } catch {
+    // Diagnostics must never replace the exact internal snapshot failure.
+  }
+  // The SDK wraps provider classification in details, not Error.cause. Never
+  // project free-form messages, causes, request IDs or payloads. Read fields
+  // independently so a hostile getter cannot hide other safe classifications.
+  const read = (value: unknown, key: string): unknown => {
+    try {
+      return value && typeof value === "object" ? Reflect.get(value, key) : undefined;
+    } catch {
+      return undefined;
+    }
+  };
+  const details = read(error, "details");
+  const providerName = read(details, "errorName");
+  if (
+    typeof providerName === "string" &&
+    [
+      "ClientError",
+      "TimeoutError",
+      "ConnectionError",
+      "AuthError",
+      "NotFoundError",
+      "InvalidError",
+      "RemoteError",
+      "AbortError",
+    ].includes(providerName)
+  )
+    fields.providerErrorName = providerName;
+  const grpcCode = read(details, "errorCode");
+  if (
+    providerName === "ClientError" &&
+    typeof grpcCode === "number" &&
+    Number.isInteger(grpcCode) &&
+    grpcCode >= 0 &&
+    grpcCode <= 16
+  )
+    fields.providerGrpcCode = grpcCode;
+  for (const key of ["status", "httpStatus", "responseStatus"]) {
+    const status = read(details, key);
+    if (typeof status === "number" && Number.isInteger(status) && status >= 100 && status <= 599) {
+      fields.providerHttpStatus = status;
+      break;
+    }
+  }
+  const retryable = read(error, "retryable");
+  if (typeof retryable === "boolean") fields.providerRetryable = retryable;
+  return fields;
+}
+
+export function warnDrainSnapshotFailure(
+  observability: Pick<Observability, "warn">,
+  message: string,
+  error: unknown,
+  lease: { workspaceId?: string; sandboxGroupId: string; leaseEpoch: number },
+): void {
+  try {
+    observability.warn(message, {
+      ...safeSnapshotError(error, "worker-lifecycle"),
+      ...(lease.workspaceId
+        ? { sandboxLeaseKey: sandboxLeaseTelemetryKey(lease.workspaceId, lease.sandboxGroupId) }
+        : {}),
+      leaseEpoch: lease.leaseEpoch,
+    });
+  } catch {
+    // Logging cannot replace the provider failure or prevent claim release.
+  }
+}

package/src/session-event-fanout.ts

@@ -0,0 +1,23 @@
+import type { SessionEvent } from "@opengeni/contracts";
+import type { EventBus } from "@opengeni/events";
+
+/** Best-effort live fanout of events that are already durable, grouped by
+ * session. Durable catch-up remains the source of truth, so failures are only
+ * reported. */
+export async function publishDurableSessionEvents(
+  bus: EventBus | null | undefined,
+  workspaceId: string,
+  events: readonly SessionEvent[] | undefined,
+  onError: (error: unknown) => void = () => undefined,
+): Promise<void> {
+  if (!bus || !events?.length) return;
+  for (const sessionId of new Set(events.map((event) => event.sessionId))) {
+    await bus
+      .publish(
+        workspaceId,
+        sessionId,
+        events.filter((event) => event.sessionId === sessionId),
+      )
+      .catch(onError);
+  }
+}

package/src/warm-capture-holder.ts

@@ -0,0 +1,15 @@
+import type { LeaseSnapshot } from "@opengeni/db";
+
+/** Physical capture retention is not permission to heartbeat or execute a closed turn. */
+export function retainsHolderForWarmCapture(
+  lease: Pick<LeaseSnapshot, "liveness" | "leaseEpoch" | "archiveCapture"> | null,
+  expectedEpoch: number,
+  now = Date.now(),
+): boolean {
+  return Boolean(
+    lease?.liveness === "warm" &&
+    lease.leaseEpoch === expectedEpoch &&
+    lease.archiveCapture &&
+    lease.archiveCapture.deadlineAt.getTime() > now,
+  );
+}

package/src/worker-service-lifecycle.ts

@@ -172,3 +172,56 @@ function safeLifecycleLog(log: () => void): void {
     // Telemetry cannot change worker lifecycle state or shutdown delivery.
   }
 }
+
+/** A cleanup stall first drains peers through WORKER_SHUTDOWN. The standalone
+ * process owns the final exit, because Temporal cannot kill JavaScript writers.
+ * Embedded hosts may supply their own termination policy at this same edge. */
+export function createWorkerCleanupContainment(input: {
+  drain: () => boolean;
+  terminate?: () => void;
+  observability: Observability;
+  timeoutMs?: number;
+}) {
+  let requested = false;
+  let finished = false;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  return {
+    request() {
+      if (requested || finished) return;
+      requested = true;
+      if (input.terminate) {
+        timer = setTimeout(() => {
+          safeLifecycleLog(() =>
+            input.observability.error("worker cleanup drain exhausted; terminating host", {
+              errorClass: "WorkerLifecycleOperationError",
+              errorCode: "worker_cleanup_drain_exhausted",
+              origin: "worker-lifecycle",
+            }),
+          );
+          input.terminate?.();
+        }, input.timeoutMs ?? 100_000);
+        timer.unref?.();
+      }
+      let accepted = false;
+      try {
+        accepted = input.drain();
+      } catch {
+        /* The host backstop remains armed. */
+      }
+      if (!accepted)
+        safeLifecycleLog(() =>
+          input.observability.error("worker cleanup drain request failed", {
+            errorClass: "WorkerLifecycleOperationError",
+            errorCode: "worker_shutdown_request_failed",
+            origin: "worker-lifecycle",
+          }),
+        );
+    },
+    finished() {
+      // Call only after Worker.run resolves. ForceShutdownError does not prove
+      // its activity promises stopped, so rejection must retain the backstop.
+      finished = true;
+      clearTimeout(timer);
+    },
+  };
+}

package/src/workflows/activities.ts

@@ -1,4 +1,9 @@
-import { ActivityCancellationType, proxyActivities } from "@temporalio/workflow";
+import {
+  ActivityCancellationType,
+  ActivityFailure,
+  ApplicationFailure,
+  proxyActivities,
+} from "@temporalio/workflow";
 import type * as activities from "../activities";
 import {
   KNOWLEDGE_SOURCE_SYNC_ACTIVITY_HEARTBEAT_TIMEOUT_MS,
@@ -10,12 +15,14 @@ type WorkflowControlActivities = Pick<
   | "enqueueGoalRetryWake"
   | "expireSessionHumanInput"
   | "expireSessionInteractionIntervention"
+  | "expireScheduledRunHumanWait"
   | "failSessionAttempt"
   | "getCodexCapacityWait"
   | "markSessionIdle"
   | "peekSessionWork"
   | "persistSessionAttemptQuiescence"
   | "reconcileSessionAttemptQuiescence"
+  | "reconcileSettledSessionAttempt"
   | "reconcileCodexCapacityWait"
   | "recoverDispatch"
   | "recoverEscapedMcpTimeout"
@@ -160,6 +167,14 @@ export const knowledgeSourceSyncActivity = proxyActivities<
 });
 
 export function workflowFailureMessage(error: unknown): string {
+  if (
+    error instanceof ActivityFailure &&
+    error.activityType === "runAgentTurn" &&
+    error.cause instanceof ApplicationFailure &&
+    error.cause.type === "TurnExecutionPolicyDefinitionMismatchError"
+  ) {
+    return error.cause.message;
+  }
   if (error instanceof Error) {
     return error.message;
   }

package/src/workflows/browser-deadline-checkpoint.ts

@@ -0,0 +1,50 @@
+import {
+  ParentClosePolicy,
+  WorkflowIdReusePolicy,
+  log,
+  proxyActivities,
+  startChild,
+  workflowInfo,
+} from "@temporalio/workflow";
+import type { BrowserDeadlineCheckpointTarget } from "@opengeni/db";
+import type * as activities from "../activities";
+
+const inventory = proxyActivities<Pick<typeof activities, "listDueBrowserCheckpoints">>({
+  startToCloseTimeout: "5 minutes",
+  heartbeatTimeout: "20 seconds",
+});
+const checkpoint = proxyActivities<Pick<typeof activities, "checkpointBrowserBeforeDeadline">>({
+  startToCloseTimeout: "25 minutes",
+  heartbeatTimeout: "20 seconds",
+  retry: { initialInterval: "1 second", maximumInterval: "30 seconds" },
+});
+
+export async function browserDeadlineCheckpointWorkflow(target: BrowserDeadlineCheckpointTarget) {
+  return await checkpoint.checkpointBrowserBeforeDeadline(target);
+}
+
+/** Runs from the existing lease-sweep dispatch. Inventory never performs provider
+ * I/O; each exact browser has an independent durable capture/cleanup child. */
+export async function browserDeadlineCheckpointSweepWorkflow(): Promise<void> {
+  const targets = await inventory.listDueBrowserCheckpoints();
+  await Promise.all(
+    targets.map(async (target) => {
+      try {
+        await startChild(browserDeadlineCheckpointWorkflow, {
+          workflowId: `browser-deadline:${target.browserSessionId}:${target.leaseEpoch}:${target.controllerGeneration}`,
+          taskQueue: workflowInfo().taskQueue,
+          workflowIdReusePolicy: WorkflowIdReusePolicy.ALLOW_DUPLICATE,
+          parentClosePolicy: ParentClosePolicy.ABANDON,
+          args: [target],
+        });
+      } catch (error) {
+        if (error instanceof Error && error.name === "WorkflowExecutionAlreadyStartedError") return;
+        // Another sweep retries start; it cannot retire the browser's authority.
+        log.warn("browser deadline checkpoint child start deferred", {
+          browserSessionId: target.browserSessionId,
+          error: error instanceof Error ? error.name : "unknown",
+        });
+      }
+    }),
+  );
+}