@opengeni/runtime 0.13.0 → 0.13.3

package/dist/index.js

@@ -29,6 +29,9 @@ import {
   REPOSITORY_DISCOVERY_LIMIT,
   RecordingError,
   RecordingUnavailableError,
+  RoutingBackendRecoveryRequiredError,
+  RoutingMutationOutcomeUnknownError,
+  RoutingRetainedProcessNotFoundError,
   RoutingSandboxSession,
   RoutingUnsupportedError,
   RunCredentialValidationError,
@@ -48,7 +51,9 @@ import {
   STREAM_TOKEN_DEFAULT_TTL_SECONDS,
   SandboxChannelAService,
   SandboxConfigError,
+  SandboxExecReadinessError,
   SandboxProviderUnavailableError,
+  SandboxReplacementProviderStateError,
   SandboxResumeStateUnavailableError,
   SelfhostedControlError,
   SelfhostedSandboxClient,
@@ -60,6 +65,9 @@ import {
   TerminalServerError,
   TerminalServerUnsupportedError,
   ToolspaceTokenPathError,
+  TurnSandboxCommandCancelledError,
+  WORKSPACE_ARCHIVE_DESCRIPTOR_VERSION,
+  WorkspaceArchiveIntegrityError,
   agentErrorToControlError,
   assertDescriptorRegistryInvariants,
   assertProviderRegistryInvariants,
@@ -69,12 +77,15 @@ import {
   buildSelfhostedBackendSession,
   buildStreamUrl,
   buildTerminalServerScript,
+  captureVerifiedWorkspaceArchive,
+  classifyProviderSandboxFailure,
   clearRunCredentials,
   clearRunCredentialsForAttempt,
   collectSandboxEnvironment,
   contentTypeForCodec,
   createSandboxClient,
   createSandboxClientForBackend,
+  createTurnToolCancellationController,
   decideSelfhostedRetry,
   decodeModalSnapshotId,
   defaultSelfhostedRetryClock,
@@ -91,9 +102,13 @@ import {
   execDeadlineHint,
   exposeStreamPort,
   extForCodec,
+  fingerprintSandboxWorkspace,
+  hasPersistableSandboxProviderIdentity,
   inspectRecordingArtifact,
   isExecSessionLostBanner,
+  isProviderSandboxGoneDuringRoutedOperation,
   isProviderSandboxNotFoundError,
+  isProviderSandboxTransientError,
   isSelfhostedProviderNotFoundError,
   localDirLazySkillSource,
   makeActiveBackendResolver,
@@ -115,10 +130,13 @@ import {
   parseNumstatZ,
   parsePorcelainV2,
   parseUnifiedPatch,
+  parseWorkspaceArchiveDescriptor,
   payloadTooLargeMessage,
+  readVerifiedWorkspaceArchive,
   readWorkspaceArchiveFromEnvelopeSessionState,
   recordingStorageKey,
   renderSelfhostedFault,
+  requirePersistableReplacementSandboxEnvelope,
   restoredSandboxSessionStateFromEntry,
   runCredentialPointerFile,
   runCredentialRoot,
@@ -130,6 +148,7 @@ import {
   selfhostedLiveness,
   selfhostedRetryBackoffMs,
   serializeEstablishedSandboxEnvelope,
+  serializeReplacementSandboxEnvelope,
   setSelfhostedApplyDiff,
   startRecording,
   stopRecording,
@@ -146,14 +165,17 @@ import {
   toolspaceTokenFileForSession,
   toolspaceTokenFileFromEnvironment,
   uploadRecordingArtifact,
+  verifyRestoredWorkspace,
+  verifySandboxExecReadiness,
   verifyStreamToken,
   withRunCredentialEnvironment,
   withRunCredentialsClient,
   withRunCredentialsSession,
   withToolspaceTokenClient,
   withToolspaceTokenEnvironment,
-  withToolspaceTokenSession
-} from "./chunk-65UDUFUO.js";
+  withToolspaceTokenSession,
+  wrapCapabilityToolsForTurnCancellation
+} from "./chunk-JGUQSAG5.js";
 import {
   getSkillLibraryEntry,
   isSkillLibraryEntryId,
@@ -172,9 +194,10 @@ import {
   boundedParallelMap,
   cancelMcpResponseBody,
   guardedMcpFetch,
+  mcpOuterConnectTimeoutMs,
   mcpSerializedSizeBytes,
   undiciFetch
-} from "./chunk-HAN5D27J.js";
+} from "./chunk-SSB7VOLE.js";
 import {
   MemorySession,
   connectMcpServers
@@ -5110,7 +5133,7 @@ var OpenAITracingExporter = class {
       const payloadItems = groupedItems.map((entry) => entry.toJSON()).filter((item) => !!item).map((item) => isRecord5(item) ? sanitizePayloadItemForTracesIngest(item) : item);
       const payload = { data: payloadItems };
       let attempts = 0;
-      let delay2 = this.#options.baseDelay;
+      let delay = this.#options.baseDelay;
       while (attempts < this.#options.maxRetries) {
         try {
           const response = await fetch(this.#options.endpoint, {
@@ -5144,13 +5167,13 @@ var OpenAITracingExporter = class {
         if (attempts >= this.#options.maxRetries) {
           break;
         }
-        const sleepTime = delay2 + Math.random() * 0.1 * delay2;
+        const sleepTime = delay + Math.random() * 0.1 * delay;
         const shouldContinue = await sleepWithAbort(sleepTime, signal);
         if (!shouldContinue) {
           logger_default.error("Tracing: request aborted");
           break;
         }
-        delay2 = Math.min(delay2 * 2, this.#options.maxDelay);
+        delay = Math.min(delay * 2, this.#options.maxDelay);
       }
       if (attempts >= this.#options.maxRetries) {
         logger_default.error(`Tracing: failed to export traces after ${this.#options.maxRetries} attempts`);
@@ -7601,668 +7624,6 @@ function safeStringify(value) {
   }
 }
 
-// src/sandbox/turn-tool-cancellation.ts
-var TURN_EXEC_YIELD_MS = 250;
-var TURN_WRITE_YIELD_MS = 250;
-var SHELL_HELPER_YIELD_MS = 1e3;
-var SHELL_GRACEFUL_POLLS = 2;
-var SHELL_POLL_MS = 100;
-var SHELL_MARKER_DIR = "/tmp/opengeni-turn-shell";
-var delay = async (milliseconds) => await new Promise((resolve) => setTimeout(resolve, milliseconds));
-var TurnSandboxCommandCancelledError = class extends Error {
-  name = "TurnSandboxCommandCancelledError";
-  constructor(reason) {
-    super(
-      reason instanceof Error ? reason.message : String(reason ?? "Turn sandbox tools were cancelled"),
-      {
-        ...reason instanceof Error ? { cause: reason } : {}
-      }
-    );
-  }
-};
-function cancellationError(reason) {
-  return new TurnSandboxCommandCancelledError(reason);
-}
-function parsedObject(input) {
-  try {
-    const parsed = JSON.parse(input);
-    return parsed !== null && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : null;
-  } catch {
-    return null;
-  }
-}
-function cappedYield(value, cap) {
-  return typeof value === "number" && Number.isInteger(value) && value >= 0 ? Math.min(value, cap) : cap;
-}
-function execOutput(raw) {
-  const marker = "\nOutput:\n";
-  const index = raw.indexOf(marker);
-  return index >= 0 ? raw.slice(index + marker.length) : "";
-}
-function nativeCommandBanner(result) {
-  if (typeof result === "string") return result;
-  if (!result || typeof result !== "object") {
-    throw new Error("Sandbox command returned an invalid result");
-  }
-  const value = result;
-  const sessionId = typeof value.sessionId === "number" ? value.sessionId : typeof value.session_id === "number" ? value.session_id : null;
-  const exitCode = typeof value.exitCode === "number" ? value.exitCode : typeof value.exit_code === "number" ? value.exit_code : null;
-  const output = [value.output, value.stderr, value.stdout].filter(
-    (candidate) => typeof candidate === "string" && candidate.length > 0
-  ).join("\n");
-  if (sessionId !== null) {
-    return `Process running with session ID ${sessionId}
-
-Output:
-${output}`;
-  }
-  if (exitCode !== null) {
-    return `Process exited with code ${exitCode}
-
-Output:
-${output}`;
-  }
-  throw new Error("Sandbox command did not report a session id or exit code");
-}
-function completedCommandBanner(exitCode, output) {
-  return `Process exited with code ${exitCode}
-
-Output:
-${output}`;
-}
-function appendBoundedOutput(current, chunk, maxOutputTokens) {
-  const maxChars = Math.max(512, maxOutputTokens * 4);
-  const combined = current + chunk;
-  return combined.length <= maxChars ? combined : combined.slice(-maxChars);
-}
-function singleQuote(value) {
-  return `'${value.replaceAll("'", `'"'"'`)}'`;
-}
-function shellMarkerPath(token) {
-  return `${SHELL_MARKER_DIR}/${token}`;
-}
-function cancellableGroupLeaderCommand(command, markerPath) {
-  const marker = singleQuote(markerPath);
-  const markerDir = singleQuote(SHELL_MARKER_DIR);
-  return [
-    `__opengeni_marker=${marker}`,
-    "umask 077",
-    `command mkdir -p ${markerDir} || exit 125`,
-    '__opengeni_pid="$$"',
-    `__opengeni_pgid="$(command ps -o pgid= -p "$__opengeni_pid" 2>/dev/null | command tr -d '[:space:]')"`,
-    'case "$__opengeni_pid:$__opengeni_pgid" in *[!0-9:]*|*:|:*) exit 125 ;; esac',
-    '[ "$__opengeni_pid" -gt 1 ] && [ "$__opengeni_pgid" -gt 1 ] || exit 125',
-    // Never signal a provider/container-wide process group. A cancellable PTY
-    // command must be its group's leader; fail the command closed otherwise.
-    '[ "$__opengeni_pid" = "$__opengeni_pgid" ] || exit 125',
-    `command printf '%s %s\\n' "$__opengeni_pid" "$__opengeni_pgid" > "$__opengeni_marker" || exit 125`,
-    `trap 'command rm -f "$__opengeni_marker"' EXIT`,
-    "(",
-    command,
-    ")",
-    "__opengeni_status=$?",
-    'exit "$__opengeni_status"'
-  ].join("\n");
-}
-function cancellableShellCommand(command, markerPath) {
-  const groupLeaderCommand = cancellableGroupLeaderCommand(command, markerPath);
-  return [
-    '__opengeni_outer_pid="$$"',
-    `__opengeni_outer_pgid="$(command ps -o pgid= -p "$__opengeni_outer_pid" 2>/dev/null | command tr -d '[:space:]')"`,
-    'case "$__opengeni_outer_pid:$__opengeni_outer_pgid" in *[!0-9:]*|*:|:*) exit 125 ;; esac',
-    'if [ "$__opengeni_outer_pid" != "$__opengeni_outer_pgid" ]; then',
-    '  __opengeni_setsid="$(command -v setsid 2>/dev/null)"',
-    '  [ -n "$__opengeni_setsid" ] || exit 125',
-    `  exec "$__opengeni_setsid" /bin/sh -c ${singleQuote(groupLeaderCommand)}`,
-    "fi",
-    groupLeaderCommand
-  ].join("\n");
-}
-function shellHelperInput(command) {
-  return JSON.stringify({
-    cmd: command,
-    login: false,
-    tty: false,
-    yield_time_ms: SHELL_HELPER_YIELD_MS,
-    max_output_tokens: 128
-  });
-}
-function safeProcessIdentity(raw) {
-  if (parseExecBannerExitCode(raw) !== 0) return null;
-  const match = execOutput(raw).trim().match(/^(\d+)\s+(\d+)$/);
-  if (!match) return null;
-  const pid = Number.parseInt(match[1], 10);
-  const processGroupId = Number.parseInt(match[2], 10);
-  return Number.isSafeInteger(pid) && Number.isSafeInteger(processGroupId) && pid > 1 && processGroupId > 1 ? { pid, processGroupId } : null;
-}
-function identityGuardScript(state, body, missingIdentityExitCode) {
-  const identity = state.identity;
-  if (!identity || !state.token) return `exit ${missingIdentityExitCode}`;
-  const token = singleQuote(state.token);
-  return [
-    `__opengeni_pid=${identity.pid}`,
-    `__opengeni_pgid=${identity.processGroupId}`,
-    `__opengeni_token=${token}`,
-    // The randomized token lands late in the wrapped command line. Procps
-    // otherwise truncates `args` to the terminal width, which can make a live
-    // group look stale and open the quiescence fence without signalling it.
-    '__opengeni_args="$(command ps -ww -o args= -p "$__opengeni_pid" 2>/dev/null)"',
-    'case "$__opengeni_args" in *"$__opengeni_token"*) ;; *) exit 0 ;; esac',
-    `__opengeni_live_pgid="$(command ps -o pgid= -p "$__opengeni_pid" 2>/dev/null | command tr -d '[:space:]')"`,
-    '[ "$__opengeni_live_pgid" = "$__opengeni_pgid" ] || exit 0',
-    body
-  ].join("\n");
-}
-function wrapComputer(computer, track) {
-  const wrapped = Object.create(Object.getPrototypeOf(computer));
-  Object.assign(wrapped, computer);
-  for (const method of [
-    "initRun",
-    "screenshot",
-    "click",
-    "doubleClick",
-    "scroll",
-    "type",
-    "wait",
-    "move",
-    "keypress",
-    "drag"
-  ]) {
-    const original = computer[method];
-    if (typeof original !== "function") continue;
-    wrapped[method] = (...args) => track(async () => await original.apply(computer, args));
-  }
-  return wrapped;
-}
-var TurnToolCancellationControllerImpl = class {
-  cancelled = false;
-  reason;
-  inFlight = /* @__PURE__ */ new Set();
-  shellSessions = /* @__PURE__ */ new Map();
-  remoteExecs = /* @__PURE__ */ new Set();
-  rawExecInvoke = null;
-  rawWriteInvoke = null;
-  drainPromise = null;
-  signal;
-  onAbort = () => this.cancel(this.signal?.reason);
-  constructor(signal) {
-    this.signal = signal;
-    if (signal?.aborted) {
-      this.cancel(signal.reason);
-    } else {
-      signal?.addEventListener("abort", this.onAbort, { once: true });
-    }
-  }
-  cancel(reason) {
-    if (!this.cancelled) {
-      this.cancelled = true;
-      this.reason = reason;
-      this.signal?.removeEventListener("abort", this.onAbort);
-    }
-    void this.ensureDrain().catch(() => void 0);
-  }
-  async waitForQuiescence() {
-    this.cancel(this.reason);
-    await this.ensureDrain();
-  }
-  runSandboxCommand(session, args) {
-    return this.track(async () => {
-      if (!session.exec && !session.execCommand) {
-        throw new Error("Sandbox session does not support command execution");
-      }
-      const lifecycleRunContext = {};
-      const invokeExec = async (_runContext, input) => {
-        const parsed = parsedObject(input);
-        if (!parsed || typeof parsed.cmd !== "string") {
-          throw new Error("Sandbox lifecycle command input was invalid");
-        }
-        const commandArgs = {
-          cmd: parsed.cmd,
-          ...typeof parsed.workdir === "string" ? { workdir: parsed.workdir } : {},
-          ...typeof parsed.shell === "string" ? { shell: parsed.shell } : {},
-          ...typeof parsed.login === "boolean" ? { login: parsed.login } : {},
-          ...typeof parsed.tty === "boolean" ? { tty: parsed.tty } : {},
-          ...typeof parsed.yield_time_ms === "number" ? { yieldTimeMs: parsed.yield_time_ms } : {},
-          ...typeof parsed.max_output_tokens === "number" ? { maxOutputTokens: parsed.max_output_tokens } : {},
-          ...typeof parsed.run_as === "string" ? { runAs: parsed.run_as } : {}
-        };
-        const result = session.supportsPty?.() !== false && session.execCommand ? await session.execCommand(commandArgs) : session.exec ? await session.exec(commandArgs) : await session.execCommand(commandArgs);
-        return nativeCommandBanner(result);
-      };
-      const invokeWrite = session.writeStdin ? async (_runContext, input) => {
-        const parsed = parsedObject(input);
-        if (!parsed || typeof parsed.session_id !== "number") {
-          throw new Error("Sandbox lifecycle stdin input was invalid");
-        }
-        return await session.writeStdin({
-          sessionId: parsed.session_id,
-          ...typeof parsed.chars === "string" ? { chars: parsed.chars } : {},
-          ...typeof parsed.yield_time_ms === "number" ? { yieldTimeMs: parsed.yield_time_ms } : {},
-          ...typeof parsed.max_output_tokens === "number" ? { maxOutputTokens: parsed.max_output_tokens } : {}
-        });
-      } : null;
-      const correlationId = `turn_lifecycle_${crypto.randomUUID()}`;
-      const useRemoteOpCancellation = Boolean(session.cancelExecCommand) && session.supportsPty?.() === false;
-      const remoteExec = session.cancelExecCommand && useRemoteOpCancellation ? this.registerRemoteExec(
-        { cancelExecCommand: session.cancelExecCommand.bind(session) },
-        `${sanitizeOpIdToken(correlationId)}:0`
-      ) : null;
-      const markerPath = shellMarkerPath(crypto.randomUUID());
-      const commandInput = JSON.stringify({
-        cmd: useRemoteOpCancellation ? args.cmd : cancellableShellCommand(args.cmd, markerPath),
-        ...args.workdir ? { workdir: args.workdir } : {},
-        ...args.shell ? { shell: args.shell } : {},
-        ...args.login !== void 0 ? { login: args.login } : {},
-        ...args.runAs ? { run_as: args.runAs } : {},
-        tty: useRemoteOpCancellation ? args.tty : true,
-        yield_time_ms: useRemoteOpCancellation ? args.yieldTimeMs : cappedYield(args.yieldTimeMs, TURN_EXEC_YIELD_MS),
-        ...args.maxOutputTokens !== void 0 ? { max_output_tokens: args.maxOutputTokens } : {}
-      });
-      let initial;
-      try {
-        initial = await runWithToolCallCorrelation(
-          correlationId,
-          async () => await invokeExec(lifecycleRunContext, commandInput, void 0)
-        );
-      } finally {
-        remoteExec?.settle();
-      }
-      if (typeof initial !== "string" || useRemoteOpCancellation) return initial;
-      const sessionId = parseExecBannerSessionId(initial);
-      if (sessionId === null) return initial;
-      if (!invokeWrite) {
-        throw new Error("Sandbox lifecycle command yielded without stdin support");
-      }
-      const token = markerPath.slice(markerPath.lastIndexOf("/") + 1);
-      const state = {
-        sessionId,
-        markerPath,
-        token,
-        runContext: lifecycleRunContext,
-        execInvoke: invokeExec,
-        writeInvoke: invokeWrite,
-        identity: null,
-        identityValidated: false,
-        cancellation: null
-      };
-      this.shellSessions.set(sessionId, state);
-      const maxOutputTokens = args.maxOutputTokens ?? 2e4;
-      const initialOutput = execOutput(initial);
-      let output = initialOutput ? appendBoundedOutput("", initialOutput, maxOutputTokens) : "";
-      while (true) {
-        if (this.cancelled) throw cancellationError(this.reason);
-        const next = await invokeWrite(
-          lifecycleRunContext,
-          JSON.stringify({
-            session_id: sessionId,
-            chars: "",
-            yield_time_ms: TURN_WRITE_YIELD_MS,
-            max_output_tokens: maxOutputTokens
-          }),
-          void 0
-        );
-        if (typeof next !== "string") {
-          throw new Error("Sandbox lifecycle stdin returned an invalid result");
-        }
-        const nextOutput = execOutput(next);
-        if (nextOutput) output = appendBoundedOutput(output, nextOutput, maxOutputTokens);
-        const exitCode = parseExecBannerExitCode(next);
-        if (exitCode !== null) {
-          this.shellSessions.delete(sessionId);
-          return completedCommandBanner(exitCode, output);
-        }
-        await delay(SHELL_POLL_MS);
-      }
-    });
-  }
-  wrapTools(tools, session) {
-    for (const tool2 of tools) {
-      if (tool2.type !== "function") continue;
-      if (tool2.name === "exec_command") this.rawExecInvoke = tool2.invoke;
-      if (tool2.name === "write_stdin") this.rawWriteInvoke = tool2.invoke;
-    }
-    return tools.map((tool2) => {
-      if (tool2.type === "function") return this.wrapFunctionTool(tool2, session);
-      if (tool2.type === "apply_patch") return this.wrapApplyPatchTool(tool2);
-      if (tool2.type === "computer") return this.wrapComputerTool(tool2);
-      return tool2;
-    });
-  }
-  wrapFunctionTool(tool2, cancellationSession) {
-    if (tool2.name === "exec_command") {
-      return {
-        ...tool2,
-        invoke: (runContext, input, details) => this.track(async () => {
-          const parsed = parsedObject(input);
-          if (!parsed || typeof parsed.cmd !== "string") {
-            return await this.correlatedInvoke(tool2.invoke, runContext, input, details);
-          }
-          const token = crypto.randomUUID();
-          const markerPath = shellMarkerPath(token);
-          const correlationId = details?.toolCall?.callId ?? `turn_exec_${token}`;
-          const cancelExecCommand = cancellationSession?.cancelExecCommand;
-          const useRemoteOpCancellation = Boolean(cancelExecCommand) && cancellationSession?.supportsPty?.() === false;
-          const remoteExec = cancelExecCommand && useRemoteOpCancellation ? this.registerRemoteExec(
-            { cancelExecCommand: cancelExecCommand.bind(cancellationSession) },
-            `${sanitizeOpIdToken(correlationId)}:0`
-          ) : null;
-          const cancellableInput = useRemoteOpCancellation ? input : JSON.stringify({
-            ...parsed,
-            cmd: cancellableShellCommand(parsed.cmd, markerPath),
-            // The SDK's only provider-neutral live-process interrupt is a PTY.
-            // The process marker provides TERM/KILL escalation when Ctrl-C is
-            // ignored; the short yield exposes the provider session promptly.
-            tty: true,
-            yield_time_ms: cappedYield(parsed.yield_time_ms, TURN_EXEC_YIELD_MS)
-          });
-          let output;
-          try {
-            output = await runWithToolCallCorrelation(
-              correlationId,
-              async () => await tool2.invoke(runContext, cancellableInput, details)
-            );
-          } finally {
-            remoteExec?.settle();
-          }
-          if (typeof output !== "string") return output;
-          const sessionId = useRemoteOpCancellation ? null : parseExecBannerSessionId(output);
-          if (sessionId !== null) {
-            this.shellSessions.set(sessionId, {
-              sessionId,
-              markerPath,
-              token,
-              runContext,
-              execInvoke: tool2.invoke,
-              writeInvoke: this.rawWriteInvoke,
-              identity: null,
-              identityValidated: false,
-              cancellation: null
-            });
-          }
-          return output;
-        })
-      };
-    }
-    if (tool2.name === "write_stdin") {
-      return {
-        ...tool2,
-        invoke: (runContext, input, details) => this.track(async () => {
-          const parsed = parsedObject(input);
-          const sessionId = parsed && typeof parsed.session_id === "number" && Number.isSafeInteger(parsed.session_id) ? parsed.session_id : null;
-          const cappedInput = parsed ? JSON.stringify({
-            ...parsed,
-            yield_time_ms: cappedYield(parsed.yield_time_ms, TURN_WRITE_YIELD_MS)
-          }) : input;
-          const output = await tool2.invoke(runContext, cappedInput, details);
-          if (sessionId !== null && typeof output === "string") {
-            if (isExecSessionLostBanner(output, sessionId)) {
-              this.shellSessions.delete(sessionId);
-            } else if (parseExecBannerSessionId(output) === sessionId) {
-              if (!this.shellSessions.has(sessionId)) {
-                this.shellSessions.set(sessionId, {
-                  sessionId,
-                  markerPath: null,
-                  token: null,
-                  runContext,
-                  execInvoke: this.rawExecInvoke ?? tool2.invoke,
-                  writeInvoke: tool2.invoke,
-                  identity: null,
-                  identityValidated: false,
-                  cancellation: null
-                });
-              }
-            } else if (parseExecBannerExitCode(output) !== null) {
-              this.shellSessions.delete(sessionId);
-            }
-          }
-          return output;
-        })
-      };
-    }
-    return {
-      ...tool2,
-      invoke: (runContext, input, details) => this.track(async () => await tool2.invoke(runContext, input, details))
-    };
-  }
-  wrapApplyPatchTool(tool2) {
-    const editor = tool2.editor;
-    const wrappedEditor = {
-      createFile: (operation, context) => this.track(async () => await editor.createFile(operation, context)),
-      updateFile: (operation, context) => this.track(async () => await editor.updateFile(operation, context)),
-      deleteFile: (operation, context) => this.track(async () => await editor.deleteFile(operation, context))
-    };
-    return { ...tool2, editor: wrappedEditor };
-  }
-  wrapComputerTool(tool2) {
-    const computer = tool2.computer;
-    if (typeof computer === "function") {
-      return {
-        ...tool2,
-        computer: async (...args) => wrapComputer(
-          await this.track(async () => await computer(...args)),
-          (operation) => this.track(operation)
-        )
-      };
-    }
-    if ("create" in computer && typeof computer.create === "function") {
-      return {
-        ...tool2,
-        computer: {
-          ...computer,
-          create: async (...args) => wrapComputer(
-            await this.track(async () => await computer.create(...args)),
-            (operation) => this.track(operation)
-          )
-        }
-      };
-    }
-    return {
-      ...tool2,
-      computer: wrapComputer(computer, (operation) => this.track(operation))
-    };
-  }
-  async correlatedInvoke(invoke, runContext, input, details) {
-    const callId = details?.toolCall?.callId;
-    return callId ? await runWithToolCallCorrelation(
-      callId,
-      async () => await invoke(runContext, input, details)
-    ) : await invoke(runContext, input, details);
-  }
-  track(operation) {
-    if (this.cancelled) return Promise.reject(cancellationError(this.reason));
-    const promise = Promise.resolve().then(async () => {
-      if (this.cancelled) throw cancellationError(this.reason);
-      return await operation();
-    });
-    this.inFlight.add(promise);
-    void promise.finally(() => {
-      this.inFlight.delete(promise);
-    }).catch(() => void 0);
-    return promise;
-  }
-  ensureDrain() {
-    this.drainPromise ??= this.drain();
-    return this.drainPromise;
-  }
-  async drain() {
-    while (true) {
-      const remoteCancellations = [...this.remoteExecs].map(async (exec) => await exec.cancel());
-      const inFlight = [...this.inFlight];
-      if (inFlight.length > 0 || remoteCancellations.length > 0) {
-        await Promise.allSettled([...inFlight, ...remoteCancellations]);
-      }
-      const sessions = [...this.shellSessions.values()];
-      if (sessions.length > 0) {
-        await Promise.all(sessions.map(async (session) => await this.cancelShellSession(session)));
-      }
-      if (this.inFlight.size === 0 && this.shellSessions.size === 0 && this.remoteExecs.size === 0)
-        return;
-    }
-  }
-  registerRemoteExec(session, opId) {
-    let resolveSettled;
-    const settledPromise = new Promise((resolve) => {
-      resolveSettled = resolve;
-    });
-    let cancellation = null;
-    const entry = {
-      settled: false,
-      settledPromise,
-      settle: () => {
-        if (entry.settled) return;
-        entry.settled = true;
-        resolveSettled();
-        this.remoteExecs.delete(entry);
-      },
-      cancel: () => {
-        cancellation ??= (async () => {
-          while (!entry.settled) {
-            const attempt = session.cancelExecCommand(opId).catch(() => false);
-            await Promise.race([attempt, entry.settledPromise]);
-            await Promise.race([entry.settledPromise, delay(SHELL_POLL_MS)]);
-          }
-        })();
-        return cancellation;
-      }
-    };
-    this.remoteExecs.add(entry);
-    return entry;
-  }
-  cancelShellSession(state) {
-    state.cancellation ??= this.cancelShellSessionOnce(state).finally(() => {
-      state.cancellation = null;
-    });
-    return state.cancellation;
-  }
-  async cancelShellSessionOnce(state) {
-    if (!state.markerPath) {
-      while (true) {
-        if (state.writeInvoke && await this.rawWrite(state, "", SHELL_POLL_MS)) {
-          this.shellSessions.delete(state.sessionId);
-          return;
-        }
-        await delay(SHELL_POLL_MS);
-      }
-    }
-    while (!state.identity) {
-      state.identity = await this.readIdentity(state);
-      if (state.identity) break;
-      if (state.writeInvoke) {
-        const completed = await this.rawWrite(state, "", SHELL_POLL_MS);
-        if (completed) {
-          this.shellSessions.delete(state.sessionId);
-          return;
-        }
-      }
-      await delay(SHELL_POLL_MS);
-    }
-    if (!await this.identityAlive(state)) {
-      this.shellSessions.delete(state.sessionId);
-      return;
-    }
-    if (state.writeInvoke) {
-      await this.rawWrite(state, "", SHELL_POLL_MS);
-      if (!await this.identityAlive(state)) {
-        this.shellSessions.delete(state.sessionId);
-        return;
-      }
-    }
-    await this.signalIdentity(state, "TERM");
-    for (let attempt = 0; attempt < SHELL_GRACEFUL_POLLS; attempt++) {
-      if (state.writeInvoke) await this.rawWrite(state, "", SHELL_POLL_MS);
-      if (!await this.identityAlive(state)) {
-        this.shellSessions.delete(state.sessionId);
-        return;
-      }
-    }
-    await this.signalIdentity(state, "KILL");
-    while (await this.identityAlive(state)) {
-      if (state.writeInvoke) await this.rawWrite(state, "", SHELL_POLL_MS);
-      await this.signalIdentity(state, "KILL");
-      await delay(SHELL_POLL_MS);
-    }
-    this.shellSessions.delete(state.sessionId);
-  }
-  async rawWrite(state, chars, yieldTimeMs) {
-    if (!state.writeInvoke) return false;
-    try {
-      const output = await state.writeInvoke(
-        state.runContext,
-        JSON.stringify({
-          session_id: state.sessionId,
-          chars,
-          yield_time_ms: yieldTimeMs,
-          max_output_tokens: 128
-        }),
-        void 0
-      );
-      if (typeof output !== "string") return false;
-      if (isExecSessionLostBanner(output, state.sessionId)) return true;
-      if (parseExecBannerSessionId(output) === state.sessionId) return false;
-      if (parseExecBannerExitCode(output) !== null) return true;
-    } catch {
-    }
-    return false;
-  }
-  async readIdentity(state) {
-    if (!state.markerPath) return null;
-    try {
-      const output = await state.execInvoke(
-        state.runContext,
-        shellHelperInput(`command cat ${singleQuote(state.markerPath)} 2>/dev/null`),
-        void 0
-      );
-      return typeof output === "string" ? safeProcessIdentity(output) : null;
-    } catch {
-      return null;
-    }
-  }
-  async identityAlive(state) {
-    const script = state.identityValidated ? [
-      `__opengeni_pgid=${state.identity?.processGroupId ?? 0}`,
-      'command kill -0 "-$__opengeni_pgid" 2>/dev/null && exit 75',
-      "exit 0"
-    ].join("\n") : identityGuardScript(
-      state,
-      'command kill -0 "-$__opengeni_pgid" 2>/dev/null && exit 75\nexit 0',
-      76
-    );
-    try {
-      const output = await state.execInvoke(state.runContext, shellHelperInput(script), void 0);
-      if (typeof output !== "string") return true;
-      const exitCode = parseExecBannerExitCode(output);
-      if (exitCode === 75) {
-        state.identityValidated = true;
-        return true;
-      }
-      return exitCode !== 0;
-    } catch {
-      return true;
-    }
-  }
-  async signalIdentity(state, signal) {
-    const script = state.identityValidated ? `command kill -${signal} "-${state.identity?.processGroupId ?? 0}" 2>/dev/null || exit 76` : identityGuardScript(
-      state,
-      `command kill -${signal} "-$__opengeni_pgid" 2>/dev/null || exit 76
-exit 0`,
-      76
-    );
-    try {
-      await state.execInvoke(state.runContext, shellHelperInput(script), void 0);
-    } catch {
-    }
-  }
-};
-function createTurnToolCancellationController(signal) {
-  return new TurnToolCancellationControllerImpl(signal);
-}
-function wrapCapabilityToolsForTurnCancellation(capability, controller) {
-  const originalTools = capability.tools;
-  capability.tools = function() {
-    return controller.wrapTools(originalTools.call(this), this._session);
-  };
-}
-
 // src/sandbox-computer.ts
 import {
   KeyAction,
@@ -10320,6 +9681,7 @@ async function connectMcpServersInBatches(servers, options) {
         await connectMcpServers(
           servers.slice(offset, offset + MCP_MAX_CONCURRENT_SERVER_OPERATIONS),
           {
+            ...options.connectTimeoutMs === void 0 ? {} : { connectTimeoutMs: options.connectTimeoutMs },
             connectInParallel: true,
             strict: options.strict
           }
@@ -10408,19 +9770,30 @@ async function prepareAgentTools(settings, tools, options = {}) {
       return {
         server,
         bestEffort,
-        optional
+        optional,
+        timeoutMs: config.timeoutMs
       };
     }
   );
-  const requiredServers = servers.filter((entry) => !entry.bestEffort).map((entry) => entry.server);
-  const bestEffortServers = servers.filter((entry) => entry.bestEffort).map((entry) => entry.server);
+  const requiredEntries = servers.filter((entry) => !entry.bestEffort);
+  const bestEffortEntries = servers.filter((entry) => entry.bestEffort);
+  const requiredServers = requiredEntries.map((entry) => entry.server);
+  const bestEffortServers = bestEffortEntries.map((entry) => entry.server);
   const optionalServerNames = new Set(
     servers.filter((entry) => entry.optional).map((entry) => entry.server.name)
   );
-  const connectedRequired = await connectMcpServersInBatches(requiredServers, { strict: true });
+  const connectedRequired = await connectMcpServersInBatches(requiredServers, {
+    strict: true,
+    connectTimeoutMs: mcpOuterConnectTimeoutMs(requiredEntries.map((entry) => entry.timeoutMs))
+  });
   let connectedBestEffort = null;
   try {
-    connectedBestEffort = bestEffortServers.length ? await connectMcpServersInBatches(bestEffortServers, { strict: false }) : null;
+    connectedBestEffort = bestEffortServers.length ? await connectMcpServersInBatches(bestEffortServers, {
+      strict: false,
+      connectTimeoutMs: mcpOuterConnectTimeoutMs(
+        bestEffortEntries.map((entry) => entry.timeoutMs)
+      )
+    }) : null;
   } catch (error) {
     await connectedRequired.close().catch(() => void 0);
     throw error;
@@ -13810,6 +13183,9 @@ export {
   REPOSITORY_DISCOVERY_LIMIT,
   RecordingError,
   RecordingUnavailableError,
+  RoutingBackendRecoveryRequiredError,
+  RoutingMutationOutcomeUnknownError,
+  RoutingRetainedProcessNotFoundError,
   RoutingSandboxSession,
   RoutingUnsupportedError,
   RunCredentialValidationError,
@@ -13832,7 +13208,9 @@ export {
   SandboxChannelAService,
   SandboxComputer,
   SandboxConfigError,
+  SandboxExecReadinessError,
   SandboxProviderUnavailableError,
+  SandboxReplacementProviderStateError,
   SandboxResumeStateUnavailableError,
   SelfhostedControlError,
   SelfhostedSandboxClient,
@@ -13847,6 +13225,8 @@ export {
   ToolspaceTokenPathError,
   TurnSandboxCommandCancelledError,
   USER_MESSAGE_TRUNCATION_MARKER,
+  WORKSPACE_ARCHIVE_DESCRIPTOR_VERSION,
+  WorkspaceArchiveIntegrityError,
   WorkspaceModelPolicyBlockedError,
   agentErrorToControlError,
   agentsErrorRunState,
@@ -13878,7 +13258,9 @@ export {
   buildSummaryItem,
   buildTerminalServerScript,
   callModelInputFilterForSettings,
+  captureVerifiedWorkspaceArchive,
   clampCompactionThresholdRatio,
+  classifyProviderSandboxFailure,
   clearRunCredentials,
   clearRunCredentialsForAttempt,
   collectSandboxEnvironment,
@@ -13925,16 +13307,20 @@ export {
   extForCodec,
   extractResponseOutputText,
   findCompactionNeededError,
+  fingerprintSandboxWorkspace,
   getSkillLibraryEntry,
   gitCredentialBindingHash,
   gitCredentialBindingTokenRefreshCommand,
   gitProviderTokenRefreshCommand,
   hasModelGeneratedItem,
+  hasPersistableSandboxProviderIdentity,
   inspectRecordingArtifact,
   isCompactionSummary,
   isEphemeralInternalContext,
   isExecSessionLostBanner,
+  isProviderSandboxGoneDuringRoutedOperation,
   isProviderSandboxNotFoundError,
+  isProviderSandboxTransientError,
   isSelfhostedProviderNotFoundError,
   isSkillLibraryEntryId,
   isUserMessage,
@@ -13975,12 +13361,14 @@ export {
   parseNumstatZ,
   parsePorcelainV2,
   parseUnifiedPatch,
+  parseWorkspaceArchiveDescriptor,
   payloadTooLargeMessage,
   pinProvidedSessionManifestEnvironment,
   prefixedMcpToolName,
   prepareAgentTools,
   prepareCompactionPromptInput,
   prepareRunInput,
+  readVerifiedWorkspaceArchive,
   readWorkspaceArchiveFromEnvelopeSessionState,
   recordingStorageKey,
   refreshGitCredentialBindingTokenFiles,
@@ -13990,6 +13378,7 @@ export {
   renderSelfhostedFault,
   repositoryCloneCommand,
   repositoryUsesSandboxClone,
+  requirePersistableReplacementSandboxEnvelope,
   resolveTurnModel,
   restoredSandboxSessionStateFromEntry,
   rigInstructions,
@@ -14021,6 +13410,7 @@ export {
   serializeApprovals,
   serializeEstablishedSandboxEnvelope,
   serializeHumanInputRequests,
+  serializeReplacementSandboxEnvelope,
   setSelfhostedApplyDiff,
   startRecording,
   stopRecording,
@@ -14042,6 +13432,8 @@ export {
   toolspaceTokenFileFromEnvironment,
   toolspaceTokenSeedCommand,
   uploadRecordingArtifact,
+  verifyRestoredWorkspace,
+  verifySandboxExecReadiness,
   verifyStreamToken,
   withManifestRefreshOnResume,
   withRunCredentialEnvironment,