@opengeni/runtime 4.6.0 → 4.7.0-canary.37090756480001

package/src/sandbox/routing/routing-session.ts

@@ -41,6 +41,7 @@ import {
   withSupervisedLaunchReservation,
   ProviderCommandStartRejectedError,
   ProviderCommandStartOutcomeUnknownError,
+  ProviderCommandInputOutcomeUnknownError,
   type ProviderCommandPersistence,
   type ProviderCommandSession,
 } from "../provider-command-session";
@@ -281,6 +282,12 @@ export interface RoutingSandboxSessionDeps {
     process: RoutingRetainedProcess;
     proof: RoutingRetainedProcessTerminalProof;
   }) => Promise<void>;
+  /** Read terminal truth for this exact copied process/backend. Missing rows,
+   * failed observations and active rows must never count as physical proof. */
+  isProcessSettled?: (input: {
+    backend: ResolvedActiveBackend;
+    process: RoutingRetainedProcess;
+  }) => Promise<boolean>;
   /** A terminal result is being returned to the model, not merely drained by
    * control/reaper work. Never invoke this for a running receipt. */
   observeProcessTerminal?: (input: {
@@ -1236,6 +1243,12 @@ export class RoutingSandboxSession implements RoutableBackendSession {
         write.call(record.backend.session, args),
       );
     } catch (error) {
+      let inputUnknown = false;
+      try {
+        inputUnknown = error instanceof ProviderCommandInputOutcomeUnknownError;
+      } catch {
+        // An unreadable provider graph cannot manufacture typed input proof.
+      }
       if (this.deps.afterProcessMutation) {
         const pending: PendingProcessMutationSettlement = {
           op,
@@ -1250,11 +1263,28 @@ export class RoutingSandboxSession implements RoutableBackendSession {
           record.pendingMutationSettlement = pending;
           throw new RoutingMutationOutcomeUnknownError(
             op,
-            `Retained-process stdin rejected at the provider but lost durable settlement; it was not replayed`,
-            { cause: settlementError },
+            inputUnknown
+              ? "Retained-process stdin acknowledgement and durable settlement unavailable; input may have been accepted. Do not resend stdin."
+              : "Retained-process stdin rejected at the provider but lost durable settlement; it was not replayed",
+            inputUnknown
+              ? {
+                  cause: new AggregateError([error, settlementError]),
+                  retainedProcess: record.process,
+                }
+              : { cause: settlementError },
           );
         }
       }
+      if (inputUnknown) {
+        // Preserve native input proof across the retained route, including its
+        // genuine byte-range cause and exact locator. Both SDK-facing rendering
+        // and direct tool faults must forbid input replay.
+        throw new RoutingMutationOutcomeUnknownError(
+          op,
+          "Provider stdin acknowledgement unavailable; input may have been accepted and was not resent. Do not resend stdin.",
+          { cause: error, retainedProcess: record.process },
+        );
+      }
       throw error;
     }
     if (this.deps.afterProcessMutation) {
@@ -2131,7 +2161,17 @@ export class RoutingSandboxSession implements RoutableBackendSession {
   async writeStdin(args: unknown): Promise<string> {
     const providerSessionId = providerSessionIdFromArgs(args);
     if (providerSessionId !== null && this.retainedProcesses.has(providerSessionId)) {
-      return await this.dispatchProcessMutation(args);
+      try {
+        return await this.dispatchProcessMutation(args);
+      } catch (error) {
+        // The SDK's write_stdin tool has no configurable errorFunction. Render
+        // genuine routing uncertainty here, before its generic retry advice.
+        // Direct/control methods still throw; this is not an output receipt or
+        // terminal proof and cannot acknowledge bytes or release the writer.
+        if (isRoutingMutationOutcomeUnknownError(error))
+          return renderRoutingMutationOutcomeUnknownToolResult(error);
+        throw error;
+      }
     }
     return this.dispatch("writeStdin", true, async (s) => {
       if (!s.writeStdin) {
@@ -2158,6 +2198,20 @@ export class RoutingSandboxSession implements RoutableBackendSession {
     );
   }
 
+  /** A reaper may have settled the command while a local capture/control receipt
+   * remained pending. Cleanup may consume that same durable proof without
+   * replaying a provider operation or accepting pending output into the model. */
+  async reconcileRetainedProcess(providerSessionId: number): Promise<boolean> {
+    const record = this.retainedProcesses.get(providerSessionId);
+    if (!record || !this.deps.isProcessSettled) return false;
+    if (!(await this.deps.isProcessSettled({ backend: record.backend, process: record.process })))
+      return false;
+    // Do not erase a rival route installed while the durable read was pending.
+    if (this.retainedProcesses.get(providerSessionId) !== record) return false;
+    this.retainedProcesses.delete(providerSessionId);
+    return true;
+  }
+
   /** Local, Docker, and OpenSandbox process ids address an in-memory table on one worker
    * session object. They are not valid durable locators for the independently
    * scheduled reaper, so their yielded handles stay turn-owned until terminal
@@ -2404,9 +2458,7 @@ export class RoutingSandboxSession implements RoutableBackendSession {
   async writePlacementPrivate(args: unknown): Promise<unknown> {
     const input = placementPrivateWrite(args);
     return this.dispatch("writePlacementPrivate", false, async (session, backend) => {
-      if (session.writePlacementPrivate) return await session.writePlacementPrivate(input);
-      if (session.writeFile) return await session.writeFile(input);
-      return await streamPlacementPrivateFile(session, input, backend.kind);
+      return await writePlacementPrivateOnBackend(session, input, backend.kind);
     });
   }
 
@@ -2414,36 +2466,8 @@ export class RoutingSandboxSession implements RoutableBackendSession {
    * control operation cannot be repurposed into a generic mutation bypass. */
   async deletePlacementPrivate(path: string, runAs?: string): Promise<void> {
     const privatePath = placementPrivatePath(path);
-    await this.dispatch("deletePlacementPrivate", false, async (session) => {
-      if (session.deletePlacementPrivate) {
-        await session.deletePlacementPrivate(privatePath, runAs);
-        return;
-      }
-      const args = {
-        cmd: `rm -f ${shellSingleQuote(privatePath)}`,
-        ...(runAs ? { runAs } : {}),
-      };
-      if (session.exec) {
-        const result = await session.exec(args);
-        if (
-          result &&
-          typeof result === "object" &&
-          typeof (result as { exitCode?: unknown }).exitCode === "number" &&
-          (result as { exitCode: number }).exitCode !== 0
-        ) {
-          throw new Error("placement-private cleanup failed");
-        }
-        return;
-      }
-      if (session.execCommand) {
-        const result = await session.execCommand(args);
-        const exitCode = parseExecBannerExitCode(result);
-        if (exitCode !== null && exitCode !== 0) {
-          throw new Error("placement-private cleanup failed");
-        }
-        return;
-      }
-      throw new RoutingUnsupportedError("deletePlacementPrivate", this.cached?.kind ?? "unknown");
+    await this.dispatch("deletePlacementPrivate", false, async (session, backend) => {
+      await deletePlacementPrivateOnBackend(session, privatePath, runAs, backend.kind);
     });
   }
 
@@ -2455,7 +2479,7 @@ export class RoutingSandboxSession implements RoutableBackendSession {
   ): Promise<WorkspaceFileImportReceipt> {
     return await this.dispatch("importWorkspaceFile", true, async (session, backend) => {
       const channel = new SandboxChannelAService({
-        session: session as ChannelASession,
+        session: withPlacementPrivateControl(session, backend.kind),
         workspaceRoot: input.workspaceRoot,
         ...(backend.kind === "selfhosted"
           ? { providerPathMode: "workspace-relative" as const }
@@ -2476,7 +2500,7 @@ export class RoutingSandboxSession implements RoutableBackendSession {
   ): Promise<readonly WorkspaceFileImportReceipt[]> {
     return await this.dispatch("importWorkspaceFiles", true, async (session, backend) => {
       const channel = new SandboxChannelAService({
-        session: session as ChannelASession,
+        session: withPlacementPrivateControl(session, backend.kind),
         workspaceRoot: input.workspaceRoot,
         ...(backend.kind === "selfhosted"
           ? { providerPathMode: "workspace-relative" as const }
@@ -2805,12 +2829,87 @@ function placementPrivateWrite(value: unknown): {
   };
 }
 
+/** Preserve private staging on the already resolved provider. Calling the
+ * routing proxy here would re-resolve between staging, import and cleanup. */
+function withPlacementPrivateControl(
+  session: RoutableBackendSession,
+  backendKind: string,
+): ChannelASession {
+  return new Proxy(session, {
+    get(target, property) {
+      if (property === "writePlacementPrivate") {
+        return async (args: unknown) =>
+          await writePlacementPrivateOnBackend(target, placementPrivateWrite(args), backendKind);
+      }
+      if (property === "deletePlacementPrivate") {
+        return async (path: string, runAs?: string) =>
+          await deletePlacementPrivateOnBackend(
+            target,
+            placementPrivatePath(path),
+            runAs,
+            backendKind,
+          );
+      }
+      const value: unknown = Reflect.get(target, property, target);
+      return typeof value === "function" ? value.bind(target) : value;
+    },
+  }) as ChannelASession;
+}
+
+async function writePlacementPrivateOnBackend(
+  session: RoutableBackendSession,
+  input: ReturnType<typeof placementPrivateWrite>,
+  backendKind: string,
+): Promise<unknown> {
+  if (session.writePlacementPrivate) return await session.writePlacementPrivate(input);
+  if (session.writeFile) return await session.writeFile(input);
+  return await streamPlacementPrivateFile(session, input, backendKind);
+}
+
+async function deletePlacementPrivateOnBackend(
+  session: RoutableBackendSession,
+  privatePath: string,
+  runAs: string | undefined,
+  backendKind: string,
+): Promise<void> {
+  if (session.deletePlacementPrivate) {
+    await session.deletePlacementPrivate(privatePath, runAs);
+    return;
+  }
+  const args = {
+    cmd: `rm -f ${shellSingleQuote(privatePath)}`,
+    ...(runAs ? { runAs } : {}),
+  };
+  if (session.exec) {
+    const result = await session.exec(args);
+    if (
+      result &&
+      typeof result === "object" &&
+      typeof (result as { exitCode?: unknown }).exitCode === "number" &&
+      (result as { exitCode: number }).exitCode !== 0
+    ) {
+      throw new Error("placement-private cleanup failed");
+    }
+    return;
+  }
+  if (session.execCommand) {
+    const result = await session.execCommand(args);
+    const exitCode = parseExecBannerExitCode(result);
+    if (exitCode !== null && exitCode !== 0) {
+      throw new Error("placement-private cleanup failed");
+    }
+    return;
+  }
+  throw new RoutingUnsupportedError("deletePlacementPrivate", backendKind);
+}
+
 async function streamPlacementPrivateFile(
   session: RoutableBackendSession,
   input: ReturnType<typeof placementPrivateWrite>,
   backendKind: string,
 ): Promise<void> {
-  if (!session.exec || !session.writeStdin) {
+  const exec = session.exec?.bind(session) ?? session.execCommand?.bind(session);
+  if (!exec || !session.writeStdin) {
     throw new RoutingUnsupportedError("writePlacementPrivate", backendKind);
   }
   const bytes = typeof input.content === "string" ? Buffer.from(input.content) : input.content;
@@ -2821,7 +2920,7 @@ async function streamPlacementPrivateFile(
     ...(input.createParents ? [`install -d -m 0700 -- ${shellSingleQuote(parent)}`] : []),
   ];
   if (bytes.byteLength === 0) {
-    const result = await session.exec({
+    const result = await exec({
       cmd: [
         ...prelude,
         `: > ${shellSingleQuote(input.path)}`,
@@ -2849,7 +2948,7 @@ async function streamPlacementPrivateFile(
     `chmod 0600 -- ${shellSingleQuote(input.path)}`,
     `printf %s ${shellSingleQuote(marker)}`,
   ].join("; ");
-  const started = await session.exec({
+  const started = await exec({
     cmd: command,
     ...(input.runAs ? { runAs: input.runAs } : {}),
     yieldTimeMs: 250,

package/src/sandbox/run-credentials.ts

@@ -18,7 +18,10 @@ const MAX_CREDENTIAL_FILES = 64;
 const MAX_CREDENTIAL_FILE_BYTES = 1024 * 1024;
 const MAX_TOTAL_MATERIAL_BYTES = 4 * 1024 * 1024;
 const MAX_AUTH_NEEDED_NOTICES = 32;
-const WRITE_CHUNK_BYTES = 24 * 1024;
+// Encoded chunks must leave room for both the cancellation shell and the SDK
+// run-as wrapper, each of which repeats and quotes the command. Keep chunks
+// base64-aligned and bounded without dropping any credential bytes.
+const WRITE_CHUNK_BYTES = 1536;
 const COMMAND_OK_MARKER = "__OPENGENI_RUN_CREDENTIAL_COMMAND_OK__";
 const AUTH_NEEDED_REASONS = new Set([
   "missing_connection",
@@ -496,27 +499,54 @@ async function runCredentialCommand(
 
 async function writeCredentialFile(
   session: RunCredentialCommandSession,
-  path: string,
+  directory: string,
+  relativePath: string,
   content: string,
   commandRunner?: RunCredentialCommandRunner,
 ): Promise<void> {
-  await runCredentialCommand(session, `: > ${shellQuote(path)}`, commandRunner);
+  await runCredentialCommand(
+    session,
+    credentialFileCommand(directory, relativePath, ': > "$_opengeni_credential_path"'),
+    commandRunner,
+  );
   const encoded = base64(content);
   for (let offset = 0; offset < encoded.length; offset += WRITE_CHUNK_BYTES) {
     const chunk = encoded.slice(offset, offset + WRITE_CHUNK_BYTES);
     await runCredentialCommand(
       session,
-      `printf %s ${shellQuote(chunk)} | ${portableBase64DecodeCommand()} >> ${shellQuote(path)}`,
+      credentialFileCommand(
+        directory,
+        relativePath,
+        `printf %s ${shellQuote(chunk)} | ${portableBase64DecodeCommand()} >> "$_opengeni_credential_path"`,
+      ),
       commandRunner,
     );
   }
   await runCredentialCommand(
     session,
-    `[ "$(wc -c < ${shellQuote(path)} | tr -d '[:space:]')" = ${shellQuote(String(byteLength(content)))} ]`,
+    credentialFileCommand(
+      directory,
+      relativePath,
+      `[ "$(wc -c < "$_opengeni_credential_path" | tr -d '[:space:]')" = ${shellQuote(String(byteLength(content)))} ]`,
+    ),
     commandRunner,
   );
 }
 
+function credentialFileCommand(directory: string, relativePath: string, command: string): string {
+  // Avoid multiplying accepted path quotes through nested shell wrappers. The
+  // existing outer Bash decodes only literal UTF-8 bytes, including trailing
+  // newlines, without evaluating host-provided text or invoking a path decoder.
+  const escapedPath = [...Buffer.from(relativePath, "utf8")]
+    .map((byte) => `\\x${byte.toString(16).padStart(2, "0")}`)
+    .join("");
+  return [
+    `printf -v _opengeni_credential_path %b ${shellQuote(escapedPath)}`,
+    `_opengeni_credential_path=${shellQuote(directory)}/"$_opengeni_credential_path"`,
+    command,
+  ].join("\n");
+}
+
 export type MaterializeRunCredentialsOptions = {
   sessionId: string;
   attemptId: string;
@@ -574,7 +604,8 @@ export async function materializeRunCredentials(
   ];
   await writeCredentialFile(
     session,
-    `${stage}/env`,
+    stage,
+    "env",
     environmentLines.length > 0 ? `${environmentLines.join("\n")}\n` : "",
     options.commandRunner,
   );
@@ -585,13 +616,20 @@ export async function materializeRunCredentials(
   );
 
   for (const file of material.files) {
-    const target = `${stage}/files/${file.path}`;
-    const parent = target.slice(0, target.lastIndexOf("/"));
-    await runCredentialCommand(session, `mkdir -p -- ${shellQuote(parent)}`, options.commandRunner);
-    await writeCredentialFile(session, target, file.content, options.commandRunner);
+    const directory = `${stage}/files`;
+    await runCredentialCommand(
+      session,
+      credentialFileCommand(directory, file.path, 'mkdir -p -- "${_opengeni_credential_path%/*}"'),
+      options.commandRunner,
+    );
+    await writeCredentialFile(session, directory, file.path, file.content, options.commandRunner);
     await runCredentialCommand(
       session,
-      `chmod -- ${shellQuote(file.mode ?? "0600")} ${shellQuote(target)}`,
+      credentialFileCommand(
+        directory,
+        file.path,
+        `chmod -- ${shellQuote(file.mode ?? "0600")} "$_opengeni_credential_path"`,
+      ),
       options.commandRunner,
     );
   }
@@ -599,25 +637,30 @@ export async function materializeRunCredentials(
   await runCredentialCommand(
     session,
     [
+      // Bind validated identifiers once rather than repeatedly expanding long
+      // paths through cancellation and SDK run-as shell quoting.
+      `_opengeni_credential_root=${shellQuote(root)}`,
+      `_opengeni_credential_version_name=${shellQuote(versionName)}`,
+      `_opengeni_credential_prefix=${shellQuote(prefix)}`,
       ...pointerLockAcquireCommands(root),
-      `previous=$(cat -- ${shellQuote(`${root}/current`)} 2>/dev/null || :)`,
-      `mv -- ${shellQuote(stage)} ${shellQuote(version)}`,
-      `printf '%s\\n' ${shellQuote(versionName)} > ${shellQuote(`${root}/.next`)}`,
-      `mv -f -- ${shellQuote(`${root}/.next`)} ${shellQuote(`${root}/current`)}`,
+      'previous=$(cat -- "$_opengeni_credential_root/current" 2>/dev/null || :)',
+      'mv -- "$_opengeni_credential_root/versions/.stage-$_opengeni_credential_version_name" "$_opengeni_credential_root/versions/$_opengeni_credential_version_name"',
+      'printf \'%s\\n\' "$_opengeni_credential_version_name" > "$_opengeni_credential_root/.next"',
+      'mv -f -- "$_opengeni_credential_root/.next" "$_opengeni_credential_root/current"',
       ...(options.prunePreviousGenerations
         ? [
-            `find ${shellQuote(`${root}/versions`)} -mindepth 1 -maxdepth 1 ! -name ${shellQuote(versionName)} -exec rm -rf -- {} +`,
+            'find "$_opengeni_credential_root/versions" -mindepth 1 -maxdepth 1 ! -name "$_opengeni_credential_version_name" -exec rm -rf -- {} +',
           ]
         : options.pruneOtherAttempts
           ? [
-              `find ${shellQuote(`${root}/versions`)} -mindepth 1 -maxdepth 1 ! -name ${shellQuote(`${attemptId}-${options.executionGeneration}-*`)} -exec rm -rf -- {} +`,
+              'find "$_opengeni_credential_root/versions" -mindepth 1 -maxdepth 1 ! -name "${_opengeni_credential_prefix}*" -exec rm -rf -- {} +',
             ]
           : options.pruneSupersededGenerations
             ? [
-                `for candidate in ${shellQuote(`${root}/versions`)}/${prefix}*; do`,
+                'for candidate in "$_opengeni_credential_root/versions/$_opengeni_credential_prefix"*; do',
                 `  [ -e "$candidate" ] || continue`,
                 `  name=\${candidate##*/}`,
-                `  [ "$name" = ${shellQuote(versionName)} ] || [ "$name" = "$previous" ] || rm -rf -- "$candidate"`,
+                '  [ "$name" = "$_opengeni_credential_version_name" ] || [ "$name" = "$previous" ] || rm -rf -- "$candidate"',
                 "done",
               ]
             : []),