@opengeni/runtime 3.0.1 → 4.0.0-canary.35585322809001

package/src/sandbox/providers/modal-legacy-command-control.ts

@@ -0,0 +1,351 @@
+import { posix } from "node:path";
+import { ModalClient } from "modal";
+import { ModalLegacyProviderCommand } from "@opengeni/contracts";
+import type { ChannelAExecArgs } from "../channel-a";
+import { shellQuote } from "@openai/agents-core/sandbox/internal";
+
+type ControlPlane = Pick<
+  ModalClient["cpClient"],
+  | "sandboxGetTaskId"
+  | "containerExec"
+  | "containerExecGetOutput"
+  | "containerExecPutInput"
+  | "taskGetCommandRouterAccess"
+>;
+
+type CommandClient = Pick<ModalClient, "cpClient" | "version"> &
+  Partial<Pick<ModalClient, "profile" | "logger">>;
+type GrpcMiddleware = NonNullable<
+  NonNullable<ConstructorParameters<typeof ModalClient>[0]>["grpcMiddleware"]
+>[number];
+/** This dedicated command client never retries a request behind the caller's
+ * back and preserves the exact cancellation signal, including streaming RPCs. */
+export const modalCommandAbortMiddleware: GrpcMiddleware = async function* (call, options) {
+  options.signal?.throwIfAborted();
+  return yield* call.next(call.request, options);
+};
+
+// Version-pinned compatibility boundary: 0.9.0's retryMiddleware drops signal
+// for streaming and retries=0. Its public custom middleware runs OUTSIDE that
+// branch, so cannot repair it. Override only this dedicated instance's factory
+// hook before construction; never mutate the SDK's global prototype or use
+// negative/fractional retry counts to manipulate its branch conditions.
+type PublicModalClient = Pick<ModalClient, keyof ModalClient>;
+const ModalClientBase: new (
+  options?: ConstructorParameters<typeof ModalClient>[0],
+) => PublicModalClient = ModalClient;
+class ModalCommandClient extends ModalClientBase {
+  retryMiddleware(): GrpcMiddleware {
+    return modalCommandAbortMiddleware;
+  }
+}
+const controlClients = new WeakMap<object, PublicModalClient>();
+
+export function commandControlPlane(client: CommandClient): ControlPlane {
+  // Minimal injected control planes are used by deterministic unit tests.
+  // Production installation requires the authenticated SDK profile below.
+  if (!client.profile) return client.cpClient;
+  if (typeof Reflect.get(ModalClient.prototype, "retryMiddleware") !== "function")
+    throw new Error("Verified Modal command cancellation contract is unavailable");
+  let controlled = controlClients.get(client);
+  if (!controlled) {
+    const profile = client.profile;
+    if (!profile.tokenId || !profile.tokenSecret)
+      throw new Error("Modal command control requires its original authenticated profile");
+    controlled = new ModalCommandClient({
+      tokenId: profile.tokenId,
+      tokenSecret: profile.tokenSecret,
+      endpoint: profile.serverUrl,
+      environment: profile.environment ?? "",
+      ...(profile.maxThrottleWaitSecs !== undefined
+        ? { maxThrottleWaitSecs: profile.maxThrottleWaitSecs }
+        : {}),
+      ...(client.logger ? { logger: client.logger } : {}),
+    });
+    controlClients.set(client, controlled);
+  }
+  return controlled.cpClient;
+}
+export type ModalCommandStream = "stdout" | "stderr";
+export type ModalProviderCommand = ModalLegacyProviderCommand;
+export type ModalProviderOutputPage = {
+  command: ModalProviderCommand;
+  chunks: Array<{ stream: ModalCommandStream; chunkId: string; text: string }>;
+  exitCode: number | null;
+  streamFidelity: "separate" | "merged";
+};
+
+// A provider batch may end halfway through a UTF-8 character. Keep only that
+// unfinished suffix in protected cursor state; invalid complete bytes still
+// use the usual replacement-character decoding policy.
+export function decodePage(prior: string, input: Uint8Array[], terminal: boolean) {
+  const bytes = Buffer.concat([Buffer.from(prior, "base64"), ...input]);
+  let boundary = bytes.length;
+  if (!terminal) {
+    for (let start = Math.max(0, bytes.length - 3); start < bytes.length; start++) {
+      const first = bytes[start]!;
+      const width =
+        first >= 0xc2 && first <= 0xdf
+          ? 2
+          : first >= 0xe0 && first <= 0xef
+            ? 3
+            : first >= 0xf0 && first <= 0xf4
+              ? 4
+              : 0;
+      if (!width || bytes.length - start >= width) continue;
+      const suffix = bytes.subarray(start + 1);
+      if (!suffix.every((byte) => byte >= 0x80 && byte <= 0xbf)) continue;
+      const second = suffix[0];
+      if (
+        second !== undefined &&
+        ((first === 0xe0 && second < 0xa0) ||
+          (first === 0xed && second > 0x9f) ||
+          (first === 0xf0 && second < 0x90) ||
+          (first === 0xf4 && second > 0x8f))
+      )
+        continue;
+      boundary = start;
+      break;
+    }
+  }
+  return {
+    text: bytes.subarray(0, boundary).toString("utf8"),
+    remainder: bytes.subarray(boundary).toString("base64"),
+  };
+}
+
+/** Version-pinned Modal control-plane boundary. Execution identity, output
+ * batches and exit status come from authenticated provider RPCs, never sandbox
+ * files or an SDK object's in-memory process map. Callers must store the
+ * returned locator/cursors outside the command's write authority. */
+export class ModalCommandControl {
+  private constructor(
+    private readonly client: ControlPlane,
+    private readonly sandboxIdentity: string | (() => string),
+    private readonly root: string,
+    private readonly environment: Record<string, string> | (() => Record<string, string>) = {},
+  ) {}
+
+  static forSandbox(
+    client: CommandClient,
+    sandboxId: string | (() => string),
+    root: string,
+    environment: Record<string, string> | (() => Record<string, string>) = {},
+  ): ModalCommandControl {
+    if (client.version() !== "0.9.0")
+      throw new Error("Modal command control requires the verified 0.9.0 SDK contract");
+    return new ModalCommandControl(commandControlPlane(client), sandboxId, root, environment);
+  }
+
+  private get sandboxId(): string {
+    return typeof this.sandboxIdentity === "function"
+      ? this.sandboxIdentity()
+      : this.sandboxIdentity;
+  }
+
+  async start(args: ChannelAExecArgs, signal?: AbortSignal): Promise<ModalProviderCommand> {
+    signal?.throwIfAborted();
+    const workdir = posix.resolve(this.root, args.workdir ?? this.root);
+    if (workdir !== this.root && !workdir.startsWith(`${this.root.replace(/\/$/u, "")}/`))
+      throw new Error("Command workdir is outside the sandbox workspace");
+    // Hydration replaces the SDK sandbox. Freeze its current identity for this
+    // start so an asynchronous replacement cannot relabel the returned locator.
+    const sandboxId = this.sandboxId;
+    const task = await this.client.sandboxGetTaskId({ sandboxId }, signal ? { signal } : undefined);
+    if (!task.taskId || task.taskResult) throw new Error("Modal command task is unavailable");
+    if (sandboxId !== this.sandboxId)
+      throw new Error("Modal sandbox changed during command preparation");
+    const login = args.shell ? (args.login ?? true) : false;
+    let command = [args.shell ?? "/bin/sh", login ? "-lc" : "-c", args.cmd];
+    if (args.runAs) {
+      const user = shellQuote(args.runAs);
+      const invocation = command.map(shellQuote).join(" ");
+      const script = `if [ "$(id -u)" = ${user} ] || [ "$(id -un 2>/dev/null)" = ${user} ]; then exec ${invocation}; elif [ "$(id -u)" = 0 ]; then exec su -s /bin/sh ${user} -c ${shellQuote(`exec ${invocation}`)}; else exec sudo -n -u ${user} -- ${invocation}; fi`;
+      command = ["/bin/sh", "-c", script];
+    }
+    const environment =
+      typeof this.environment === "function" ? this.environment() : this.environment;
+    if (Object.keys(environment).length) {
+      command = [
+        "/usr/bin/env",
+        "--",
+        ...Object.entries(environment).map(([key, value]) => `${key}=${value}`),
+        ...command,
+      ];
+    }
+    // Do not retry this mutating start on an ambiguous transport error: this
+    // provider API assigns the execution id in its response, not in our request.
+    const result = await this.client.containerExec(
+      {
+        taskId: task.taskId,
+        command,
+        terminateContainerOnExit: false,
+        runtimeDebug: false,
+        stdoutOutput: 2,
+        stderrOutput: 2,
+        timeoutSecs: 0,
+        workdir,
+        secretIds: [],
+        ...(args.tty
+          ? {
+              ptyInfo: {
+                enabled: true,
+                winszRows: 24,
+                winszCols: 80,
+                envTerm: "xterm",
+                envColorterm: "",
+                envTermProgram: "",
+                ptyType: 1,
+                noTerminateOnIdleStdin: true,
+              },
+            }
+          : {}),
+      },
+      { retries: 0, ...(signal ? { signal } : {}) },
+    );
+    if (!result.execId) throw new Error("Modal command start returned no execution identity");
+    return {
+      kind: "modal-control-v1",
+      sandboxId,
+      taskId: task.taskId,
+      execId: result.execId,
+      ...(args.tty ? { pty: true } : {}),
+      streams: {
+        stdout: { batchIndex: 0, utf8Remainder: "", exitCode: null },
+        stderr: { batchIndex: 0, utf8Remainder: "", exitCode: null },
+      },
+    };
+  }
+
+  async read(
+    command: ModalProviderCommand,
+    yieldTimeMs: number,
+    signal?: AbortSignal,
+  ): Promise<ModalProviderOutputPage> {
+    return this.readOutput(command, yieldTimeMs, signal);
+  }
+
+  /** Fixed internal probes own their cancellation controller and must drain
+   * both reads before relinquishing the surrounding provider-operation gate.
+   * Retained-command readers keep their existing signal/receipt contract.
+   */
+  async readProbe(
+    command: ModalProviderCommand,
+    yieldTimeMs: number,
+    cancellation: AbortController,
+  ): Promise<ModalProviderOutputPage> {
+    return this.readOutput(command, yieldTimeMs, cancellation.signal, (error) =>
+      cancellation.abort(error),
+    );
+  }
+
+  private async readOutput(
+    command: ModalProviderCommand,
+    yieldTimeMs: number,
+    signal?: AbortSignal,
+    cancelSiblings?: (error: unknown) => void,
+  ): Promise<ModalProviderOutputPage> {
+    signal?.throwIfAborted();
+    this.assertIdentity(command);
+    const next = structuredClone(command);
+    let failure: { error: unknown } | undefined;
+    const reads = (["stdout", "stderr"] as const)
+      .map(async (stream) => {
+        const cursor = command.streams[stream];
+        if (cursor.exitCode !== null) return null;
+        const descriptor = stream === "stdout" ? 1 : 2;
+        for await (const batch of this.client.containerExecGetOutput(
+          {
+            execId: command.execId,
+            timeout: Math.max(0, yieldTimeMs) / 1000,
+            lastBatchIndex: cursor.batchIndex,
+            fileDescriptor: descriptor,
+            getRawBytes: true,
+          },
+          signal ? { signal } : undefined,
+        )) {
+          if (batch.batchIndex <= cursor.batchIndex) continue;
+          if (!Number.isSafeInteger(batch.batchIndex))
+            throw new Error("Invalid Modal output cursor");
+          const terminal = batch.exitCode !== undefined;
+          if (terminal && !Number.isSafeInteger(batch.exitCode))
+            throw new Error("Invalid Modal exit status");
+          const decoded = decodePage(
+            cursor.utf8Remainder,
+            batch.items
+              .filter((item) => item.fileDescriptor === descriptor)
+              .map((item) => item.messageBytes),
+            terminal,
+          );
+          next.streams[stream] = {
+            batchIndex: batch.batchIndex,
+            utf8Remainder: decoded.remainder,
+            exitCode: terminal ? batch.exitCode! : null,
+          };
+          return {
+            stream,
+            chunkId: `modal:${command.execId}:${stream}:${cursor.batchIndex}:${batch.batchIndex}`,
+            text: decoded.text,
+          };
+        }
+        return null;
+      })
+      .map((read) =>
+        !cancelSiblings
+          ? read
+          : read.catch((error: unknown) => {
+              // A logical read owns both output streams. Promise.all alone releases
+              // its caller on the first rejection while a sibling RPC may still be
+              // running. Cancel that sibling, retain the original error, and drain
+              // both reads before returning control to mutation/capture ownership.
+              failure ??= { error };
+              cancelSiblings(error);
+              throw error;
+            }),
+      );
+    const pages = cancelSiblings
+      ? await Promise.allSettled(reads).then((results) => {
+          if (failure) throw failure.error;
+          return results.map((result) => {
+            if (result.status === "rejected") throw result.reason;
+            return result.value;
+          });
+        })
+      : await Promise.all(reads);
+    const stdoutExit = next.streams.stdout.exitCode;
+    const stderrExit = next.streams.stderr.exitCode;
+    if (stdoutExit !== null && stderrExit !== null && stdoutExit !== stderrExit)
+      throw new Error("Modal output streams disagree about command exit status");
+    return {
+      command: next,
+      streamFidelity: command.pty ? "merged" : "separate",
+      chunks: pages.filter((page): page is NonNullable<typeof page> => page !== null),
+      exitCode: stdoutExit !== null && stderrExit !== null ? stdoutExit : null,
+    };
+  }
+
+  async write(command: ModalProviderCommand, chars: string, messageIndex: number): Promise<void> {
+    this.assertIdentity(command);
+    if (!chars) return;
+    if (!Number.isSafeInteger(messageIndex) || messageIndex < 1)
+      throw new Error("Modal stdin requires a protected monotonic message index");
+    await this.client.containerExecPutInput(
+      {
+        execId: command.execId,
+        input: { message: Buffer.from(chars), messageIndex, eof: false },
+      },
+      { retries: 0 },
+    );
+  }
+
+  private assertIdentity(command: ModalProviderCommand): void {
+    ModalLegacyProviderCommand.parse(command);
+    if (
+      command.kind !== "modal-control-v1" ||
+      command.sandboxId !== this.sandboxId ||
+      !command.taskId ||
+      !command.execId
+    )
+      throw new Error("Modal command locator does not match the original sandbox");
+  }
+}

package/src/sandbox/providers/modal-materialization-verification.ts

@@ -0,0 +1,122 @@
+import {
+  SandboxMaterializationVerificationError,
+  retainMaterializationVerificationDiagnostic,
+  type MaterializationVerificationDiagnostic,
+} from "../materialization-verification-error";
+import type { ModalCommandControl, ModalProviderCommand } from "./modal-command-control";
+
+const MARKER = "__OPENGENI_MATERIALIZED_PATH_VISIBLE__";
+const OUTPUT_LIMIT = 16 * 1024;
+
+/** A fixed read-only probe, not an admitted user command. Advance its private
+ * provider cursor locally; never borrow the surrounding mutation's retained
+ * handle or relax retained-command persistence requirements. The caller keeps
+ * the original backend and provider-operation gate for the entire observation.
+ */
+export async function verifyModalMaterializedPath(
+  control: Pick<ModalCommandControl, "start" | "readProbe">,
+  path: string,
+  workdir: string,
+  pending: Set<AbortController>,
+  observationTimeoutMs = 30_000,
+): Promise<void> {
+  const quote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`;
+  const command = `test -e ${quote(path)} && printf %s ${quote(MARKER)}`;
+  const cancellation = new AbortController();
+  const deadlineReason = new Error("Materialization visibility observation deadline exceeded");
+  let providerCommand: ModalProviderCommand | undefined;
+  let stdout = "";
+  let stderr = "";
+  const diagnostic = (
+    reason: MaterializationVerificationDiagnostic["reason"],
+    exitCode: number | null = null,
+  ): MaterializationVerificationDiagnostic => ({
+    reason,
+    path,
+    workdir,
+    command,
+    output: (stdout + stderr).slice(0, OUTPUT_LIMIT),
+    exitCode,
+    providerSessionId: null,
+    ...(providerCommand
+      ? {
+          providerExecution: {
+            sandboxId: providerCommand.sandboxId,
+            taskId: providerCommand.taskId,
+            execId: providerCommand.execId,
+          },
+        }
+      : {}),
+  });
+  const timer = setTimeout(() => {
+    cancellation.abort(deadlineReason);
+  }, observationTimeoutMs);
+  timer.unref();
+  pending.add(cancellation);
+  try {
+    providerCommand = await control.start(
+      { cmd: command, workdir, shell: "sh", login: false, tty: false },
+      cancellation.signal,
+    );
+    for (;;) {
+      cancellation.signal.throwIfAborted();
+      const page = await control.readProbe(providerCommand, 1_000, cancellation);
+      // Cancellation ends observation, not the process. Never accept a late
+      // success after the attempt requested cancellation.
+      cancellation.signal.throwIfAborted();
+      if (
+        page.command.sandboxId !== providerCommand.sandboxId ||
+        page.command.taskId !== providerCommand.taskId ||
+        page.command.execId !== providerCommand.execId
+      ) {
+        throw new SandboxMaterializationVerificationError(diagnostic("invalid_response"));
+      }
+      providerCommand = page.command;
+      for (const chunk of page.chunks) {
+        if (chunk.stream === "stdout") stdout += chunk.text;
+        else stderr += chunk.text;
+      }
+      if (stdout.length + stderr.length > OUTPUT_LIMIT) {
+        throw new SandboxMaterializationVerificationError(diagnostic("invalid_response"));
+      }
+      if (page.exitCode === null) {
+        // An empty provider page may return immediately. Yield to cancellation
+        // and the deadline instead of starving timers with resolved promises.
+        await new Promise<void>((resolve) => setImmediate(resolve));
+        continue;
+      }
+      if (page.exitCode === 0 && stdout === MARKER) return;
+      throw new SandboxMaterializationVerificationError(
+        diagnostic(
+          page.exitCode === 1
+            ? "path_not_visible"
+            : page.exitCode === 0
+              ? "invalid_response"
+              : "command_failed",
+          page.exitCode,
+        ),
+      );
+    }
+  } catch (error) {
+    if (cancellation.signal.reason === deadlineReason) {
+      throw new SandboxMaterializationVerificationError(
+        {
+          ...diagnostic("command_pending"),
+          causeMessage: "Observation deadline exceeded; provider process completion is unconfirmed",
+        },
+        { cause: error },
+      );
+    }
+    if (!(error instanceof SandboxMaterializationVerificationError)) {
+      retainMaterializationVerificationDiagnostic(error, {
+        ...diagnostic("command_error"),
+        causeMessage: error instanceof Error ? error.message : String(error),
+      });
+    }
+    throw error;
+  } finally {
+    cancellation.abort(new Error("Materialization visibility observation finished"));
+    clearTimeout(timer);
+    pending.delete(cancellation);
+  }
+}

package/src/sandbox/providers/modal.ts

@@ -796,7 +796,7 @@ type ModalClientLike = InstanceType<ModalModule["ModalClient"]>;
 // a `ModalImageSelector.fromImage(...)`. `build` is synchronous and modal is imported
 // lazily (never loaded for non-modal backends), so resolution can't happen inside
 // `build`; the worker awaits `ensureModalRegistryImage` at boot for global refs and
-// at turn time for pack-scoped refs, then `build` reads the settled result. Modal
+
 // images are lazy, workspace-scoped definitions, so an image built by this module's
 // client is usable by the ModalSandboxClient's own client.
 
@@ -819,14 +819,6 @@ function registryImageCacheKey(settings: Settings): string {
   ].join("|");
 }
 
-/**
- * Resolve + cache the private-registry Modal image. No-op unless BOTH
- * `modalImageRef` and `modalImageRegistrySecret` are set. Memoized per
- * (imageRef, secret, environment) so it runs once per worker process. Awaited at
- * worker boot for the deployment-global image and at turn time for pack-scoped
- * images BEFORE the first sandbox using that ref is created; `build` then reads the
- * resolved image and otherwise falls back to the public `fromTag` path.
- */
 export async function ensureModalRegistryImage(
   settings: Settings,
   loadModal: ModalModuleLoader = defaultModalLoader,