@opengeni/runtime 3.0.0-canary.0 → 3.0.1-canary.0

package/src/sandbox/providers/modal-command-control.ts

@@ -217,12 +217,36 @@ export class ModalCommandControl {
     command: ModalProviderCommand,
     yieldTimeMs: number,
     signal?: AbortSignal,
+  ): Promise<ModalProviderOutputPage> {
+    return this.readOutput(command, yieldTimeMs, signal);
+  }
+
+  /** Fixed internal probes own their cancellation controller and must drain
+   * both reads before relinquishing the surrounding provider-operation gate.
+   * Retained-command readers keep their existing signal/receipt contract.
+   */
+  async readProbe(
+    command: ModalProviderCommand,
+    yieldTimeMs: number,
+    cancellation: AbortController,
+  ): Promise<ModalProviderOutputPage> {
+    return this.readOutput(command, yieldTimeMs, cancellation.signal, (error) =>
+      cancellation.abort(error),
+    );
+  }
+
+  private async readOutput(
+    command: ModalProviderCommand,
+    yieldTimeMs: number,
+    signal?: AbortSignal,
+    cancelSiblings?: (error: unknown) => void,
   ): Promise<ModalProviderOutputPage> {
     signal?.throwIfAborted();
     this.assertIdentity(command);
     const next = structuredClone(command);
-    const pages = await Promise.all(
-      (["stdout", "stderr"] as const).map(async (stream) => {
+    let failure: { error: unknown } | undefined;
+    const reads = (["stdout", "stderr"] as const)
+      .map(async (stream) => {
         const cursor = command.streams[stream];
         if (cursor.exitCode !== null) return null;
         const descriptor = stream === "stdout" ? 1 : 2;
@@ -261,8 +285,29 @@ export class ModalCommandControl {
           };
         }
         return null;
-      }),
-    );
+      })
+      .map((read) =>
+        !cancelSiblings
+          ? read
+          : read.catch((error: unknown) => {
+              // A logical read owns both output streams. Promise.all alone releases
+              // its caller on the first rejection while a sibling RPC may still be
+              // running. Cancel that sibling, retain the original error, and drain
+              // both reads before returning control to mutation/capture ownership.
+              failure ??= { error };
+              cancelSiblings(error);
+              throw error;
+            }),
+      );
+    const pages = cancelSiblings
+      ? await Promise.allSettled(reads).then((results) => {
+          if (failure) throw failure.error;
+          return results.map((result) => {
+            if (result.status === "rejected") throw result.reason;
+            return result.value;
+          });
+        })
+      : await Promise.all(reads);
     const stdoutExit = next.streams.stdout.exitCode;
     const stderrExit = next.streams.stderr.exitCode;
     if (stdoutExit !== null && stderrExit !== null && stdoutExit !== stderrExit)

package/src/sandbox/providers/modal-command-session.ts

@@ -11,6 +11,7 @@ import {
   admittedProviderCommandHandle,
 } from "../provider-command-session";
 import type { ModalCommandControl, ModalProviderCommand } from "./modal-command-control";
+import { verifyModalMaterializedPath } from "./modal-materialization-verification";
 
 type Entry = {
   command: ModalProviderCommand;
@@ -33,6 +34,7 @@ export function installModalCommandSession(
   session: ProviderCommandSession & {
     execCommand?: ChannelASession["execCommand"];
     writeStdin?: ChannelASession["writeStdin"];
+    verifyMaterializedPath?: (path: string, workdir: string) => Promise<void>;
   },
   control: ModalCommandControl,
 ): void {
@@ -41,6 +43,8 @@ export function installModalCommandSession(
   const originalWrite = session.writeStdin?.bind(session);
   const cancelLegacyStart = session.cancelPendingExecCommand?.bind(session);
   const pendingStarts = new Set<AbortController>();
+  session.verifyMaterializedPath = (path, workdir) =>
+    verifyModalMaterializedPath(control, path, workdir, pendingStarts);
   const entries = new Map<number, Entry>();
   // Current SDK setup commands are not retained/admitted commands. Give their
   // live observer handles a disjoint, adapter-local range; never use a missing

package/src/sandbox/providers/modal-materialization-verification.ts

@@ -0,0 +1,122 @@
+import {
+  SandboxMaterializationVerificationError,
+  retainMaterializationVerificationDiagnostic,
+  type MaterializationVerificationDiagnostic,
+} from "../materialization-verification-error";
+import type { ModalCommandControl, ModalProviderCommand } from "./modal-command-control";
+
+const MARKER = "__OPENGENI_MATERIALIZED_PATH_VISIBLE__";
+const OUTPUT_LIMIT = 16 * 1024;
+
+/** A fixed read-only probe, not an admitted user command. Advance its private
+ * provider cursor locally; never borrow the surrounding mutation's retained
+ * handle or relax retained-command persistence requirements. The caller keeps
+ * the original backend and provider-operation gate for the entire observation.
+ */
+export async function verifyModalMaterializedPath(
+  control: Pick<ModalCommandControl, "start" | "readProbe">,
+  path: string,
+  workdir: string,
+  pending: Set<AbortController>,
+  observationTimeoutMs = 30_000,
+): Promise<void> {
+  const quote = (value: string) => `'${value.replaceAll("'", `'"'"'`)}'`;
+  const command = `test -e ${quote(path)} && printf %s ${quote(MARKER)}`;
+  const cancellation = new AbortController();
+  const deadlineReason = new Error("Materialization visibility observation deadline exceeded");
+  let providerCommand: ModalProviderCommand | undefined;
+  let stdout = "";
+  let stderr = "";
+  const diagnostic = (
+    reason: MaterializationVerificationDiagnostic["reason"],
+    exitCode: number | null = null,
+  ): MaterializationVerificationDiagnostic => ({
+    reason,
+    path,
+    workdir,
+    command,
+    output: (stdout + stderr).slice(0, OUTPUT_LIMIT),
+    exitCode,
+    providerSessionId: null,
+    ...(providerCommand
+      ? {
+          providerExecution: {
+            sandboxId: providerCommand.sandboxId,
+            taskId: providerCommand.taskId,
+            execId: providerCommand.execId,
+          },
+        }
+      : {}),
+  });
+  const timer = setTimeout(() => {
+    cancellation.abort(deadlineReason);
+  }, observationTimeoutMs);
+  timer.unref();
+  pending.add(cancellation);
+  try {
+    providerCommand = await control.start(
+      { cmd: command, workdir, shell: "sh", login: false, tty: false },
+      cancellation.signal,
+    );
+    for (;;) {
+      cancellation.signal.throwIfAborted();
+      const page = await control.readProbe(providerCommand, 1_000, cancellation);
+      // Cancellation ends observation, not the process. Never accept a late
+      // success after the attempt requested cancellation.
+      cancellation.signal.throwIfAborted();
+      if (
+        page.command.sandboxId !== providerCommand.sandboxId ||
+        page.command.taskId !== providerCommand.taskId ||
+        page.command.execId !== providerCommand.execId
+      ) {
+        throw new SandboxMaterializationVerificationError(diagnostic("invalid_response"));
+      }
+      providerCommand = page.command;
+      for (const chunk of page.chunks) {
+        if (chunk.stream === "stdout") stdout += chunk.text;
+        else stderr += chunk.text;
+      }
+      if (stdout.length + stderr.length > OUTPUT_LIMIT) {
+        throw new SandboxMaterializationVerificationError(diagnostic("invalid_response"));
+      }
+      if (page.exitCode === null) {
+        // An empty provider page may return immediately. Yield to cancellation
+        // and the deadline instead of starving timers with resolved promises.
+        await new Promise<void>((resolve) => setImmediate(resolve));
+        continue;
+      }
+      if (page.exitCode === 0 && stdout === MARKER) return;
+      throw new SandboxMaterializationVerificationError(
+        diagnostic(
+          page.exitCode === 1
+            ? "path_not_visible"
+            : page.exitCode === 0
+              ? "invalid_response"
+              : "command_failed",
+          page.exitCode,
+        ),
+      );
+    }
+  } catch (error) {
+    if (cancellation.signal.reason === deadlineReason) {
+      throw new SandboxMaterializationVerificationError(
+        {
+          ...diagnostic("command_pending"),
+          causeMessage: "Observation deadline exceeded; provider process completion is unconfirmed",
+        },
+        { cause: error },
+      );
+    }
+    if (!(error instanceof SandboxMaterializationVerificationError)) {
+      retainMaterializationVerificationDiagnostic(error, {
+        ...diagnostic("command_error"),
+        causeMessage: error instanceof Error ? error.message : String(error),
+      });
+    }
+    throw error;
+  } finally {
+    cancellation.abort(new Error("Materialization visibility observation finished"));
+    clearTimeout(timer);
+    pending.delete(cancellation);
+  }
+}

package/src/sandbox/routing/routing-session.ts

@@ -34,7 +34,13 @@ import {
   type ProviderCommandSession,
 } from "../provider-command-session";
 import type { SandboxProviderCommand } from "@opengeni/contracts";
-import { hasTypedExecHandleLoss } from "../exec-banner";
+import { hasTypedExecHandleLoss, parseExecResponseBanner } from "../exec-banner";
+import {
+  SandboxMaterializationVerificationError,
+  materializationVerificationDiagnostic,
+  retainMaterializationVerificationDiagnostic,
+  type MaterializationFailureReason,
+} from "../materialization-verification-error";
 import { CAPABILITY_DESCRIPTORS, type SandboxBackend } from "@opengeni/contracts";
 import { SelfhostedControlError } from "../selfhosted/control-rpc";
 import {
@@ -102,6 +108,8 @@ export interface RoutableBackendSession extends ProviderCommandSession {
   pathExists?(path: string, runAs?: string): Promise<boolean>;
   viewImage?(args: unknown): Promise<unknown>;
   materializeEntry?(args: unknown): Promise<void>;
+  /** Provider-owned fixed read-only probe; never an agent command surface. */
+  verifyMaterializedPath?(path: string, workdir: string): Promise<void>;
   supportsPty?(): boolean;
   resolveExposedPort?(port: number): Promise<ExposedPortEndpoint>;
   serializeSessionState?(): Promise<unknown>;
@@ -299,6 +307,7 @@ export type RoutingSandboxOperationObservation = {
   op: string;
   outcome: "ok" | "not_found" | "failed";
   durationMs: number;
+  materializationFailureReason?: MaterializationFailureReason;
 };
 
 export type RoutingSandboxOperationObserver = (
@@ -1689,11 +1698,13 @@ export class RoutingSandboxSession implements RoutableBackendSession {
   ): Promise<T> {
     const startedAt = performance.now();
     let outcome: RoutingSandboxOperationObservation["outcome"] = "failed";
+    let materializationFailureReason: MaterializationFailureReason | undefined;
     try {
       const result = await withSandboxProviderOperation(backend.session, fn, onCaptureWait);
       outcome = "ok";
       return result;
     } catch (error) {
+      materializationFailureReason = materializationVerificationDiagnostic(error)?.reason;
       if (
         (op === "readFile" || op === "listDir" || op === "pathExists" || op === "viewImage") &&
         isDefinitePathNotFoundError(error)
@@ -1708,6 +1719,7 @@ export class RoutingSandboxSession implements RoutableBackendSession {
           op,
           outcome,
           durationMs: Math.max(0, performance.now() - startedAt),
+          ...(materializationFailureReason ? { materializationFailureReason } : {}),
         });
       } catch {
         // Telemetry is never part of provider or durable-settlement authority.
@@ -2289,6 +2301,9 @@ async function assertProviderCanSeeMaterializedPath(
   session: RoutableBackendSession,
   path: string,
 ): Promise<void> {
+  if (session.verifyMaterializedPath) {
+    return session.verifyMaterializedPath(path, providerManifestRoot(session) ?? "/workspace");
+  }
   const command = `test -e ${shellSingleQuote(path)} && printf %s ${shellSingleQuote(
     MATERIALIZED_PATH_MARKER,
   )}`;
@@ -2300,18 +2315,68 @@ async function assertProviderCanSeeMaterializedPath(
     tty: false,
   };
   let output: string;
-  if (session.execCommand) {
-    output = await session.execCommand(args);
-  } else if (session.exec) {
-    output = formatExecResult(await session.exec(args));
-  } else {
+  let result: unknown;
+  if (!session.execCommand && !session.exec) {
     throw new RoutingUnsupportedError("materializeEntry.verify", "unknown");
   }
-  if (!output.includes(MATERIALIZED_PATH_MARKER)) {
-    throw new Error(
-      `Sandbox materialization completed but the provider cannot read the destination path: ${path}`,
-    );
+  try {
+    result = session.execCommand ? await session.execCommand(args) : await session.exec!(args);
+  } catch (cause) {
+    retainMaterializationVerificationDiagnostic(cause, {
+      reason: "command_error",
+      path,
+      workdir: args.workdir,
+      command,
+      output: null,
+      exitCode: null,
+      providerSessionId: null,
+      causeMessage: cause instanceof Error ? cause.message : String(cause),
+    });
+    throw cause;
+  }
+  try {
+    output = formatExecResult(result);
+  } catch (cause) {
+    const record = result && typeof result === "object" ? (result as Record<string, unknown>) : {};
+    const returnedOutput = [record.output, record.stderr, record.stdout]
+      .filter((value): value is string => typeof value === "string")
+      .join("\n");
+    throw new SandboxMaterializationVerificationError({
+      reason: "invalid_response",
+      path,
+      workdir: args.workdir,
+      command,
+      output: returnedOutput || null,
+      exitCode: null,
+      providerSessionId: null,
+      causeMessage: cause instanceof Error ? cause.message : String(cause),
+    });
+  }
+  const status = parseExecResponseBanner(output);
+  const markerPresent = stripExecBanner(output).includes(MATERIALIZED_PATH_MARKER);
+  if (
+    markerPresent &&
+    (status.kind === "absent" || (status.kind === "exited" && status.exitCode === 0))
+  ) {
+    return;
   }
+  const reason: MaterializationFailureReason =
+    status.kind === "running"
+      ? "command_pending"
+      : status.kind === "exited" && status.exitCode === 1
+        ? "path_not_visible"
+        : status.kind === "exited" && status.exitCode !== 0
+          ? "command_failed"
+          : "invalid_response";
+  throw new SandboxMaterializationVerificationError({
+    reason,
+    path,
+    workdir: args.workdir,
+    command,
+    output,
+    exitCode: status.kind === "exited" ? status.exitCode : null,
+    providerSessionId: status.kind === "running" ? status.sessionId : null,
+  });
 }
 
 function providerManifestRoot(session: RoutableBackendSession): string | null {