@agent-compose/sdk 0.5.9 → 0.7.0

package/src/step-invocation/invoker.ts

@@ -24,8 +24,8 @@
 import { randomBytes } from "node:crypto";
 import { z } from "zod";
 import { SandboxUnavailableError } from "../sandbox-errors.js";
-import type { SandboxProvider } from "../types/sandbox.js";
-import { RUNNER_COMMAND, STEP_ENV, stepResultLinePrefix, stepPauseLinePrefix, requestContextPath, stepInputPath, stepResultFilePath } from "./protocol.js";
+import type { SandboxProvider, SandboxBackgroundProcess } from "../types/sandbox.js";
+import { RUNNER_COMMAND, STEP_ENV, stepResultLinePrefix, stepPauseLinePrefix, requestContextPath, stepInputPath, stepResultFilePath, stepLogFilePath } from "./protocol.js";
 import { StepPauseRequestSchema } from "./types.js";
 import type { StepRequest, StepResult } from "./types.js";
 import type { StepObservability } from "../workflow-steps/observability.js";
@@ -194,44 +194,50 @@ export interface InvokeStepOptions {
    *  the caller's job; the activity batches and inserts at step completion. */
   onStdout?: (line: string) => void;
   onStderr?: (line: string) => void;
+  /** Called when the LIVE output stream fails mid-run (e.g. E2B's connect-web
+   *  transport throws `received unsupported compressed output` on a large
+   *  compressed frame) and the invoker falls back to recovering the result +
+   *  full logs from the durable token-keyed files. The run is NOT failed — this
+   *  is the hook to emit a structured alert so the degradation is visible/paged.
+   *  Best-effort: keep it cheap and non-throwing. */
+  onStreamDegraded?: (info: { error: unknown; runnerPid: number; resultToken: string }) => void;
 }
 
-export async function invokeStep<TOutput = unknown>(
-  sandbox: SandboxProvider,
-  request: StepRequest,
-  opts?: InvokeStepOptions,
-): Promise<StepResult<TOutput>> {
-  const resultToken = randomBytes(16).toString("hex");
-
-  await Promise.all([
-    sandbox.files.write(stepInputPath(request.stepIndex),      JSON.stringify(request.input)),
-    sandbox.files.write(requestContextPath(request.stepIndex), JSON.stringify(request.requestContext)),
-  ]);
-
-  const envs = {
-    ...(opts?.envs ?? {}),
-    ...buildStepEnvs({
-      runId:       request.runId,
-      stepIndex:   request.stepIndex,
-      resultToken,
-      isResume:    opts?.isResume,
-    }),
-  };
+/** A step running as a background command (ADR-0028). The activity races its
+ *  `wait()` against a server pause request; on a pause it freezes the VM
+ *  (`pauseProcess`) and persists `runnerPid` + `resultToken` so the resume
+ *  activity can `reconnectStep(...)` to the SAME process — no re-run. */
+export interface RunningStep<TOutput = unknown> {
+  /** OS pid of the background runner inside the VM — the reconnect handle. */
+  runnerPid: number;
+  /** Per-invocation token keying the durable result file the runner writes. */
+  resultToken: string;
+  /** Await the runner's exit and classify its output into a `StepResult`. */
+  wait(): Promise<StepResult<TOutput>>;
+}
 
-  // The sandbox emits stdout as raw chunks, not lines. Buffer between
-  // emissions so a `console.log` split across two chunks (or a partial
-  // trailing line) is delivered to onStdout/onStderr as one logical line.
-  // Sentinel lines (carrying `resultToken`) are filtered out so callers
-  // never see protocol bytes in user-log capture. Both the result and
-  // pause prefixes are built from the same shared helpers `parseStepResult`
-  // uses, so the filter and the parser can't drift apart.
+/** Buffer the sandbox's raw stdout/stderr chunks into whole lines, filtering
+ *  the protocol sentinel out of `onStdout` so user-log capture never sees
+ *  protocol bytes. Shared by the foreground (`invokeStep`), background
+ *  (`launchStep`), and resume (`reconnectStep`) paths so the filter and the
+ *  result parser can't drift. Returns the per-stream chunk handlers (undefined
+ *  when the caller passed no sink) plus a `flush` for trailing newline-less
+ *  output. */
+function makeStreamSplitters(
+  opts: Pick<InvokeStepOptions, "onStdout" | "onStderr"> | undefined,
+  resultToken: string,
+): { onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void; flush: () => void; stdoutConsumedChars: () => number } {
   const resultSentinel = stepResultLinePrefix(resultToken);
   const pauseSentinel  = stepPauseLinePrefix(resultToken);
-  const isSentinel = (line: string) =>
-    line.startsWith(resultSentinel) || line.startsWith(pauseSentinel);
-  const makeLineSplitter = (sink: ((line: string) => void) | undefined, filterSentinel: boolean) => {
-    if (!sink) return { onChunk: undefined, flush: () => {} };
+  const isSentinel = (line: string) => line.startsWith(resultSentinel) || line.startsWith(pauseSentinel);
+  const make = (sink: ((line: string) => void) | undefined, filterSentinel: boolean) => {
+    if (!sink) return { onChunk: undefined, flush: () => {}, consumed: () => 0 };
     let buf = "";
+    // Chars consumed as WHOLE lines (incl. each trailing "\n"), counting sentinel
+    // lines too — this is the offset into the tee'd log file up to which the live
+    // stream has already delivered. The recovery backfill starts exactly here, so
+    // an in-flight partial line is re-read whole from the file (no split, no dup).
+    let consumed = 0;
     return {
       onChunk: (chunk: string) => {
         buf += chunk;
@@ -239,33 +245,220 @@ export async function invokeStep<TOutput = unknown>(
         while ((nl = buf.indexOf("\n")) !== -1) {
           const line = buf.slice(0, nl);
           buf = buf.slice(nl + 1);
+          consumed += line.length + 1;
           if (filterSentinel && isSentinel(line)) continue;
           sink(line);
         }
       },
-      // Drain any remaining buffered output that ended without a newline.
-      // Called after `sandbox.commands.run` resolves so a runner that
-      // exits with `process.stdout.write("final")` (no trailing \n)
-      // doesn't silently drop its last line.
       flush: () => {
         if (buf.length === 0) return;
         const line = buf;
         buf = "";
+        consumed += line.length;
         if (filterSentinel && isSentinel(line)) return;
         sink(line);
       },
+      consumed: () => consumed,
+    };
+  };
+  const stdout = make(opts?.onStdout, true);
+  const stderr = make(opts?.onStderr, false);
+  return {
+    ...(stdout.onChunk ? { onStdout: stdout.onChunk } : {}),
+    ...(stderr.onChunk ? { onStderr: stderr.onChunk } : {}),
+    flush: () => { stdout.flush(); stderr.flush(); },
+    stdoutConsumedChars: () => stdout.consumed(),
+  };
+}
+
+/** Turn the runner's terminal output into a kinded `StepResult`. stdout is the
+ *  fast path; the runner also persists the sentinel to a durable token-keyed
+ *  file, so when stdout carries no sentinel (providers drop the tail of heavy
+ *  streams; a reconnect captures only post-resume output) we read the file
+ *  back. Shared by all three invocation paths. */
+async function classifyRunnerOutcome<TOutput>(
+  sandbox: SandboxProvider,
+  output: { stdout: string; stderr: string; exitCode: number },
+  resultToken: string,
+): Promise<StepResult<TOutput>> {
+  const parsed = parseStepResult<TOutput>(output.stdout, resultToken);
+  if (parsed) return parsed;
+
+  // No sentinel on stdout — read the durable token-keyed file the runner wrote
+  // BEFORE its stdout emit. A runner that exited 1 after a user-step throw
+  // still wrote `{ok:false, kind:"user-step"}`, which beats a generic exit.
+  const fileRead = await sandbox.commands.run(
+    `cat ${stepResultFilePath(resultToken)} 2>/dev/null || true`,
+  ).catch(() => null);
+  if (fileRead?.stdout) {
+    const fromFile = parseStepResult<TOutput>(fileRead.stdout, resultToken);
+    if (fromFile) return fromFile;
+  }
+
+  // Distinguish "runner exited badly before emitting" (runner-exit) from
+  // "runner exited cleanly but didn't speak the protocol" (protocol).
+  if (output.exitCode !== 0) {
+    const stderrTail = output.stderr.trim().slice(-2000);
+    const stdoutTail = output.stdout.trim().slice(-2000);
+    const details = [
+      stderrTail ? `stderr:\n${stderrTail}` : "",
+      stdoutTail ? `stdout:\n${stdoutTail}` : "",
+    ].filter(Boolean).join("\n");
+    return {
+      ok: false,
+      error: {
+        kind:     "runner-exit",
+        message:  `runner subprocess exited ${output.exitCode} before emitting a step result${details ? `\n${details}` : ""}`,
+        exitCode: output.exitCode,
+      },
     };
+  }
+  const tail = output.stdout.trim().slice(-500);
+  return {
+    ok: false,
+    error: {
+      kind: "protocol",
+      message: `no tokenised step result on stdout or in the result file (runner exited 0 without emitting)${tail ? `\nstdout tail:\n${tail}` : ""}`,
+    },
   };
-  const stdoutSplitter = makeLineSplitter(opts?.onStdout, true);
-  const stderrSplitter = makeLineSplitter(opts?.onStderr, false);
+}
+
+/** Poll the runner's durable result file (token-keyed, written immediately before
+ *  exit on EVERY path — success, failure, pause) as the AUTHORITATIVE completion
+ *  signal, using `/proc/<pid>` liveness to tell work-in-progress from a crash.
+ *  Used wherever the live stream cannot be trusted for completion: the native
+ *  resume path (a reconnected handle's `wait()` can resolve early after a VM
+ *  pause/resume) AND the launch path after a live-stream transport fault.
+ *  `cat`-ing the result file is itself immune to the large-frame compression bug
+ *  — it is a single small JSON line, far below any compression threshold. The
+ *  activity's `startToCloseTimeout` is the real upper bound; `DEADLINE` is a
+ *  backstop so a wedged runner can't leak this loop in the worker forever. */
+async function pollDurableResult<TOutput>(
+  sandbox: SandboxProvider,
+  runnerPid: number,
+  resultToken: string,
+  signal?: AbortSignal,
+): Promise<StepResult<TOutput>> {
+  const resultFile = stepResultFilePath(resultToken);
+  const POLL_MS = 1000;
+  const DEADLINE = Date.now() + 45 * 60_000;
+  const readResult = async (): Promise<StepResult<TOutput> | null> => {
+    const r = await sandbox.commands.run(`cat ${resultFile} 2>/dev/null || true`).catch(() => null);
+    return r?.stdout ? (parseStepResult<TOutput>(r.stdout, resultToken) ?? null) : null;
+  };
+  for (let attempt = 1; ; attempt++) {
+    // Aborted = the activity already resolved via the pause watcher (a re-pause);
+    // this poll's result is now unobserved. Return (never throw — the promise is
+    // no longer awaited) so it settles cleanly with no unhandled rejection.
+    if (signal?.aborted) {
+      return { ok: false, error: { kind: "runner-exit", message: "reconnectStep aborted (run re-paused)", exitCode: 1 } };
+    }
+    const fromFile = await readResult();
+    if (fromFile) return fromFile;
+
+    const probe = await sandbox.commands
+      .run(`test -d /proc/${runnerPid} && echo alive || echo dead`)
+      .catch(() => null);
+    const dead = (probe?.stdout ?? "").includes("dead");
+    process.stderr.write(
+      `[recover] poll ${attempt}: result-file=absent runner=${dead ? "dead" : "alive"} pid=${runnerPid}\n`,
+    );
+    if (dead) {
+      // Close the write-then-exit race with one final read, else classify a crash.
+      const finalRead = await readResult();
+      if (finalRead) return finalRead;
+      return {
+        ok: false,
+        error: { kind: "runner-exit", message: `runner pid ${runnerPid} exited without writing a result file`, exitCode: 1 },
+      };
+    }
+    if (Date.now() > DEADLINE) {
+      return {
+        ok: false,
+        error: { kind: "runner-exit", message: `timed out after 45m waiting for runner pid ${runnerPid} to emit a result`, exitCode: 1 },
+      };
+    }
+    await new Promise((r) => setTimeout(r, POLL_MS));
+  }
+}
+
+/** Recover a step's result AND its full logs after the live output stream died
+ *  mid-run (the dominant cause being E2B's connect-web transport throwing on a
+ *  compressed large frame — a LOG-TRANSPORT fault, not a runner failure). Waits
+ *  for the runner to actually finish (`pollDurableResult`), then reads the tee'd
+ *  log file back over the HTTP file transport (compression-immune) and backfills
+ *  only the lines the dead stream never delivered — everything past the last
+ *  whole line already emitted live (`stdoutConsumedChars`), so no duplication and
+ *  no split lines. Logs BEFORE the fault are already persisted by the activity's
+ *  live flush; this restores the tail so "what happened" survives intact. */
+async function recoverLogsAndResult<TOutput>(
+  sandbox: SandboxProvider,
+  runnerPid: number,
+  resultToken: string,
+  liveSplitters: { stdoutConsumedChars: () => number },
+  opts: Pick<InvokeStepOptions, "onStdout" | "onStderr"> | undefined,
+): Promise<StepResult<TOutput>> {
+  const result = await pollDurableResult<TOutput>(sandbox, runnerPid, resultToken);
+  // Runner has exited → the tee'd log file is complete. Best-effort: a failed
+  // readback just means the recovered run keeps the live logs it already had.
+  if (sandbox.files.read && opts?.onStdout) {
+    const fullLog = await sandbox.files.read(stepLogFilePath(resultToken)).catch(() => null);
+    if (fullLog != null) {
+      const already = liveSplitters.stdoutConsumedChars();
+      const tail = fullLog.length > already ? fullLog.slice(already) : "";
+      if (tail.length > 0) {
+        const backfill = makeStreamSplitters(opts, resultToken);
+        backfill.onStdout?.(tail);
+        backfill.flush();
+      }
+    }
+  }
+  return result;
+}
+
+/** Write the step's input + request-context files and build the runner env.
+ *  Shared by the foreground and background launch paths. Returns the
+ *  per-invocation `resultToken` + the env map. */
+async function prepareStepLaunch(
+  sandbox: SandboxProvider,
+  request: StepRequest,
+  opts?: InvokeStepOptions,
+): Promise<{ resultToken: string; envs: Record<string, string> }> {
+  const resultToken = randomBytes(16).toString("hex");
+  await Promise.all([
+    sandbox.files.write(stepInputPath(request.stepIndex),      JSON.stringify(request.input)),
+    sandbox.files.write(requestContextPath(request.stepIndex), JSON.stringify(request.requestContext)),
+  ]);
+  const envs = {
+    ...(opts?.envs ?? {}),
+    ...buildStepEnvs({ runId: request.runId, stepIndex: request.stepIndex, resultToken, isResume: opts?.isResume }),
+  };
+  return { resultToken, envs };
+}
+
+export async function invokeStep<TOutput = unknown>(
+  sandbox: SandboxProvider,
+  request: StepRequest,
+  opts?: InvokeStepOptions,
+): Promise<StepResult<TOutput>> {
+  const { resultToken, envs } = await prepareStepLaunch(sandbox, request, opts);
+  const splitters = makeStreamSplitters(opts, resultToken);
 
   let result: { stdout: string; stderr: string; exitCode: number };
   try {
+    // Run the agent as ROOT (ADR-0019 follow-up). The factory drive (Archil)
+    // presents its S3-synced files root-owned and exposes no uid-mapped mount,
+    // so a non-root agent EACCESes on every shared file — the reason for the
+    // expensive boot-time `chmod -R` walk. As root the agent writes them
+    // directly: the walk disappears entirely. Egress is edge-enforced with NO
+    // root exemption (see sandbox.ts), so root is confined exactly like the
+    // non-root user. `HOME=/root` so the spawned `claude` finds root's skills.
     result = await sandbox.commands.run(RUNNER_COMMAND, {
-      envs,
+      envs: { ...envs, HOME: "/root", IS_SANDBOX: "1" },
+      sudo: true,
       timeoutMs: 0,
-      ...(stdoutSplitter.onChunk ? { onStdout: stdoutSplitter.onChunk } : {}),
-      ...(stderrSplitter.onChunk ? { onStderr: stderrSplitter.onChunk } : {}),
+      ...(splitters.onStdout ? { onStdout: splitters.onStdout } : {}),
+      ...(splitters.onStderr ? { onStderr: splitters.onStderr } : {}),
     });
   } catch (e) {
     // Typed sandbox-infrastructure failure: the engine's recovery contract
@@ -286,55 +479,135 @@ export async function invokeStep<TOutput = unknown>(
       exitCode: ce.exitCode ?? ce.result?.exitCode ?? 1,
     };
   }
-  stdoutSplitter.flush();
-  stderrSplitter.flush();
+  splitters.flush();
+  return classifyRunnerOutcome<TOutput>(sandbox, result, resultToken);
+}
 
-  const parsed = parseStepResult<TOutput>(result.stdout, resultToken);
-  if (parsed) return parsed;
+/**
+ * Launch the step runner as a BACKGROUND command (ADR-0028) and return a handle
+ * the activity drives: it races `wait()` against a server pause request and, on
+ * a pause, freezes the VM (`pauseProcess`) and persists `runnerPid` +
+ * `resultToken` so `reconnectStep` can continue the SAME process — no re-run.
+ * Requires a provider with `commands.runBackground` (E2B); the foreground
+ * `invokeStep` is the path for providers without it (Vercel).
+ */
+export async function launchStep<TOutput = unknown>(
+  sandbox: SandboxProvider,
+  request: StepRequest,
+  opts?: InvokeStepOptions,
+): Promise<RunningStep<TOutput>> {
+  if (!sandbox.commands.runBackground) {
+    throw new Error("launchStep requires a provider with background-command support (commands.runBackground)");
+  }
+  const { resultToken, envs } = await prepareStepLaunch(sandbox, request, opts);
+  const splitters = makeStreamSplitters(opts, resultToken);
+  // Tee the runner's stdout to a durable, token-keyed LOG file (stderr stays a
+  // separate live stream). The live output rides E2B's connect-web command
+  // stream, which THROWS on a compressed large frame (gRPC-web cannot decode
+  // message compression) and kills the feed mid-run. The tee'd file, read back
+  // over the envd HTTP transport (compression-immune), lets `wait()` recover the
+  // full logs + result instead of failing a run that actually completed. `tee`
+  // runs in the same `bash -c` pipeline E2B already wraps the command in, so the
+  // background pid (the pause / reconnect-by-pid handle) is unchanged; the result
+  // sentinel still rides stdout (tee passes it through) and the result FILE is
+  // written by the runner directly, so the happy path is untouched. `set -o
+  // pipefail` is REQUIRED: without it the pipeline's exit code is tee's (0),
+  // masking a non-zero runner exit and breaking the runner-exit classification;
+  // pipefail propagates the runner's code (E2B execs via `bash -c`).
+  //
+  // Run the agent as ROOT — see invokeStep above. `sudo:true` → user:"root";
+  // `HOME=/root` so the spawned `claude` finds root's skills.
+  const proc = await sandbox.commands.runBackground(`set -o pipefail; ${RUNNER_COMMAND} | tee ${stepLogFilePath(resultToken)}`, {
+    envs: { ...envs, HOME: "/root", IS_SANDBOX: "1" },
+    sudo: true,
+    timeoutMs: 0,
+    ...(splitters.onStdout ? { onStdout: splitters.onStdout } : {}),
+    ...(splitters.onStderr ? { onStderr: splitters.onStderr } : {}),
+  });
+  return {
+    runnerPid:   proc.pid,
+    resultToken,
+    async wait() {
+      try {
+        const result = await proc.wait();
+        splitters.flush();
+        return classifyRunnerOutcome<TOutput>(sandbox, result, resultToken);
+      } catch (e) {
+        // A genuine infra death must propagate so the engine re-provisions.
+        if (e instanceof SandboxUnavailableError) throw e;
+        // Otherwise the runner's LIVE output stream died while the runner itself
+        // is alive and writing its durable result + tee'd log files. The dominant
+        // cause is E2B's connect-web transport throwing on a compressed large
+        // frame ("received unsupported compressed output") — a LOG-TRANSPORT
+        // fault, NEVER a reason to fail a run that completed. Degrade the live
+        // feed, alert, and recover the result + full logs over the HTTP file
+        // transport. We do NOT flush the live splitter here — the recovery
+        // backfills from the last whole line, re-reading any in-flight partial
+        // line whole from the durable log (avoids a split/duplicated line).
+        opts?.onStreamDegraded?.({ error: e, runnerPid: proc.pid, resultToken });
+        return recoverLogsAndResult<TOutput>(sandbox, proc.pid, resultToken, splitters, opts);
+      }
+    },
+  };
+}
 
-  // No sentinel on stdout. The runner also persists the sentinel line to a
-  // token-keyed file before emitting — providers drop the tail of heavy
-  // stdout streams, so read the durable copy back. Tried before exit-code
-  // classification: a runner that exited 1 after a user-step throw still
-  // wrote the file, and its `{ok:false, kind:"user-step"}` beats a generic
-  // runner-exit error. A runner killed before emitting wrote no file and
-  // falls through.
-  const fileRead = await sandbox.commands.run(
-    `cat ${stepResultFilePath(resultToken)} 2>/dev/null || true`,
-  ).catch(() => null);
-  if (fileRead?.stdout) {
-    const fromFile = parseStepResult<TOutput>(fileRead.stdout, resultToken);
-    if (fromFile) return fromFile;
+/**
+ * Resume a previously-paused background runner (ADR-0028) and return a
+ * `RunningStep` handle — uniform with `launchStep` so the activity can race
+ * `wait()` against a fresh pause request (a resumed step can pause again). After
+ * the workflow reconnects the suspended VM (`Sandbox.connect` auto-resumes it),
+ * this re-attaches to the still-running runner by `runnerPid`; `wait()` awaits
+ * its exit (event-driven — no polling) and classifies the output (the durable
+ * result file is authoritative on this path). If the runner already exited
+ * during resume, the re-attach fails and `wait()` classifies from the file.
+ */
+export async function reconnectStep<TOutput = unknown>(
+  sandbox: SandboxProvider,
+  resume: { runnerPid: number; resultToken: string; stepIndex: number },
+  opts?: Pick<InvokeStepOptions, "onStdout" | "onStderr"> & { signal?: AbortSignal },
+): Promise<RunningStep<TOutput>> {
+  if (!sandbox.commands.connectProcess) {
+    throw new Error("reconnectStep requires a provider with background-command support (commands.connectProcess)");
   }
+  const { runnerPid, resultToken } = resume;
+  const splitters = makeStreamSplitters(opts, resultToken);
+  const signal = opts?.signal;
 
-  // No tokenised sentinel on stdout. Distinguish "runner exited badly
-  // before emitting" (runner-exit) from "runner exited cleanly but didn't
-  // speak the protocol" (protocol) — the exit code is the evidence.
-  if (result.exitCode !== 0) {
-    const stderrTail = result.stderr.trim().slice(-2000);
-    const stdoutTail = result.stdout.trim().slice(-2000);
-    const details = [
-      stderrTail ? `stderr:\n${stderrTail}` : "",
-      stdoutTail ? `stdout:\n${stdoutTail}` : "",
-    ].filter(Boolean).join("\n");
-    return {
-      ok: false,
-      error: {
-        kind:     "runner-exit",
-        message:  `runner subprocess exited ${result.exitCode} before emitting a step result${details ? `\n${details}` : ""}`,
-        exitCode: result.exitCode,
-      },
-    };
+  // Re-attach to the suspended runner for LIVE stdout streaming only. This is
+  // best-effort: a reconnect that throws after the native resume just means no
+  // live feed for the dashboard — completion is read from the durable result
+  // file below, NOT from this handle.
+  let attached: SandboxBackgroundProcess | null = null;
+  try {
+    attached = await sandbox.commands.connectProcess(runnerPid, {
+      ...(splitters.onStdout ? { onStdout: splitters.onStdout } : {}),
+      ...(splitters.onStderr ? { onStderr: splitters.onStderr } : {}),
+    });
+  } catch (e) {
+    if (e instanceof SandboxUnavailableError) throw e;
   }
-  // Carry the stdout tail: "no tokenised step result" alone is useless to
-  // an operator — the tail usually shows whether the runner finished its
-  // work (output truncated by the provider) or never got there.
-  const tail = result.stdout.trim().slice(-500);
+
   return {
-    ok: false,
-    error: {
-      kind: "protocol",
-      message: `no tokenised step result on stdout or in the result file (runner exited 0 without emitting)${tail ? `\nstdout tail:\n${tail}` : ""}`,
+    runnerPid,
+    resultToken,
+    async wait() {
+      // CRITICAL: do NOT trust `connect(pid).wait()` on the resume path. After a
+      // native VM pause/resume, E2B's re-attached handle can resolve its wait()
+      // EARLY — the reconnected stdout stream closes while the runner process is
+      // still alive and working — returning a default `{exitCode:0}`. That made
+      // the worker report "exited 0 without emitting" and force-kill a runner
+      // that was mid-iteration. So ignore the handle for completion and poll the
+      // runner's durable result file + `/proc` liveness instead — the same
+      // authoritative signal the launch-path recovery uses (`pollDurableResult`).
+      try {
+        const result = await pollDurableResult<TOutput>(sandbox, runnerPid, resultToken, signal);
+        splitters.flush();
+        return result;
+      } finally {
+        // Release the streaming handle (keeps it un-GC'd for the duration of the
+        // poll above; a dangling reconnect after a re-pause is cleaned up here).
+        await attached?.kill().catch(() => {});
+      }
     },
   };
 }

package/src/step-invocation/protocol.ts

@@ -80,3 +80,14 @@ export function requestContextPath(stepIndex: number): string {
 export function stepResultFilePath(token: string): string {
   return `/tmp/wf/step-result-${token}.json`;
 }
+
+/** Sandbox-side path where the runner's stdout is tee'd as a durable LOG file,
+ *  keyed by the per-invocation token. The live output rides E2B's connect-web
+ *  command stream, which THROWS on a compressed large frame (gRPC-web cannot
+ *  decode message compression) and kills the feed mid-run. This file, read back
+ *  over the envd HTTP file transport (compression-immune), lets the invoker
+ *  recover the FULL logs after such a fault instead of losing the tail — the
+ *  log-side analogue of `stepResultFilePath` for the result. */
+export function stepLogFilePath(token: string): string {
+  return `/tmp/wf/step-log-${token}.log`;
+}

package/src/types/runtime.ts

@@ -32,6 +32,13 @@ export interface RuntimeOptions {
   /** Agent id and label for processor context / adapter logs. */
   agentId?: string;
   iteration?: number;
+  /** The platform manual (file conventions, connectors & access, how to pause).
+   *  `agent()` builds it per-run (`buildAgentContextDoc`) and threads it here so
+   *  a runtime that supports a system-prompt append (the claude runtime) injects
+   *  it directly — instead of relying on the agent to `cat` the on-disk
+   *  AGENTS.md/CLAUDE.md, which the Agent SDK doesn't auto-load and which can
+   *  fail to write on a read-only/degraded working dir. */
+  agentManual?: string;
   /** The run's pause boundary, threaded from the agent loop so a runtime-driven
    *  pre-tool gate (e.g. the ACP `session/request_permission` path through
    *  `gateToolCall`) can raise a human-approval `ctx.pause`. The runtime binds it

package/src/types/sandbox.ts

@@ -25,6 +25,30 @@ export interface SandboxCommandResult {
   stderr: string;
 }
 
+/** A long-running command launched in the background (ADR-0028). Unlike
+ *  `commands.run` (which awaits completion on one connection), a background
+ *  command keeps running inside the VM independent of the launching
+ *  connection: it survives `pauseProcess()`/resume and is re-attachable by
+ *  `pid` after a fresh `Sandbox.connect`. This is what lets the platform
+ *  freeze an agent mid-turn for a human-in-the-loop pause and continue the
+ *  SAME process on resume — no re-run.
+ *
+ *  Implemented ONLY by process-resume-capable providers (E2B); the presence
+ *  of `commands.runBackground` IS the capability flag, paired with
+ *  `pauseProcess`. Providers without it leave both undefined and pause via
+ *  `snapshot()` + re-run instead. */
+export interface SandboxBackgroundProcess {
+  /** OS pid inside the VM — the durable handle used to reconnect after a
+   *  pause/resume cycle via `commands.connectProcess(pid)`. */
+  pid: number;
+  /** Resolve when the process exits, with its buffered result. Live output
+   *  streams to the `onStdout`/`onStderr` passed at launch / connect time.
+   *  Does NOT throw on a non-zero exit — the result carries `exitCode`. */
+  wait(): Promise<SandboxCommandResult>;
+  /** Force-terminate the process. */
+  kill(): Promise<void>;
+}
+
 /** A spawned long-lived command with a writable stdin and readable stdout,
  *  exposed as byte web-streams. Unlike `commands.run` (which buffers to
  *  completion and exposes stdout only via an `onStdout` callback), a duplex
@@ -85,9 +109,28 @@ export interface SandboxProvider {
      *  view). The vercel/e2b providers (server→sandbox) leave it undefined; an
      *  ACP caller that finds it absent falls back to the JSONL transport. */
     spawnDuplex?(cmd: string, opts?: SandboxSpawnDuplexOptions): SandboxDuplexProcess;
+    /** Launch a command in the background and return immediately with a
+     *  reconnectable handle (ADR-0028). The process survives the launching
+     *  connection dropping AND a `pauseProcess()`/resume cycle. OPTIONAL —
+     *  only process-resume providers (E2B) implement it; its presence (paired
+     *  with `pauseProcess`) is the native-pause capability flag. */
+    runBackground?(cmd: string, opts?: SandboxCommandRunOptions): Promise<SandboxBackgroundProcess>;
+    /** Re-attach to a background command by `pid` after a fresh
+     *  `Sandbox.connect` (the resume half of `runBackground`). OPTIONAL,
+     *  E2B-only. Throws if no process with that pid is running. */
+    connectProcess?(pid: number, opts?: Pick<SandboxCommandRunOptions, "onStdout" | "onStderr" | "timeoutMs">): Promise<SandboxBackgroundProcess>;
   };
   files: {
     write(path: string, content: string): Promise<void>;
+    /** Read a file's text content over the provider's FILE transport. On E2B this
+     *  is the envd HTTP API (`Sandbox.files.read`), a DIFFERENT transport from
+     *  `commands` — so a large readback is immune to the connect-web gRPC
+     *  message-compression that can abort `commands.run` output on a big frame
+     *  ("received unsupported compressed output"). This is what lets `launchStep`
+     *  recover the full logs + result after a live-stream fault. OPTIONAL —
+     *  implemented where durable file-readback is needed (E2B, local); providers
+     *  that never drive the recovery path (Vercel — foreground only) may omit it. */
+    read?(path: string): Promise<string>;
   };
   kill(): Promise<void>;
   /** Capture the running sandbox's state as a reusable snapshot. Vercel and E2B
@@ -99,6 +142,16 @@ export interface SandboxProvider {
    *  be omitted when the provider doesn't expose it; the server stores
    *  `null` for missing values rather than estimating. */
   snapshot?(): Promise<{ snapshotId: string; sizeBytes?: number }>;
+  /** Suspend the live VM in place and return a handle to resume it (ADR-0027).
+   *  Present ONLY on process-resume-capable providers (E2B via `sandbox.pause()`,
+   *  returning the sandbox id; resume is `Sandbox.connect(handle)`, which
+   *  auto-resumes the paused VM). Unlike `snapshot()` — which captures an FS
+   *  image, kills the origin, and re-runs the step from a fresh sandbox — a
+   *  process-resume pause FREEZES the live process (zero compute) and continues
+   *  it exactly where it blocked. The presence of this method IS the capability
+   *  flag: providers without native VM-suspend leave it undefined and fall back
+   *  to `snapshot()` + re-run. */
+  pauseProcess?(): Promise<{ resumeHandle: string }>;
   /** Replace the live sandbox's egress policy in place — so the server can
    *  push a freshly resolved policy (with re-minted connector access tokens)
    *  before each step instead of relying on the policy baked at create.

package/src/types/workflow.ts

@@ -110,6 +110,12 @@ export interface WorkflowDefinition<
   /** Same as `input`, for the workflow's return value. Captured into
    *  `outputSchema` metadata and rendered in the IO panel. */
   output?: z.ZodType<TOutput>;
+  /**
+   * @deprecated Legacy run-form. Prefer step-form — the
+   * `.step(defineStep(...))` builder — for per-step durability/replay and
+   * working pause. A run-form body compiles to one opaque step
+   * (`compileRunForm`), so any failure/resume re-runs the whole body.
+   */
   run: WorkflowFn<TOutput, TInput>;
   /**
    * All snapshot config — boot source plus capture mode.
@@ -298,6 +304,12 @@ function compileRunForm<TOutput, TInput extends Record<string, unknown>>(
  * stores the step plan; runner subprocesses execute one step at a time
  * via the StepInvocation seam.
  */
+/**
+ * @deprecated Run-form is legacy. Use the step-form overload —
+ * `defineWorkflow({ id, input, output }).step(defineStep(...)).build()` — for
+ * durable, replayable steps and working pause. Run-form compiles to a single
+ * opaque step (`compileRunForm`); there is no per-step replay.
+ */
 export function defineWorkflow<
   TOutput = unknown,
   TInput extends Record<string, unknown> = Record<string, unknown>,