@agent-compose/sdk 0.6.0 → 0.7.0

package/src/step-invocation/invoker.ts

@@ -25,7 +25,7 @@ import { randomBytes } from "node:crypto";
 import { z } from "zod";
 import { SandboxUnavailableError } from "../sandbox-errors.js";
 import type { SandboxProvider, SandboxBackgroundProcess } from "../types/sandbox.js";
-import { RUNNER_COMMAND, STEP_ENV, stepResultLinePrefix, stepPauseLinePrefix, requestContextPath, stepInputPath, stepResultFilePath } from "./protocol.js";
+import { RUNNER_COMMAND, STEP_ENV, stepResultLinePrefix, stepPauseLinePrefix, requestContextPath, stepInputPath, stepResultFilePath, stepLogFilePath } from "./protocol.js";
 import { StepPauseRequestSchema } from "./types.js";
 import type { StepRequest, StepResult } from "./types.js";
 import type { StepObservability } from "../workflow-steps/observability.js";
@@ -194,6 +194,13 @@ export interface InvokeStepOptions {
    *  the caller's job; the activity batches and inserts at step completion. */
   onStdout?: (line: string) => void;
   onStderr?: (line: string) => void;
+  /** Called when the LIVE output stream fails mid-run (e.g. E2B's connect-web
+   *  transport throws `received unsupported compressed output` on a large
+   *  compressed frame) and the invoker falls back to recovering the result +
+   *  full logs from the durable token-keyed files. The run is NOT failed — this
+   *  is the hook to emit a structured alert so the degradation is visible/paged.
+   *  Best-effort: keep it cheap and non-throwing. */
+  onStreamDegraded?: (info: { error: unknown; runnerPid: number; resultToken: string }) => void;
 }
 
 /** A step running as a background command (ADR-0028). The activity races its
@@ -219,13 +226,18 @@ export interface RunningStep<TOutput = unknown> {
 function makeStreamSplitters(
   opts: Pick<InvokeStepOptions, "onStdout" | "onStderr"> | undefined,
   resultToken: string,
-): { onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void; flush: () => void } {
+): { onStdout?: (chunk: string) => void; onStderr?: (chunk: string) => void; flush: () => void; stdoutConsumedChars: () => number } {
   const resultSentinel = stepResultLinePrefix(resultToken);
   const pauseSentinel  = stepPauseLinePrefix(resultToken);
   const isSentinel = (line: string) => line.startsWith(resultSentinel) || line.startsWith(pauseSentinel);
   const make = (sink: ((line: string) => void) | undefined, filterSentinel: boolean) => {
-    if (!sink) return { onChunk: undefined, flush: () => {} };
+    if (!sink) return { onChunk: undefined, flush: () => {}, consumed: () => 0 };
     let buf = "";
+    // Chars consumed as WHOLE lines (incl. each trailing "\n"), counting sentinel
+    // lines too — this is the offset into the tee'd log file up to which the live
+    // stream has already delivered. The recovery backfill starts exactly here, so
+    // an in-flight partial line is re-read whole from the file (no split, no dup).
+    let consumed = 0;
     return {
       onChunk: (chunk: string) => {
         buf += chunk;
@@ -233,6 +245,7 @@ function makeStreamSplitters(
         while ((nl = buf.indexOf("\n")) !== -1) {
           const line = buf.slice(0, nl);
           buf = buf.slice(nl + 1);
+          consumed += line.length + 1;
           if (filterSentinel && isSentinel(line)) continue;
           sink(line);
         }
@@ -241,9 +254,11 @@ function makeStreamSplitters(
         if (buf.length === 0) return;
         const line = buf;
         buf = "";
+        consumed += line.length;
         if (filterSentinel && isSentinel(line)) return;
         sink(line);
       },
+      consumed: () => consumed,
     };
   };
   const stdout = make(opts?.onStdout, true);
@@ -252,6 +267,7 @@ function makeStreamSplitters(
     ...(stdout.onChunk ? { onStdout: stdout.onChunk } : {}),
     ...(stderr.onChunk ? { onStderr: stderr.onChunk } : {}),
     flush: () => { stdout.flush(); stderr.flush(); },
+    stdoutConsumedChars: () => stdout.consumed(),
   };
 }
 
@@ -307,6 +323,99 @@ async function classifyRunnerOutcome<TOutput>(
   };
 }
 
+/** Poll the runner's durable result file (token-keyed, written immediately before
+ *  exit on EVERY path — success, failure, pause) as the AUTHORITATIVE completion
+ *  signal, using `/proc/<pid>` liveness to tell work-in-progress from a crash.
+ *  Used wherever the live stream cannot be trusted for completion: the native
+ *  resume path (a reconnected handle's `wait()` can resolve early after a VM
+ *  pause/resume) AND the launch path after a live-stream transport fault.
+ *  `cat`-ing the result file is itself immune to the large-frame compression bug
+ *  — it is a single small JSON line, far below any compression threshold. The
+ *  activity's `startToCloseTimeout` is the real upper bound; `DEADLINE` is a
+ *  backstop so a wedged runner can't leak this loop in the worker forever. */
+async function pollDurableResult<TOutput>(
+  sandbox: SandboxProvider,
+  runnerPid: number,
+  resultToken: string,
+  signal?: AbortSignal,
+): Promise<StepResult<TOutput>> {
+  const resultFile = stepResultFilePath(resultToken);
+  const POLL_MS = 1000;
+  const DEADLINE = Date.now() + 45 * 60_000;
+  const readResult = async (): Promise<StepResult<TOutput> | null> => {
+    const r = await sandbox.commands.run(`cat ${resultFile} 2>/dev/null || true`).catch(() => null);
+    return r?.stdout ? (parseStepResult<TOutput>(r.stdout, resultToken) ?? null) : null;
+  };
+  for (let attempt = 1; ; attempt++) {
+    // Aborted = the activity already resolved via the pause watcher (a re-pause);
+    // this poll's result is now unobserved. Return (never throw — the promise is
+    // no longer awaited) so it settles cleanly with no unhandled rejection.
+    if (signal?.aborted) {
+      return { ok: false, error: { kind: "runner-exit", message: "reconnectStep aborted (run re-paused)", exitCode: 1 } };
+    }
+    const fromFile = await readResult();
+    if (fromFile) return fromFile;
+
+    const probe = await sandbox.commands
+      .run(`test -d /proc/${runnerPid} && echo alive || echo dead`)
+      .catch(() => null);
+    const dead = (probe?.stdout ?? "").includes("dead");
+    process.stderr.write(
+      `[recover] poll ${attempt}: result-file=absent runner=${dead ? "dead" : "alive"} pid=${runnerPid}\n`,
+    );
+    if (dead) {
+      // Close the write-then-exit race with one final read, else classify a crash.
+      const finalRead = await readResult();
+      if (finalRead) return finalRead;
+      return {
+        ok: false,
+        error: { kind: "runner-exit", message: `runner pid ${runnerPid} exited without writing a result file`, exitCode: 1 },
+      };
+    }
+    if (Date.now() > DEADLINE) {
+      return {
+        ok: false,
+        error: { kind: "runner-exit", message: `timed out after 45m waiting for runner pid ${runnerPid} to emit a result`, exitCode: 1 },
+      };
+    }
+    await new Promise((r) => setTimeout(r, POLL_MS));
+  }
+}
+
+/** Recover a step's result AND its full logs after the live output stream died
+ *  mid-run (the dominant cause being E2B's connect-web transport throwing on a
+ *  compressed large frame — a LOG-TRANSPORT fault, not a runner failure). Waits
+ *  for the runner to actually finish (`pollDurableResult`), then reads the tee'd
+ *  log file back over the HTTP file transport (compression-immune) and backfills
+ *  only the lines the dead stream never delivered — everything past the last
+ *  whole line already emitted live (`stdoutConsumedChars`), so no duplication and
+ *  no split lines. Logs BEFORE the fault are already persisted by the activity's
+ *  live flush; this restores the tail so "what happened" survives intact. */
+async function recoverLogsAndResult<TOutput>(
+  sandbox: SandboxProvider,
+  runnerPid: number,
+  resultToken: string,
+  liveSplitters: { stdoutConsumedChars: () => number },
+  opts: Pick<InvokeStepOptions, "onStdout" | "onStderr"> | undefined,
+): Promise<StepResult<TOutput>> {
+  const result = await pollDurableResult<TOutput>(sandbox, runnerPid, resultToken);
+  // Runner has exited → the tee'd log file is complete. Best-effort: a failed
+  // readback just means the recovered run keeps the live logs it already had.
+  if (sandbox.files.read && opts?.onStdout) {
+    const fullLog = await sandbox.files.read(stepLogFilePath(resultToken)).catch(() => null);
+    if (fullLog != null) {
+      const already = liveSplitters.stdoutConsumedChars();
+      const tail = fullLog.length > already ? fullLog.slice(already) : "";
+      if (tail.length > 0) {
+        const backfill = makeStreamSplitters(opts, resultToken);
+        backfill.onStdout?.(tail);
+        backfill.flush();
+      }
+    }
+  }
+  return result;
+}
+
 /** Write the step's input + request-context files and build the runner env.
  *  Shared by the foreground and background launch paths. Returns the
  *  per-invocation `resultToken` + the env map. */
@@ -337,8 +446,16 @@ export async function invokeStep<TOutput = unknown>(
 
   let result: { stdout: string; stderr: string; exitCode: number };
   try {
+    // Run the agent as ROOT (ADR-0019 follow-up). The factory drive (Archil)
+    // presents its S3-synced files root-owned and exposes no uid-mapped mount,
+    // so a non-root agent EACCESes on every shared file — the reason for the
+    // expensive boot-time `chmod -R` walk. As root the agent writes them
+    // directly: the walk disappears entirely. Egress is edge-enforced with NO
+    // root exemption (see sandbox.ts), so root is confined exactly like the
+    // non-root user. `HOME=/root` so the spawned `claude` finds root's skills.
     result = await sandbox.commands.run(RUNNER_COMMAND, {
-      envs,
+      envs: { ...envs, HOME: "/root", IS_SANDBOX: "1" },
+      sudo: true,
       timeoutMs: 0,
       ...(splitters.onStdout ? { onStdout: splitters.onStdout } : {}),
       ...(splitters.onStderr ? { onStderr: splitters.onStderr } : {}),
@@ -384,8 +501,25 @@ export async function launchStep<TOutput = unknown>(
   }
   const { resultToken, envs } = await prepareStepLaunch(sandbox, request, opts);
   const splitters = makeStreamSplitters(opts, resultToken);
-  const proc = await sandbox.commands.runBackground(RUNNER_COMMAND, {
-    envs,
+  // Tee the runner's stdout to a durable, token-keyed LOG file (stderr stays a
+  // separate live stream). The live output rides E2B's connect-web command
+  // stream, which THROWS on a compressed large frame (gRPC-web cannot decode
+  // message compression) and kills the feed mid-run. The tee'd file, read back
+  // over the envd HTTP transport (compression-immune), lets `wait()` recover the
+  // full logs + result instead of failing a run that actually completed. `tee`
+  // runs in the same `bash -c` pipeline E2B already wraps the command in, so the
+  // background pid (the pause / reconnect-by-pid handle) is unchanged; the result
+  // sentinel still rides stdout (tee passes it through) and the result FILE is
+  // written by the runner directly, so the happy path is untouched. `set -o
+  // pipefail` is REQUIRED: without it the pipeline's exit code is tee's (0),
+  // masking a non-zero runner exit and breaking the runner-exit classification;
+  // pipefail propagates the runner's code (E2B execs via `bash -c`).
+  //
+  // Run the agent as ROOT — see invokeStep above. `sudo:true` → user:"root";
+  // `HOME=/root` so the spawned `claude` finds root's skills.
+  const proc = await sandbox.commands.runBackground(`set -o pipefail; ${RUNNER_COMMAND} | tee ${stepLogFilePath(resultToken)}`, {
+    envs: { ...envs, HOME: "/root", IS_SANDBOX: "1" },
+    sudo: true,
     timeoutMs: 0,
     ...(splitters.onStdout ? { onStdout: splitters.onStdout } : {}),
     ...(splitters.onStderr ? { onStderr: splitters.onStderr } : {}),
@@ -394,9 +528,25 @@ export async function launchStep<TOutput = unknown>(
     runnerPid:   proc.pid,
     resultToken,
     async wait() {
-      const result = await proc.wait();
-      splitters.flush();
-      return classifyRunnerOutcome<TOutput>(sandbox, result, resultToken);
+      try {
+        const result = await proc.wait();
+        splitters.flush();
+        return classifyRunnerOutcome<TOutput>(sandbox, result, resultToken);
+      } catch (e) {
+        // A genuine infra death must propagate so the engine re-provisions.
+        if (e instanceof SandboxUnavailableError) throw e;
+        // Otherwise the runner's LIVE output stream died while the runner itself
+        // is alive and writing its durable result + tee'd log files. The dominant
+        // cause is E2B's connect-web transport throwing on a compressed large
+        // frame ("received unsupported compressed output") — a LOG-TRANSPORT
+        // fault, NEVER a reason to fail a run that completed. Degrade the live
+        // feed, alert, and recover the result + full logs over the HTTP file
+        // transport. We do NOT flush the live splitter here — the recovery
+        // backfills from the last whole line, re-reading any in-flight partial
+        // line whole from the durable log (avoids a split/duplicated line).
+        opts?.onStreamDegraded?.({ error: e, runnerPid: proc.pid, resultToken });
+        return recoverLogsAndResult<TOutput>(sandbox, proc.pid, resultToken, splitters, opts);
+      }
     },
   };
 }
@@ -414,38 +564,50 @@ export async function launchStep<TOutput = unknown>(
 export async function reconnectStep<TOutput = unknown>(
   sandbox: SandboxProvider,
   resume: { runnerPid: number; resultToken: string; stepIndex: number },
-  opts?: Pick<InvokeStepOptions, "onStdout" | "onStderr">,
+  opts?: Pick<InvokeStepOptions, "onStdout" | "onStderr"> & { signal?: AbortSignal },
 ): Promise<RunningStep<TOutput>> {
   if (!sandbox.commands.connectProcess) {
     throw new Error("reconnectStep requires a provider with background-command support (commands.connectProcess)");
   }
   const { runnerPid, resultToken } = resume;
   const splitters = makeStreamSplitters(opts, resultToken);
-  let proc: SandboxBackgroundProcess | null = null;
+  const signal = opts?.signal;
+
+  // Re-attach to the suspended runner for LIVE stdout streaming only. This is
+  // best-effort: a reconnect that throws after the native resume just means no
+  // live feed for the dashboard — completion is read from the durable result
+  // file below, NOT from this handle.
+  let attached: SandboxBackgroundProcess | null = null;
   try {
-    proc = await sandbox.commands.connectProcess(runnerPid, {
+    attached = await sandbox.commands.connectProcess(runnerPid, {
       ...(splitters.onStdout ? { onStdout: splitters.onStdout } : {}),
       ...(splitters.onStderr ? { onStderr: splitters.onStderr } : {}),
     });
   } catch (e) {
-    // A sandbox-infrastructure failure must propagate intact (recovery
-    // contract). Anything else means the runner already exited during resume
-    // (we re-attached too late) — its durable result file is written, so the
-    // handle's wait() classifies from the file below.
     if (e instanceof SandboxUnavailableError) throw e;
   }
-  const attached = proc;
+
   return {
     runnerPid,
     resultToken,
     async wait() {
-      let output: { stdout: string; stderr: string; exitCode: number } = { stdout: "", stderr: "", exitCode: 0 };
-      if (attached) {
-        try { output = await attached.wait(); }
-        catch (e) { if (e instanceof SandboxUnavailableError) throw e; }
+      // CRITICAL: do NOT trust `connect(pid).wait()` on the resume path. After a
+      // native VM pause/resume, E2B's re-attached handle can resolve its wait()
+      // EARLY — the reconnected stdout stream closes while the runner process is
+      // still alive and working — returning a default `{exitCode:0}`. That made
+      // the worker report "exited 0 without emitting" and force-kill a runner
+      // that was mid-iteration. So ignore the handle for completion and poll the
+      // runner's durable result file + `/proc` liveness instead — the same
+      // authoritative signal the launch-path recovery uses (`pollDurableResult`).
+      try {
+        const result = await pollDurableResult<TOutput>(sandbox, runnerPid, resultToken, signal);
+        splitters.flush();
+        return result;
+      } finally {
+        // Release the streaming handle (keeps it un-GC'd for the duration of the
+        // poll above; a dangling reconnect after a re-pause is cleaned up here).
+        await attached?.kill().catch(() => {});
       }
-      splitters.flush();
-      return classifyRunnerOutcome<TOutput>(sandbox, output, resultToken);
     },
   };
 }

package/src/step-invocation/protocol.ts

@@ -80,3 +80,14 @@ export function requestContextPath(stepIndex: number): string {
 export function stepResultFilePath(token: string): string {
   return `/tmp/wf/step-result-${token}.json`;
 }
+
+/** Sandbox-side path where the runner's stdout is tee'd as a durable LOG file,
+ *  keyed by the per-invocation token. The live output rides E2B's connect-web
+ *  command stream, which THROWS on a compressed large frame (gRPC-web cannot
+ *  decode message compression) and kills the feed mid-run. This file, read back
+ *  over the envd HTTP file transport (compression-immune), lets the invoker
+ *  recover the FULL logs after such a fault instead of losing the tail — the
+ *  log-side analogue of `stepResultFilePath` for the result. */
+export function stepLogFilePath(token: string): string {
+  return `/tmp/wf/step-log-${token}.log`;
+}

package/src/types/sandbox.ts

@@ -122,6 +122,15 @@ export interface SandboxProvider {
   };
   files: {
     write(path: string, content: string): Promise<void>;
+    /** Read a file's text content over the provider's FILE transport. On E2B this
+     *  is the envd HTTP API (`Sandbox.files.read`), a DIFFERENT transport from
+     *  `commands` — so a large readback is immune to the connect-web gRPC
+     *  message-compression that can abort `commands.run` output on a big frame
+     *  ("received unsupported compressed output"). This is what lets `launchStep`
+     *  recover the full logs + result after a live-stream fault. OPTIONAL —
+     *  implemented where durable file-readback is needed (E2B, local); providers
+     *  that never drive the recovery path (Vercel — foreground only) may omit it. */
+    read?(path: string): Promise<string>;
   };
   kill(): Promise<void>;
   /** Capture the running sandbox's state as a reusable snapshot. Vercel and E2B

package/src/types/workflow.ts

@@ -110,6 +110,12 @@ export interface WorkflowDefinition<
   /** Same as `input`, for the workflow's return value. Captured into
    *  `outputSchema` metadata and rendered in the IO panel. */
   output?: z.ZodType<TOutput>;
+  /**
+   * @deprecated Legacy run-form. Prefer step-form — the
+   * `.step(defineStep(...))` builder — for per-step durability/replay and
+   * working pause. A run-form body compiles to one opaque step
+   * (`compileRunForm`), so any failure/resume re-runs the whole body.
+   */
   run: WorkflowFn<TOutput, TInput>;
   /**
    * All snapshot config — boot source plus capture mode.
@@ -298,6 +304,12 @@ function compileRunForm<TOutput, TInput extends Record<string, unknown>>(
  * stores the step plan; runner subprocesses execute one step at a time
  * via the StepInvocation seam.
  */
+/**
+ * @deprecated Run-form is legacy. Use the step-form overload —
+ * `defineWorkflow({ id, input, output }).step(defineStep(...)).build()` — for
+ * durable, replayable steps and working pause. Run-form compiles to a single
+ * opaque step (`compileRunForm`); there is no per-step replay.
+ */
 export function defineWorkflow<
   TOutput = unknown,
   TInput extends Record<string, unknown> = Record<string, unknown>,

package/src/utils/errors.ts

@@ -1,4 +1,21 @@
-/** Convert any thrown value to a string message. */
+/** Convert any thrown value to a string message, INCLUDING its `.cause` chain.
+ *
+ *  Many wrapped errors carry the real reason on `.cause` and only a generic
+ *  summary on `.message` — the Temporal SDK's "Failed to start Workflow" is the
+ *  canonical example (its `.cause` is the actual gRPC rejection, e.g. "search
+ *  attribute X is not defined"). Returning `.message` alone swallowed that, so
+ *  failures surfaced as opaque one-liners. Walk the chain and join the messages
+ *  so the root cause is always visible. Cycle-guarded against self-referential
+ *  `cause` links. */
 export function formatError(err: unknown): string {
-  return err instanceof Error ? err.message : String(err);
+  if (!(err instanceof Error)) return String(err);
+  const parts: string[] = [err.message];
+  const seen = new Set<unknown>([err]);
+  let cause: unknown = (err as { cause?: unknown }).cause;
+  while (cause != null && !seen.has(cause)) {
+    seen.add(cause);
+    parts.push(cause instanceof Error ? cause.message : String(cause));
+    cause = cause instanceof Error ? (cause as { cause?: unknown }).cause : undefined;
+  }
+  return parts.join(": ");
 }