@agent-compose/sdk 0.8.1 → 0.8.2

package/src/runtimes/_cli-agent.ts

@@ -24,7 +24,7 @@
  * values set via `agentc secrets set` are visible to the CLI.
  */
 
-import type { AgentMessage, ModelExecutionContract, RuntimeOptions, SandboxCommandResult, SandboxProvider, ToolCallGateResult } from "../index.js";
+import type { AgentMessage, ModelExecutionContract, RuntimeOptions, SandboxProvider, ToolCallGateResult } from "../index.js";
 import { defineRuntime } from "../types/runtime.js";
 import { AsyncQueue } from "../agent/async-queue.js";
 import { formatError } from "../utils/errors.js";
@@ -35,12 +35,65 @@ import { RequestContext } from "../request-context/request-context.js";
 import { AcpClientPeer, ACP_PROTOCOL_VERSION } from "./_acp-client.js";
 import { isPauseSignal, boundProcessorPause } from "../pause/pause-core.js";
 import {
-  JSONL_GUARD_LINE_CAP_BYTES, JSONL_GUARD_KEEP_BYTES,
+  JSONL_GUARD_LINE_CAP_BYTES, JSONL_GUARD_KEEP_BYTES, JSONL_GUARD_NO_SENTINEL_EXIT,
   jsonlGuardScript, wrapJsonlCommand,
 } from "./_jsonl-guard.js";
+import type { SandboxBackgroundProcess } from "../types/sandbox.js";
 
 function now(): string { return new Date().toISOString(); }
 
+/** Backoff between tail re-attach attempts after a mid-turn stream fault
+ *  (durable detached transport). Short: the runner is alive and producing,
+ *  and each retry costs one exec; the executor's evidence machinery — not
+ *  this loop — bounds a truly dead sandbox. Exported for tests. */
+export const TAIL_REATTACH_DELAY_MS = 2_000;
+
+/** Provider deadline on the detached LAUNCH exec. The launch shell only
+ *  truncates the durable files, forks the detached runner (stdio fully
+ *  redirected — see the launch command), writes the pidfile, and echoes the
+ *  pid — sub-second work, so 30s is generous headroom for a slow envd, not a
+ *  budget the runner ever consumes. Exported for tests. */
+export const LAUNCH_EXEC_TIMEOUT_MS = 30_000;
+
+/** How many times (and how spaced) a THROWN launch exec falls back to reading
+ *  the durable pidfile before the fault is surfaced. A deadlined/canceled
+ *  launch STREAM does not mean the launch failed — the detached tree may be
+ *  up and working — so the pid is recovered over the envd HTTP file
+ *  transport (immune to the command-stream fault) and the turn proceeds.
+ *  Only "no pidfile after these attempts" is a real launch failure.
+ *  Exported for tests. */
+export const LAUNCH_PID_RECOVERY_ATTEMPTS = 3;
+export const LAUNCH_PID_RECOVERY_DELAY_MS = 1_000;
+
+/** Explicit deadline on the `command -v` install probe. E2B's default command
+ *  deadline (60s) throws the deadline_exceeded TimeoutError — every provider
+ *  call in the turn path carries an explicit timeout so no SDK default can
+ *  decide a turn's fate. Exported for tests. */
+export const INSTALL_PROBE_TIMEOUT_MS = 30_000;
+
+/** Parse a pid out of captured stdout (last non-empty line). NaN when absent. */
+function parsePid(raw: string): number {
+  const pid = Number.parseInt(raw.trim().split("\n").pop() ?? "", 10);
+  return Number.isInteger(pid) && pid > 0 ? pid : Number.NaN;
+}
+
+/** Recover the detached runner's pid from its durable pidfile after the
+ *  launch exec's STREAM died (deadline_exceeded, canceled context, any wire
+ *  fault). Reads over the file transport with a short retry so the race
+ *  where the launch shell is still writing the pidfile is absorbed. */
+async function recoverLaunchPid(read: (path: string) => Promise<string>, pidPath: string): Promise<number> {
+  for (let attempt = 0; attempt < LAUNCH_PID_RECOVERY_ATTEMPTS; attempt++) {
+    try {
+      const pid = parsePid(await read(pidPath));
+      if (!Number.isNaN(pid)) return pid;
+    } catch { /* file plane briefly unreachable — retried */ }
+    if (attempt < LAUNCH_PID_RECOVERY_ATTEMPTS - 1) {
+      await new Promise((r) => setTimeout(r, LAUNCH_PID_RECOVERY_DELAY_MS));
+    }
+  }
+  return Number.NaN;
+}
+
 /** Single-quote a value for safe interpolation into a `sh -c` command line. */
 export function shellQuote(value: string): string {
   return `'${value.replace(/'/g, `'\\''`)}'`;
@@ -302,7 +355,8 @@ export class CliAgentRunner implements ModelExecutionContract {
    *  install command runs exactly once: on the first run of a content hash. */
   private async ensureInstalled(): Promise<void> {
     if (this.installed) return;
-    const probe = await this.sandbox.commands.run(`command -v ${this.spec.bin}`);
+    const probe = await this.sandbox.commands.run(
+      `command -v ${this.spec.bin}`, { timeoutMs: INSTALL_PROBE_TIMEOUT_MS });
     if (probe.exitCode === 0) { this.installed = true; return; }
     const res = await this.sandbox.commands.run(this.spec.install, { timeoutMs: 300_000 });
     if (res.exitCode !== 0) {
@@ -640,53 +694,224 @@ export class CliAgentRunner implements ModelExecutionContract {
         keepBytes: JSONL_GUARD_KEEP_BYTES,
       });
 
-      // Bridge the streaming stdout callback into an async-iterable of complete
-      // JSONL lines. `onStdout` chunks aren't line-aligned, so buffer + split.
+      // Bridge the transport's output into an async-iterable of complete
+      // JSONL lines. Two transports feed it (chosen below):
+      //
+      //   - DURABLE DETACHED (providers with `runBackground` + `files.read`,
+      //     i.e. E2B — every cloud session): the CLI runs as a DETACHED
+      //     in-guest process whose stdout is a durable file; the server only
+      //     ever TAILS that file, and a dead tail is re-attached (or the file
+      //     re-read over the envd HTTP file transport) without the runner
+      //     noticing. The exit code travels IN-BAND as a final sentinel line.
+      //     This exists because of the 2026-08-14 prod incident (conversation
+      //     8bc082f0, turn bfcb318f): the turn's single streaming exec was the
+      //     CLI's lifeline — when the connect-web stream's context was
+      //     canceled at the wire 50s in (envd logged `context canceled`), envd
+      //     tore down the exec's stdout, the guard died on EPIPE, the CLI took
+      //     SIGPIPE mid-work, the finished output was lost, and `wait()` never
+      //     settled — the turn wedged silently until the next dispatch
+      //     superseded it as a phantom "session restarted".
+      //
+      //   - SINGLE-EXEC STREAMING (everything else: local child_process,
+      //     Vercel): the pre-incident shape — one exec, stdout streamed,
+      //     exit code from the exec result.
       const lines = new AsyncQueue<string>();
-      let buf = "";
-      const onStdout = (data: string) => {
-        buf += data;
-        let nl: number;
-        while ((nl = buf.indexOf("\n")) >= 0) {
-          const line = buf.slice(0, nl).trim();
-          buf = buf.slice(nl + 1);
-          if (line) lines.push(line);
-        }
-      };
-      const runOpts = {
-        ...(this.options.cwd ? { cwd: this.options.cwd } : {}),
-        onStdout,
-        timeoutMs: 0,   // no provider deadline — see the doc comment above
-      };
-
-      // The command resolves when the process exits. Kick it off (don't await
-      // yet); flush the trailing buffer + close the queue on completion so the
-      // for-await below drains and we can read the exit code. Prefer the
-      // kill-capable background handle so abort/early-exit can terminate the
-      // process itself, not just stop reading its stream.
-      let exited = false;
-      const settle = (res: SandboxCommandResult): SandboxCommandResult => {
-        exited = true;
-        const tail = buf.trim();
-        if (tail) lines.push(tail);
-        lines.close();
-        return res;
-      };
-      const fail = (err: unknown): never => { exited = true; lines.close(); throw err; };
-      let kill: (() => Promise<void>) | undefined;
-      let runPromise: Promise<SandboxCommandResult>;
+      // How the turn ended: the CLI's exit code once known (null while
+      // running), plus a best-effort stderr tail for the error message.
+      let exitCode: number | null = null;
+      let exitStderr = "";
+      let transportError: unknown = null;
+      let consumerStopped = false;
+      let reap: () => void = () => { /* set per transport */ };
+      let transport: Promise<void>;
+
+      const durableRead = this.sandbox.files.read?.bind(this.sandbox.files);
       if (this.sandbox.commands.runBackground) {
-        const handle = await this.sandbox.commands.runBackground(guardedCmd, runOpts);
-        kill = () => handle.kill();
-        runPromise = handle.wait().then(settle, fail);
+        // ── Durable detached transport ─────────────────────────────────────
+        const outPath = `${promptPath}.out`;
+        const errPath = `${promptPath}.err`;
+        // The exit sentinel is appended by the DETACHED script itself after
+        // the guarded pipeline exits, so it lands in the durable file even if
+        // every server↔sandbox stream is dead by then. In node-guard mode the
+        // pipeline's exit code is already the CLI's (the guard re-raises the
+        // inner sentinel, including NO_SENTINEL_EXIT for a mid-pipe death).
+        const detachedScript =
+          `{ ${guardedCmd}; } >> ${shellQuote(outPath)} 2>> ${shellQuote(errPath)} </dev/null; `
+          + `printf '\\n%s %s\\n' ${shellQuote(sentinel)} "$?" >> ${shellQuote(outPath)}`;
+        // `setsid` detaches the runner into its own session/process group so
+        // it survives the launch exec ending AND gives `kill -- -pid` a whole
+        // tree to terminate. The fallback keeps working where setsid is
+        // absent; a runner that dies anyway surfaces as NO_SENTINEL below.
+        //
+        // The `>/dev/null 2>&1 </dev/null` on the forked child is LOAD-BEARING,
+        // not hygiene (2026-08-14 prod regression, v0.10.39): without it the
+        // detached `sh` inherits the LAUNCH exec's stdout/stderr pipes, and
+        // envd only ends a command's event stream on pipe EOF — so the launch
+        // exec's wait() couldn't settle until the entire detached CLI tree
+        // exited, the 30s launch deadline fired first, and EVERY turn died
+        // with E2B's `[deadline_exceeded] the operation timed out: … exceeding
+        // 'timeoutMs' …` while the orphaned runner kept working unobserved.
+        // (The runner's real stdio is redirected INSIDE detachedScript — into
+        // the durable out/err files — so the child needs nothing from the
+        // launch exec's pipes.)
+        //
+        // The pid ALSO lands in a durable pidfile so a launch whose STREAM
+        // dies (the same wire faults the tail survives) is recovered over the
+        // file transport instead of surfacing as turn death — see the catch
+        // below.
+        const pidPath = `${promptPath}.pid`;
+        const launchCmd =
+          `: > ${shellQuote(outPath)}; : > ${shellQuote(errPath)}; `
+          + `if command -v setsid >/dev/null 2>&1; then setsid sh -c ${shellQuote(detachedScript)} >/dev/null 2>&1 </dev/null & `
+          + `else sh -c ${shellQuote(detachedScript)} >/dev/null 2>&1 </dev/null & fi; `
+          + `echo "$!" > ${shellQuote(pidPath)}; echo "$!"`;
+        let pid = Number.NaN;
+        let launched: { exitCode: number; stdout?: string; stderr?: string } | null = null;
+        let launchErr: unknown = null;
+        try {
+          launched = await this.sandbox.commands.run(launchCmd, {
+            ...(this.options.cwd ? { cwd: this.options.cwd } : {}), timeoutMs: LAUNCH_EXEC_TIMEOUT_MS });
+        } catch (err) { launchErr = err; }
+        if (launched) {
+          // The exec ROUND-TRIPPED: its exit code and stdout are authoritative.
+          pid = parsePid(launched.stdout ?? "");
+          if (launched.exitCode !== 0 || Number.isNaN(pid)) {
+            throw new Error(`failed to launch ${this.spec.kind}: `
+              + `exit ${launched.exitCode}${launched.stderr ? `: ${launched.stderr.slice(-500)}` : ""}`);
+          }
+        } else {
+          // Structural rule of this transport: a dead STREAM is never treated
+          // as a dead RUNNER. The launch may well have run to completion with
+          // only its exec stream lost (deadline_exceeded, canceled context) —
+          // recover the pid from the durable pidfile and continue; only an
+          // absent pidfile makes the original fault real.
+          pid = durableRead ? await recoverLaunchPid(durableRead, pidPath) : Number.NaN;
+          if (Number.isNaN(pid)) throw launchErr;
+          console.warn(`[cli-agent] ${this.spec.kind} launch exec stream died but the detached runner is up `
+            + `(pid ${pid} via ${pidPath}) — continuing on the durable transport: ${formatError(launchErr)}`);
+        }
+
+        // Bytes of COMPLETE lines already consumed from outPath — a re-attach
+        // tails from here, so a dead stream never duplicates or drops lines.
+        let offset = 0;
+        let currentTail: SandboxBackgroundProcess | null = null;
+        const consumeLine = (raw: string): void => {
+          offset += Buffer.byteLength(raw, "utf8") + 1;
+          const line = raw.trim();
+          if (!line) return;
+          if (line.startsWith(sentinel)) {
+            const code = Number(line.slice(sentinel.length).trim());
+            exitCode = Number.isFinite(code) ? code : 0;
+            return;
+          }
+          lines.push(line);
+        };
+
+        reap = () => {
+          // Kill the detached tree (group first — setsid made pid the group
+          // leader), then whatever tail is currently attached. Fresh execs,
+          // deliberately independent of any possibly-dead stream.
+          void this.sandbox.commands.run(
+            `kill -TERM -- -${pid} 2>/dev/null; kill -TERM ${pid} 2>/dev/null; true`,
+            { timeoutMs: 10_000 }).catch(() => { /* best-effort */ });
+          void currentTail?.kill().catch(() => { /* already gone */ });
+        };
+
+        transport = (async () => {
+          while (exitCode === null && !consumerStopped && !opts.signal?.aborted) {
+            let buf = "";
+            let handle: SandboxBackgroundProcess | null = null;
+            const onStdout = (data: string): void => {
+              buf += data;
+              let nl: number;
+              while ((nl = buf.indexOf("\n")) >= 0) {
+                consumeLine(buf.slice(0, nl));
+                buf = buf.slice(nl + 1);
+                if (exitCode !== null) { void handle?.kill().catch(() => { /* racing exit */ }); return; }
+              }
+            };
+            try {
+              handle = await this.sandbox.commands.runBackground!(
+                `tail -c +${offset + 1} -f ${shellQuote(outPath)}`,
+                { timeoutMs: 0, onStdout });
+            } catch { /* attach failed at the wire — durable catch-up below */ }
+            if (handle) {
+              if (exitCode !== null) void handle.kill().catch(() => { /* done */ });
+              currentTail = handle;
+              await handle.wait().catch(() => undefined);
+              currentTail = null;
+            }
+            if (exitCode !== null || consumerStopped || opts.signal?.aborted) break;
+            // The tail stream died without delivering the exit line. Catch up
+            // over the FILE transport (envd HTTP — immune to the command-
+            // stream fault): this is also what recovers an already-finished
+            // turn's result when every stream is dead.
+            if (durableRead) {
+              try {
+                const bytes = Buffer.from(await durableRead(outPath), "utf8");
+                let rest = bytes.subarray(offset).toString("utf8");
+                let nl: number;
+                while (exitCode === null && (nl = rest.indexOf("\n")) >= 0) {
+                  consumeLine(rest.slice(0, nl));
+                  rest = rest.slice(nl + 1);
+                }
+              } catch { /* file plane briefly unreachable — retried below */ }
+            }
+            if (exitCode !== null) break;
+            // Runner still alive? A FRESH exec answers; unknown (wire fault)
+            // keeps retrying — the executor's evidence machinery bounds a
+            // truly dead sandbox, not this loop.
+            let alive: boolean | null = null;
+            try {
+              const probe = await this.sandbox.commands.run(
+                `kill -0 ${pid} 2>/dev/null`, { timeoutMs: 10_000 });
+              alive = probe.exitCode === 0;
+            } catch { alive = null; }
+            if (alive === false) {
+              // Dead with no exit line even in the durable file: abnormal
+              // death — surfaces as an error, never a clean done.
+              exitCode = JSONL_GUARD_NO_SENTINEL_EXIT;
+              break;
+            }
+            console.warn(`[cli-agent] ${this.spec.kind} turn stream lost with the runner alive — re-tailing ${outPath} from byte ${offset}`);
+            await new Promise((r) => setTimeout(r, TAIL_REATTACH_DELAY_MS));
+          }
+          if (exitCode !== null && exitCode !== 0 && durableRead) {
+            try { exitStderr = (await durableRead(errPath)).slice(-2000); }
+            catch { /* best-effort */ }
+          }
+        })().finally(() => lines.close());
       } else {
-        runPromise = this.sandbox.commands.run(guardedCmd, runOpts).then(settle, fail);
+        // ── Single-exec streaming transport (no runBackground/files.read) ──
+        let buf = "";
+        const onStdout = (data: string) => {
+          buf += data;
+          let nl: number;
+          while ((nl = buf.indexOf("\n")) >= 0) {
+            const line = buf.slice(0, nl).trim();
+            buf = buf.slice(nl + 1);
+            if (line) lines.push(line);
+          }
+        };
+        const runOpts = {
+          ...(this.options.cwd ? { cwd: this.options.cwd } : {}),
+          onStdout,
+          timeoutMs: 0,   // no provider deadline — see the doc comment above
+        };
+        transport = this.sandbox.commands.run(guardedCmd, runOpts).then(
+          (res) => {
+            const tail = buf.trim();
+            if (tail) lines.push(tail);
+            exitCode = res.exitCode;
+            exitStderr = (res.stderr ?? "").slice(-2000);
+          },
+          (err) => { transportError = err; },
+        ).finally(() => lines.close());
       }
-      // An early generator exit stops awaiting runPromise — keep its rejection
+      // An early generator exit stops awaiting the transport — keep it
       // observed so an aborted turn can never surface an unhandled rejection.
-      runPromise.catch(() => { /* observed via `await runPromise` when consumed */ });
+      void transport.catch(() => { /* observed via `await transport` when consumed */ });
 
-      const reap = () => { if (!exited) void kill?.().catch(() => { /* already gone */ }); };
       if (opts.signal) {
         if (opts.signal.aborted) reap();
         else opts.signal.addEventListener("abort", reap, { once: true });
@@ -707,26 +932,38 @@ export class CliAgentRunner implements ModelExecutionContract {
           }
         }
 
-        const res = await runPromise;
-        if (res.exitCode !== 0 && !sawError) {
-          const tail = (res.stderr ?? "").slice(-2000);
-          yield { type: "error", text: `${this.spec.kind} exited with code ${res.exitCode}${tail ? `: ${tail}` : ""}`, timestamp: now() };
+        await transport;
+        if (transportError) throw transportError;
+        if (exitCode === null) {
+          // Aborted/reaped before an exit line existed — honest terminal,
+          // never a hang and never a clean done.
+          if (!sawError) {
+            yield { type: "error", text: `${this.spec.kind} aborted — the in-sandbox runner was terminated`, timestamp: now() };
+          }
+          return;
+        }
+        if (exitCode !== 0 && !sawError) {
+          const detail = exitCode === JSONL_GUARD_NO_SENTINEL_EXIT
+            ? " (the runner died without reporting an exit code)" : "";
+          yield { type: "error", text: `${this.spec.kind} exited with code ${exitCode}${detail}${exitStderr ? `: ${exitStderr}` : ""}`, timestamp: now() };
           return;
         }
         if (!sawError) yield { type: "done", sessionId: sessionId ?? "", timestamp: now() };
       } finally {
         // Runs on completion AND on early generator exit (the caller broke
         // out of its for-await: abort, turn closed under the executor,
-        // supersede). `reap` is a no-op once the process has exited.
+        // supersede). Reap only while the runner hasn't reported an exit.
         opts.signal?.removeEventListener("abort", reap);
-        reap();
-        // Opportunistic prompt-file + guard-script cleanup: paths are never
-        // reused (see uniquePromptPath), so this is hygiene, not correctness —
-        // fire and forget, and a dead sandbox / failed rm is fine. The shell
-        // holds an open fd on the redirect, so unlinking under a still-exiting
-        // CLI is harmless.
+        consumerStopped = true;
+        if (exitCode === null) reap();
+        // Opportunistic prompt-file + guard-script + transcript cleanup:
+        // paths are never reused (see uniquePromptPath), so this is hygiene,
+        // not correctness — fire and forget, and a dead sandbox / failed rm
+        // is fine. The shell holds an open fd on the redirect, so unlinking
+        // under a still-exiting CLI is harmless.
         void this.sandbox.commands.run(
-          `rm -f ${shellQuote(promptPath)} ${shellQuote(guardPath)}`, { timeoutMs: 10_000 })
+          `rm -f ${shellQuote(promptPath)} ${shellQuote(guardPath)} ${shellQuote(`${promptPath}.out`)} ${shellQuote(`${promptPath}.err`)} ${shellQuote(`${promptPath}.pid`)}`,
+          { timeoutMs: 10_000 })
           .catch(() => { /* best-effort */ });
       }
     } catch (err) {

package/src/runtimes/codex.ts

@@ -95,14 +95,20 @@ export const codexSpec: CliAgentSpec = {
       // backstop for a caller that bypasses that gate. The value comes from
       // the closed CliReasoningEffort set, so it is shell-safe unquoted.
       ...(effort ? ["-c", `model_reasoning_effort=${effort === "max" ? "xhigh" : effort}`] : []),
-      ...(cwd ? ["-C", shellQuote(cwd)] : []),
     ].join(" ");
+    // Working directory via a shell `cd` — the idiom EVERY other runtime uses
+    // (claude-code/cursor/droid/opencode) — NOT codex's `-C` flag. `-C` is
+    // accepted by `codex exec` but REJECTED by `codex exec resume`
+    // ("unexpected argument '-C'"), so a fresh turn worked and every follow-up
+    // died. `cd` sets the cwd identically for both paths; promptPath is
+    // absolute (/tmp/…), so the `< prompt` redirect survives the cd.
+    const cd = cwd ? `cd ${shellQuote(cwd)} && ` : "";
     // Fresh turn: `codex exec <flags> - < prompt`. Continue a thread:
     // `codex exec resume <id> <flags> - < prompt`. (`-` = read prompt from stdin.)
     const exec = sessionId
       ? `codex exec resume ${shellQuote(sessionId)} ${flags}`
       : `codex exec ${flags}`;
-    return `${exec} - < ${shellQuote(promptPath)}`;
+    return `${cd}${exec} - < ${shellQuote(promptPath)}`;
   },
   extractSessionId: (p) =>
     p.type === "thread.started" && typeof p.thread_id === "string" ? p.thread_id : undefined,

package/src/sandbox/providers/e2b.ts

@@ -237,11 +237,15 @@ function makeE2bSandboxProvider(sb: Sandbox): SandboxProvider {
       return { resumeHandle: sb.sandboxId };
     },
     // The provider kill-clock seam: e2b's `setTimeout` REPLACES the deadline
-    // (extend or reduce) relative to now. The server's session lifecycle
-    // keeps this behind its own suspend clock so the deferred pause always
-    // wins the race against the create-time timeout.
+    // (extend or reduce) relative to now, in MILLISECONDS. The server's
+    // session lifecycle uses it two ways: a FAR-HORIZON push while the
+    // session is active (the kill-clock is an orphan backstop, never
+    // load-bearing during a live turn) and a short re-arm when the deferred
+    // pause is about to park the VM. Clamped to the plan cap exactly like
+    // the create timeout — an over-cap push would 400 and leave the OLD
+    // (possibly short) deadline standing.
     async extendLifetime(ms) {
-      await sb.setTimeout(ms);
+      await sb.setTimeout(Math.min(ms, e2bMaxSandboxMs()));
     },
     // Push a freshly-resolved egress policy onto the live sandbox via E2B's
     // native `updateNetwork` — the E2B analogue of Vercel's `update({