@agent-compose/sdk 0.7.0 → 0.8.0

package/src/step-invocation/invoker.ts

@@ -25,7 +25,7 @@ import { randomBytes } from "node:crypto";
 import { z } from "zod";
 import { SandboxUnavailableError } from "../sandbox-errors.js";
 import type { SandboxProvider, SandboxBackgroundProcess } from "../types/sandbox.js";
-import { RUNNER_COMMAND, STEP_ENV, stepResultLinePrefix, stepPauseLinePrefix, requestContextPath, stepInputPath, stepResultFilePath, stepLogFilePath } from "./protocol.js";
+import { RUNNER_COMMAND, STEP_ENV, stepResultLinePrefix, stepPauseLinePrefix, requestContextPath, stepInputPath, stepResultFilePath, stepLogFilePath, stepPidFilePath } from "./protocol.js";
 import { StepPauseRequestSchema } from "./types.js";
 import type { StepRequest, StepResult } from "./types.js";
 import type { StepObservability } from "../workflow-steps/observability.js";
@@ -201,6 +201,11 @@ export interface InvokeStepOptions {
    *  is the hook to emit a structured alert so the degradation is visible/paged.
    *  Best-effort: keep it cheap and non-throwing. */
   onStreamDegraded?: (info: { error: unknown; runnerPid: number; resultToken: string }) => void;
+  /** Ceiling on the durable-result recovery poll after a live-stream fault
+   *  (default 45m — the wedged-runner backstop). The activity passes its step
+   *  ceiling so a stream fault on a step with hours of legitimate work left
+   *  recovers instead of timing out; startToClose is the real wall. */
+  recoveryDeadlineMs?: number;
 }
 
 /** A step running as a background command (ADR-0028). The activity races its
@@ -331,20 +336,43 @@ async function classifyRunnerOutcome<TOutput>(
  *  pause/resume) AND the launch path after a live-stream transport fault.
  *  `cat`-ing the result file is itself immune to the large-frame compression bug
  *  — it is a single small JSON line, far below any compression threshold. The
- *  activity's `startToCloseTimeout` is the real upper bound; `DEADLINE` is a
- *  backstop so a wedged runner can't leak this loop in the worker forever. */
+ *  activity's `startToCloseTimeout` is the real upper bound; the deadline here
+ *  is a backstop so a wedged runner can't leak this loop in the worker forever
+ *  (default 45m; a worker-death recovery reconnecting mid-step passes the full
+ *  step ceiling instead — hours of legitimate work may remain). */
 async function pollDurableResult<TOutput>(
   sandbox: SandboxProvider,
   runnerPid: number,
   resultToken: string,
-  signal?: AbortSignal,
+  opts?: { signal?: AbortSignal; deadlineMs?: number },
 ): Promise<StepResult<TOutput>> {
+  const signal = opts?.signal;
   const resultFile = stepResultFilePath(resultToken);
   const POLL_MS = 1000;
-  const DEADLINE = Date.now() + 45 * 60_000;
-  const readResult = async (): Promise<StepResult<TOutput> | null> => {
-    const r = await sandbox.commands.run(`cat ${resultFile} 2>/dev/null || true`).catch(() => null);
-    return r?.stdout ? (parseStepResult<TOutput>(r.stdout, resultToken) ?? null) : null;
+  const deadlineMs = opts?.deadlineMs ?? 45 * 60_000;
+  const DEADLINE = Date.now() + deadlineMs;
+  // Both internal commands carry an explicit budget. Required for Vercel:
+  // without `timeoutMs` there is no hard client-side deadline in the provider's
+  // `commands.run`, so a wedged VM hangs one `cat` forever and the poll
+  // deadline never advances. A timeout throws → the `.catch(() => null)`
+  // treats it as a failed read/probe and the loop continues (a hung probe
+  // never misclassifies as dead). Harmless on E2B — a 1-line cat is ms-fast.
+  const CMD_TIMEOUT_MS = 10_000;
+  // A destroyed sandbox (e.g. run cancelled → sandbox killed mid-recovery)
+  // rejects EVERY command, which a lone `.catch(() => null)` renders
+  // indistinguishable from a live runner — the loop would spin until the
+  // deadline (hours) against a VM that no longer exists. Track consecutive
+  // iterations where BOTH the result read and the liveness probe rejected and
+  // exit with an honest runner-exit once the streak is unambiguous. 20
+  // iterations ≈ 30s–7min of continuous rejections (each command may burn its
+  // 10s budget) — long enough to ride out a transient provider-API blip,
+  // short enough that a dead sandbox doesn't hold the worker slot for hours.
+  const UNREACHABLE_LIMIT = 20;
+  let unreachableStreak = 0;
+  const readResult = async (): Promise<StepResult<TOutput> | "unreachable" | null> => {
+    const r = await sandbox.commands.run(`cat ${resultFile} 2>/dev/null || true`, { timeoutMs: CMD_TIMEOUT_MS }).catch(() => "unreachable" as const);
+    if (r === "unreachable") return "unreachable";
+    return r.stdout ? (parseStepResult<TOutput>(r.stdout, resultToken) ?? null) : null;
   };
   for (let attempt = 1; ; attempt++) {
     // Aborted = the activity already resolved via the pause watcher (a re-pause);
@@ -354,11 +382,26 @@ async function pollDurableResult<TOutput>(
       return { ok: false, error: { kind: "runner-exit", message: "reconnectStep aborted (run re-paused)", exitCode: 1 } };
     }
     const fromFile = await readResult();
-    if (fromFile) return fromFile;
+    if (fromFile !== null && fromFile !== "unreachable") return fromFile;
 
     const probe = await sandbox.commands
-      .run(`test -d /proc/${runnerPid} && echo alive || echo dead`)
+      .run(`test -d /proc/${runnerPid} && echo alive || echo dead`, { timeoutMs: CMD_TIMEOUT_MS })
       .catch(() => null);
+    if (fromFile === "unreachable" && probe === null) {
+      unreachableStreak++;
+      if (unreachableStreak >= UNREACHABLE_LIMIT) {
+        return {
+          ok: false,
+          error: {
+            kind: "runner-exit",
+            message: `sandbox stopped responding while waiting for runner pid ${runnerPid} to emit a result (${UNREACHABLE_LIMIT} consecutive failed polls)`,
+            exitCode: 1,
+          },
+        };
+      }
+    } else {
+      unreachableStreak = 0;
+    }
     const dead = (probe?.stdout ?? "").includes("dead");
     process.stderr.write(
       `[recover] poll ${attempt}: result-file=absent runner=${dead ? "dead" : "alive"} pid=${runnerPid}\n`,
@@ -366,7 +409,7 @@ async function pollDurableResult<TOutput>(
     if (dead) {
       // Close the write-then-exit race with one final read, else classify a crash.
       const finalRead = await readResult();
-      if (finalRead) return finalRead;
+      if (finalRead !== null && finalRead !== "unreachable") return finalRead;
       return {
         ok: false,
         error: { kind: "runner-exit", message: `runner pid ${runnerPid} exited without writing a result file`, exitCode: 1 },
@@ -375,7 +418,11 @@ async function pollDurableResult<TOutput>(
     if (Date.now() > DEADLINE) {
       return {
         ok: false,
-        error: { kind: "runner-exit", message: `timed out after 45m waiting for runner pid ${runnerPid} to emit a result`, exitCode: 1 },
+        error: {
+          kind: "runner-exit",
+          message: `timed out after ${Math.round(deadlineMs / 60_000)}m waiting for runner pid ${runnerPid} to emit a result`,
+          exitCode: 1,
+        },
       };
     }
     await new Promise((r) => setTimeout(r, POLL_MS));
@@ -397,8 +444,9 @@ async function recoverLogsAndResult<TOutput>(
   resultToken: string,
   liveSplitters: { stdoutConsumedChars: () => number },
   opts: Pick<InvokeStepOptions, "onStdout" | "onStderr"> | undefined,
+  pollOpts?: { deadlineMs?: number },
 ): Promise<StepResult<TOutput>> {
-  const result = await pollDurableResult<TOutput>(sandbox, runnerPid, resultToken);
+  const result = await pollDurableResult<TOutput>(sandbox, runnerPid, resultToken, pollOpts);
   // Runner has exited → the tee'd log file is complete. Best-effort: a failed
   // readback just means the recovered run keeps the live logs it already had.
   if (sandbox.files.read && opts?.onStdout) {
@@ -416,6 +464,38 @@ async function recoverLogsAndResult<TOutput>(
   return result;
 }
 
+/** Read the foreground launch's pidfile back — the /proc liveness handle for
+ *  the recovery poll. Returns `null` when recovery is impossible, for either
+ *  reason: the sandbox cannot run ANY command (genuinely dead VM), or the
+ *  pidfile never appeared. The pidfile write is the FIRST statement of the
+ *  runner pipeline, so a still-absent pid after the retries means the runner
+ *  never started and no result file will ever exist — recovery would poll
+ *  blind for the full step ceiling and then misclassify; the caller must
+ *  rethrow the original terminal error instead (fast, honest). The retries
+ *  cover both a transiently-unreachable sandbox and the launch race where the
+ *  stream fault fired before the VM executed the pidfile write. */
+async function readRunnerPid(
+  sandbox: SandboxProvider,
+  resultToken: string,
+): Promise<number | null> {
+  const ATTEMPTS = 3;
+  const RETRY_DELAY_MS = 2_000;
+  for (let attempt = 1; attempt <= ATTEMPTS; attempt++) {
+    try {
+      const r = await sandbox.commands.run(
+        `cat ${stepPidFilePath(resultToken)} 2>/dev/null || true`,
+        { timeoutMs: 10_000 },
+      );
+      const pid = Number.parseInt(r.stdout.trim(), 10);
+      if (Number.isInteger(pid) && pid > 0) return pid;
+    } catch {
+      // Sandbox unreachable this attempt — fall through to the retry delay.
+    }
+    if (attempt < ATTEMPTS) await new Promise((r) => setTimeout(r, RETRY_DELAY_MS));
+  }
+  return null;
+}
+
 /** Write the step's input + request-context files and build the runner env.
  *  Shared by the foreground and background launch paths. Returns the
  *  per-invocation `resultToken` + the env map. */
@@ -453,21 +533,53 @@ export async function invokeStep<TOutput = unknown>(
     // directly: the walk disappears entirely. Egress is edge-enforced with NO
     // root exemption (see sandbox.ts), so root is confined exactly like the
     // non-root user. `HOME=/root` so the spawned `claude` finds root's skills.
-    result = await sandbox.commands.run(RUNNER_COMMAND, {
-      envs: { ...envs, HOME: "/root", IS_SANDBOX: "1" },
-      sudo: true,
-      timeoutMs: 0,
-      ...(splitters.onStdout ? { onStdout: splitters.onStdout } : {}),
-      ...(splitters.onStderr ? { onStderr: splitters.onStderr } : {}),
-    });
+    //
+    // Same pipeline pattern as `launchStep`: tee the runner's stdout to the
+    // durable token-keyed log file so a live-stream fault can backfill the
+    // undelivered tail, and write the wrapping shell's pid (`$$`) to a pidfile
+    // first — the shell waits on the pipeline, so `/proc/<pid>` liveness is a
+    // correct "runner may still write a result" proxy for the recovery poll.
+    // `set -o pipefail` is REQUIRED: without it the pipeline's exit code is
+    // tee's (0), masking a non-zero runner exit and breaking the runner-exit
+    // classification (the provider execs via `sh -c`, where pipefail works).
+    result = await sandbox.commands.run(
+      `set -o pipefail; echo $$ > ${stepPidFilePath(resultToken)}; ${RUNNER_COMMAND} | tee ${stepLogFilePath(resultToken)}`,
+      {
+        envs: { ...envs, HOME: "/root", IS_SANDBOX: "1" },
+        sudo: true,
+        timeoutMs: 0,
+        ...(splitters.onStdout ? { onStdout: splitters.onStdout } : {}),
+        ...(splitters.onStderr ? { onStderr: splitters.onStderr } : {}),
+      },
+    );
   } catch (e) {
     // Typed sandbox-infrastructure failure: the engine's recovery contract
-    // (re-provision when retryable, honest terminal classification otherwise)
-    // keys on this error propagating INTACT — its `[sandbox-unavailable:*]`
-    // message prefix must reach the workflow as the failure leaf, not ride an
-    // embedded stderr tail that a later slice(-2000) can drop. Rethrow before
-    // the CommandExitError downgrade below.
-    if (e instanceof SandboxUnavailableError) throw e;
+    // keys on retryability. Its `[sandbox-unavailable:*]` message prefix must
+    // reach the workflow as the failure leaf, not ride an embedded stderr tail
+    // that a later slice(-2000) can drop. Handle before the CommandExitError
+    // downgrade below.
+    if (e instanceof SandboxUnavailableError) {
+      // Pre-launch (retryable): nothing user-side ran — propagate INTACT so the
+      // workflow layer re-provisions (identity + prefix contract; pinned test).
+      if (e.retryable) throw e;
+      // Post-launch terminal: the LIVE stream/wait call died (e.g. Vercel's raw
+      // "The operation timed out.") while the runner is very likely still alive
+      // and will write its durable result file. Recover instead of failing —
+      // NEVER re-launch the runner. A null pid means recovery is impossible
+      // (sandbox can't run ANY command, or the pidfile never appeared because
+      // the runner never actually started): rethrow the original terminal
+      // classification fast rather than polling blind for the step ceiling.
+      // We do NOT flush the live splitter here — the recovery backfills from
+      // the last whole line, re-reading any in-flight partial line whole from
+      // the durable log (avoids a split/duplicated line).
+      const runnerPid = await readRunnerPid(sandbox, resultToken);
+      if (runnerPid === null) throw e;
+      opts?.onStreamDegraded?.({ error: e, runnerPid, resultToken });
+      return recoverLogsAndResult<TOutput>(
+        sandbox, runnerPid, resultToken, splitters, opts,
+        opts?.recoveryDeadlineMs !== undefined ? { deadlineMs: opts.recoveryDeadlineMs } : undefined,
+      );
+    }
     // Some providers (E2B) throw a CommandExitError on a non-zero exit instead of
     // returning it. Recover stdout/stderr/exitCode from the error so we can STILL
     // parse the runner's structured `__AC_STEP_RESULT__` payload — otherwise the
@@ -564,7 +676,14 @@ export async function launchStep<TOutput = unknown>(
 export async function reconnectStep<TOutput = unknown>(
   sandbox: SandboxProvider,
   resume: { runnerPid: number; resultToken: string; stepIndex: number },
-  opts?: Pick<InvokeStepOptions, "onStdout" | "onStderr"> & { signal?: AbortSignal },
+  opts?: Pick<InvokeStepOptions, "onStdout" | "onStderr"> & {
+    signal?: AbortSignal;
+    /** Ceiling on the durable-result poll (default 45m — the wedged-runner
+     *  backstop). A worker-death recovery reconnecting MID-step passes the
+     *  step ceiling instead: hours of legitimate work may remain, and the
+     *  activity's per-attempt startToClose already bounds it server-side. */
+    pollDeadlineMs?: number;
+  },
 ): Promise<RunningStep<TOutput>> {
   if (!sandbox.commands.connectProcess) {
     throw new Error("reconnectStep requires a provider with background-command support (commands.connectProcess)");
@@ -572,6 +691,7 @@ export async function reconnectStep<TOutput = unknown>(
   const { runnerPid, resultToken } = resume;
   const splitters = makeStreamSplitters(opts, resultToken);
   const signal = opts?.signal;
+  const pollDeadlineMs = opts?.pollDeadlineMs;
 
   // Re-attach to the suspended runner for LIVE stdout streaming only. This is
   // best-effort: a reconnect that throws after the native resume just means no
@@ -600,7 +720,10 @@ export async function reconnectStep<TOutput = unknown>(
       // runner's durable result file + `/proc` liveness instead — the same
       // authoritative signal the launch-path recovery uses (`pollDurableResult`).
       try {
-        const result = await pollDurableResult<TOutput>(sandbox, runnerPid, resultToken, signal);
+        const result = await pollDurableResult<TOutput>(sandbox, runnerPid, resultToken, {
+          ...(signal ? { signal } : {}),
+          ...(pollDeadlineMs !== undefined ? { deadlineMs: pollDeadlineMs } : {}),
+        });
         splitters.flush();
         return result;
       } finally {

package/src/step-invocation/protocol.ts

@@ -81,6 +81,14 @@ export function stepResultFilePath(token: string): string {
   return `/tmp/wf/step-result-${token}.json`;
 }
 
+/** Sandbox-side pidfile the FOREGROUND launch writes (`echo $$ > …`) before
+ *  spawning the runner pipeline — the wrapping shell's pid. The recovery path
+ *  reads it back to probe `/proc/<pid>` liveness when the live stream/wait
+ *  call dies mid-step (providers with no background-process pid — Vercel). */
+export function stepPidFilePath(token: string): string {
+  return `/tmp/wf/step-pid-${token}.pid`;
+}
+
 /** Sandbox-side path where the runner's stdout is tee'd as a durable LOG file,
  *  keyed by the per-invocation token. The live output rides E2B's connect-web
  *  command stream, which THROWS on a compressed large frame (gRPC-web cannot

package/src/types/api-compliance.ts

@@ -0,0 +1,79 @@
+/**
+ * Break-glass compliance session wire types (ADR-0051).
+ *
+ * A team admin has no default access to a scoped (Private/Shared) document.
+ * To view one they don't hold a grant on, they open a READ-ONLY,
+ * owner-approved, time-boxed compliance session over a scope (one factory, or
+ * the whole team). These are the shapes the server's `/api/v1/compliance/*`
+ * routes emit — `client.ts` is a thin typed wrapper over them.
+ */
+
+export type ComplianceScopeKind = "factory" | "team";
+export type ComplianceStatus = "pending" | "active" | "expired" | "revoked";
+
+/** One break-glass compliance session. `expiresAt` is null while pending —
+ *  the TTL clock starts at approval, not request. */
+export interface ComplianceSession {
+  id: string;
+  teamId: string;
+  requestedByUserId: string | null;
+  requestedByLabel: string | null;
+  approvedByUserId: string | null;
+  approvedByLabel: string | null;
+  reason: string;
+  scopeKind: ComplianceScopeKind;
+  /** The factory this session covers (factory scope); null for team scope. */
+  scopeRef: string | null;
+  status: ComplianceStatus;
+  ttlSeconds: number;
+  expiresAt: string | null;
+  revokedByUserId: string | null;
+  revokedByLabel: string | null;
+  createdAt: string;
+  approvedAt: string | null;
+  revokedAt: string | null;
+}
+
+/** One per-document access recorded under an active session — the individual
+ *  audit trail the session's reasoned entry authorizes. */
+export interface ComplianceAccess {
+  id: string;
+  sessionId: string;
+  /** Nulled when the accessed file is later deleted (ON DELETE SET NULL);
+   *  `filePath` is the durable snapshot that survives. */
+  fileId: string | null;
+  filePath: string;
+  accessedByUserId: string | null;
+  accessedByLabel: string | null;
+  accessedAt: string;
+}
+
+export interface RequestComplianceSessionInput {
+  scopeKind: ComplianceScopeKind;
+  /** Required for factory scope; must be omitted for team scope. */
+  scopeRef?: string | null;
+  reason: string;
+  /** Session lifetime once approved — 15 min floor, 7 day ceiling. */
+  ttlSeconds: number;
+}
+
+export interface ListComplianceSessionsOptions {
+  status?: ComplianceStatus;
+  limit?: number;
+  cursor?: string;
+}
+
+export interface ComplianceSessionsPage {
+  sessions: ComplianceSession[];
+  nextCursor: string | null;
+}
+
+export interface ListComplianceAccessesOptions {
+  limit?: number;
+  cursor?: string;
+}
+
+export interface ComplianceAccessesPage {
+  accesses: ComplianceAccess[];
+  nextCursor: string | null;
+}