@foldrun/core 0.2.0 → 0.4.0

package/src/run-container.ts

@@ -25,6 +25,7 @@ import crypto from "node:crypto";
 import { spawn, spawnSync } from "node:child_process";
 import type { McpServerConfig } from "@anthropic-ai/claude-agent-sdk";
 import { isPlatformPath, type ApiSpec, type Effort } from "./store.ts";
+import { divertedSummary, isDivertedPath } from "./test-mode.ts";
 import type { EventExtra } from "./step-exec.ts";
 import { isFileValue, fileContent } from "./secrets.ts";
 import { safeTenantSegment, type RuntimeSpec } from "./runtime.ts";
@@ -61,10 +62,14 @@ export interface ContainerStepInput {
   /** The step's spend ceiling and the model's per-token price — see
    *  step-exec.ts. Values, so the driver can enforce them in the pod. */
   budgetUsd?: number | null;
+  budgetNote?: string;
   price?: { input: number; output: number } | null;
   verify?: string;
   /** `output: json` — checked inside, where the reply is. */
   output?: "json";
+  /** `schema:` and `max_turns:`, resolved host-side — see ExecOptions. */
+  schema?: Record<string, unknown> | boolean;
+  maxTurns?: number;
   /** tools: [search] — the directories to search, as paths INSIDE the
    *  container (/workspace/…, /library/…). */
   search?: SearchRoot[];
@@ -106,6 +111,10 @@ export interface StepActuals {
 export interface ContainerStepOutcome {
   status: "completed" | "failed";
   result: string | null;
+  /** Why the sandbox ended when the step did not: what the cluster said
+   *  (OOMKilled, Evicted, the pod gone). Absent when the driver finished
+   *  and said so itself. A retry policy reads this. */
+  reason?: string | null;
   /** The model's final text block — see StepRecord.conclusion. Crosses the
    *  boundary with the rest of the outcome; the driver spreads it. */
   conclusion?: string | null;
@@ -186,10 +195,70 @@ export function allowedBack(rel: string): boolean {
  * the step did not touch it, so it is not ours to write back. Concurrent
  * edits survive, and a step that genuinely changed a file still wins.
  */
+/**
+ * What a workspace copy looked like when it went in: the sha256 of every
+ * file the step could hand back. Small enough to ride into the sandbox
+ * beside the input, so a driver that inherits the step can tell what the
+ * step changed without the staging copy the first driver held in /tmp.
+ */
+export function hashTree(dir: string): Record<string, string> {
+  const out: Record<string, string> = {};
+  const walk = (d: string) => {
+    for (const entry of fs.readdirSync(d, { withFileTypes: true })) {
+      const abs = path.join(d, entry.name);
+      const rel = path.relative(dir, abs).replaceAll("\\", "/");
+      if (entry.isSymbolicLink()) continue;
+      if (entry.isDirectory()) {
+        if (allowedBack(rel + "/")) walk(abs);
+        continue;
+      }
+      if (!allowedBack(rel)) continue;
+      out[rel] = crypto.createHash("sha256").update(fs.readFileSync(abs)).digest("hex");
+    }
+  };
+  if (fs.existsSync(dir)) walk(dir);
+  return out;
+}
+
+const sha = (b: Buffer) => crypto.createHash("sha256").update(b).digest("hex");
+
+/**
+ * Both sides changed a file the step was handed. When each is the handed-in
+ * content with lines added on the end — a ledger, history.md, a .jsonl — the
+ * answer is the base, then the host's additions, then the step's. Only the
+ * base's hash may be known, so the base is found as the longest shared
+ * leading run of whole lines whose hash matches. Anything else is null: a
+ * real conflict, which the caller must not settle by dropping one side.
+ */
+export function mergeAppends(was: Buffer | string | null, host: Buffer, next: Buffer): Buffer | null {
+  // Both sides created the same new file: no shared base to append to, and
+  // joining two JSON documents is corruption, not a merge.
+  if (was === null) return null;
+  const h = host.toString("utf8");
+  const n = next.toString("utf8");
+  let common = 0;
+  const max = Math.min(h.length, n.length);
+  while (common < max && h.charCodeAt(common) === n.charCodeAt(common)) common++;
+  for (let end = common; end >= 0; end--) {
+    if (end > 0 && h[end - 1] !== "\n") continue;
+    const prefix = h.slice(0, end);
+    const matches =
+      typeof was === "string" ? sha(Buffer.from(prefix, "utf8")) === was : Buffer.from(prefix, "utf8").equals(was);
+    if (!matches) continue;
+    const hostAdded = h.slice(end);
+    const stepAdded = n.slice(end);
+    const joined = hostAdded && !hostAdded.endsWith("\n") ? hostAdded + "\n" : hostAdded;
+    return Buffer.from(prefix + joined + stepAdded, "utf8");
+  }
+  return null;
+}
+
 export function applyContainerChanges(
   hostWs: string,
   containerWs: string,
-  baseline?: string,
+  baseline?: string | Record<string, string>,
+  note?: (message: string) => void,
+  divert?: WriteDivert,
 ): string[] {
   const applied: string[] = [];
   const walk = (dir: string) => {
@@ -208,15 +277,60 @@ export function applyContainerChanges(
         continue;
       }
       if (!allowedBack(rel)) continue;
-      const next = fs.readFileSync(abs);
+      let next: Buffer = fs.readFileSync(abs);
+      // What the step was handed: its bytes (a staging dir) or its hash.
+      // null when the file did not exist then; undefined when no baseline.
+      let was: Buffer | string | null | undefined;
+      if (typeof baseline === "string") {
+        const p = path.join(baseline, rel);
+        was = fs.existsSync(p) ? fs.readFileSync(p) : null;
+      } else if (baseline) {
+        was = baseline[rel.replaceAll("\\", "/")] ?? null;
+      }
+      const same = (b: Buffer) =>
+        was === null || was === undefined ? false : typeof was === "string" ? sha(b) === was : b.equals(was);
       // Untouched by this step: whatever the host says now is more current
       // than what we handed in, including a change made while it ran.
-      if (baseline) {
-        const was = path.join(baseline, rel);
-        if (fs.existsSync(was) && fs.readFileSync(was).equals(next)) continue;
-      }
+      if (same(next)) continue;
       const target = path.join(hostWs, rel);
-      if (fs.existsSync(target) && fs.readFileSync(target).equals(next)) continue;
+      const hostNow = fs.existsSync(target) ? fs.readFileSync(target) : null;
+      if (hostNow && hostNow.equals(next)) continue;
+      // The host moved too since the step was handed its copy — another run
+      // or a person wrote it. Writing the step's copy over it erased the
+      // other run's ledger rows on 2026-09-14. Merge appends; keep both
+      // sides of anything else.
+      const hostMoved = was !== undefined && hostNow !== null && !same(hostNow);
+      if (hostMoved) {
+        const merged = mergeAppends(was ?? null, hostNow, next);
+        if (merged) {
+          next = merged;
+        } else {
+          const ext = path.extname(rel);
+          const stem = rel.slice(0, rel.length - ext.length);
+          const conflictRel = `${stem}.conflict-${Date.now().toString(36)}${ext}`;
+          const conflictAbs = path.join(hostWs, conflictRel);
+          fs.mkdirSync(path.dirname(conflictAbs), { recursive: true });
+          fs.writeFileSync(conflictAbs, next);
+          applied.push(conflictRel);
+          note?.(
+            `write-back conflict on ${rel.replaceAll("\\", "/")}: it changed on the workspace while this step ran, ` +
+              `and not only by appending — kept the workspace's version; this step's version is ${conflictRel.replaceAll("\\", "/")}`,
+          );
+          continue;
+        }
+      }
+      // A test run's state/ and storage/ writes land under the run, not on
+      // the workspace: the step ran against the real files and its output
+      // is kept where a person can read it, but the next real run starts
+      // from state a test never touched. Decided after the merge so the
+      // diverted copy is what WOULD have been written, conflicts included.
+      if (divert && isDivertedPath(rel)) {
+        const aside = path.join(divert.to, rel);
+        fs.mkdirSync(path.dirname(aside), { recursive: true });
+        fs.writeFileSync(aside, next);
+        divert.note(rel.replaceAll("\\", "/"), divertedSummary(rel.replaceAll("\\", "/"), hostNow, next));
+        continue;
+      }
       fs.mkdirSync(path.dirname(target), { recursive: true });
       fs.writeFileSync(target, next);
       applied.push(rel);
@@ -346,9 +460,9 @@ try {
 
   // API and script tools, rebuilt in here from their specs. Secrets were
   // substituted into API headers before the input crossed the boundary, so
-  // buildApiTools gets an already-resolved environment. (verify: needs no
-  // separate env either — the container's own environment already carries
-  // the declared secrets, which is why executeStep gets verifyEnv: {}.)
+  // buildApiTools gets an already-resolved environment. verify: gets the
+  // container's own environment, whole: it is the boundary, assembled on
+  // the host from an allowlist, and the declared secrets are in it.
   const api = buildApiTools("", input.apis, undefined, { env, missing: [] });
   const script = buildScriptTools(
     agentDir,
@@ -393,10 +507,13 @@ try {
     env,
     timeoutSec: input.timeoutSec,
     budgetUsd: input.budgetUsd,
+    budgetNote: input.budgetNote,
     price: input.price,
     verify: input.verify,
-    verifyEnv: {},
+    verifyEnv: env,
     output: input.output,
+    schema: input.schema,
+    maxTurns: input.maxTurns,
     // The container is the boundary; the SDK's own bash sandbox here would
     // only block declared network use (SSH, curl) for no added safety.
     sandboxBash: false,
@@ -504,8 +621,17 @@ export function runnerImageTag(): string {
  * has not been built before. `npm pack` of the installed core, so the image
  * runs the same bytes the server does.
  */
-export function ensureRunnerImage(): { tag: string; log: string[] } {
-  const tag = process.env.FOLDRUN_RUNNER_IMAGE ?? runnerImageTag();
+/**
+ * `platform`: build for another architecture — "linux/arm64" on the amd64
+ * box, through QEMU — and name the result with the arch appended so it never
+ * shadows the native image in the daemon. Only the publisher asks for this;
+ * a step always runs the native image. Needs binfmt registered on the host
+ * (tonistiigi/binfmt); the publisher does that itself, every time, because
+ * the registration does not survive a reboot.
+ */
+export function ensureRunnerImage(opts: { platform?: string } = {}): { tag: string; log: string[] } {
+  const arch = opts.platform ? `-${opts.platform.split("/").pop()}` : "";
+  const tag = process.env.FOLDRUN_RUNNER_IMAGE ?? runnerImageTag() + arch;
   const log: string[] = [];
   if (process.env.FOLDRUN_RUNNER_IMAGE) return { tag, log };
 
@@ -524,7 +650,8 @@ export function ensureRunnerImage(): { tag: string; log: string[] } {
     fs.writeFileSync(path.join(build, "entry.sh"), ENTRY);
     fs.writeFileSync(path.join(build, "Dockerfile"), DOCKERFILE);
     log.push(`building runner image ${tag} (first run only)`);
-    const out = spawnSync(cli(), ["build", "-t", tag, build], { encoding: "utf8" });
+    const args = ["build", ...(opts.platform ? ["--platform", opts.platform] : []), "-t", tag, build];
+    const out = spawnSync(cli(), args, { encoding: "utf8" });
     if (out.status !== 0) {
       throw new Error(`runner image build failed:\n${(out.stderr || out.stdout).slice(-2000)}`);
     }
@@ -611,6 +738,8 @@ function ensureRunNetwork(): string {
 export interface RunInContainerArgs {
   workspaceRoot: string;
   libraryRoot: string;
+  /** The platform's gallery, staged under /library BEFORE the account library so the account's copies win. */
+  galleryRoot?: string;
   /** The account this step belongs to — the key for its dependency cache.
    *  Optional so an embedder that has no tenancy simply gets no cache. */
   tenant?: string;
@@ -625,6 +754,75 @@ export interface RunInContainerArgs {
   runId?: string;
   /** The reservation class — which limits this step's sandbox holds. */
   size?: "small" | "large" | "heavy";
+  /** Re-attach to a sandbox a previous driver started, instead of creating
+   *  one: its handle, and how many output lines that driver had applied.
+   *  The executor replays nothing before `consumed`, and reads the step's
+   *  files back from the sandbox's own copy-in snapshot. */
+  resume?: { ref: string; consumed: number } | null;
+  /** Called by an executor that can be resumed: when its sandbox exists
+   *  (consumed 0), and before each output line it applies. The runner
+   *  records these on the step, so the next driver knows where to attach. */
+  checkpoint?: (ref: string, consumed: number) => void;
+  /** A test run: where state/ and storage/ changes go instead of the
+   *  workspace, and who to tell about each. See test-mode.ts. */
+  divert?: WriteDivert;
+  /** The run is a test run. An isolated executor labels the sandbox so the
+   *  cluster can cut its network down to the egress proxy: a script tool
+   *  calls the network directly, not through the proxy where the test-mode
+   *  policy lives, so without this a test run's script could still send. */
+  test?: boolean;
+}
+
+/** What a test run does with a write it must not apply. */
+export interface WriteDivert {
+  /** The directory the diverted files are written under, by their
+   *  workspace-relative path: runs/<id>/test-writes/. */
+  to: string;
+  note: (rel: string, summary: string) => void;
+}
+
+/**
+ * What crosses into the container as its environment, and how.
+ *
+ * A docker --env-file is one `KEY=value` per line and cannot carry a
+ * newline. @file secrets are multi-line by nature (a PEM key, a cert), so
+ * they are staged as files NOW, into the copied-in workspace, and the env
+ * carries the container path — the one place a path is stable across the
+ * boundary. A plain secret whose value happens to hold a newline gets the
+ * same treatment, and the trace says so: it used to be dropped from the
+ * env file without a word, and the step failed later, elsewhere, on a
+ * variable that read as unset. Everything else goes through the env file.
+ */
+export function stageContainerEnv(
+  env: Record<string, string>,
+  wsIn: string,
+  agentRel: string,
+  emit: (type: "info" | "error", text: string) => void,
+): Record<string, string> {
+  const containerEnv: Record<string, string> = {};
+  const stage = (k: string, content: string) => {
+    const rel = path.join(agentRel, ".secret-files", k.toLowerCase());
+    const abs = path.join(wsIn, rel);
+    fs.mkdirSync(path.dirname(abs), { recursive: true, mode: 0o700 });
+    fs.writeFileSync(abs, content, { mode: 0o600 });
+    return `/workspace/${rel.replaceAll("\\", "/")}`;
+  };
+  for (const [k, v] of Object.entries(env)) {
+    if (typeof v !== "string") continue;
+    if (isFileValue(v)) {
+      containerEnv[k] = stage(k, fileContent(v));
+    } else if (/[\r\n]/.test(v)) {
+      containerEnv[k] = stage(k, v);
+      emit(
+        "info",
+        `${k} holds a line break and cannot cross as an environment variable — ` +
+          `passed as a file instead; $${k} is its path in the sandbox`,
+      );
+    } else {
+      containerEnv[k] = v;
+    }
+  }
+  return containerEnv;
 }
 
 /** The limits a size class reserves. Large is the install's configured
@@ -718,6 +916,7 @@ export async function runStepInContainer(args: RunInContainerArgs): Promise<Cont
       filter: (src) => !isPlatformPath(path.relative(args.workspaceRoot, src)),
     });
     const libIn = path.join(staging, "library");
+    if (args.galleryRoot && fs.existsSync(args.galleryRoot)) fs.cpSync(args.galleryRoot, libIn, { recursive: true });
     if (fs.existsSync(args.libraryRoot)) fs.cpSync(args.libraryRoot, libIn, { recursive: true });
     else fs.mkdirSync(libIn, { recursive: true });
 
@@ -725,26 +924,7 @@ export async function runStepInContainer(args: RunInContainerArgs): Promise<Cont
     fs.mkdirSync(jobIn);
     fs.writeFileSync(path.join(jobIn, "input.json"), JSON.stringify(args.input, null, 2));
 
-    // @file secrets are multi-line (a PEM key, a cert), and a docker
-    // --env-file cannot carry a newline. So they are staged as files *now*,
-    // into the copied-in workspace, and the env carries the container path —
-    // the one place a path is stable across the boundary. Everything else
-    // goes through the env file. (Without this the key silently vanished:
-    // the old filter dropped any value with a newline, and ssh got an empty
-    // -i path.)
-    const containerEnv: Record<string, string> = {};
-    for (const [k, v] of Object.entries(args.env)) {
-      if (typeof v !== "string") continue;
-      if (isFileValue(v)) {
-        const rel = path.join(args.input.agentRel, ".secret-files", k.toLowerCase());
-        const abs = path.join(wsIn, rel);
-        fs.mkdirSync(path.dirname(abs), { recursive: true, mode: 0o700 });
-        fs.writeFileSync(abs, fileContent(v), { mode: 0o600 });
-        containerEnv[k] = `/workspace/${rel.replaceAll("\\", "/")}`;
-      } else if (!v.includes("\n")) {
-        containerEnv[k] = v;
-      }
-    }
+    const containerEnv = stageContainerEnv(args.env, wsIn, args.input.agentRel, args.emit);
 
     const envFile = path.join(staging, "env");
     fs.writeFileSync(
@@ -862,7 +1042,7 @@ export async function runStepInContainer(args: RunInContainerArgs): Promise<Cont
     fs.mkdirSync(wsOut);
     const back = spawnSync(cli(), ["cp", `${containerId}:/workspace/.`, wsOut], { encoding: "utf8" });
     if (back.status === 0) {
-      applyContainerChanges(args.workspaceRoot, wsOut, wsIn);
+      applyContainerChanges(args.workspaceRoot, wsOut, wsIn, (m) => args.emit("error", m), args.divert);
     } else {
       args.emit("error", `copy-out failed — the step's file changes were lost:\n${back.stderr.slice(0, 500)}`);
     }