@foldrun/core 0.2.0 → 0.4.0

package/dist/src/run-container.js

@@ -23,6 +23,7 @@ import path from "node:path";
 import crypto from "node:crypto";
 import { spawn, spawnSync } from "node:child_process";
 import { isPlatformPath } from "./store.js";
+import { divertedSummary, isDivertedPath } from "./test-mode.js";
 import { isFileValue, fileContent } from "./secrets.js";
 import { safeTenantSegment } from "./runtime.js";
 import { dataRoot } from "./paths.js";
@@ -95,7 +96,69 @@ export function allowedBack(rel) {
  * the step did not touch it, so it is not ours to write back. Concurrent
  * edits survive, and a step that genuinely changed a file still wins.
  */
-export function applyContainerChanges(hostWs, containerWs, baseline) {
+/**
+ * What a workspace copy looked like when it went in: the sha256 of every
+ * file the step could hand back. Small enough to ride into the sandbox
+ * beside the input, so a driver that inherits the step can tell what the
+ * step changed without the staging copy the first driver held in /tmp.
+ */
+export function hashTree(dir) {
+    const out = {};
+    const walk = (d) => {
+        for (const entry of fs.readdirSync(d, { withFileTypes: true })) {
+            const abs = path.join(d, entry.name);
+            const rel = path.relative(dir, abs).replaceAll("\\", "/");
+            if (entry.isSymbolicLink())
+                continue;
+            if (entry.isDirectory()) {
+                if (allowedBack(rel + "/"))
+                    walk(abs);
+                continue;
+            }
+            if (!allowedBack(rel))
+                continue;
+            out[rel] = crypto.createHash("sha256").update(fs.readFileSync(abs)).digest("hex");
+        }
+    };
+    if (fs.existsSync(dir))
+        walk(dir);
+    return out;
+}
+const sha = (b) => crypto.createHash("sha256").update(b).digest("hex");
+/**
+ * Both sides changed a file the step was handed. When each is the handed-in
+ * content with lines added on the end — a ledger, history.md, a .jsonl — the
+ * answer is the base, then the host's additions, then the step's. Only the
+ * base's hash may be known, so the base is found as the longest shared
+ * leading run of whole lines whose hash matches. Anything else is null: a
+ * real conflict, which the caller must not settle by dropping one side.
+ */
+export function mergeAppends(was, host, next) {
+    // Both sides created the same new file: no shared base to append to, and
+    // joining two JSON documents is corruption, not a merge.
+    if (was === null)
+        return null;
+    const h = host.toString("utf8");
+    const n = next.toString("utf8");
+    let common = 0;
+    const max = Math.min(h.length, n.length);
+    while (common < max && h.charCodeAt(common) === n.charCodeAt(common))
+        common++;
+    for (let end = common; end >= 0; end--) {
+        if (end > 0 && h[end - 1] !== "\n")
+            continue;
+        const prefix = h.slice(0, end);
+        const matches = typeof was === "string" ? sha(Buffer.from(prefix, "utf8")) === was : Buffer.from(prefix, "utf8").equals(was);
+        if (!matches)
+            continue;
+        const hostAdded = h.slice(end);
+        const stepAdded = n.slice(end);
+        const joined = hostAdded && !hostAdded.endsWith("\n") ? hostAdded + "\n" : hostAdded;
+        return Buffer.from(prefix + joined + stepAdded, "utf8");
+    }
+    return null;
+}
+export function applyContainerChanges(hostWs, containerWs, baseline, note, divert) {
     const applied = [];
     const walk = (dir) => {
         for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
@@ -116,17 +179,61 @@ export function applyContainerChanges(hostWs, containerWs, baseline) {
             }
             if (!allowedBack(rel))
                 continue;
-            const next = fs.readFileSync(abs);
+            let next = fs.readFileSync(abs);
+            // What the step was handed: its bytes (a staging dir) or its hash.
+            // null when the file did not exist then; undefined when no baseline.
+            let was;
+            if (typeof baseline === "string") {
+                const p = path.join(baseline, rel);
+                was = fs.existsSync(p) ? fs.readFileSync(p) : null;
+            }
+            else if (baseline) {
+                was = baseline[rel.replaceAll("\\", "/")] ?? null;
+            }
+            const same = (b) => was === null || was === undefined ? false : typeof was === "string" ? sha(b) === was : b.equals(was);
             // Untouched by this step: whatever the host says now is more current
             // than what we handed in, including a change made while it ran.
-            if (baseline) {
-                const was = path.join(baseline, rel);
-                if (fs.existsSync(was) && fs.readFileSync(was).equals(next))
+            if (same(next))
+                continue;
+            const target = path.join(hostWs, rel);
+            const hostNow = fs.existsSync(target) ? fs.readFileSync(target) : null;
+            if (hostNow && hostNow.equals(next))
+                continue;
+            // The host moved too since the step was handed its copy — another run
+            // or a person wrote it. Writing the step's copy over it erased the
+            // other run's ledger rows on 2026-09-14. Merge appends; keep both
+            // sides of anything else.
+            const hostMoved = was !== undefined && hostNow !== null && !same(hostNow);
+            if (hostMoved) {
+                const merged = mergeAppends(was ?? null, hostNow, next);
+                if (merged) {
+                    next = merged;
+                }
+                else {
+                    const ext = path.extname(rel);
+                    const stem = rel.slice(0, rel.length - ext.length);
+                    const conflictRel = `${stem}.conflict-${Date.now().toString(36)}${ext}`;
+                    const conflictAbs = path.join(hostWs, conflictRel);
+                    fs.mkdirSync(path.dirname(conflictAbs), { recursive: true });
+                    fs.writeFileSync(conflictAbs, next);
+                    applied.push(conflictRel);
+                    note?.(`write-back conflict on ${rel.replaceAll("\\", "/")}: it changed on the workspace while this step ran, ` +
+                        `and not only by appending — kept the workspace's version; this step's version is ${conflictRel.replaceAll("\\", "/")}`);
                     continue;
+                }
             }
-            const target = path.join(hostWs, rel);
-            if (fs.existsSync(target) && fs.readFileSync(target).equals(next))
+            // A test run's state/ and storage/ writes land under the run, not on
+            // the workspace: the step ran against the real files and its output
+            // is kept where a person can read it, but the next real run starts
+            // from state a test never touched. Decided after the merge so the
+            // diverted copy is what WOULD have been written, conflicts included.
+            if (divert && isDivertedPath(rel)) {
+                const aside = path.join(divert.to, rel);
+                fs.mkdirSync(path.dirname(aside), { recursive: true });
+                fs.writeFileSync(aside, next);
+                divert.note(rel.replaceAll("\\", "/"), divertedSummary(rel.replaceAll("\\", "/"), hostNow, next));
                 continue;
+            }
             fs.mkdirSync(path.dirname(target), { recursive: true });
             fs.writeFileSync(target, next);
             applied.push(rel);
@@ -253,9 +360,9 @@ try {
 
   // API and script tools, rebuilt in here from their specs. Secrets were
   // substituted into API headers before the input crossed the boundary, so
-  // buildApiTools gets an already-resolved environment. (verify: needs no
-  // separate env either — the container's own environment already carries
-  // the declared secrets, which is why executeStep gets verifyEnv: {}.)
+  // buildApiTools gets an already-resolved environment. verify: gets the
+  // container's own environment, whole: it is the boundary, assembled on
+  // the host from an allowlist, and the declared secrets are in it.
   const api = buildApiTools("", input.apis, undefined, { env, missing: [] });
   const script = buildScriptTools(
     agentDir,
@@ -300,10 +407,13 @@ try {
     env,
     timeoutSec: input.timeoutSec,
     budgetUsd: input.budgetUsd,
+    budgetNote: input.budgetNote,
     price: input.price,
     verify: input.verify,
-    verifyEnv: {},
+    verifyEnv: env,
     output: input.output,
+    schema: input.schema,
+    maxTurns: input.maxTurns,
     // The container is the boundary; the SDK's own bash sandbox here would
     // only block declared network use (SSH, curl) for no added safety.
     sandboxBash: false,
@@ -409,8 +519,17 @@ export function runnerImageTag() {
  * has not been built before. `npm pack` of the installed core, so the image
  * runs the same bytes the server does.
  */
-export function ensureRunnerImage() {
-    const tag = process.env.FOLDRUN_RUNNER_IMAGE ?? runnerImageTag();
+/**
+ * `platform`: build for another architecture — "linux/arm64" on the amd64
+ * box, through QEMU — and name the result with the arch appended so it never
+ * shadows the native image in the daemon. Only the publisher asks for this;
+ * a step always runs the native image. Needs binfmt registered on the host
+ * (tonistiigi/binfmt); the publisher does that itself, every time, because
+ * the registration does not survive a reboot.
+ */
+export function ensureRunnerImage(opts = {}) {
+    const arch = opts.platform ? `-${opts.platform.split("/").pop()}` : "";
+    const tag = process.env.FOLDRUN_RUNNER_IMAGE ?? runnerImageTag() + arch;
     const log = [];
     if (process.env.FOLDRUN_RUNNER_IMAGE)
         return { tag, log };
@@ -430,7 +549,8 @@ export function ensureRunnerImage() {
         fs.writeFileSync(path.join(build, "entry.sh"), ENTRY);
         fs.writeFileSync(path.join(build, "Dockerfile"), DOCKERFILE);
         log.push(`building runner image ${tag} (first run only)`);
-        const out = spawnSync(cli(), ["build", "-t", tag, build], { encoding: "utf8" });
+        const args = ["build", ...(opts.platform ? ["--platform", opts.platform] : []), "-t", tag, build];
+        const out = spawnSync(cli(), args, { encoding: "utf8" });
         if (out.status !== 0) {
             throw new Error(`runner image build failed:\n${(out.stderr || out.stdout).slice(-2000)}`);
         }
@@ -505,6 +625,44 @@ function ensureRunNetwork() {
     }
     return RUN_NETWORK;
 }
+/**
+ * What crosses into the container as its environment, and how.
+ *
+ * A docker --env-file is one `KEY=value` per line and cannot carry a
+ * newline. @file secrets are multi-line by nature (a PEM key, a cert), so
+ * they are staged as files NOW, into the copied-in workspace, and the env
+ * carries the container path — the one place a path is stable across the
+ * boundary. A plain secret whose value happens to hold a newline gets the
+ * same treatment, and the trace says so: it used to be dropped from the
+ * env file without a word, and the step failed later, elsewhere, on a
+ * variable that read as unset. Everything else goes through the env file.
+ */
+export function stageContainerEnv(env, wsIn, agentRel, emit) {
+    const containerEnv = {};
+    const stage = (k, content) => {
+        const rel = path.join(agentRel, ".secret-files", k.toLowerCase());
+        const abs = path.join(wsIn, rel);
+        fs.mkdirSync(path.dirname(abs), { recursive: true, mode: 0o700 });
+        fs.writeFileSync(abs, content, { mode: 0o600 });
+        return `/workspace/${rel.replaceAll("\\", "/")}`;
+    };
+    for (const [k, v] of Object.entries(env)) {
+        if (typeof v !== "string")
+            continue;
+        if (isFileValue(v)) {
+            containerEnv[k] = stage(k, fileContent(v));
+        }
+        else if (/[\r\n]/.test(v)) {
+            containerEnv[k] = stage(k, v);
+            emit("info", `${k} holds a line break and cannot cross as an environment variable — ` +
+                `passed as a file instead; $${k} is its path in the sandbox`);
+        }
+        else {
+            containerEnv[k] = v;
+        }
+    }
+    return containerEnv;
+}
 /** The limits a size class reserves. Large is the install's configured
  *  limits unchanged, so existing installs bill and behave identically;
  *  small defaults to a quarter-ish slice and is env-tunable. */
@@ -595,6 +753,8 @@ export async function runStepInContainer(args) {
             filter: (src) => !isPlatformPath(path.relative(args.workspaceRoot, src)),
         });
         const libIn = path.join(staging, "library");
+        if (args.galleryRoot && fs.existsSync(args.galleryRoot))
+            fs.cpSync(args.galleryRoot, libIn, { recursive: true });
         if (fs.existsSync(args.libraryRoot))
             fs.cpSync(args.libraryRoot, libIn, { recursive: true });
         else
@@ -602,28 +762,7 @@ export async function runStepInContainer(args) {
         const jobIn = path.join(staging, "job");
         fs.mkdirSync(jobIn);
         fs.writeFileSync(path.join(jobIn, "input.json"), JSON.stringify(args.input, null, 2));
-        // @file secrets are multi-line (a PEM key, a cert), and a docker
-        // --env-file cannot carry a newline. So they are staged as files *now*,
-        // into the copied-in workspace, and the env carries the container path —
-        // the one place a path is stable across the boundary. Everything else
-        // goes through the env file. (Without this the key silently vanished:
-        // the old filter dropped any value with a newline, and ssh got an empty
-        // -i path.)
-        const containerEnv = {};
-        for (const [k, v] of Object.entries(args.env)) {
-            if (typeof v !== "string")
-                continue;
-            if (isFileValue(v)) {
-                const rel = path.join(args.input.agentRel, ".secret-files", k.toLowerCase());
-                const abs = path.join(wsIn, rel);
-                fs.mkdirSync(path.dirname(abs), { recursive: true, mode: 0o700 });
-                fs.writeFileSync(abs, fileContent(v), { mode: 0o600 });
-                containerEnv[k] = `/workspace/${rel.replaceAll("\\", "/")}`;
-            }
-            else if (!v.includes("\n")) {
-                containerEnv[k] = v;
-            }
-        }
+        const containerEnv = stageContainerEnv(args.env, wsIn, args.input.agentRel, args.emit);
         const envFile = path.join(staging, "env");
         fs.writeFileSync(envFile, Object.entries(containerEnv).map(([k, v]) => `${k}=${v}`).join("\n"), { mode: 0o600 });
         const flags = [
@@ -737,7 +876,7 @@ export async function runStepInContainer(args) {
         fs.mkdirSync(wsOut);
         const back = spawnSync(cli(), ["cp", `${containerId}:/workspace/.`, wsOut], { encoding: "utf8" });
         if (back.status === 0) {
-            applyContainerChanges(args.workspaceRoot, wsOut, wsIn);
+            applyContainerChanges(args.workspaceRoot, wsOut, wsIn, (m) => args.emit("error", m), args.divert);
         }
         else {
             args.emit("error", `copy-out failed — the step's file changes were lost:\n${back.stderr.slice(0, 500)}`);

package/dist/src/runner.d.ts

@@ -1,4 +1,4 @@
-import { type FlowStep, type RunRecord } from "./store.ts";
+import { type FlowStep, type RunRecord, type StepRecord } from "./store.ts";
 /** How much of the earlier groups' output a step is handed, in characters.
  *  Newest groups are kept when the total is over it. */
 export declare const CONTEXT_CHARS = 30000;
@@ -12,6 +12,17 @@ export declare const DESK_RUNS_CAP = 100;
  * step is most likely to be about.
  */
 export declare function joinEarlierGroups(groupResults: (string | null)[], gi: number, cap?: number): string | null;
+/**
+ * `parallel: N` — a pool of N slots the instances of one fan-out share.
+ * Fan-out always ran every instance at once, and still does unless the
+ * step says otherwise: twenty browser steps against one site, or one
+ * vendor's rate limit, is what the cap is for. Nothing is rented while an
+ * instance waits for a slot.
+ */
+export declare function slotPool(size: number): {
+    acquire: () => Promise<void>;
+    release: () => void;
+};
 export interface DiscoveredSkill {
     name: string;
     description: string;
@@ -102,6 +113,20 @@ export declare function applicableSkills<T extends {
  */
 export declare function saysUntilMarker(result: string | null, until: string): boolean;
 export declare function resolveDocLinks(text: string, workspaceRoot: string): string;
+/**
+ * Write one attempt onto the step's history and re-total the step.
+ *
+ * runStep reports the attempt it just ran — its cost, tokens and sandbox
+ * seconds — by assigning the step's fields. Those become this attempt's row;
+ * the step's fields then become the sum of every row, which is what the
+ * run's spend, the flow's budget: and the platform's bill read. A retried
+ * step used to carry only its last attempt's cost, so a `retry: 2` step
+ * could spend three shares while reporting one.
+ */
+export declare function recordAttempt(step: StepRecord, n: number, status: "completed" | "failed", startedAt: string, eventsBefore: number, 
+/** The attempt re-attached to a sandbox a previous driver left: same
+ *  attempt, so its row is replaced rather than repeated. */
+reattach?: boolean): void;
 /**
  * Copy a directory tree by reading and writing the bytes.
  *
@@ -132,8 +157,19 @@ export declare const OVERLOAD_RETRIES = 3;
  */
 export declare function isTransientOverload(text: string): boolean;
 export declare function backoffMs(attempt: number, text?: string, random?: () => number): number;
-export declare function createFlowRun(tenant: string, workspace: string, steps: FlowStep[], flowName: string, status: "queued" | "running", tags?: string[]): RunRecord;
-export declare function startFlowRun(tenant: string, workspace: string, steps: FlowStep[], flowName: string, modelOverride?: string | null, tags?: string[], effortOverride?: string | null): RunRecord;
+/**
+ * `schema: <path>` — the file, read and parsed, or null. Agent-relative like
+ * every other path in a flow (`../../schemas/lead.json`), confined to the
+ * workspace: a schema is the author's document, never something a step
+ * can point outside.
+ */
+export declare function readSchemaFile(workspaceRoot: string, rel: string): Record<string, unknown> | boolean | null;
+export declare function createFlowRun(tenant: string, workspace: string, steps: FlowStep[], flowName: string, status: "queued" | "running", tags?: string[], startedBy?: string | null, opts?: {
+    test?: boolean;
+}): RunRecord;
+export declare function startFlowRun(tenant: string, workspace: string, steps: FlowStep[], flowName: string, modelOverride?: string | null, tags?: string[], effortOverride?: string | null, startedBy?: string | null, opts?: {
+    test?: boolean;
+}): RunRecord;
 /**
  * Drive a run to completion: groups in order, approval gates, retries.
  *
@@ -206,7 +242,7 @@ export interface Reconciliation {
  * outlives any process. A run whose person already answered is not waiting,
  * and is picked back up — see below.
  */
-export declare function reconcileRuns(tenant: string, now?: number): Reconciliation[];
+export declare function reconcileRuns(tenant: string, now?: number, live?: ReadonlySet<string>): Reconciliation[];
 /**
  * The same, for every account this installation holds.
  *
@@ -214,7 +250,7 @@ export declare function reconcileRuns(tenant: string, now?: number): Reconciliat
  * one hard-coded account left every other account's interrupted runs marked
  * `running` forever, with no process left to finish them.
  */
-export declare function reconcileAllRuns(now?: number): Reconciliation[];
+export declare function reconcileAllRuns(now?: number, live?: ReadonlySet<string>): Reconciliation[];
 /**
  * Wait for a run to stop being in progress.
  *