@foldrun/core 0.2.0 → 0.4.0

package/dist/src/runner.js

@@ -5,11 +5,15 @@
 import fs from "node:fs";
 import path from "node:path";
 import matter from "gray-matter";
-import { executeStep, extractJson, stepCeiling } from "./step-exec.js";
+import { knownPrice, executeStep, extractJson, stepCeiling, stepCeilingFor } from "./step-exec.js";
 import { eventUrl } from "./webhook.js";
 import { runStepInContainer, sizeLimits, killRunSandboxes } from "./run-container.js";
+import { hostSafeEnv } from "./host-env.js";
+import { validateSchema, describeSchemaErrors, looksLikeSchema } from "./json-schema.js";
 import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute } from "./egress.js";
 import { platform } from "./platform.js";
+import { healthKey } from "./secret-health.js";
+import { TEST_WRITES_DIR, restoreDivertedDirs, snapshotDivertedDirs, testHeadline, testModeEnv, withholdSecrets } from "./test-mode.js";
 /** Does this process run steps in a sandbox — the container core ships, or
  *  one the platform registered (a pod)? */
 const isolatedRun = () => {
@@ -19,7 +23,7 @@ const isolatedRun = () => {
 import { gatherConsults, buildConsultTools } from "./agent-tools.js";
 import { TOOL_MAP, BUILTIN_TOOLS, ownToolNames, toolRefs, legacyUseNames, legacyUseError } from "./tool-names.js";
 import { refNames } from "./refs.js";
-import { accountDir, workspaceDir, buildMemoryIndex, workspaceMemoryIndex, knowledgeIndex, workspaceKnowledgeIndex, workspaceTools, brokenToolReport, listWorkspaces, listTenants, listRuns, syncBundleFor, parseToolDef, parseFlow, parseApis, resolveModel, parseProvider, providerEnvFor, resolveEffort, isEffortWord, EFFORT_LEVELS, checkFormatVersion, readRun, writeRun, runSummary, STORAGE_DIR, adoptLegacyVersionKey, runCost, } from "./store.js";
+import { accountDir, workspaceDir, buildMemoryIndex, workspaceMemoryIndex, knowledgeIndex, workspaceKnowledgeIndex, workspaceTools, brokenToolReport, listWorkspaces, listTenants, listRuns, syncBundleFor, parseToolDef, readFlow, markerPresent, parseApis, resolveModel, parseProvider, providerEnvFor, resolveEffort, isEffortWord, EFFORT_LEVELS, checkFormatVersion, readRun, writeRun, runSummary, sumAttempts, parseSchemaText, STORAGE_DIR, adoptLegacyVersionKey, listAgents, } from "./store.js";
 import { loadCatalog, checkModel, clampEffort, catalogCost, findModel } from "./catalog.js";
 import { resolveSecrets, getSecret, materializeSecrets } from "./secrets.js";
 import { materializeFileSecrets, cleanupFileSecrets } from "./secret-files.js";
@@ -34,6 +38,7 @@ import { mergeRuntimes, parseRuntime, prepareRuntime } from "./runtime.js";
 import { materializeFiles, harvestFiles } from "./storage.js";
 import { chooseExecutor, ensureImage } from "./container.js";
 import { stampBundle } from "./okf.js";
+import { trimChars } from "./paths.js";
 // An MCP tool definition becomes an SDK server config. ${SECRET} placeholders
 // in env and headers resolve server-side, so a credential reaches the server
 // process without ever entering the model's context.
@@ -84,6 +89,34 @@ export function joinEarlierGroups(groupResults, gi, cap = CONTEXT_CHARS) {
         parts.shift();
     return parts.join("\n\n");
 }
+/**
+ * `parallel: N` — a pool of N slots the instances of one fan-out share.
+ * Fan-out always ran every instance at once, and still does unless the
+ * step says otherwise: twenty browser steps against one site, or one
+ * vendor's rate limit, is what the cap is for. Nothing is rented while an
+ * instance waits for a slot.
+ */
+export function slotPool(size) {
+    let free = Math.max(1, size);
+    const waiting = [];
+    return {
+        acquire: () => new Promise((resolve) => {
+            if (free > 0) {
+                free -= 1;
+                resolve();
+            }
+            else
+                waiting.push(resolve);
+        }),
+        release: () => {
+            const next = waiting.shift();
+            if (next)
+                next();
+            else
+                free += 1;
+        },
+    };
+}
 const toTags = (v) => Array.isArray(v) ? v.map(String) : typeof v === "string" && v.trim() ? [v.trim()] : [];
 // gray-matter throws on invalid YAML. Skills authored for another client
 // sometimes ship YAML that that client's parser happened to accept — most
@@ -311,10 +344,9 @@ export function saysUntilMarker(result, until) {
         .some((line) => 
     // Strip markdown emphasis and punctuation from BOTH ends: "**APPROVED.**"
     // is a decision written by someone using bold, not a sentence.
-    line
-        .trim()
-        .replace(/^[#>\-*_`\s]+/, "")
-        .replace(/[.!:,*_`\s]+$/, "")
+    // trimChars, not a `[…]+$` regex: that one is quadratic on a
+    // line of tabs, and this is a line an agent wrote.
+    trimChars(trimChars(line.trim(), "#>-*_`"), ".!:,*_`").trim()
         .toLowerCase() === marker);
 }
 export function resolveDocLinks(text, workspaceRoot) {
@@ -408,7 +440,10 @@ export function resolveDocLinks(text, workspaceRoot) {
             map.set(norm(fwd), `../../${STORAGE_DIR}/${fwd}`);
         }
     }
-    return text.replace(/\[\[([^\]\n]+)\]\]/g, (whole, name) => {
+    // `[` excluded from the name: with it allowed, a run of `[[` makes the
+    // engine re-scan to the end from every opening — quadratic on a result
+    // an agent wrote.
+    return text.replace(/\[\[([^\]\[\n]+)\]\]/g, (whole, name) => {
         const hit = map.get(norm(name.trim()));
         return hit ? `\`${hit}\`` : whole;
     });
@@ -1072,6 +1107,42 @@ function repriced(catalog, model, usage, sdkCost, push) {
     }
     return known;
 }
+/**
+ * Write one attempt onto the step's history and re-total the step.
+ *
+ * runStep reports the attempt it just ran — its cost, tokens and sandbox
+ * seconds — by assigning the step's fields. Those become this attempt's row;
+ * the step's fields then become the sum of every row, which is what the
+ * run's spend, the flow's budget: and the platform's bill read. A retried
+ * step used to carry only its last attempt's cost, so a `retry: 2` step
+ * could spend three shares while reporting one.
+ */
+export function recordAttempt(step, n, status, startedAt, eventsBefore, 
+/** The attempt re-attached to a sandbox a previous driver left: same
+ *  attempt, so its row is replaced rather than repeated. */
+reattach = false) {
+    const tries = step.tries ?? [];
+    const lastError = step.events
+        .slice(eventsBefore)
+        .filter((e) => e.type === "error")
+        .at(-1)?.text;
+    const row = {
+        n,
+        status,
+        costUsd: step.costUsd,
+        tokens: step.tokens ?? null,
+        computeSecs: step.computeSecs ?? null,
+        startedAt,
+        finishedAt: step.finishedAt ?? new Date().toISOString(),
+        ...(status === "failed" && lastError ? { error: lastError.slice(0, 500) } : {}),
+    };
+    const kept = reattach && tries.at(-1)?.n === n ? tries.slice(0, -1) : tries;
+    step.tries = [...kept, row];
+    const totals = sumAttempts(step.tries);
+    step.costUsd = totals.costUsd;
+    step.tokens = totals.tokens;
+    step.computeSecs = totals.computeSecs;
+}
 async function runStep(agentDir, tenant, step, context, save, modelOverride, tags = [], effortOverride, runId, 
 /** What the previous group returned as DATA (an `output: json` step's
  *  value), beside the prose in `context`. Undefined when nothing did. */
@@ -1079,7 +1150,16 @@ priorData,
 /** The most this step may spend before it is stopped mid-turn: the run's
  *  remaining `budget:`, shared across the steps launched with it. Null
  *  means the flow set no budget. */
-stepBudgetUsd = null) {
+stepBudgetUsd = null, 
+/** Which line set that ceiling, for the error that names it. */
+stepBudgetNote, 
+/** The run is a test run — see test-mode.ts. Read off the run record by
+ *  the caller, never from anything the sandbox could have written. */
+testRun = false, 
+/** Has a person asked for the run to stop? Read on a clock by the
+ *  in-process model loop, so a stop lands mid-step and not at the next
+ *  group. The isolated path needs none: stopRun destroys the sandbox. */
+stopRequested) {
     // Secret values are injected into scripts as environment variables and
     // substituted into API headers, so a model that reads one back — from a
     // script that prints it, an API error that echoes it, or its own `env` —
@@ -1103,6 +1183,22 @@ stepBudgetUsd = null) {
             out = out.split(value).join(`[redacted:${name}]`);
         return out;
     };
+    // The reply, the conclusion and an `output: json` value land on the same
+    // record the events do, and a model that quotes a script's stdout puts
+    // the credential in its answer rather than in a tool event. Same scrub,
+    // same place. The JSON value goes through its text form: a secret inside
+    // a nested field is still a secret, and the key name survives either way.
+    const redactText = (text) => (typeof text === "string" ? redact(text) : text);
+    const redactData = (value) => {
+        if (value === undefined || redactions.length === 0)
+            return value;
+        try {
+            return JSON.parse(redact(JSON.stringify(value)));
+        }
+        catch {
+            return value;
+        }
+    };
     const push = (type, text, extra) => {
         step.events.push({ t: new Date().toISOString(), type, text: redact(text), ...extra });
         save();
@@ -1140,8 +1236,15 @@ stepBudgetUsd = null) {
         // the failure of a step that returned no JSON are testable at no cost.
         if (step.output === "json") {
             const extracted = extractJson(step.result);
-            if (extracted.ok)
+            if (extracted.ok) {
                 step.data = extracted.value;
+                const schema = step.schema ?? (step.schemaPath ? readSchemaFile(path.resolve(agentDir, "..", ".."), step.schemaPath) : null);
+                const errors = schema && looksLikeSchema(schema) ? validateSchema(step.data, schema) : [];
+                if (errors.length) {
+                    push("error", `schema: the value does not fit — ${describeSchemaErrors(errors)}`);
+                    step.status = "failed";
+                }
+            }
             else {
                 push("error", `output: json — ${extracted.reason}`);
                 step.status = "failed";
@@ -1158,14 +1261,17 @@ stepBudgetUsd = null) {
         // step: the generic HTTP tool still works without the typed ones.
         for (const w of await prefetchOpenApi(tenant, openApiSources(agentDir, tenant)))
             push("error", `openapi: ${w}`);
-        const { front, clockEnv, systemPrompt, allowed, disabled, apiTools, scriptTools, secretEnv, secretScopes, missingSecrets, missingTools, runtime, unknownTools, shadowed, legacyUse, mcpServers, mcpNames, apiSpecs, scriptSpecs, brokenTools, size, providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning, fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools, translator, fallbackTranslator, } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent });
+        const { front, clockEnv, systemPrompt, allowed, disabled, apiTools, scriptTools, secretEnv, secretScopes, missingSecrets, missingTools, runtime, unknownTools, shadowed, legacyUse, mcpServers, mcpNames, apiSpecs, scriptSpecs, brokenTools, size: agentSize, providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning, fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools, translator, fallbackTranslator, } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent });
+        // A retry that moved the step up a class (an evicted or OOM-killed
+        // attempt) wins over the agent's own `size:` for the attempts after.
+        const size = step.sizeUp ?? agentSize;
         // oauth2 secrets become live access tokens here — the one async moment
         // in secret resolution, at the last host-side point before anything is
         // injected or substituted. The refresh recipes never leave this process:
         // what crosses into containers, headers and env files is the token. A
         // failed refresh fails the step naming the secret and the provider's
         // reason, which beats a 401 three layers later.
-        const liveSecrets = await materializeSecrets(secretEnv);
+        const liveSecrets = await materializeSecrets(secretEnv, { tenant, workspace: path.basename(workspaceRootOf(agentDir)) });
         // @file values (SSH keys, certs) still carry their content at this point.
         // The in-process branch materialises them to 0600 paths on this host
         // below; the container branch passes the content across and the driver
@@ -1289,6 +1395,11 @@ stepBudgetUsd = null) {
                 push("info", fitted.note);
             effort = fitted.effort;
         }
+        // No catalogue — the platform's own Anthropic key, or a gateway that
+        // publishes no prices — but the model's NAME is often enough. Without
+        // this the in-pod meter priced haiku and sonnet at Opus rates and a
+        // step was stopped at up to fifteen times what it had spent.
+        tokenPrice ??= knownPrice(wireModel);
         fs.mkdirSync(path.join(agentDir, "outputs"), { recursive: true });
         fs.mkdirSync(path.join(agentDir, "memory"), { recursive: true });
         // Expose the workspace's shared scripts/ as ./shared so every agent reaches
@@ -1299,6 +1410,12 @@ stepBudgetUsd = null) {
         // The confinement boundary: agents in a workspace are one team, but the
         // workspace is where isolation is enforced.
         const workspaceRoot = path.resolve(agentDir, "..", "..");
+        // Where a test run's state/ and storage/ writes go, and the line each
+        // one leaves: under the run, beside its archived outputs.
+        const divertFor = (id) => ({
+            to: path.join(workspaceRoot, "runs", id, TEST_WRITES_DIR),
+            note: (rel, summary) => push("info", `test: ${summary}`, { effect: { kind: "diverted", path: rel, summary } }),
+        });
         let prompt = step.instruction
             ? resolveDocLinks(step.instruction, workspaceRoot)
             : "Begin your run now, following your instructions.";
@@ -1315,11 +1432,18 @@ stepBudgetUsd = null) {
         if (step.output === "json") {
             // The contract the runner parses back out — stated in the prompt so
             // the instruction and the parser agree by construction, like delegate:.
+            // The schema, when there is one, is quoted whole: the model that has
+            // to satisfy it should not have to guess it from the field names.
+            const schema = step.schema ?? (step.schemaPath ? readSchemaFile(path.resolve(agentDir, "..", ".."), step.schemaPath) : null);
             prompt +=
                 `\n\n<output>\nThis step returns data. End your reply with exactly one JSON value inside a ` +
                     "```json fenced block — the value the next step will compute on. Prose before it is " +
                     `welcome (lead with a one-line headline); nothing may follow the block. If the instruction ` +
-                    `names fields, use exactly those names.\n</output>`;
+                    `names fields, use exactly those names.` +
+                    (schema && looksLikeSchema(schema)
+                        ? ` The value must satisfy this JSON Schema, and the step fails if it does not:\n${JSON.stringify(schema, null, 2).slice(0, 20_000)}`
+                        : "") +
+                    `\n</output>`;
         }
         if (step.waitFor === "event" && step.eventPayload !== undefined) {
             // Not resolved for links, like the previous step's results: it is
@@ -1384,6 +1508,16 @@ stepBudgetUsd = null) {
         // fallback), else the platform's, and only for a step riding the
         // platform credential.
         const secondSupply = fallbackEnv ?? (Object.keys(providerEnv).length === 0 ? platformFallbackEnv() : null);
+        // `schema:` — inline on the step, or a file under the workspace read
+        // now, host-side, so both executors are handed the same document. A
+        // file that cannot be read is a failed step before any model runs:
+        // the contract the author wrote down cannot be checked.
+        const resolvedSchema = step.schema ?? (step.schemaPath ? readSchemaFile(workspaceRoot, step.schemaPath) : null);
+        if (step.schemaPath && resolvedSchema === null) {
+            throw new Error(`schema: ${step.schemaPath} — not a readable JSON or YAML schema under the workspace`);
+        }
+        if (step.maxTurns)
+            push("info", `max_turns: ${step.maxTurns}`);
         const isolation = process.env.FOLDRUN_RUN_ISOLATION;
         if (isolatedRun()) {
             // The isolated path: the whole loop — model, built-in tools, scripts —
@@ -1412,12 +1546,17 @@ stepBudgetUsd = null) {
                 if (!host)
                     continue;
                 for (const v of [...Object.values(api.headers), ...Object.values(api.query)]) {
-                    for (const name of placeholderNames(v))
-                        if (name in liveSecrets)
-                            addGrant(grant, name, liveSecrets[name], host);
+                    for (const name of placeholderNames(v)) {
+                        if (!(name in liveSecrets))
+                            continue;
+                        addGrant(grant, name, liveSecrets[name], host);
+                        // Which vault entry this is, so the proxy's record of what the
+                        // far end said lands on the right credential.
+                        (grant.healthKeys ??= {})[name] = healthKey(name, secretScopes[name] === "workspace" ? "workspace" : "account", path.basename(path.resolve(agentDir, "..", "..")));
+                    }
                 }
             }
-            const lease = await platform.egress.lease({ tenant, runId: runId ?? "adhoc", grant });
+            const lease = await platform.egress.lease({ tenant, runId: runId ?? "adhoc", grant, test: testRun });
             // Which secrets must still be real inside the sandbox: a script tool
             // reads them from its environment, and unless it says `secrets:
             // proxied` it needs the value, not the name. bash is a script by
@@ -1437,14 +1576,35 @@ stepBudgetUsd = null) {
             }
             // Sandbox env for the declared secrets: the value, or its own name as
             // a placeholder that the proxy fills on the way out.
-            const sandboxSecrets = materialise
-                ? liveSecrets
-                : Object.fromEntries(declared.map((name) => [name, `\${${name}}`]));
+            const placeholders = Object.fromEntries(declared.map((name) => [name, `\${${name}}`]));
+            let sandboxSecrets = materialise ? liveSecrets : placeholders;
+            // A test run hands the sandbox no send-capable value. The env var is
+            // still there, reading TEST_MODE_WITHHELD, so a script that never
+            // checked FOLDRUN_TEST_MODE fails at the provider's door with an auth
+            // error — and sends nothing. The script tools say whether the step
+            // has a declared sender (everything withheld) or a declared reader
+            // (nothing withheld); see test-mode.ts for the order.
+            if (testRun && materialise) {
+                const held = withholdSecrets(liveSecrets, {
+                    outward: scriptSpecs.some((sc) => sc.outward),
+                    allow: scriptSpecs.some((sc) => sc.testMode === "allow"),
+                });
+                sandboxSecrets = held.env;
+                for (const name of held.withheld) {
+                    push("info", `test: ${name} withheld from the sandbox — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from the sandbox` } });
+                }
+            }
+            // On a test run the http tools always go through the proxy as
+            // placeholders, even when a script made the step materialise: the
+            // proxy is where the outward-write policy lives, and a header
+            // carrying TEST_MODE_WITHHELD would only fail where it should have
+            // been sunk or refused.
+            const apiValues = testRun && lease ? placeholders : sandboxSecrets;
             const substitutedApis = apiSpecs.map((api) => ({
                 ...api,
                 headers: Object.fromEntries(Object.entries(api.headers).map(([k, v]) => [
                     k,
-                    v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => sandboxSecrets[name] ?? whole),
+                    v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => apiValues[name] ?? whole),
                 ])),
             }));
             // The provider's own key and headers, when the step has a lease: the
@@ -1472,6 +1632,7 @@ stepBudgetUsd = null) {
             const isolatedArgs = (modelEnv, keyName = MODEL_KEY_NAME) => ({
                 workspaceRoot,
                 libraryRoot: libraryDir(tenant),
+                galleryRoot: platform.galleryDir() ?? undefined,
                 // Keys this step's dependency cache. Per-account by design — see
                 // run-container.ts#runtimeCacheDir for why it is not shared.
                 tenant,
@@ -1492,9 +1653,12 @@ stepBudgetUsd = null) {
                     consults,
                     timeoutSec: step.timeout,
                     budgetUsd: stepBudgetUsd,
+                    budgetNote: stepBudgetNote,
                     price: tokenPrice,
                     verify: step.verify,
                     output: step.output,
+                    schema: resolvedSchema ?? undefined,
+                    maxTurns: step.maxTurns,
                     translator: keyName === MODEL_KEY_NAME ? primaryTranslator : secondTranslator,
                     // The container sees the workspace at /workspace and the library
                     // at /library; the roots are named host-side and moved here.
@@ -1524,12 +1688,29 @@ stepBudgetUsd = null) {
                     FOLDRUN_RUN_ID: runId,
                     FOLDRUN_AGENT: step.agent,
                     FOLDRUN_WORKSPACE: path.basename(workspaceRoot),
+                    ...(testRun ? testModeEnv() : {}),
                     ...sandboxSecrets,
                 }).filter((entry) => typeof entry[1] === "string")),
                 emit: pushWatching,
                 // Stamps the sandbox, so stopping this run can destroy it.
                 runId,
                 size,
+                // A test run's state/ and storage/ changes come out under the run.
+                divert: testRun && runId ? divertFor(runId) : undefined,
+                test: testRun,
+                // Resume: a sandbox a previous driver left running, recorded on the
+                // step. Consumed once — a fresh attempt after this one (a retry, a
+                // fallback) starts its own sandbox.
+                resume: step.sandbox && step.sandbox.kind === isolation ? { ref: step.sandbox.ref, consumed: step.sandbox.consumed } : null,
+                // Checkpoint: where the executor is, on the record, so a driver
+                // that inherits this run can attach where this one left off. The
+                // executor calls it before applying each line; the line's own emit
+                // saves the record, so this only updates the fields.
+                checkpoint: (ref, consumed) => {
+                    step.sandbox = { kind: isolation, ref, consumed, since: step.sandbox?.since ?? new Date().toISOString() };
+                    if (consumed === 0)
+                        save();
+                },
             });
             // A busy provider gets another moment on the SAME supply before the
             // fallback is considered: under a fan-out, twenty steps against one
@@ -1539,6 +1720,7 @@ stepBudgetUsd = null) {
             const firstArgs = isolatedArgs(platformModelEnv());
             await lease?.commit();
             let outcome = await runIsolated(firstArgs);
+            step.sandbox = null; // whatever happens next starts its own
             for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
                 const wait = backoffMs(n, lastRefusal);
                 push("info", `provider busy (${lastRefusal.slice(0, 60)}) — waiting ${(wait / 1000).toFixed(1)}s, attempt ${n} of ${OVERLOAD_RETRIES}`);
@@ -1579,16 +1761,28 @@ stepBudgetUsd = null) {
                 outcome = withEarlierTiming(outcome, first);
             }
             if (lease) {
-                for (const line of await lease.drainLog())
-                    push("info", line);
+                for (const line of await lease.drainLog()) {
+                    if (typeof line === "string")
+                        push("info", line);
+                    else
+                        push("info", line.text, { effect: line.effect });
+                }
                 await lease.release();
             }
-            publishPublicDir(tenant, path.basename(workspaceRoot), push);
+            // Nothing a test run left in storage/public/ reached storage/, so
+            // there is nothing to publish — and a share link is an outward act.
+            if (!testRun)
+                publishPublicDir(tenant, path.basename(workspaceRoot), push);
             step.status = outcome.status;
-            step.result = outcome.result;
-            step.conclusion = outcome.conclusion;
+            step.result = redactText(outcome.result);
+            step.conclusion = redactText(outcome.conclusion);
+            step.sandbox = null;
+            // The cluster's word on why a sandbox ended without the driver saying
+            // so — OOMKilled, Evicted — on the record, where the retry reads it.
+            if (outcome.status === "failed" && outcome.reason)
+                push("error", `sandbox ended: ${outcome.reason}`);
             if (outcome.data !== undefined)
-                step.data = outcome.data;
+                step.data = redactData(outcome.data);
             step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push);
             step.tokens = outcome.usage
                 ? { input: outcome.usage.inputTokens, output: outcome.usage.outputTokens }
@@ -1619,10 +1813,23 @@ stepBudgetUsd = null) {
             step.reserved = { cpus: Number(lim.cpus) || 2, memGiB: gib };
         }
         else {
-            const consultTools = buildConsultTools(consults, { ...process.env, ...liveSecrets, ...providerEnv }, (type, text) => push(type, text));
+            // Same withholding as the sandbox path — the in-process step is what
+            // `foldrun run --test` drives on a laptop, where there is no proxy
+            // and this is the only guard there is.
+            const held = testRun
+                ? withholdSecrets(liveSecrets, {
+                    outward: scriptSpecs.some((sc) => sc.outward),
+                    allow: scriptSpecs.some((sc) => sc.testMode === "allow"),
+                })
+                : { env: liveSecrets, withheld: [] };
+            for (const name of held.withheld) {
+                push("info", `test: ${name} withheld from scripts — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from scripts` } });
+            }
+            const stepSecrets = held.env;
+            const consultTools = buildConsultTools(consults, { ...hostSafeEnv(), ...stepSecrets, ...providerEnv }, (type, text) => push(type, text));
             // @file secrets become 0600 paths here (host run), cleaned up in the
             // finally — once, however many attempts follow.
-            const mat = materializeFileSecrets(agentDir, liveSecrets);
+            const mat = materializeFileSecrets(agentDir, stepSecrets);
             fileDir = mat.dir;
             const attemptInProcess = (modelEnv) => executeStep({
                 agentDir,
@@ -1644,13 +1851,20 @@ stepBudgetUsd = null) {
                     ...mcpServers,
                 },
                 // Declared secrets reach the agent's scripts as env vars; the model
-                // only ever sees the variable names, not the values.
-                env: { ...process.env, ...clockEnv, ...mat.env, ...modelEnv },
+                // only ever sees the variable names, not the values. The base is
+                // the allowlisted host environment, not process.env: the SDK's
+                // Bash tool runs with this, and on a platform process.env holds
+                // every tenant's root key and the datastore credentials.
+                env: { ...hostSafeEnv(), ...clockEnv, ...(testRun ? testModeEnv() : {}), ...mat.env, ...modelEnv },
                 timeoutSec: step.timeout,
                 budgetUsd: stepBudgetUsd,
+                budgetNote: stepBudgetNote,
                 price: tokenPrice,
                 verify: step.verify,
                 output: step.output,
+                schema: resolvedSchema ?? undefined,
+                maxTurns: step.maxTurns,
+                stopRequested,
                 // What the scripts saw, the verify sees: a flow can then check that
                 // the proof a step left names THIS run, not one that came before —
                 // copy-back never propagates a deletion, so a marker from an earlier
@@ -1660,23 +1874,33 @@ stepBudgetUsd = null) {
                     ...(runId ? { FOLDRUN_RUN_ID: runId } : {}),
                     FOLDRUN_AGENT: step.agent,
                     FOLDRUN_WORKSPACE: path.basename(path.resolve(agentDir, "..", "..")),
-                    ...liveSecrets,
+                    ...(testRun ? testModeEnv() : {}),
+                    ...stepSecrets,
                 },
                 emit: pushWatching,
             });
             // With a translator the step's endpoint is the translator: started
             // here, pointed at by the SDK's env, closed when the attempt ends.
+            // A test run brackets each attempt with a snapshot of state/ and
+            // storage/: the step writes them for real, because there is no
+            // sandbox to write instead, and afterwards what changed is moved
+            // under the run and the originals come back.
             const attemptThrough = async (modelEnv, spec) => {
-                if (!spec)
-                    return attemptInProcess(modelEnv);
-                const t = await startTranslator(spec);
+                const before = testRun ? snapshotDivertedDirs(workspaceRoot) : null;
+                const t = spec ? await startTranslator(spec) : null;
                 try {
-                    return await attemptInProcess({ ...modelEnv, ...t.env });
+                    return await attemptInProcess(t ? { ...modelEnv, ...t.env } : modelEnv);
                 }
                 finally {
-                    for (const line of t.drainLog())
-                        push("info", line.startsWith("translator") ? line : `translator: ${line}`);
-                    await t.close();
+                    if (t) {
+                        for (const line of t.drainLog())
+                            push("info", line.startsWith("translator") ? line : `translator: ${line}`);
+                        await t.close();
+                    }
+                    if (before && runId) {
+                        const d = divertFor(runId);
+                        restoreDivertedDirs(workspaceRoot, before, d.to, d.note);
+                    }
                 }
             };
             let outcome = await attemptThrough(providerEnv, translator);
@@ -1695,12 +1919,13 @@ stepBudgetUsd = null) {
                 push("info", `primary model supply refused (${lastRefusal.slice(0, 80)}) — retrying this step on the fallback provider`);
                 outcome = await attemptThrough(secondSupply, fallbackEnv && secondSupply === fallbackEnv ? fallbackTranslator : null);
             }
-            publishPublicDir(tenant, path.basename(workspaceRoot), push);
+            if (!testRun)
+                publishPublicDir(tenant, path.basename(workspaceRoot), push);
             step.status = outcome.status;
-            step.result = outcome.result;
-            step.conclusion = outcome.conclusion;
+            step.result = redactText(outcome.result);
+            step.conclusion = redactText(outcome.conclusion);
             if (outcome.data !== undefined)
-                step.data = outcome.data;
+                step.data = redactData(outcome.data);
             // A consult's spend belongs to the step that asked.
             const consultCost = consultTools.drainCost();
             const stepCost = repriced(catalog, wireModel, outcome.usage, outcome.costUsd, push);
@@ -2006,12 +2231,35 @@ function csvItems(workspaceDir, step, take) {
     }
     return data.slice(0, take).map((r) => `${header}\n${r}`);
 }
-export function createFlowRun(tenant, workspace, steps, flowName, status, tags = []) {
+/**
+ * `schema: <path>` — the file, read and parsed, or null. Agent-relative like
+ * every other path in a flow (`../../schemas/lead.json`), confined to the
+ * workspace: a schema is the author's document, never something a step
+ * can point outside.
+ */
+export function readSchemaFile(workspaceRoot, rel) {
+    const root = path.resolve(workspaceRoot);
+    // Written agent-relative (`../../schemas/x.json`) or workspace-relative
+    // (`schemas/x.json`); both land on the same file.
+    const candidates = [path.resolve(root, rel.replace(/^(\.\.\/){2}/, "")), path.resolve(root, "agents", "x", rel)];
+    for (const abs of candidates) {
+        if (abs !== root && !abs.startsWith(root + path.sep))
+            continue;
+        if (!fs.existsSync(abs) || !fs.statSync(abs).isFile())
+            continue;
+        const parsed = parseSchemaText(fs.readFileSync(abs, "utf8"));
+        return "problem" in parsed ? null : parsed.schema;
+    }
+    return null;
+}
+export function createFlowRun(tenant, workspace, steps, flowName, status, tags = [], startedBy = null, opts = {}) {
     const run = {
         id: `run-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`,
         flow: flowName,
         tags,
         status,
+        ...(startedBy ? { startedBy: startedBy.trim().toLowerCase() } : {}),
+        ...(opts.test ? { test: true } : {}),
         startedAt: new Date().toISOString(),
         finishedAt: null,
         steps: steps.map((s) => ({
@@ -2041,6 +2289,10 @@ export function createFlowRun(tenant, workspace, steps, flowName, status, tags =
             preview: s.preview,
             delegate: s.delegate,
             output: s.output,
+            schema: s.schema,
+            schemaPath: s.schemaPath,
+            parallel: s.parallel,
+            maxTurns: s.maxTurns,
             attempts: 0,
             status: "pending",
             events: [],
@@ -2066,8 +2318,8 @@ export function createFlowRun(tenant, workspace, steps, flowName, status, tags =
     }
     return run;
 }
-export function startFlowRun(tenant, workspace, steps, flowName, modelOverride, tags = [], effortOverride) {
-    const run = createFlowRun(tenant, workspace, steps, flowName, "running", tags);
+export function startFlowRun(tenant, workspace, steps, flowName, modelOverride, tags = [], effortOverride, startedBy = null, opts = {}) {
+    const run = createFlowRun(tenant, workspace, steps, flowName, "running", tags, startedBy, opts);
     void driveRun(tenant, workspace, run, modelOverride, tags, { effortOverride });
     return run;
 }
@@ -2262,16 +2514,26 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
             // never happened, which reconcile can then never see. Reset it to
             // pending and it simply runs again; driveRun's whole contract is that
             // re-driving a partly-finished record is safe.
+            // Unless the step's sandbox outlived its driver AND the executor can
+            // re-attach to it: then the record carries the pod and how far the
+            // last driver got, runStep passes that as `resume`, and the step
+            // carries on from where it was — a deploy no longer costs the run.
             let orphaned = false;
             for (const step of run.steps) {
                 if (step.status !== "running")
                     continue;
-                orphaned = true;
+                const resumable = !!step.sandbox && platform.sandboxResumable(step.sandbox.kind);
+                if (!resumable) {
+                    orphaned = true;
+                    step.sandbox = null;
+                }
                 step.status = "pending";
                 step.events.push({
                     t: new Date().toISOString(),
                     type: "info",
-                    text: "interrupted mid-step (platform restart) — running again from the start of the step",
+                    text: resumable
+                        ? `driver changed mid-step — re-attaching to the running sandbox (${step.sandbox.ref}, ${step.sandbox.consumed} lines already applied)`
+                        : "interrupted mid-step (platform restart) — running again from the start of the step",
                 });
             }
             if (orphaned) {
@@ -2344,6 +2606,19 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
             // is assembled) keeps the newest: when the total is over it, the
             // oldest groups are dropped whole before anything is cut mid-text.
             const contextBefore = (gi) => joinEarlierGroups(groupResults, gi);
+            // What `when:` and `case:` read: the PREVIOUS group's result alone —
+            // the nearest earlier group that produced one, so a routed-past group
+            // between is looked through rather than read as silence. The prompt
+            // gets every earlier group joined (contextBefore); routing must not,
+            // because a verdict two groups back would still open a gate a later
+            // group had overruled: a classifier's BUG line matched a `case: BUG`
+            // three groups on, after the debugger had already answered it.
+            const routingBefore = (gi) => {
+                for (let i = gi - 1; i >= 0; i--)
+                    if (groupResults[i])
+                        return groupResults[i];
+                return null;
+            };
             // The data half of the same handoff: what the nearest earlier group
             // returned through `output: json`. Rebuilt from the record rather than
             // kept only in memory, so a resumed run hands step 3 the value step 1
@@ -2407,8 +2682,11 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                 const group = ordered[gi];
                 context = contextBefore(gi);
                 ctxData = dataBefore(gi);
-                if (run.budgetUsd && runCost(run) >= run.budgetUsd && group.some((s) => s.status === "pending")) {
-                    const spent = runCost(run);
+                // What the run has spent, not just what its tokens cost: on an
+                // install that prices sandbox seconds, a step that called no model
+                // still spent money, and a cap that ignored it capped nothing.
+                if (run.budgetUsd && platform.runSpend(run) >= run.budgetUsd && group.some((s) => s.status === "pending")) {
+                    const spent = platform.runSpend(run);
                     const last = run.steps.findLast((s) => s.status === "completed" || s.status === "failed") ?? run.steps[0];
                     last?.events.push({
                         t: new Date().toISOString(),
@@ -2494,6 +2772,7 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                         t: new Date().toISOString(),
                         type: "info",
                         text: `fan-out: ${used.length} instance${used.length === 1 ? "" : "s"}` +
+                            (step.parallel && step.parallel < used.length ? `, ${step.parallel} at a time (parallel:)` : "") +
                             (dropped ? ` (more items existed — capped at ${cap})` : ""),
                     });
                     const at = run.steps.indexOf(step);
@@ -2508,6 +2787,7 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                         result: null,
                         costUsd: null,
                         attempts: 0,
+                        tries: undefined,
                     }));
                     run.steps.splice(at + 1, 0, ...instances);
                     ordered = orderedGroups();
@@ -2520,12 +2800,25 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                 // this is what keeps the *next* group from starting.
                 const stopCheck = readRun(tenant, workspace, run.id);
                 if (stopCheck?.stopRequested) {
+                    let skipped = 0;
                     for (const s of run.steps) {
                         if (s.status === "pending" || s.status === "running") {
                             s.status = "skipped";
                             s.skipReason = "run stopped";
+                            skipped += 1;
                         }
                     }
+                    // Say so on the trail. stopRun wrote "run stopped" onto the
+                    // record's first step, but the save() below writes this process's
+                    // copy of the record over it, so the only witness was the
+                    // stopRequested flag — and a run that "failed" with no error line
+                    // anywhere took a code read to explain (rank-desk, 2026-09-13).
+                    const witness = run.steps.findLast((s) => s.status === "completed" || s.status === "failed") ?? run.steps[0];
+                    witness?.events.push({
+                        t: new Date().toISOString(),
+                        type: "info",
+                        text: `run stopped by a person between groups — ${skipped} remaining step${skipped === 1 ? "" : "s"} skipped`,
+                    });
                     run.status = "failed";
                     run.stopRequested = true;
                     save();
@@ -2560,39 +2853,51 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                     break;
                 }
                 const ctx = context;
-                // `when:` — skip a step whose condition isn't met by prior results.
+                // The previous group's result — see routingBefore.
+                const routing = routingBefore(gi);
+                // `when:` — skip a step whose condition isn't met by the previous
+                // group's result.
                 for (const step of freshGroup) {
                     if (step.status !== "pending" || !step.when)
                         continue;
-                    const met = (ctx ?? "").toLowerCase().includes(step.when.toLowerCase());
+                    // A marker at the start of a line, not a word anywhere in the
+                    // prose — see markerPresent. "There are no BLOCKED items" must
+                    // not open a gate keyed on BLOCKED.
+                    const met = markerPresent(routing, step.when);
                     if (!met) {
                         step.status = "skipped";
                         step.skipReason = `condition not met: when "${step.when}"`;
                         step.events.push({
                             t: new Date().toISOString(),
                             type: "info",
-                            text: `skipped — previous results do not mention "${step.when}"`,
+                            text: `skipped — the previous group's result has no line beginning "${step.when}" (when: reads a marker at the start of a line of the previous result, not a word in a sentence)`,
                         });
                     }
                 }
                 save();
                 // `case:`/`else:` — exclusive routing. Of this group's case steps,
-                // the FIRST whose text appears in the previous results runs; the
-                // rest are routed past. `else:` runs only when no case matched.
+                // the FIRST whose marker leads a line of the previous results runs;
+                // the rest are routed past. `else:` runs only when no case matched.
                 // (`when:` above stays independent — every matching when runs —
                 // which is why routing is its own vocabulary instead of a mode.)
+                //
+                // Same marker rule as `when:`, and it matters more here: routing is
+                // exclusive, so a label matched out of a sentence does not merely
+                // run an extra step, it sends the flow down the wrong branch and
+                // skips the right one. "This is not a COMPLAINT, it is a QUESTION"
+                // used to route to the complaints handler.
                 const caseSteps = freshGroup.filter((s) => s.status === "pending" && s.case);
                 if (caseSteps.length || freshGroup.some((s) => s.status === "pending" && s.else)) {
                     let matchedCase = null;
                     for (const step of caseSteps) {
-                        if (!matchedCase && (ctx ?? "").toLowerCase().includes(step.case.toLowerCase())) {
+                        if (!matchedCase && markerPresent(routing, step.case)) {
                             matchedCase = step;
                             continue;
                         }
                         step.status = "skipped";
                         step.skipReason = matchedCase
                             ? `routed past — "${matchedCase.case}" matched first`
-                            : `case not matched: "${step.case}"`;
+                            : `case not matched: the previous group's result has no line beginning "${step.case}"`;
                         step.events.push({
                             t: new Date().toISOString(),
                             type: "info",
@@ -2634,6 +2939,20 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                         });
                     }
                     run.status = "awaiting-approval";
+                    // approve_within: — the deadline is stamped when the gate opens,
+                    // not counted from the run's start, so "two days to answer" means
+                    // two days from being asked however long the work before it took.
+                    // Only a gate that asks a PERSON gets one: an external event has
+                    // no one to chase.
+                    const within = readFlow(tenant, workspace, run.flow)?.approveWithin ?? null;
+                    // Always assigned, never only set: a run with two gates would
+                    // otherwise carry the FIRST gate's deadline into the second, and
+                    // a deadline already in the past expires a gate the moment it
+                    // opens. Null when this gate has none.
+                    run.approveBy =
+                        within && needsApproval.some((s) => s.waitFor !== "event")
+                            ? new Date(Date.now() + within * 1000).toISOString()
+                            : null;
                     save();
                     // The gate's declared preview, resolved now against what the run
                     // has actually produced — a glob at park time, a list of files on
@@ -2706,18 +3025,88 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                 // twenty steps over — the shares sum to what was left. Between
                 // groups the check above still refuses to start the next one.
                 const launching = freshGroup.filter((s) => s.status === "pending");
-                const ceilingUsd = stepCeiling(run.budgetUsd, runCost(run), launching.length);
+                // An agent's own `budget:` — the most it may spend in one run — is
+                // read at launch, so a cap raised mid-run applies to the next step.
+                const agentBudgets = new Map(listAgents(tenant, workspace).map((a) => [a.name, a.budget]));
+                // The ceiling for one attempt of one step, read NOW: what is left of
+                // the run's budget shared across the group, or the agent's own cap
+                // less what its steps have cost. Computed before every attempt, not
+                // once before the loop — a retry's share is what is left after the
+                // failed attempt spent, and the step's own figure already carries
+                // that spend (it is the sum of its tries).
+                const ceilingFor = (step) => stepCeilingFor(stepCeiling(run.budgetUsd, platform.runSpend(run), launching.length), agentBudgets.get(step.agent), run.steps.filter((s) => s !== step && s.agent === step.agent).reduce((sum, s) => sum + (s.costUsd ?? 0), 0), step.agent);
+                // parallel: — the instances of one fan-out template share a pool.
+                // Keyed by what the instances share (agent + instruction), since
+                // the template itself is behind them in the record as "expanded".
+                const pools = new Map();
+                const poolFor = (step) => {
+                    if (!step.parallel || !step.item)
+                        return null;
+                    const key = `${step.agent}\u0000${step.instruction}`;
+                    let pool = pools.get(key);
+                    if (!pool)
+                        pools.set(key, (pool = slotPool(step.parallel)));
+                    return pool;
+                };
                 await Promise.all(launching
                     .map(async (step) => {
+                    const pool = poolFor(step);
+                    if (pool)
+                        await pool.acquire();
+                    try {
+                        await launchStep(step);
+                    }
+                    finally {
+                        pool?.release();
+                    }
+                }));
+                async function launchStep(step) {
                     const attempts = (step.retry ?? 0) + 1;
-                    for (let attempt = 1; attempt <= attempts; attempt++) {
+                    // A step re-attaching to its sandbox is still on the attempt
+                    // that started it; a step re-run after a backoff picks up the
+                    // count it left. Only a step nobody has touched starts at 1.
+                    const interruptedInBackoff = !step.sandbox && !!step.attempts && /retrying in \d+s/.test(step.events[step.events.length - 1]?.text ?? "");
+                    const from = step.sandbox ? Math.max(1, step.attempts ?? 1) : interruptedInBackoff ? Math.min(attempts, step.attempts + 1) : 1;
+                    for (let attempt = from; attempt <= attempts; attempt++) {
+                        const ceiling = ceilingFor(step);
+                        // Nothing left is a refusal, not "no ceiling": a zero passed
+                        // down would read as unset, and the one step that must not
+                        // run would run with no cap at all. On a retry this is what
+                        // stops the second attempt spending a share the first one
+                        // already spent.
+                        if (ceiling.ceilingUsd === 0) {
+                            step.status = "failed";
+                            step.events.push({
+                                t: new Date().toISOString(),
+                                type: "error",
+                                text: attempt === from
+                                    ? `over budget — nothing left to spend before this step (${ceiling.note}); it was not started`
+                                    : `over budget — nothing left to spend for attempt ${attempt} (${ceiling.note}); not retried`,
+                            });
+                            save();
+                            return;
+                        }
+                        // Re-attaching to a sandbox is the same attempt, so its row
+                        // is replaced; anything else is a new row, a loop cycle's
+                        // re-run included.
+                        const reattach = !!step.sandbox;
+                        const attemptStartedAt = new Date().toISOString();
+                        const eventsBefore = step.events.length;
                         step.attempts = attempt;
                         step.status = "running";
                         save();
-                        await runStep(path.join(pDir, "agents", step.agent), tenant, step, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData, ceilingUsd);
+                        await runStep(path.join(pDir, "agents", step.agent), tenant, step, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData, ceiling.ceilingUsd, ceiling.note, run.test === true, 
+                        // save() merges a stop written by another process into
+                        // this record; the model loop polls this on a clock.
+                        () => {
+                            save();
+                            return run.stopRequested === true;
+                        });
                         // runStep mutates step.status; read it through a widened local
                         // so TS doesn't keep the "running" narrowing from above.
                         const outcome = step.status;
+                        recordAttempt(step, attempt, outcome === "completed" ? "completed" : "failed", attemptStartedAt, eventsBefore, reattach);
+                        save();
                         if (outcome !== "failed" || attempt === attempts)
                             break;
                         // A stop destroys the step's sandbox, which reads here as a
@@ -2727,14 +3116,45 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                         save();
                         if (run.stopRequested)
                             break;
+                        // The retry policy. An attempt the cluster ended for want of
+                        // memory or disk is retried one size class up — the same
+                        // size would end the same way. Every retry waits: 15 s, 30 s,
+                        // 1 m, 2 m, 4 m, jittered, so a provider or a site that was
+                        // struggling gets a moment, and twenty fan-out steps do not
+                        // hammer it in lock-step. The wait holds a worker slot, not
+                        // a sandbox — nothing is rented while it waits — and a stop
+                        // ends it. The step is `pending` meanwhile, so a driver that
+                        // inherits the run mid-wait simply runs the attempt.
+                        const last = [...step.events].reverse().find((e) => e.type === "error")?.text ?? "";
+                        let sizeNote = "";
+                        if (/OOMKilled|Evicted|ephemeral-storage|out of memory/i.test(last)) {
+                            const up = step.sizeUp === "large" || (!step.sizeUp && step.size === "large") ? "heavy" : step.sizeUp === "heavy" || step.size === "heavy" ? null : "large";
+                            if (up) {
+                                step.sizeUp = up;
+                                sizeNote = ` at size: ${up}`;
+                            }
+                        }
+                        const waitMs = Math.round(Math.min(retryBaseMs() * 16, retryBaseMs() * 2 ** (attempt - 1)) * (0.75 + Math.random() * 0.5));
+                        step.status = "pending";
                         step.events.push({
                             t: new Date().toISOString(),
                             type: "info",
-                            text: `attempt ${attempt} failed — retrying (${attempts - attempt} left)`,
+                            text: `attempt ${attempt} failed — retrying in ${Math.round(waitMs / 1000)}s${sizeNote} (${attempts - attempt} left)`,
                         });
                         save();
+                        const until = Date.now() + waitMs;
+                        while (Date.now() < until) {
+                            await sleep(Math.min(1000, until - Date.now()));
+                            save();
+                            if (run.stopRequested)
+                                break;
+                        }
+                        if (run.stopRequested) {
+                            step.status = "failed";
+                            break;
+                        }
                     }
-                }));
+                }
                 // on-fail: another agent takes the step over. Sequential and rare:
                 // the rescue is a second full attempt at the step's job with the
                 // failure in front of it, and its success is the step's success —
@@ -2786,7 +3206,7 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                     rescue.attempts = 1;
                     rescue.status = "running";
                     save();
-                    await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData);
+                    await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData, null, undefined, run.test === true);
                     // runStep mutates rescue.status; widen past TS's "running" narrowing.
                     const rescueOutcome = rescue.status;
                     if (rescueOutcome === "completed") {
@@ -2919,16 +3339,25 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
         }
         catch (err) {
             run.status = "failed";
+            const text = err instanceof Error ? err.message : String(err);
+            let told = false;
             run.steps.forEach((s) => {
                 if (s.status === "running") {
                     s.status = "failed";
-                    s.events.push({
-                        t: new Date().toISOString(),
-                        type: "error",
-                        text: err instanceof Error ? err.message : String(err),
-                    });
+                    s.events.push({ t: new Date().toISOString(), type: "error", text });
+                    told = true;
                 }
             });
+            // Thrown between groups, with nothing running: the error still has to
+            // land on some step's trail, or the run reads as failed for no reason.
+            if (!told) {
+                const witness = run.steps.findLast((s) => s.status !== "pending") ?? run.steps[0];
+                witness?.events.push({
+                    t: new Date().toISOString(),
+                    type: "error",
+                    text: `run failed between steps: ${text}`,
+                });
+            }
         }
         finally {
             if (!parked) {
@@ -2939,6 +3368,10 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                 // learn, and the runs list and the notification stop having to say
                 // only "completed".
                 run.summary = runSummary(run);
+                // A test run's headline says so wherever the headline goes — the
+                // runs list, the history tool, a later run's recall.
+                if (run.test && run.summary)
+                    run.summary = testHeadline(run.summary);
                 // Archive before the next run resets outputs/ — a failed run's
                 // artifacts are usually the most interesting ones, so this runs on
                 // failure too.
@@ -2976,17 +3409,6 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
         }
     })();
 }
-function readFlow(tenant, workspace, flowName) {
-    const dir = path.join(workspaceDir(tenant, workspace), "flows");
-    if (!fs.existsSync(dir))
-        return null;
-    for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".md"))) {
-        const flow = parseFlow(f, fs.readFileSync(path.join(dir, f), "utf8"));
-        if (flow.name === flowName)
-            return flow;
-    }
-    return null;
-}
 // Splice `[[flow:other]]` steps into the parent's step list, renumbering
 // groups so the nested flow's own parallelism survives. Depth-limited, and
 // a cycle raises rather than expanding forever.
@@ -3051,6 +3473,9 @@ export function loadFlow(tenant, workspace, flowName) {
 // once it has been quiet for two minutes — the margin is for a record
 // written a beat before its driver registered, not a judgement about pace.
 const ABANDONED_AFTER_MS = 2 * 60 * 1000;
+/** The first retry's wait; each one after doubles it, to sixteen times
+ *  this (15 s → 4 min by default). An env, so a test can make it small. */
+const retryBaseMs = () => Number(process.env.FOLDRUN_RETRY_BASE_MS) || 15_000;
 /** The most recent moment a run showed any sign of life. */
 function lastActivity(run) {
     let latest = Date.parse(run.startedAt);
@@ -3082,12 +3507,19 @@ function lastActivity(run) {
  * outlives any process. A run whose person already answered is not waiting,
  * and is picked back up — see below.
  */
-export function reconcileRuns(tenant, now = Date.now()) {
+export function reconcileRuns(tenant, now = Date.now(), live) {
     const closed = [];
     for (const workspace of listWorkspaces(tenant)) {
         for (const summary of listRuns(tenant, workspace.name)) {
             if (drivingRuns.has(summary.id))
                 continue;
+            // Driven by another process — a second worker pod, on a platform with
+            // more than one. The idle window protects a quiet step from THIS
+            // process's judgement only; a worker that boots beside a live one
+            // would otherwise close every long step its neighbour is driving.
+            // The caller says which runs those are (a heartbeated claim).
+            if (live?.has(summary.id))
+                continue;
             if (now - lastActivity(summary) < ABANDONED_AFTER_MS)
                 continue;
             // Approved, then abandoned. Approval only writes to the run record; the
@@ -3168,8 +3600,8 @@ export function reconcileRuns(tenant, now = Date.now()) {
  * one hard-coded account left every other account's interrupted runs marked
  * `running` forever, with no process left to finish them.
  */
-export function reconcileAllRuns(now = Date.now()) {
-    return listTenants().flatMap((tenant) => reconcileRuns(tenant, now));
+export function reconcileAllRuns(now = Date.now(), live) {
+    return listTenants().flatMap((tenant) => reconcileRuns(tenant, now, live));
 }
 /**
  * Wait for a run to stop being in progress.