@foldrun/core 0.2.0 → 0.4.0

package/src/runner.ts

@@ -7,11 +7,16 @@ import fs from "node:fs";
 import path from "node:path";
 import matter from "gray-matter";
 import { spawn } from "node:child_process";
-import { executeStep, extractJson, stepCeiling, type EventExtra } from "./step-exec.ts";
+import {
+  knownPrice, executeStep, extractJson, stepCeiling, stepCeilingFor, type EventExtra } from "./step-exec.ts";
 import { eventUrl } from "./webhook.ts";
 import { runStepInContainer, sizeLimits, killRunSandboxes, type StepTiming } from "./run-container.ts";
+import { hostSafeEnv } from "./host-env.ts";
+import { validateSchema, describeSchemaErrors, looksLikeSchema } from "./json-schema.ts";
 import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute, type EgressGrant } from "./egress.ts";
 import { platform } from "./platform.ts";
+import { healthKey } from "./secret-health.ts";
+import { TEST_WRITES_DIR, restoreDivertedDirs, snapshotDivertedDirs, testHeadline, testModeEnv, withholdSecrets } from "./test-mode.ts";
 
 /** Does this process run steps in a sandbox — the container core ships, or
  *  one the platform registered (a pod)? */
@@ -37,6 +42,8 @@ import {
   syncBundleFor,
   parseToolDef,
   parseFlow,
+  readFlow,
+  markerPresent,
   parseApis,
   resolveModel,
   parseProvider,
@@ -52,8 +59,12 @@ import {
   type McpSpec,
   type RunRecord,
   type StepRecord,
+  type StepAttempt,
+  sumAttempts,
+  parseSchemaText,
   STORAGE_DIR,
   adoptLegacyVersionKey,
+  listAgents,
   runCost,
 } from "./store.ts";
 import { loadCatalog, checkModel, clampEffort, catalogCost, findModel, type Catalog } from "./catalog.ts";
@@ -71,6 +82,7 @@ import { materializeFiles, harvestFiles } from "./storage.ts";
 import { chooseExecutor, ensureImage } from "./container.ts";
 import { stampBundle } from "./okf.ts";
 import type { McpServerConfig } from "@anthropic-ai/claude-agent-sdk";
+import { trimChars } from "./paths.ts";
 
 // An MCP tool definition becomes an SDK server config. ${SECRET} placeholders
 // in env and headers resolve server-side, so a credential reaches the server
@@ -122,6 +134,32 @@ export function joinEarlierGroups(groupResults: (string | null)[], gi: number, c
   return parts.join("\n\n");
 }
 
+/**
+ * `parallel: N` — a pool of N slots the instances of one fan-out share.
+ * Fan-out always ran every instance at once, and still does unless the
+ * step says otherwise: twenty browser steps against one site, or one
+ * vendor's rate limit, is what the cap is for. Nothing is rented while an
+ * instance waits for a slot.
+ */
+export function slotPool(size: number): { acquire: () => Promise<void>; release: () => void } {
+  let free = Math.max(1, size);
+  const waiting: (() => void)[] = [];
+  return {
+    acquire: () =>
+      new Promise<void>((resolve) => {
+        if (free > 0) {
+          free -= 1;
+          resolve();
+        } else waiting.push(resolve);
+      }),
+    release: () => {
+      const next = waiting.shift();
+      if (next) next();
+      else free += 1;
+    },
+  };
+}
+
 export interface DiscoveredSkill {
   name: string;
   description: string;
@@ -377,10 +415,9 @@ export function saysUntilMarker(result: string | null, until: string): boolean {
       (line) =>
         // Strip markdown emphasis and punctuation from BOTH ends: "**APPROVED.**"
         // is a decision written by someone using bold, not a sentence.
-        line
-          .trim()
-          .replace(/^[#>\-*_`\s]+/, "")
-          .replace(/[.!:,*_`\s]+$/, "")
+        // trimChars, not a `[…]+$` regex: that one is quadratic on a
+        // line of tabs, and this is a line an agent wrote.
+        trimChars(trimChars(line.trim(), "#>-*_`"), ".!:,*_`").trim()
           .toLowerCase() === marker,
     );
 }
@@ -470,7 +507,10 @@ export function resolveDocLinks(text: string, workspaceRoot: string): string {
       map.set(norm(fwd), `../../${STORAGE_DIR}/${fwd}`);
     }
   }
-  return text.replace(/\[\[([^\]\n]+)\]\]/g, (whole, name: string) => {
+  // `[` excluded from the name: with it allowed, a run of `[[` makes the
+  // engine re-scan to the end from every opening — quadratic on a result
+  // an agent wrote.
+  return text.replace(/\[\[([^\]\[\n]+)\]\]/g, (whole, name: string) => {
     const hit = map.get(norm(name.trim()));
     return hit ? `\`${hit}\`` : whole;
   });
@@ -1196,6 +1236,49 @@ function repriced(
   return known;
 }
 
+/**
+ * Write one attempt onto the step's history and re-total the step.
+ *
+ * runStep reports the attempt it just ran — its cost, tokens and sandbox
+ * seconds — by assigning the step's fields. Those become this attempt's row;
+ * the step's fields then become the sum of every row, which is what the
+ * run's spend, the flow's budget: and the platform's bill read. A retried
+ * step used to carry only its last attempt's cost, so a `retry: 2` step
+ * could spend three shares while reporting one.
+ */
+export function recordAttempt(
+  step: StepRecord,
+  n: number,
+  status: "completed" | "failed",
+  startedAt: string,
+  eventsBefore: number,
+  /** The attempt re-attached to a sandbox a previous driver left: same
+   *  attempt, so its row is replaced rather than repeated. */
+  reattach = false,
+) {
+  const tries = step.tries ?? [];
+  const lastError = step.events
+    .slice(eventsBefore)
+    .filter((e) => e.type === "error")
+    .at(-1)?.text;
+  const row: StepAttempt = {
+    n,
+    status,
+    costUsd: step.costUsd,
+    tokens: step.tokens ?? null,
+    computeSecs: step.computeSecs ?? null,
+    startedAt,
+    finishedAt: step.finishedAt ?? new Date().toISOString(),
+    ...(status === "failed" && lastError ? { error: lastError.slice(0, 500) } : {}),
+  };
+  const kept = reattach && tries.at(-1)?.n === n ? tries.slice(0, -1) : tries;
+  step.tries = [...kept, row];
+  const totals = sumAttempts(step.tries);
+  step.costUsd = totals.costUsd;
+  step.tokens = totals.tokens;
+  step.computeSecs = totals.computeSecs;
+}
+
 async function runStep(
   agentDir: string,
   tenant: string,
@@ -1213,6 +1296,15 @@ async function runStep(
    *  remaining `budget:`, shared across the steps launched with it. Null
    *  means the flow set no budget. */
   stepBudgetUsd: number | null = null,
+  /** Which line set that ceiling, for the error that names it. */
+  stepBudgetNote?: string,
+  /** The run is a test run — see test-mode.ts. Read off the run record by
+   *  the caller, never from anything the sandbox could have written. */
+  testRun = false,
+  /** Has a person asked for the run to stop? Read on a clock by the
+   *  in-process model loop, so a stop lands mid-step and not at the next
+   *  group. The isolated path needs none: stopRun destroys the sandbox. */
+  stopRequested?: () => boolean,
 ) {
   // Secret values are injected into scripts as environment variables and
   // substituted into API headers, so a model that reads one back — from a
@@ -1237,6 +1329,22 @@ async function runStep(
     return out;
   };
 
+  // The reply, the conclusion and an `output: json` value land on the same
+  // record the events do, and a model that quotes a script's stdout puts
+  // the credential in its answer rather than in a tool event. Same scrub,
+  // same place. The JSON value goes through its text form: a secret inside
+  // a nested field is still a secret, and the key name survives either way.
+  const redactText = <T extends string | null | undefined>(text: T): T =>
+    (typeof text === "string" ? redact(text) : text) as T;
+  const redactData = (value: unknown): unknown => {
+    if (value === undefined || redactions.length === 0) return value;
+    try {
+      return JSON.parse(redact(JSON.stringify(value)));
+    } catch {
+      return value;
+    }
+  };
+
   const push = (type: StepRecord["events"][number]["type"], text: string, extra?: EventExtra) => {
     step.events.push({ t: new Date().toISOString(), type, text: redact(text), ...extra });
     save();
@@ -1276,8 +1384,15 @@ async function runStep(
     // the failure of a step that returned no JSON are testable at no cost.
     if (step.output === "json") {
       const extracted = extractJson(step.result);
-      if (extracted.ok) step.data = extracted.value;
-      else {
+      if (extracted.ok) {
+        step.data = extracted.value;
+        const schema = step.schema ?? (step.schemaPath ? readSchemaFile(path.resolve(agentDir, "..", ".."), step.schemaPath) : null);
+        const errors = schema && looksLikeSchema(schema) ? validateSchema(step.data, schema) : [];
+        if (errors.length) {
+          push("error", `schema: the value does not fit — ${describeSchemaErrors(errors)}`);
+          step.status = "failed";
+        }
+      } else {
         push("error", `output: json — ${extracted.reason}`);
         step.status = "failed";
       }
@@ -1297,11 +1412,14 @@ async function runStep(
       front, clockEnv, systemPrompt, allowed, disabled, apiTools, scriptTools,
       secretEnv, secretScopes, missingSecrets, missingTools, runtime,
       unknownTools, shadowed, legacyUse, mcpServers, mcpNames,
-      apiSpecs, scriptSpecs, brokenTools, size,
+      apiSpecs, scriptSpecs, brokenTools, size: agentSize,
       providerEnv, providerLabel, providerSecrets, providerWarnings, formatWarning,
       fallbackEnv, apiWarnings, searchRoots, historyDigest, deskDigest, searchTools, historyTools, deskTools,
       translator, fallbackTranslator,
     } = agentContext(agentDir, tenant, tags, { runId, agent: step.agent });
+    // A retry that moved the step up a class (an evicted or OOM-killed
+    // attempt) wins over the agent's own `size:` for the attempts after.
+    const size = step.sizeUp ?? agentSize;
 
     // oauth2 secrets become live access tokens here — the one async moment
     // in secret resolution, at the last host-side point before anything is
@@ -1309,7 +1427,7 @@ async function runStep(
     // what crosses into containers, headers and env files is the token. A
     // failed refresh fails the step naming the secret and the provider's
     // reason, which beats a 401 three layers later.
-    const liveSecrets = await materializeSecrets(secretEnv);
+    const liveSecrets = await materializeSecrets(secretEnv, { tenant, workspace: path.basename(workspaceRootOf(agentDir)) });
     // @file values (SSH keys, certs) still carry their content at this point.
     // The in-process branch materialises them to 0600 paths on this host
     // below; the container branch passes the content across and the driver
@@ -1439,6 +1557,11 @@ async function runStep(
       if (fitted.note) push("info", fitted.note);
       effort = fitted.effort;
     }
+    // No catalogue — the platform's own Anthropic key, or a gateway that
+    // publishes no prices — but the model's NAME is often enough. Without
+    // this the in-pod meter priced haiku and sonnet at Opus rates and a
+    // step was stopped at up to fifteen times what it had spent.
+    tokenPrice ??= knownPrice(wireModel);
 
     fs.mkdirSync(path.join(agentDir, "outputs"), { recursive: true });
     fs.mkdirSync(path.join(agentDir, "memory"), { recursive: true });
@@ -1452,6 +1575,13 @@ async function runStep(
     // The confinement boundary: agents in a workspace are one team, but the
     // workspace is where isolation is enforced.
     const workspaceRoot = path.resolve(agentDir, "..", "..");
+    // Where a test run's state/ and storage/ writes go, and the line each
+    // one leaves: under the run, beside its archived outputs.
+    const divertFor = (id: string) => ({
+      to: path.join(workspaceRoot, "runs", id, TEST_WRITES_DIR),
+      note: (rel: string, summary: string) =>
+        push("info", `test: ${summary}`, { effect: { kind: "diverted" as const, path: rel, summary } }),
+    });
 
     let prompt = step.instruction
       ? resolveDocLinks(step.instruction, workspaceRoot)
@@ -1468,11 +1598,18 @@ async function runStep(
     if (step.output === "json") {
       // The contract the runner parses back out — stated in the prompt so
       // the instruction and the parser agree by construction, like delegate:.
+      // The schema, when there is one, is quoted whole: the model that has
+      // to satisfy it should not have to guess it from the field names.
+      const schema = step.schema ?? (step.schemaPath ? readSchemaFile(path.resolve(agentDir, "..", ".."), step.schemaPath) : null);
       prompt +=
         `\n\n<output>\nThis step returns data. End your reply with exactly one JSON value inside a ` +
         "```json fenced block — the value the next step will compute on. Prose before it is " +
         `welcome (lead with a one-line headline); nothing may follow the block. If the instruction ` +
-        `names fields, use exactly those names.\n</output>`;
+        `names fields, use exactly those names.` +
+        (schema && looksLikeSchema(schema)
+          ? ` The value must satisfy this JSON Schema, and the step fails if it does not:\n${JSON.stringify(schema, null, 2).slice(0, 20_000)}`
+          : "") +
+        `\n</output>`;
     }
     if (step.waitFor === "event" && step.eventPayload !== undefined) {
       // Not resolved for links, like the previous step's results: it is
@@ -1539,6 +1676,16 @@ async function runStep(
     // platform credential.
     const secondSupply = fallbackEnv ?? (Object.keys(providerEnv).length === 0 ? platformFallbackEnv() : null);
 
+    // `schema:` — inline on the step, or a file under the workspace read
+    // now, host-side, so both executors are handed the same document. A
+    // file that cannot be read is a failed step before any model runs:
+    // the contract the author wrote down cannot be checked.
+    const resolvedSchema: Record<string, unknown> | boolean | null = step.schema ?? (step.schemaPath ? readSchemaFile(workspaceRoot, step.schemaPath) : null);
+    if (step.schemaPath && resolvedSchema === null) {
+      throw new Error(`schema: ${step.schemaPath} — not a readable JSON or YAML schema under the workspace`);
+    }
+    if (step.maxTurns) push("info", `max_turns: ${step.maxTurns}`);
+
     const isolation = process.env.FOLDRUN_RUN_ISOLATION;
     if (isolatedRun()) {
       // The isolated path: the whole loop — model, built-in tools, scripts —
@@ -1566,10 +1713,20 @@ async function runStep(
         const host = hostOf(api.base);
         if (!host) continue;
         for (const v of [...Object.values(api.headers), ...Object.values(api.query)]) {
-          for (const name of placeholderNames(v)) if (name in liveSecrets) addGrant(grant, name, liveSecrets[name], host);
+          for (const name of placeholderNames(v)) {
+            if (!(name in liveSecrets)) continue;
+            addGrant(grant, name, liveSecrets[name], host);
+            // Which vault entry this is, so the proxy's record of what the
+            // far end said lands on the right credential.
+            (grant.healthKeys ??= {})[name] = healthKey(
+              name,
+              secretScopes[name] === "workspace" ? "workspace" : "account",
+              path.basename(path.resolve(agentDir, "..", "..")),
+            );
+          }
         }
       }
-      const lease = await platform.egress.lease({ tenant, runId: runId ?? "adhoc", grant });
+      const lease = await platform.egress.lease({ tenant, runId: runId ?? "adhoc", grant, test: testRun });
       // Which secrets must still be real inside the sandbox: a script tool
       // reads them from its environment, and unless it says `secrets:
       // proxied` it needs the value, not the name. bash is a script by
@@ -1592,15 +1749,36 @@ async function runStep(
       }
       // Sandbox env for the declared secrets: the value, or its own name as
       // a placeholder that the proxy fills on the way out.
-      const sandboxSecrets: Record<string, string> = materialise
-        ? liveSecrets
-        : Object.fromEntries(declared.map((name) => [name, `\${${name}}`]));
+      const placeholders = Object.fromEntries(declared.map((name) => [name, `\${${name}}`]));
+      let sandboxSecrets: Record<string, string> = materialise ? liveSecrets : placeholders;
+      // A test run hands the sandbox no send-capable value. The env var is
+      // still there, reading TEST_MODE_WITHHELD, so a script that never
+      // checked FOLDRUN_TEST_MODE fails at the provider's door with an auth
+      // error — and sends nothing. The script tools say whether the step
+      // has a declared sender (everything withheld) or a declared reader
+      // (nothing withheld); see test-mode.ts for the order.
+      if (testRun && materialise) {
+        const held = withholdSecrets(liveSecrets, {
+          outward: scriptSpecs.some((sc) => sc.outward),
+          allow: scriptSpecs.some((sc) => sc.testMode === "allow"),
+        });
+        sandboxSecrets = held.env;
+        for (const name of held.withheld) {
+          push("info", `test: ${name} withheld from the sandbox — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from the sandbox` } });
+        }
+      }
+      // On a test run the http tools always go through the proxy as
+      // placeholders, even when a script made the step materialise: the
+      // proxy is where the outward-write policy lives, and a header
+      // carrying TEST_MODE_WITHHELD would only fail where it should have
+      // been sunk or refused.
+      const apiValues = testRun && lease ? placeholders : sandboxSecrets;
       const substitutedApis = apiSpecs.map((api) => ({
         ...api,
         headers: Object.fromEntries(
           Object.entries(api.headers).map(([k, v]) => [
             k,
-            v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => sandboxSecrets[name] ?? whole),
+            v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => apiValues[name] ?? whole),
           ]),
         ),
       }));
@@ -1630,6 +1808,7 @@ async function runStep(
       const isolatedArgs = (modelEnv: Record<string, string | undefined>, keyName = MODEL_KEY_NAME) => ({
         workspaceRoot,
         libraryRoot: libraryDir(tenant),
+        galleryRoot: platform.galleryDir() ?? undefined,
         // Keys this step's dependency cache. Per-account by design — see
         // run-container.ts#runtimeCacheDir for why it is not shared.
         tenant,
@@ -1650,9 +1829,12 @@ async function runStep(
           consults,
           timeoutSec: step.timeout,
           budgetUsd: stepBudgetUsd,
+          budgetNote: stepBudgetNote,
           price: tokenPrice,
           verify: step.verify,
           output: step.output,
+          schema: resolvedSchema ?? undefined,
+          maxTurns: step.maxTurns,
           translator: keyName === MODEL_KEY_NAME ? primaryTranslator : secondTranslator,
           // The container sees the workspace at /workspace and the library
           // at /library; the roots are named host-side and moved here.
@@ -1683,6 +1865,7 @@ async function runStep(
             FOLDRUN_RUN_ID: runId,
             FOLDRUN_AGENT: step.agent,
             FOLDRUN_WORKSPACE: path.basename(workspaceRoot),
+            ...(testRun ? testModeEnv() : {}),
             ...sandboxSecrets,
           }).filter((entry): entry is [string, string] => typeof entry[1] === "string"),
         ),
@@ -1690,6 +1873,21 @@ async function runStep(
         // Stamps the sandbox, so stopping this run can destroy it.
         runId,
         size,
+        // A test run's state/ and storage/ changes come out under the run.
+        divert: testRun && runId ? divertFor(runId) : undefined,
+        test: testRun,
+        // Resume: a sandbox a previous driver left running, recorded on the
+        // step. Consumed once — a fresh attempt after this one (a retry, a
+        // fallback) starts its own sandbox.
+        resume: step.sandbox && step.sandbox.kind === isolation ? { ref: step.sandbox.ref, consumed: step.sandbox.consumed } : null,
+        // Checkpoint: where the executor is, on the record, so a driver
+        // that inherits this run can attach where this one left off. The
+        // executor calls it before applying each line; the line's own emit
+        // saves the record, so this only updates the fields.
+        checkpoint: (ref: string, consumed: number) => {
+          step.sandbox = { kind: isolation!, ref, consumed, since: step.sandbox?.since ?? new Date().toISOString() };
+          if (consumed === 0) save();
+        },
       });
 
       // A busy provider gets another moment on the SAME supply before the
@@ -1700,6 +1898,7 @@ async function runStep(
       const firstArgs = isolatedArgs(platformModelEnv());
       await lease?.commit();
       let outcome = await runIsolated(firstArgs);
+      step.sandbox = null; // whatever happens next starts its own
       for (let n = 1; n <= OVERLOAD_RETRIES && outcome.status === "failed" && isTransientOverload(lastRefusal); n++) {
         const wait = backoffMs(n, lastRefusal);
         push("info", `provider busy (${lastRefusal.slice(0, 60)}) — waiting ${(wait / 1000).toFixed(1)}s, attempt ${n} of ${OVERLOAD_RETRIES}`);
@@ -1745,14 +1944,23 @@ async function runStep(
         outcome = withEarlierTiming(outcome, first);
       }
       if (lease) {
-        for (const line of await lease.drainLog()) push("info", line);
+        for (const line of await lease.drainLog()) {
+          if (typeof line === "string") push("info", line);
+          else push("info", line.text, { effect: line.effect });
+        }
         await lease.release();
       }
-      publishPublicDir(tenant, path.basename(workspaceRoot), push);
+      // Nothing a test run left in storage/public/ reached storage/, so
+      // there is nothing to publish — and a share link is an outward act.
+      if (!testRun) publishPublicDir(tenant, path.basename(workspaceRoot), push);
       step.status = outcome.status;
-      step.result = outcome.result;
-      step.conclusion = outcome.conclusion;
-      if (outcome.data !== undefined) step.data = outcome.data;
+      step.result = redactText(outcome.result);
+      step.conclusion = redactText(outcome.conclusion);
+      step.sandbox = null;
+      // The cluster's word on why a sandbox ended without the driver saying
+      // so — OOMKilled, Evicted — on the record, where the retry reads it.
+      if (outcome.status === "failed" && outcome.reason) push("error", `sandbox ended: ${outcome.reason}`);
+      if (outcome.data !== undefined) step.data = redactData(outcome.data);
       step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push);
       step.tokens = outcome.usage
         ? { input: outcome.usage.inputTokens, output: outcome.usage.outputTokens }
@@ -1785,14 +1993,27 @@ async function runStep(
       })();
       step.reserved = { cpus: Number(lim.cpus) || 2, memGiB: gib };
     } else {
+      // Same withholding as the sandbox path — the in-process step is what
+      // `foldrun run --test` drives on a laptop, where there is no proxy
+      // and this is the only guard there is.
+      const held = testRun
+        ? withholdSecrets(liveSecrets, {
+            outward: scriptSpecs.some((sc) => sc.outward),
+            allow: scriptSpecs.some((sc) => sc.testMode === "allow"),
+          })
+        : { env: liveSecrets, withheld: [] as string[] };
+      for (const name of held.withheld) {
+        push("info", `test: ${name} withheld from scripts — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from scripts` } });
+      }
+      const stepSecrets = held.env;
       const consultTools = buildConsultTools(
         consults,
-        { ...process.env, ...liveSecrets, ...providerEnv },
+        { ...hostSafeEnv(), ...stepSecrets, ...providerEnv },
         (type, text) => push(type, text),
       );
       // @file secrets become 0600 paths here (host run), cleaned up in the
       // finally — once, however many attempts follow.
-      const mat = materializeFileSecrets(agentDir, liveSecrets);
+      const mat = materializeFileSecrets(agentDir, stepSecrets);
       fileDir = mat.dir;
       const attemptInProcess = (modelEnv: Record<string, string | undefined>) => executeStep({
         agentDir,
@@ -1814,13 +2035,20 @@ async function runStep(
           ...mcpServers,
         },
         // Declared secrets reach the agent's scripts as env vars; the model
-        // only ever sees the variable names, not the values.
-        env: { ...process.env, ...clockEnv, ...mat.env, ...modelEnv },
+        // only ever sees the variable names, not the values. The base is
+        // the allowlisted host environment, not process.env: the SDK's
+        // Bash tool runs with this, and on a platform process.env holds
+        // every tenant's root key and the datastore credentials.
+        env: { ...hostSafeEnv(), ...clockEnv, ...(testRun ? testModeEnv() : {}), ...mat.env, ...modelEnv },
         timeoutSec: step.timeout,
         budgetUsd: stepBudgetUsd,
+        budgetNote: stepBudgetNote,
         price: tokenPrice,
         verify: step.verify,
         output: step.output,
+        schema: resolvedSchema ?? undefined,
+        maxTurns: step.maxTurns,
+        stopRequested,
         // What the scripts saw, the verify sees: a flow can then check that
         // the proof a step left names THIS run, not one that came before —
         // copy-back never propagates a deletion, so a marker from an earlier
@@ -1830,20 +2058,31 @@ async function runStep(
           ...(runId ? { FOLDRUN_RUN_ID: runId } : {}),
           FOLDRUN_AGENT: step.agent,
           FOLDRUN_WORKSPACE: path.basename(path.resolve(agentDir, "..", "..")),
-          ...liveSecrets,
+          ...(testRun ? testModeEnv() : {}),
+          ...stepSecrets,
         },
         emit: pushWatching,
       });
       // With a translator the step's endpoint is the translator: started
       // here, pointed at by the SDK's env, closed when the attempt ends.
+      // A test run brackets each attempt with a snapshot of state/ and
+      // storage/: the step writes them for real, because there is no
+      // sandbox to write instead, and afterwards what changed is moved
+      // under the run and the originals come back.
       const attemptThrough = async (modelEnv: Record<string, string | undefined>, spec: TranslatorSpec | null) => {
-        if (!spec) return attemptInProcess(modelEnv);
-        const t = await startTranslator(spec);
+        const before = testRun ? snapshotDivertedDirs(workspaceRoot) : null;
+        const t = spec ? await startTranslator(spec) : null;
         try {
-          return await attemptInProcess({ ...modelEnv, ...t.env });
+          return await attemptInProcess(t ? { ...modelEnv, ...t.env } : modelEnv);
         } finally {
-          for (const line of t.drainLog()) push("info", line.startsWith("translator") ? line : `translator: ${line}`);
-          await t.close();
+          if (t) {
+            for (const line of t.drainLog()) push("info", line.startsWith("translator") ? line : `translator: ${line}`);
+            await t.close();
+          }
+          if (before && runId) {
+            const d = divertFor(runId);
+            restoreDivertedDirs(workspaceRoot, before, d.to, d.note);
+          }
         }
       };
       let outcome = await attemptThrough(providerEnv, translator);
@@ -1865,11 +2104,11 @@ async function runStep(
         );
         outcome = await attemptThrough(secondSupply, fallbackEnv && secondSupply === fallbackEnv ? fallbackTranslator : null);
       }
-      publishPublicDir(tenant, path.basename(workspaceRoot), push);
+      if (!testRun) publishPublicDir(tenant, path.basename(workspaceRoot), push);
       step.status = outcome.status;
-      step.result = outcome.result;
-      step.conclusion = outcome.conclusion;
-      if (outcome.data !== undefined) step.data = outcome.data;
+      step.result = redactText(outcome.result);
+      step.conclusion = redactText(outcome.conclusion);
+      if (outcome.data !== undefined) step.data = redactData(outcome.data);
       // A consult's spend belongs to the step that asked.
       const consultCost = consultTools.drainCost();
       const stepCost = repriced(catalog, wireModel, outcome.usage, outcome.costUsd, push);
@@ -2163,6 +2402,26 @@ function csvItems(workspaceDir: string, step: StepRecord, take: number): string[
   return data.slice(0, take).map((r) => `${header}\n${r}`);
 }
 
+/**
+ * `schema: <path>` — the file, read and parsed, or null. Agent-relative like
+ * every other path in a flow (`../../schemas/lead.json`), confined to the
+ * workspace: a schema is the author's document, never something a step
+ * can point outside.
+ */
+export function readSchemaFile(workspaceRoot: string, rel: string): Record<string, unknown> | boolean | null {
+  const root = path.resolve(workspaceRoot);
+  // Written agent-relative (`../../schemas/x.json`) or workspace-relative
+  // (`schemas/x.json`); both land on the same file.
+  const candidates = [path.resolve(root, rel.replace(/^(\.\.\/){2}/, "")), path.resolve(root, "agents", "x", rel)];
+  for (const abs of candidates) {
+    if (abs !== root && !abs.startsWith(root + path.sep)) continue;
+    if (!fs.existsSync(abs) || !fs.statSync(abs).isFile()) continue;
+    const parsed = parseSchemaText(fs.readFileSync(abs, "utf8"));
+    return "problem" in parsed ? null : parsed.schema;
+  }
+  return null;
+}
+
 export function createFlowRun(
   tenant: string,
   workspace: string,
@@ -2170,12 +2429,16 @@ export function createFlowRun(
   flowName: string,
   status: "queued" | "running",
   tags: string[] = [],
+  startedBy: string | null = null,
+  opts: { test?: boolean } = {},
 ): RunRecord {
   const run: RunRecord = {
     id: `run-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`,
     flow: flowName,
     tags,
     status,
+    ...(startedBy ? { startedBy: startedBy.trim().toLowerCase() } : {}),
+    ...(opts.test ? { test: true } : {}),
     startedAt: new Date().toISOString(),
     finishedAt: null,
     steps: steps.map((s) => ({
@@ -2205,6 +2468,10 @@ export function createFlowRun(
       preview: s.preview,
       delegate: s.delegate,
       output: s.output,
+      schema: s.schema,
+      schemaPath: s.schemaPath,
+      parallel: s.parallel,
+      maxTurns: s.maxTurns,
       attempts: 0,
       status: "pending",
       events: [],
@@ -2241,8 +2508,10 @@ export function startFlowRun(
   modelOverride?: string | null,
   tags: string[] = [],
   effortOverride?: string | null,
+  startedBy: string | null = null,
+  opts: { test?: boolean } = {},
 ): RunRecord {
-  const run = createFlowRun(tenant, workspace, steps, flowName, "running", tags);
+  const run = createFlowRun(tenant, workspace, steps, flowName, "running", tags, startedBy, opts);
   void driveRun(tenant, workspace, run, modelOverride, tags, { effortOverride });
   return run;
 }
@@ -2450,15 +2719,25 @@ function driveRunInner(
       // never happened, which reconcile can then never see. Reset it to
       // pending and it simply runs again; driveRun's whole contract is that
       // re-driving a partly-finished record is safe.
+      // Unless the step's sandbox outlived its driver AND the executor can
+      // re-attach to it: then the record carries the pod and how far the
+      // last driver got, runStep passes that as `resume`, and the step
+      // carries on from where it was — a deploy no longer costs the run.
       let orphaned = false;
       for (const step of run.steps) {
         if (step.status !== "running") continue;
-        orphaned = true;
+        const resumable = !!step.sandbox && platform.sandboxResumable(step.sandbox.kind);
+        if (!resumable) {
+          orphaned = true;
+          step.sandbox = null;
+        }
         step.status = "pending";
         step.events.push({
           t: new Date().toISOString(),
           type: "info",
-          text: "interrupted mid-step (platform restart) — running again from the start of the step",
+          text: resumable
+            ? `driver changed mid-step — re-attaching to the running sandbox (${step.sandbox!.ref}, ${step.sandbox!.consumed} lines already applied)`
+            : "interrupted mid-step (platform restart) — running again from the start of the step",
         });
       }
       if (orphaned) {
@@ -2530,6 +2809,17 @@ function driveRunInner(
       // is assembled) keeps the newest: when the total is over it, the
       // oldest groups are dropped whole before anything is cut mid-text.
       const contextBefore = (gi: number) => joinEarlierGroups(groupResults, gi);
+      // What `when:` and `case:` read: the PREVIOUS group's result alone —
+      // the nearest earlier group that produced one, so a routed-past group
+      // between is looked through rather than read as silence. The prompt
+      // gets every earlier group joined (contextBefore); routing must not,
+      // because a verdict two groups back would still open a gate a later
+      // group had overruled: a classifier's BUG line matched a `case: BUG`
+      // three groups on, after the debugger had already answered it.
+      const routingBefore = (gi: number): string | null => {
+        for (let i = gi - 1; i >= 0; i--) if (groupResults[i]) return groupResults[i];
+        return null;
+      };
       // The data half of the same handoff: what the nearest earlier group
       // returned through `output: json`. Rebuilt from the record rather than
       // kept only in memory, so a resumed run hands step 3 the value step 1
@@ -2594,8 +2884,11 @@ function driveRunInner(
         context = contextBefore(gi);
         ctxData = dataBefore(gi);
 
-        if (run.budgetUsd && runCost(run) >= run.budgetUsd && group.some((s) => s.status === "pending")) {
-          const spent = runCost(run);
+        // What the run has spent, not just what its tokens cost: on an
+        // install that prices sandbox seconds, a step that called no model
+        // still spent money, and a cap that ignored it capped nothing.
+        if (run.budgetUsd && platform.runSpend(run) >= run.budgetUsd && group.some((s) => s.status === "pending")) {
+          const spent = platform.runSpend(run);
           const last = run.steps.findLast((s) => s.status === "completed" || s.status === "failed") ?? run.steps[0];
           last?.events.push({
             t: new Date().toISOString(),
@@ -2684,6 +2977,7 @@ function driveRunInner(
             type: "info",
             text:
               `fan-out: ${used.length} instance${used.length === 1 ? "" : "s"}` +
+              (step.parallel && step.parallel < used.length ? `, ${step.parallel} at a time (parallel:)` : "") +
               (dropped ? ` (more items existed — capped at ${cap})` : ""),
           });
           const at = run.steps.indexOf(step);
@@ -2698,6 +2992,7 @@ function driveRunInner(
             result: null,
             costUsd: null,
             attempts: 0,
+            tries: undefined,
           }));
           run.steps.splice(at + 1, 0, ...instances);
           ordered = orderedGroups();
@@ -2711,12 +3006,25 @@ function driveRunInner(
         // this is what keeps the *next* group from starting.
         const stopCheck = readRun(tenant, workspace, run.id);
         if (stopCheck?.stopRequested) {
+          let skipped = 0;
           for (const s of run.steps) {
             if (s.status === "pending" || s.status === "running") {
               s.status = "skipped";
               s.skipReason = "run stopped";
+              skipped += 1;
             }
           }
+          // Say so on the trail. stopRun wrote "run stopped" onto the
+          // record's first step, but the save() below writes this process's
+          // copy of the record over it, so the only witness was the
+          // stopRequested flag — and a run that "failed" with no error line
+          // anywhere took a code read to explain (rank-desk, 2026-09-13).
+          const witness = run.steps.findLast((s) => s.status === "completed" || s.status === "failed") ?? run.steps[0];
+          witness?.events.push({
+            t: new Date().toISOString(),
+            type: "info",
+            text: `run stopped by a person between groups — ${skipped} remaining step${skipped === 1 ? "" : "s"} skipped`,
+          });
           run.status = "failed";
           run.stopRequested = true;
           save();
@@ -2752,40 +3060,52 @@ function driveRunInner(
         }
 
         const ctx = context;
+        // The previous group's result — see routingBefore.
+        const routing = routingBefore(gi);
 
-        // `when:` — skip a step whose condition isn't met by prior results.
+        // `when:` — skip a step whose condition isn't met by the previous
+        // group's result.
         for (const step of freshGroup) {
           if (step.status !== "pending" || !step.when) continue;
-          const met = (ctx ?? "").toLowerCase().includes(step.when.toLowerCase());
+          // A marker at the start of a line, not a word anywhere in the
+          // prose — see markerPresent. "There are no BLOCKED items" must
+          // not open a gate keyed on BLOCKED.
+          const met = markerPresent(routing, step.when);
           if (!met) {
             step.status = "skipped";
             step.skipReason = `condition not met: when "${step.when}"`;
             step.events.push({
               t: new Date().toISOString(),
               type: "info",
-              text: `skipped — previous results do not mention "${step.when}"`,
+              text: `skipped — the previous group's result has no line beginning "${step.when}" (when: reads a marker at the start of a line of the previous result, not a word in a sentence)`,
             });
           }
         }
         save();
 
         // `case:`/`else:` — exclusive routing. Of this group's case steps,
-        // the FIRST whose text appears in the previous results runs; the
-        // rest are routed past. `else:` runs only when no case matched.
+        // the FIRST whose marker leads a line of the previous results runs;
+        // the rest are routed past. `else:` runs only when no case matched.
         // (`when:` above stays independent — every matching when runs —
         // which is why routing is its own vocabulary instead of a mode.)
+        //
+        // Same marker rule as `when:`, and it matters more here: routing is
+        // exclusive, so a label matched out of a sentence does not merely
+        // run an extra step, it sends the flow down the wrong branch and
+        // skips the right one. "This is not a COMPLAINT, it is a QUESTION"
+        // used to route to the complaints handler.
         const caseSteps = freshGroup.filter((s) => s.status === "pending" && s.case);
         if (caseSteps.length || freshGroup.some((s) => s.status === "pending" && s.else)) {
           let matchedCase: StepRecord | null = null;
           for (const step of caseSteps) {
-            if (!matchedCase && (ctx ?? "").toLowerCase().includes(step.case!.toLowerCase())) {
+            if (!matchedCase && markerPresent(routing, step.case!)) {
               matchedCase = step;
               continue;
             }
             step.status = "skipped";
             step.skipReason = matchedCase
               ? `routed past — "${matchedCase.case}" matched first`
-              : `case not matched: "${step.case}"`;
+              : `case not matched: the previous group's result has no line beginning "${step.case}"`;
             step.events.push({
               t: new Date().toISOString(),
               type: "info",
@@ -2829,6 +3149,20 @@ function driveRunInner(
             });
           }
           run.status = "awaiting-approval";
+          // approve_within: — the deadline is stamped when the gate opens,
+          // not counted from the run's start, so "two days to answer" means
+          // two days from being asked however long the work before it took.
+          // Only a gate that asks a PERSON gets one: an external event has
+          // no one to chase.
+          const within = readFlow(tenant, workspace, run.flow)?.approveWithin ?? null;
+          // Always assigned, never only set: a run with two gates would
+          // otherwise carry the FIRST gate's deadline into the second, and
+          // a deadline already in the past expires a gate the moment it
+          // opens. Null when this gate has none.
+          run.approveBy =
+            within && needsApproval.some((s) => s.waitFor !== "event")
+              ? new Date(Date.now() + within * 1000).toISOString()
+              : null;
           save();
 
           // The gate's declared preview, resolved now against what the run
@@ -2903,12 +3237,79 @@ function driveRunInner(
         // twenty steps over — the shares sum to what was left. Between
         // groups the check above still refuses to start the next one.
         const launching = freshGroup.filter((s) => s.status === "pending");
-        const ceilingUsd = stepCeiling(run.budgetUsd, runCost(run), launching.length);
+        // An agent's own `budget:` — the most it may spend in one run — is
+        // read at launch, so a cap raised mid-run applies to the next step.
+        const agentBudgets = new Map(listAgents(tenant, workspace).map((a) => [a.name, a.budget]));
+        // The ceiling for one attempt of one step, read NOW: what is left of
+        // the run's budget shared across the group, or the agent's own cap
+        // less what its steps have cost. Computed before every attempt, not
+        // once before the loop — a retry's share is what is left after the
+        // failed attempt spent, and the step's own figure already carries
+        // that spend (it is the sum of its tries).
+        const ceilingFor = (step: StepRecord) =>
+          stepCeilingFor(
+            stepCeiling(run.budgetUsd, platform.runSpend(run), launching.length),
+            agentBudgets.get(step.agent),
+            run.steps.filter((s) => s !== step && s.agent === step.agent).reduce((sum, s) => sum + (s.costUsd ?? 0), 0),
+            step.agent,
+          );
+        // parallel: — the instances of one fan-out template share a pool.
+        // Keyed by what the instances share (agent + instruction), since
+        // the template itself is behind them in the record as "expanded".
+        const pools = new Map<string, ReturnType<typeof slotPool>>();
+        const poolFor = (step: StepRecord) => {
+          if (!step.parallel || !step.item) return null;
+          const key = `${step.agent}\u0000${step.instruction}`;
+          let pool = pools.get(key);
+          if (!pool) pools.set(key, (pool = slotPool(step.parallel)));
+          return pool;
+        };
         await Promise.all(
           launching
             .map(async (step) => {
+              const pool = poolFor(step);
+              if (pool) await pool.acquire();
+              try {
+                await launchStep(step);
+              } finally {
+                pool?.release();
+              }
+            }),
+        );
+        async function launchStep(step: StepRecord) {
               const attempts = (step.retry ?? 0) + 1;
-              for (let attempt = 1; attempt <= attempts; attempt++) {
+              // A step re-attaching to its sandbox is still on the attempt
+              // that started it; a step re-run after a backoff picks up the
+              // count it left. Only a step nobody has touched starts at 1.
+              const interruptedInBackoff =
+                !step.sandbox && !!step.attempts && /retrying in \d+s/.test(step.events[step.events.length - 1]?.text ?? "");
+              const from = step.sandbox ? Math.max(1, step.attempts ?? 1) : interruptedInBackoff ? Math.min(attempts, step.attempts! + 1) : 1;
+              for (let attempt = from; attempt <= attempts; attempt++) {
+                const ceiling = ceilingFor(step);
+                // Nothing left is a refusal, not "no ceiling": a zero passed
+                // down would read as unset, and the one step that must not
+                // run would run with no cap at all. On a retry this is what
+                // stops the second attempt spending a share the first one
+                // already spent.
+                if (ceiling.ceilingUsd === 0) {
+                  step.status = "failed";
+                  step.events.push({
+                    t: new Date().toISOString(),
+                    type: "error",
+                    text:
+                      attempt === from
+                        ? `over budget — nothing left to spend before this step (${ceiling.note}); it was not started`
+                        : `over budget — nothing left to spend for attempt ${attempt} (${ceiling.note}); not retried`,
+                  });
+                  save();
+                  return;
+                }
+                // Re-attaching to a sandbox is the same attempt, so its row
+                // is replaced; anything else is a new row, a loop cycle's
+                // re-run included.
+                const reattach = !!step.sandbox;
+                const attemptStartedAt = new Date().toISOString();
+                const eventsBefore = step.events.length;
                 step.attempts = attempt;
                 step.status = "running";
                 save();
@@ -2923,11 +3324,21 @@ function driveRunInner(
                   effortOverride,
                   run.id,
                   ctxData,
-                  ceilingUsd,
+                  ceiling.ceilingUsd,
+                  ceiling.note,
+                  run.test === true,
+                  // save() merges a stop written by another process into
+                  // this record; the model loop polls this on a clock.
+                  () => {
+                    save();
+                    return run.stopRequested === true;
+                  },
                 );
                 // runStep mutates step.status; read it through a widened local
                 // so TS doesn't keep the "running" narrowing from above.
                 const outcome: string = step.status;
+                recordAttempt(step, attempt, outcome === "completed" ? "completed" : "failed", attemptStartedAt, eventsBefore, reattach);
+                save();
                 if (outcome !== "failed" || attempt === attempts) break;
                 // A stop destroys the step's sandbox, which reads here as a
                 // failed attempt — and a failed attempt used to be retried in
@@ -2935,15 +3346,44 @@ function driveRunInner(
                 // more. save() merges the flag from the record.
                 save();
                 if (run.stopRequested) break;
+                // The retry policy. An attempt the cluster ended for want of
+                // memory or disk is retried one size class up — the same
+                // size would end the same way. Every retry waits: 15 s, 30 s,
+                // 1 m, 2 m, 4 m, jittered, so a provider or a site that was
+                // struggling gets a moment, and twenty fan-out steps do not
+                // hammer it in lock-step. The wait holds a worker slot, not
+                // a sandbox — nothing is rented while it waits — and a stop
+                // ends it. The step is `pending` meanwhile, so a driver that
+                // inherits the run mid-wait simply runs the attempt.
+                const last = [...step.events].reverse().find((e) => e.type === "error")?.text ?? "";
+                let sizeNote = "";
+                if (/OOMKilled|Evicted|ephemeral-storage|out of memory/i.test(last)) {
+                  const up = step.sizeUp === "large" || (!step.sizeUp && step.size === "large") ? "heavy" : step.sizeUp === "heavy" || step.size === "heavy" ? null : "large";
+                  if (up) {
+                    step.sizeUp = up;
+                    sizeNote = ` at size: ${up}`;
+                  }
+                }
+                const waitMs = Math.round(Math.min(retryBaseMs() * 16, retryBaseMs() * 2 ** (attempt - 1)) * (0.75 + Math.random() * 0.5));
+                step.status = "pending";
                 step.events.push({
                   t: new Date().toISOString(),
                   type: "info",
-                  text: `attempt ${attempt} failed — retrying (${attempts - attempt} left)`,
+                  text: `attempt ${attempt} failed — retrying in ${Math.round(waitMs / 1000)}s${sizeNote} (${attempts - attempt} left)`,
                 });
                 save();
+                const until = Date.now() + waitMs;
+                while (Date.now() < until) {
+                  await sleep(Math.min(1000, until - Date.now()));
+                  save();
+                  if (run.stopRequested) break;
+                }
+                if (run.stopRequested) {
+                  step.status = "failed";
+                  break;
+                }
               }
-            }),
-        );
+        }
 
         // on-fail: another agent takes the step over. Sequential and rare:
         // the rescue is a second full attempt at the step's job with the
@@ -2995,7 +3435,7 @@ function driveRunInner(
           rescue.attempts = 1;
           rescue.status = "running";
           save();
-          await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData);
+          await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData, null, undefined, run.test === true);
           // runStep mutates rescue.status; widen past TS's "running" narrowing.
           const rescueOutcome: string = rescue.status;
           if (rescueOutcome === "completed") {
@@ -3125,16 +3565,25 @@ function driveRunInner(
       for (const s of run.steps) if (s.status === "pending") s.status = "skipped";
     } catch (err) {
       run.status = "failed";
+      const text = err instanceof Error ? err.message : String(err);
+      let told = false;
       run.steps.forEach((s) => {
         if (s.status === "running") {
           s.status = "failed";
-          s.events.push({
-            t: new Date().toISOString(),
-            type: "error",
-            text: err instanceof Error ? err.message : String(err),
-          });
+          s.events.push({ t: new Date().toISOString(), type: "error", text });
+          told = true;
         }
       });
+      // Thrown between groups, with nothing running: the error still has to
+      // land on some step's trail, or the run reads as failed for no reason.
+      if (!told) {
+        const witness = run.steps.findLast((s) => s.status !== "pending") ?? run.steps[0];
+        witness?.events.push({
+          t: new Date().toISOString(),
+          type: "error",
+          text: `run failed between steps: ${text}`,
+        });
+      }
     } finally {
       if (!parked) {
         run.finishedAt = new Date().toISOString();
@@ -3144,6 +3593,9 @@ function driveRunInner(
         // learn, and the runs list and the notification stop having to say
         // only "completed".
         run.summary = runSummary(run);
+        // A test run's headline says so wherever the headline goes — the
+        // runs list, the history tool, a later run's recall.
+        if (run.test && run.summary) run.summary = testHeadline(run.summary);
         // Archive before the next run resets outputs/ — a failed run's
         // artifacts are usually the most interesting ones, so this runs on
         // failure too.
@@ -3184,16 +3636,6 @@ function driveRunInner(
   })();
 }
 
-function readFlow(tenant: string, workspace: string, flowName: string) {
-  const dir = path.join(workspaceDir(tenant, workspace), "flows");
-  if (!fs.existsSync(dir)) return null;
-  for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".md"))) {
-    const flow = parseFlow(f, fs.readFileSync(path.join(dir, f), "utf8"));
-    if (flow.name === flowName) return flow;
-  }
-  return null;
-}
-
 // Splice `[[flow:other]]` steps into the parent's step list, renumbering
 // groups so the nested flow's own parallelism survives. Depth-limited, and
 // a cycle raises rather than expanding forever.
@@ -3270,6 +3712,10 @@ export function loadFlow(tenant: string, workspace: string, flowName: string) {
 // written a beat before its driver registered, not a judgement about pace.
 const ABANDONED_AFTER_MS = 2 * 60 * 1000;
 
+/** The first retry's wait; each one after doubles it, to sixteen times
+ *  this (15 s → 4 min by default). An env, so a test can make it small. */
+const retryBaseMs = () => Number(process.env.FOLDRUN_RETRY_BASE_MS) || 15_000;
+
 /** The most recent moment a run showed any sign of life. */
 function lastActivity(run: RunRecord): number {
   let latest = Date.parse(run.startedAt);
@@ -3311,12 +3757,18 @@ export interface Reconciliation {
  * outlives any process. A run whose person already answered is not waiting,
  * and is picked back up — see below.
  */
-export function reconcileRuns(tenant: string, now = Date.now()): Reconciliation[] {
+export function reconcileRuns(tenant: string, now = Date.now(), live?: ReadonlySet<string>): Reconciliation[] {
   const closed: Reconciliation[] = [];
 
   for (const workspace of listWorkspaces(tenant)) {
     for (const summary of listRuns(tenant, workspace.name)) {
       if (drivingRuns.has(summary.id)) continue;
+      // Driven by another process — a second worker pod, on a platform with
+      // more than one. The idle window protects a quiet step from THIS
+      // process's judgement only; a worker that boots beside a live one
+      // would otherwise close every long step its neighbour is driving.
+      // The caller says which runs those are (a heartbeated claim).
+      if (live?.has(summary.id)) continue;
       if (now - lastActivity(summary) < ABANDONED_AFTER_MS) continue;
 
       // Approved, then abandoned. Approval only writes to the run record; the
@@ -3396,8 +3848,8 @@ export function reconcileRuns(tenant: string, now = Date.now()): Reconciliation[
  * one hard-coded account left every other account's interrupted runs marked
  * `running` forever, with no process left to finish them.
  */
-export function reconcileAllRuns(now = Date.now()): Reconciliation[] {
-  return listTenants().flatMap((tenant) => reconcileRuns(tenant, now));
+export function reconcileAllRuns(now = Date.now(), live?: ReadonlySet<string>): Reconciliation[] {
+  return listTenants().flatMap((tenant) => reconcileRuns(tenant, now, live));
 }
 
 /**