@foldrun/core 0.3.0 → 0.4.0

package/src/runner.ts

@@ -8,11 +8,15 @@ import path from "node:path";
 import matter from "gray-matter";
 import { spawn } from "node:child_process";
 import {
-  knownPrice, executeStep, extractJson, stepCeiling, type EventExtra } from "./step-exec.ts";
+  knownPrice, executeStep, extractJson, stepCeiling, stepCeilingFor, type EventExtra } from "./step-exec.ts";
 import { eventUrl } from "./webhook.ts";
 import { runStepInContainer, sizeLimits, killRunSandboxes, type StepTiming } from "./run-container.ts";
+import { hostSafeEnv } from "./host-env.ts";
+import { validateSchema, describeSchemaErrors, looksLikeSchema } from "./json-schema.ts";
 import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute, type EgressGrant } from "./egress.ts";
 import { platform } from "./platform.ts";
+import { healthKey } from "./secret-health.ts";
+import { TEST_WRITES_DIR, restoreDivertedDirs, snapshotDivertedDirs, testHeadline, testModeEnv, withholdSecrets } from "./test-mode.ts";
 
 /** Does this process run steps in a sandbox — the container core ships, or
  *  one the platform registered (a pod)? */
@@ -38,6 +42,8 @@ import {
   syncBundleFor,
   parseToolDef,
   parseFlow,
+  readFlow,
+  markerPresent,
   parseApis,
   resolveModel,
   parseProvider,
@@ -53,8 +59,12 @@ import {
   type McpSpec,
   type RunRecord,
   type StepRecord,
+  type StepAttempt,
+  sumAttempts,
+  parseSchemaText,
   STORAGE_DIR,
   adoptLegacyVersionKey,
+  listAgents,
   runCost,
 } from "./store.ts";
 import { loadCatalog, checkModel, clampEffort, catalogCost, findModel, type Catalog } from "./catalog.ts";
@@ -72,6 +82,7 @@ import { materializeFiles, harvestFiles } from "./storage.ts";
 import { chooseExecutor, ensureImage } from "./container.ts";
 import { stampBundle } from "./okf.ts";
 import type { McpServerConfig } from "@anthropic-ai/claude-agent-sdk";
+import { trimChars } from "./paths.ts";
 
 // An MCP tool definition becomes an SDK server config. ${SECRET} placeholders
 // in env and headers resolve server-side, so a credential reaches the server
@@ -123,6 +134,32 @@ export function joinEarlierGroups(groupResults: (string | null)[], gi: number, c
   return parts.join("\n\n");
 }
 
+/**
+ * `parallel: N` — a pool of N slots the instances of one fan-out share.
+ * Fan-out always ran every instance at once, and still does unless the
+ * step says otherwise: twenty browser steps against one site, or one
+ * vendor's rate limit, is what the cap is for. Nothing is rented while an
+ * instance waits for a slot.
+ */
+export function slotPool(size: number): { acquire: () => Promise<void>; release: () => void } {
+  let free = Math.max(1, size);
+  const waiting: (() => void)[] = [];
+  return {
+    acquire: () =>
+      new Promise<void>((resolve) => {
+        if (free > 0) {
+          free -= 1;
+          resolve();
+        } else waiting.push(resolve);
+      }),
+    release: () => {
+      const next = waiting.shift();
+      if (next) next();
+      else free += 1;
+    },
+  };
+}
+
 export interface DiscoveredSkill {
   name: string;
   description: string;
@@ -378,10 +415,9 @@ export function saysUntilMarker(result: string | null, until: string): boolean {
       (line) =>
         // Strip markdown emphasis and punctuation from BOTH ends: "**APPROVED.**"
         // is a decision written by someone using bold, not a sentence.
-        line
-          .trim()
-          .replace(/^[#>\-*_`\s]+/, "")
-          .replace(/[.!:,*_`\s]+$/, "")
+        // trimChars, not a `[…]+$` regex: that one is quadratic on a
+        // line of tabs, and this is a line an agent wrote.
+        trimChars(trimChars(line.trim(), "#>-*_`"), ".!:,*_`").trim()
           .toLowerCase() === marker,
     );
 }
@@ -471,7 +507,10 @@ export function resolveDocLinks(text: string, workspaceRoot: string): string {
       map.set(norm(fwd), `../../${STORAGE_DIR}/${fwd}`);
     }
   }
-  return text.replace(/\[\[([^\]\n]+)\]\]/g, (whole, name: string) => {
+  // `[` excluded from the name: with it allowed, a run of `[[` makes the
+  // engine re-scan to the end from every opening — quadratic on a result
+  // an agent wrote.
+  return text.replace(/\[\[([^\]\[\n]+)\]\]/g, (whole, name: string) => {
     const hit = map.get(norm(name.trim()));
     return hit ? `\`${hit}\`` : whole;
   });
@@ -1197,6 +1236,49 @@ function repriced(
   return known;
 }
 
+/**
+ * Write one attempt onto the step's history and re-total the step.
+ *
+ * runStep reports the attempt it just ran — its cost, tokens and sandbox
+ * seconds — by assigning the step's fields. Those become this attempt's row;
+ * the step's fields then become the sum of every row, which is what the
+ * run's spend, the flow's budget: and the platform's bill read. A retried
+ * step used to carry only its last attempt's cost, so a `retry: 2` step
+ * could spend three shares while reporting one.
+ */
+export function recordAttempt(
+  step: StepRecord,
+  n: number,
+  status: "completed" | "failed",
+  startedAt: string,
+  eventsBefore: number,
+  /** The attempt re-attached to a sandbox a previous driver left: same
+   *  attempt, so its row is replaced rather than repeated. */
+  reattach = false,
+) {
+  const tries = step.tries ?? [];
+  const lastError = step.events
+    .slice(eventsBefore)
+    .filter((e) => e.type === "error")
+    .at(-1)?.text;
+  const row: StepAttempt = {
+    n,
+    status,
+    costUsd: step.costUsd,
+    tokens: step.tokens ?? null,
+    computeSecs: step.computeSecs ?? null,
+    startedAt,
+    finishedAt: step.finishedAt ?? new Date().toISOString(),
+    ...(status === "failed" && lastError ? { error: lastError.slice(0, 500) } : {}),
+  };
+  const kept = reattach && tries.at(-1)?.n === n ? tries.slice(0, -1) : tries;
+  step.tries = [...kept, row];
+  const totals = sumAttempts(step.tries);
+  step.costUsd = totals.costUsd;
+  step.tokens = totals.tokens;
+  step.computeSecs = totals.computeSecs;
+}
+
 async function runStep(
   agentDir: string,
   tenant: string,
@@ -1214,6 +1296,15 @@ async function runStep(
    *  remaining `budget:`, shared across the steps launched with it. Null
    *  means the flow set no budget. */
   stepBudgetUsd: number | null = null,
+  /** Which line set that ceiling, for the error that names it. */
+  stepBudgetNote?: string,
+  /** The run is a test run — see test-mode.ts. Read off the run record by
+   *  the caller, never from anything the sandbox could have written. */
+  testRun = false,
+  /** Has a person asked for the run to stop? Read on a clock by the
+   *  in-process model loop, so a stop lands mid-step and not at the next
+   *  group. The isolated path needs none: stopRun destroys the sandbox. */
+  stopRequested?: () => boolean,
 ) {
   // Secret values are injected into scripts as environment variables and
   // substituted into API headers, so a model that reads one back — from a
@@ -1238,6 +1329,22 @@ async function runStep(
     return out;
   };
 
+  // The reply, the conclusion and an `output: json` value land on the same
+  // record the events do, and a model that quotes a script's stdout puts
+  // the credential in its answer rather than in a tool event. Same scrub,
+  // same place. The JSON value goes through its text form: a secret inside
+  // a nested field is still a secret, and the key name survives either way.
+  const redactText = <T extends string | null | undefined>(text: T): T =>
+    (typeof text === "string" ? redact(text) : text) as T;
+  const redactData = (value: unknown): unknown => {
+    if (value === undefined || redactions.length === 0) return value;
+    try {
+      return JSON.parse(redact(JSON.stringify(value)));
+    } catch {
+      return value;
+    }
+  };
+
   const push = (type: StepRecord["events"][number]["type"], text: string, extra?: EventExtra) => {
     step.events.push({ t: new Date().toISOString(), type, text: redact(text), ...extra });
     save();
@@ -1277,8 +1384,15 @@ async function runStep(
     // the failure of a step that returned no JSON are testable at no cost.
     if (step.output === "json") {
       const extracted = extractJson(step.result);
-      if (extracted.ok) step.data = extracted.value;
-      else {
+      if (extracted.ok) {
+        step.data = extracted.value;
+        const schema = step.schema ?? (step.schemaPath ? readSchemaFile(path.resolve(agentDir, "..", ".."), step.schemaPath) : null);
+        const errors = schema && looksLikeSchema(schema) ? validateSchema(step.data, schema) : [];
+        if (errors.length) {
+          push("error", `schema: the value does not fit — ${describeSchemaErrors(errors)}`);
+          step.status = "failed";
+        }
+      } else {
         push("error", `output: json — ${extracted.reason}`);
         step.status = "failed";
       }
@@ -1313,7 +1427,7 @@ async function runStep(
     // what crosses into containers, headers and env files is the token. A
     // failed refresh fails the step naming the secret and the provider's
     // reason, which beats a 401 three layers later.
-    const liveSecrets = await materializeSecrets(secretEnv);
+    const liveSecrets = await materializeSecrets(secretEnv, { tenant, workspace: path.basename(workspaceRootOf(agentDir)) });
     // @file values (SSH keys, certs) still carry their content at this point.
     // The in-process branch materialises them to 0600 paths on this host
     // below; the container branch passes the content across and the driver
@@ -1461,6 +1575,13 @@ async function runStep(
     // The confinement boundary: agents in a workspace are one team, but the
     // workspace is where isolation is enforced.
     const workspaceRoot = path.resolve(agentDir, "..", "..");
+    // Where a test run's state/ and storage/ writes go, and the line each
+    // one leaves: under the run, beside its archived outputs.
+    const divertFor = (id: string) => ({
+      to: path.join(workspaceRoot, "runs", id, TEST_WRITES_DIR),
+      note: (rel: string, summary: string) =>
+        push("info", `test: ${summary}`, { effect: { kind: "diverted" as const, path: rel, summary } }),
+    });
 
     let prompt = step.instruction
       ? resolveDocLinks(step.instruction, workspaceRoot)
@@ -1477,11 +1598,18 @@ async function runStep(
     if (step.output === "json") {
       // The contract the runner parses back out — stated in the prompt so
       // the instruction and the parser agree by construction, like delegate:.
+      // The schema, when there is one, is quoted whole: the model that has
+      // to satisfy it should not have to guess it from the field names.
+      const schema = step.schema ?? (step.schemaPath ? readSchemaFile(path.resolve(agentDir, "..", ".."), step.schemaPath) : null);
       prompt +=
         `\n\n<output>\nThis step returns data. End your reply with exactly one JSON value inside a ` +
         "```json fenced block — the value the next step will compute on. Prose before it is " +
         `welcome (lead with a one-line headline); nothing may follow the block. If the instruction ` +
-        `names fields, use exactly those names.\n</output>`;
+        `names fields, use exactly those names.` +
+        (schema && looksLikeSchema(schema)
+          ? ` The value must satisfy this JSON Schema, and the step fails if it does not:\n${JSON.stringify(schema, null, 2).slice(0, 20_000)}`
+          : "") +
+        `\n</output>`;
     }
     if (step.waitFor === "event" && step.eventPayload !== undefined) {
       // Not resolved for links, like the previous step's results: it is
@@ -1548,6 +1676,16 @@ async function runStep(
     // platform credential.
     const secondSupply = fallbackEnv ?? (Object.keys(providerEnv).length === 0 ? platformFallbackEnv() : null);
 
+    // `schema:` — inline on the step, or a file under the workspace read
+    // now, host-side, so both executors are handed the same document. A
+    // file that cannot be read is a failed step before any model runs:
+    // the contract the author wrote down cannot be checked.
+    const resolvedSchema: Record<string, unknown> | boolean | null = step.schema ?? (step.schemaPath ? readSchemaFile(workspaceRoot, step.schemaPath) : null);
+    if (step.schemaPath && resolvedSchema === null) {
+      throw new Error(`schema: ${step.schemaPath} — not a readable JSON or YAML schema under the workspace`);
+    }
+    if (step.maxTurns) push("info", `max_turns: ${step.maxTurns}`);
+
     const isolation = process.env.FOLDRUN_RUN_ISOLATION;
     if (isolatedRun()) {
       // The isolated path: the whole loop — model, built-in tools, scripts —
@@ -1575,10 +1713,20 @@ async function runStep(
         const host = hostOf(api.base);
         if (!host) continue;
         for (const v of [...Object.values(api.headers), ...Object.values(api.query)]) {
-          for (const name of placeholderNames(v)) if (name in liveSecrets) addGrant(grant, name, liveSecrets[name], host);
+          for (const name of placeholderNames(v)) {
+            if (!(name in liveSecrets)) continue;
+            addGrant(grant, name, liveSecrets[name], host);
+            // Which vault entry this is, so the proxy's record of what the
+            // far end said lands on the right credential.
+            (grant.healthKeys ??= {})[name] = healthKey(
+              name,
+              secretScopes[name] === "workspace" ? "workspace" : "account",
+              path.basename(path.resolve(agentDir, "..", "..")),
+            );
+          }
         }
       }
-      const lease = await platform.egress.lease({ tenant, runId: runId ?? "adhoc", grant });
+      const lease = await platform.egress.lease({ tenant, runId: runId ?? "adhoc", grant, test: testRun });
       // Which secrets must still be real inside the sandbox: a script tool
       // reads them from its environment, and unless it says `secrets:
       // proxied` it needs the value, not the name. bash is a script by
@@ -1601,15 +1749,36 @@ async function runStep(
       }
       // Sandbox env for the declared secrets: the value, or its own name as
       // a placeholder that the proxy fills on the way out.
-      const sandboxSecrets: Record<string, string> = materialise
-        ? liveSecrets
-        : Object.fromEntries(declared.map((name) => [name, `\${${name}}`]));
+      const placeholders = Object.fromEntries(declared.map((name) => [name, `\${${name}}`]));
+      let sandboxSecrets: Record<string, string> = materialise ? liveSecrets : placeholders;
+      // A test run hands the sandbox no send-capable value. The env var is
+      // still there, reading TEST_MODE_WITHHELD, so a script that never
+      // checked FOLDRUN_TEST_MODE fails at the provider's door with an auth
+      // error — and sends nothing. The script tools say whether the step
+      // has a declared sender (everything withheld) or a declared reader
+      // (nothing withheld); see test-mode.ts for the order.
+      if (testRun && materialise) {
+        const held = withholdSecrets(liveSecrets, {
+          outward: scriptSpecs.some((sc) => sc.outward),
+          allow: scriptSpecs.some((sc) => sc.testMode === "allow"),
+        });
+        sandboxSecrets = held.env;
+        for (const name of held.withheld) {
+          push("info", `test: ${name} withheld from the sandbox — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from the sandbox` } });
+        }
+      }
+      // On a test run the http tools always go through the proxy as
+      // placeholders, even when a script made the step materialise: the
+      // proxy is where the outward-write policy lives, and a header
+      // carrying TEST_MODE_WITHHELD would only fail where it should have
+      // been sunk or refused.
+      const apiValues = testRun && lease ? placeholders : sandboxSecrets;
       const substitutedApis = apiSpecs.map((api) => ({
         ...api,
         headers: Object.fromEntries(
           Object.entries(api.headers).map(([k, v]) => [
             k,
-            v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => sandboxSecrets[name] ?? whole),
+            v.replace(/\$\{([A-Z][A-Z0-9_]*)\}/g, (whole, name) => apiValues[name] ?? whole),
           ]),
         ),
       }));
@@ -1639,6 +1808,7 @@ async function runStep(
       const isolatedArgs = (modelEnv: Record<string, string | undefined>, keyName = MODEL_KEY_NAME) => ({
         workspaceRoot,
         libraryRoot: libraryDir(tenant),
+        galleryRoot: platform.galleryDir() ?? undefined,
         // Keys this step's dependency cache. Per-account by design — see
         // run-container.ts#runtimeCacheDir for why it is not shared.
         tenant,
@@ -1659,9 +1829,12 @@ async function runStep(
           consults,
           timeoutSec: step.timeout,
           budgetUsd: stepBudgetUsd,
+          budgetNote: stepBudgetNote,
           price: tokenPrice,
           verify: step.verify,
           output: step.output,
+          schema: resolvedSchema ?? undefined,
+          maxTurns: step.maxTurns,
           translator: keyName === MODEL_KEY_NAME ? primaryTranslator : secondTranslator,
           // The container sees the workspace at /workspace and the library
           // at /library; the roots are named host-side and moved here.
@@ -1692,6 +1865,7 @@ async function runStep(
             FOLDRUN_RUN_ID: runId,
             FOLDRUN_AGENT: step.agent,
             FOLDRUN_WORKSPACE: path.basename(workspaceRoot),
+            ...(testRun ? testModeEnv() : {}),
             ...sandboxSecrets,
           }).filter((entry): entry is [string, string] => typeof entry[1] === "string"),
         ),
@@ -1699,6 +1873,9 @@ async function runStep(
         // Stamps the sandbox, so stopping this run can destroy it.
         runId,
         size,
+        // A test run's state/ and storage/ changes come out under the run.
+        divert: testRun && runId ? divertFor(runId) : undefined,
+        test: testRun,
         // Resume: a sandbox a previous driver left running, recorded on the
         // step. Consumed once — a fresh attempt after this one (a retry, a
         // fallback) starts its own sandbox.
@@ -1767,18 +1944,23 @@ async function runStep(
         outcome = withEarlierTiming(outcome, first);
       }
       if (lease) {
-        for (const line of await lease.drainLog()) push("info", line);
+        for (const line of await lease.drainLog()) {
+          if (typeof line === "string") push("info", line);
+          else push("info", line.text, { effect: line.effect });
+        }
         await lease.release();
       }
-      publishPublicDir(tenant, path.basename(workspaceRoot), push);
+      // Nothing a test run left in storage/public/ reached storage/, so
+      // there is nothing to publish — and a share link is an outward act.
+      if (!testRun) publishPublicDir(tenant, path.basename(workspaceRoot), push);
       step.status = outcome.status;
-      step.result = outcome.result;
-      step.conclusion = outcome.conclusion;
+      step.result = redactText(outcome.result);
+      step.conclusion = redactText(outcome.conclusion);
       step.sandbox = null;
       // The cluster's word on why a sandbox ended without the driver saying
       // so — OOMKilled, Evicted — on the record, where the retry reads it.
       if (outcome.status === "failed" && outcome.reason) push("error", `sandbox ended: ${outcome.reason}`);
-      if (outcome.data !== undefined) step.data = outcome.data;
+      if (outcome.data !== undefined) step.data = redactData(outcome.data);
       step.costUsd = repriced(catalog, wireModel, outcome.usage ?? null, outcome.costUsd, push);
       step.tokens = outcome.usage
         ? { input: outcome.usage.inputTokens, output: outcome.usage.outputTokens }
@@ -1811,14 +1993,27 @@ async function runStep(
       })();
       step.reserved = { cpus: Number(lim.cpus) || 2, memGiB: gib };
     } else {
+      // Same withholding as the sandbox path — the in-process step is what
+      // `foldrun run --test` drives on a laptop, where there is no proxy
+      // and this is the only guard there is.
+      const held = testRun
+        ? withholdSecrets(liveSecrets, {
+            outward: scriptSpecs.some((sc) => sc.outward),
+            allow: scriptSpecs.some((sc) => sc.testMode === "allow"),
+          })
+        : { env: liveSecrets, withheld: [] as string[] };
+      for (const name of held.withheld) {
+        push("info", `test: ${name} withheld from scripts — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from scripts` } });
+      }
+      const stepSecrets = held.env;
       const consultTools = buildConsultTools(
         consults,
-        { ...process.env, ...liveSecrets, ...providerEnv },
+        { ...hostSafeEnv(), ...stepSecrets, ...providerEnv },
         (type, text) => push(type, text),
       );
       // @file secrets become 0600 paths here (host run), cleaned up in the
       // finally — once, however many attempts follow.
-      const mat = materializeFileSecrets(agentDir, liveSecrets);
+      const mat = materializeFileSecrets(agentDir, stepSecrets);
       fileDir = mat.dir;
       const attemptInProcess = (modelEnv: Record<string, string | undefined>) => executeStep({
         agentDir,
@@ -1840,13 +2035,20 @@ async function runStep(
           ...mcpServers,
         },
         // Declared secrets reach the agent's scripts as env vars; the model
-        // only ever sees the variable names, not the values.
-        env: { ...process.env, ...clockEnv, ...mat.env, ...modelEnv },
+        // only ever sees the variable names, not the values. The base is
+        // the allowlisted host environment, not process.env: the SDK's
+        // Bash tool runs with this, and on a platform process.env holds
+        // every tenant's root key and the datastore credentials.
+        env: { ...hostSafeEnv(), ...clockEnv, ...(testRun ? testModeEnv() : {}), ...mat.env, ...modelEnv },
         timeoutSec: step.timeout,
         budgetUsd: stepBudgetUsd,
+        budgetNote: stepBudgetNote,
         price: tokenPrice,
         verify: step.verify,
         output: step.output,
+        schema: resolvedSchema ?? undefined,
+        maxTurns: step.maxTurns,
+        stopRequested,
         // What the scripts saw, the verify sees: a flow can then check that
         // the proof a step left names THIS run, not one that came before —
         // copy-back never propagates a deletion, so a marker from an earlier
@@ -1856,20 +2058,31 @@ async function runStep(
           ...(runId ? { FOLDRUN_RUN_ID: runId } : {}),
           FOLDRUN_AGENT: step.agent,
           FOLDRUN_WORKSPACE: path.basename(path.resolve(agentDir, "..", "..")),
-          ...liveSecrets,
+          ...(testRun ? testModeEnv() : {}),
+          ...stepSecrets,
         },
         emit: pushWatching,
       });
       // With a translator the step's endpoint is the translator: started
       // here, pointed at by the SDK's env, closed when the attempt ends.
+      // A test run brackets each attempt with a snapshot of state/ and
+      // storage/: the step writes them for real, because there is no
+      // sandbox to write instead, and afterwards what changed is moved
+      // under the run and the originals come back.
       const attemptThrough = async (modelEnv: Record<string, string | undefined>, spec: TranslatorSpec | null) => {
-        if (!spec) return attemptInProcess(modelEnv);
-        const t = await startTranslator(spec);
+        const before = testRun ? snapshotDivertedDirs(workspaceRoot) : null;
+        const t = spec ? await startTranslator(spec) : null;
         try {
-          return await attemptInProcess({ ...modelEnv, ...t.env });
+          return await attemptInProcess(t ? { ...modelEnv, ...t.env } : modelEnv);
         } finally {
-          for (const line of t.drainLog()) push("info", line.startsWith("translator") ? line : `translator: ${line}`);
-          await t.close();
+          if (t) {
+            for (const line of t.drainLog()) push("info", line.startsWith("translator") ? line : `translator: ${line}`);
+            await t.close();
+          }
+          if (before && runId) {
+            const d = divertFor(runId);
+            restoreDivertedDirs(workspaceRoot, before, d.to, d.note);
+          }
         }
       };
       let outcome = await attemptThrough(providerEnv, translator);
@@ -1891,11 +2104,11 @@ async function runStep(
         );
         outcome = await attemptThrough(secondSupply, fallbackEnv && secondSupply === fallbackEnv ? fallbackTranslator : null);
       }
-      publishPublicDir(tenant, path.basename(workspaceRoot), push);
+      if (!testRun) publishPublicDir(tenant, path.basename(workspaceRoot), push);
       step.status = outcome.status;
-      step.result = outcome.result;
-      step.conclusion = outcome.conclusion;
-      if (outcome.data !== undefined) step.data = outcome.data;
+      step.result = redactText(outcome.result);
+      step.conclusion = redactText(outcome.conclusion);
+      if (outcome.data !== undefined) step.data = redactData(outcome.data);
       // A consult's spend belongs to the step that asked.
       const consultCost = consultTools.drainCost();
       const stepCost = repriced(catalog, wireModel, outcome.usage, outcome.costUsd, push);
@@ -2189,6 +2402,26 @@ function csvItems(workspaceDir: string, step: StepRecord, take: number): string[
   return data.slice(0, take).map((r) => `${header}\n${r}`);
 }
 
+/**
+ * `schema: <path>` — the file, read and parsed, or null. Agent-relative like
+ * every other path in a flow (`../../schemas/lead.json`), confined to the
+ * workspace: a schema is the author's document, never something a step
+ * can point outside.
+ */
+export function readSchemaFile(workspaceRoot: string, rel: string): Record<string, unknown> | boolean | null {
+  const root = path.resolve(workspaceRoot);
+  // Written agent-relative (`../../schemas/x.json`) or workspace-relative
+  // (`schemas/x.json`); both land on the same file.
+  const candidates = [path.resolve(root, rel.replace(/^(\.\.\/){2}/, "")), path.resolve(root, "agents", "x", rel)];
+  for (const abs of candidates) {
+    if (abs !== root && !abs.startsWith(root + path.sep)) continue;
+    if (!fs.existsSync(abs) || !fs.statSync(abs).isFile()) continue;
+    const parsed = parseSchemaText(fs.readFileSync(abs, "utf8"));
+    return "problem" in parsed ? null : parsed.schema;
+  }
+  return null;
+}
+
 export function createFlowRun(
   tenant: string,
   workspace: string,
@@ -2196,12 +2429,16 @@ export function createFlowRun(
   flowName: string,
   status: "queued" | "running",
   tags: string[] = [],
+  startedBy: string | null = null,
+  opts: { test?: boolean } = {},
 ): RunRecord {
   const run: RunRecord = {
     id: `run-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`,
     flow: flowName,
     tags,
     status,
+    ...(startedBy ? { startedBy: startedBy.trim().toLowerCase() } : {}),
+    ...(opts.test ? { test: true } : {}),
     startedAt: new Date().toISOString(),
     finishedAt: null,
     steps: steps.map((s) => ({
@@ -2231,6 +2468,10 @@ export function createFlowRun(
       preview: s.preview,
       delegate: s.delegate,
       output: s.output,
+      schema: s.schema,
+      schemaPath: s.schemaPath,
+      parallel: s.parallel,
+      maxTurns: s.maxTurns,
       attempts: 0,
       status: "pending",
       events: [],
@@ -2267,8 +2508,10 @@ export function startFlowRun(
   modelOverride?: string | null,
   tags: string[] = [],
   effortOverride?: string | null,
+  startedBy: string | null = null,
+  opts: { test?: boolean } = {},
 ): RunRecord {
-  const run = createFlowRun(tenant, workspace, steps, flowName, "running", tags);
+  const run = createFlowRun(tenant, workspace, steps, flowName, "running", tags, startedBy, opts);
   void driveRun(tenant, workspace, run, modelOverride, tags, { effortOverride });
   return run;
 }
@@ -2566,6 +2809,17 @@ function driveRunInner(
       // is assembled) keeps the newest: when the total is over it, the
       // oldest groups are dropped whole before anything is cut mid-text.
       const contextBefore = (gi: number) => joinEarlierGroups(groupResults, gi);
+      // What `when:` and `case:` read: the PREVIOUS group's result alone —
+      // the nearest earlier group that produced one, so a routed-past group
+      // between is looked through rather than read as silence. The prompt
+      // gets every earlier group joined (contextBefore); routing must not,
+      // because a verdict two groups back would still open a gate a later
+      // group had overruled: a classifier's BUG line matched a `case: BUG`
+      // three groups on, after the debugger had already answered it.
+      const routingBefore = (gi: number): string | null => {
+        for (let i = gi - 1; i >= 0; i--) if (groupResults[i]) return groupResults[i];
+        return null;
+      };
       // The data half of the same handoff: what the nearest earlier group
       // returned through `output: json`. Rebuilt from the record rather than
       // kept only in memory, so a resumed run hands step 3 the value step 1
@@ -2630,8 +2884,11 @@ function driveRunInner(
         context = contextBefore(gi);
         ctxData = dataBefore(gi);
 
-        if (run.budgetUsd && runCost(run) >= run.budgetUsd && group.some((s) => s.status === "pending")) {
-          const spent = runCost(run);
+        // What the run has spent, not just what its tokens cost: on an
+        // install that prices sandbox seconds, a step that called no model
+        // still spent money, and a cap that ignored it capped nothing.
+        if (run.budgetUsd && platform.runSpend(run) >= run.budgetUsd && group.some((s) => s.status === "pending")) {
+          const spent = platform.runSpend(run);
           const last = run.steps.findLast((s) => s.status === "completed" || s.status === "failed") ?? run.steps[0];
           last?.events.push({
             t: new Date().toISOString(),
@@ -2720,6 +2977,7 @@ function driveRunInner(
             type: "info",
             text:
               `fan-out: ${used.length} instance${used.length === 1 ? "" : "s"}` +
+              (step.parallel && step.parallel < used.length ? `, ${step.parallel} at a time (parallel:)` : "") +
               (dropped ? ` (more items existed — capped at ${cap})` : ""),
           });
           const at = run.steps.indexOf(step);
@@ -2734,6 +2992,7 @@ function driveRunInner(
             result: null,
             costUsd: null,
             attempts: 0,
+            tries: undefined,
           }));
           run.steps.splice(at + 1, 0, ...instances);
           ordered = orderedGroups();
@@ -2747,12 +3006,25 @@ function driveRunInner(
         // this is what keeps the *next* group from starting.
         const stopCheck = readRun(tenant, workspace, run.id);
         if (stopCheck?.stopRequested) {
+          let skipped = 0;
           for (const s of run.steps) {
             if (s.status === "pending" || s.status === "running") {
               s.status = "skipped";
               s.skipReason = "run stopped";
+              skipped += 1;
             }
           }
+          // Say so on the trail. stopRun wrote "run stopped" onto the
+          // record's first step, but the save() below writes this process's
+          // copy of the record over it, so the only witness was the
+          // stopRequested flag — and a run that "failed" with no error line
+          // anywhere took a code read to explain (rank-desk, 2026-09-13).
+          const witness = run.steps.findLast((s) => s.status === "completed" || s.status === "failed") ?? run.steps[0];
+          witness?.events.push({
+            t: new Date().toISOString(),
+            type: "info",
+            text: `run stopped by a person between groups — ${skipped} remaining step${skipped === 1 ? "" : "s"} skipped`,
+          });
           run.status = "failed";
           run.stopRequested = true;
           save();
@@ -2788,40 +3060,52 @@ function driveRunInner(
         }
 
         const ctx = context;
+        // The previous group's result — see routingBefore.
+        const routing = routingBefore(gi);
 
-        // `when:` — skip a step whose condition isn't met by prior results.
+        // `when:` — skip a step whose condition isn't met by the previous
+        // group's result.
         for (const step of freshGroup) {
           if (step.status !== "pending" || !step.when) continue;
-          const met = (ctx ?? "").toLowerCase().includes(step.when.toLowerCase());
+          // A marker at the start of a line, not a word anywhere in the
+          // prose — see markerPresent. "There are no BLOCKED items" must
+          // not open a gate keyed on BLOCKED.
+          const met = markerPresent(routing, step.when);
           if (!met) {
             step.status = "skipped";
             step.skipReason = `condition not met: when "${step.when}"`;
             step.events.push({
               t: new Date().toISOString(),
               type: "info",
-              text: `skipped — previous results do not mention "${step.when}"`,
+              text: `skipped — the previous group's result has no line beginning "${step.when}" (when: reads a marker at the start of a line of the previous result, not a word in a sentence)`,
             });
           }
         }
         save();
 
         // `case:`/`else:` — exclusive routing. Of this group's case steps,
-        // the FIRST whose text appears in the previous results runs; the
-        // rest are routed past. `else:` runs only when no case matched.
+        // the FIRST whose marker leads a line of the previous results runs;
+        // the rest are routed past. `else:` runs only when no case matched.
         // (`when:` above stays independent — every matching when runs —
         // which is why routing is its own vocabulary instead of a mode.)
+        //
+        // Same marker rule as `when:`, and it matters more here: routing is
+        // exclusive, so a label matched out of a sentence does not merely
+        // run an extra step, it sends the flow down the wrong branch and
+        // skips the right one. "This is not a COMPLAINT, it is a QUESTION"
+        // used to route to the complaints handler.
         const caseSteps = freshGroup.filter((s) => s.status === "pending" && s.case);
         if (caseSteps.length || freshGroup.some((s) => s.status === "pending" && s.else)) {
           let matchedCase: StepRecord | null = null;
           for (const step of caseSteps) {
-            if (!matchedCase && (ctx ?? "").toLowerCase().includes(step.case!.toLowerCase())) {
+            if (!matchedCase && markerPresent(routing, step.case!)) {
               matchedCase = step;
               continue;
             }
             step.status = "skipped";
             step.skipReason = matchedCase
               ? `routed past — "${matchedCase.case}" matched first`
-              : `case not matched: "${step.case}"`;
+              : `case not matched: the previous group's result has no line beginning "${step.case}"`;
             step.events.push({
               t: new Date().toISOString(),
               type: "info",
@@ -2865,6 +3149,20 @@ function driveRunInner(
             });
           }
           run.status = "awaiting-approval";
+          // approve_within: — the deadline is stamped when the gate opens,
+          // not counted from the run's start, so "two days to answer" means
+          // two days from being asked however long the work before it took.
+          // Only a gate that asks a PERSON gets one: an external event has
+          // no one to chase.
+          const within = readFlow(tenant, workspace, run.flow)?.approveWithin ?? null;
+          // Always assigned, never only set: a run with two gates would
+          // otherwise carry the FIRST gate's deadline into the second, and
+          // a deadline already in the past expires a gate the moment it
+          // opens. Null when this gate has none.
+          run.approveBy =
+            within && needsApproval.some((s) => s.waitFor !== "event")
+              ? new Date(Date.now() + within * 1000).toISOString()
+              : null;
           save();
 
           // The gate's declared preview, resolved now against what the run
@@ -2939,10 +3237,46 @@ function driveRunInner(
         // twenty steps over — the shares sum to what was left. Between
         // groups the check above still refuses to start the next one.
         const launching = freshGroup.filter((s) => s.status === "pending");
-        const ceilingUsd = stepCeiling(run.budgetUsd, runCost(run), launching.length);
+        // An agent's own `budget:` — the most it may spend in one run — is
+        // read at launch, so a cap raised mid-run applies to the next step.
+        const agentBudgets = new Map(listAgents(tenant, workspace).map((a) => [a.name, a.budget]));
+        // The ceiling for one attempt of one step, read NOW: what is left of
+        // the run's budget shared across the group, or the agent's own cap
+        // less what its steps have cost. Computed before every attempt, not
+        // once before the loop — a retry's share is what is left after the
+        // failed attempt spent, and the step's own figure already carries
+        // that spend (it is the sum of its tries).
+        const ceilingFor = (step: StepRecord) =>
+          stepCeilingFor(
+            stepCeiling(run.budgetUsd, platform.runSpend(run), launching.length),
+            agentBudgets.get(step.agent),
+            run.steps.filter((s) => s !== step && s.agent === step.agent).reduce((sum, s) => sum + (s.costUsd ?? 0), 0),
+            step.agent,
+          );
+        // parallel: — the instances of one fan-out template share a pool.
+        // Keyed by what the instances share (agent + instruction), since
+        // the template itself is behind them in the record as "expanded".
+        const pools = new Map<string, ReturnType<typeof slotPool>>();
+        const poolFor = (step: StepRecord) => {
+          if (!step.parallel || !step.item) return null;
+          const key = `${step.agent}\u0000${step.instruction}`;
+          let pool = pools.get(key);
+          if (!pool) pools.set(key, (pool = slotPool(step.parallel)));
+          return pool;
+        };
         await Promise.all(
           launching
             .map(async (step) => {
+              const pool = poolFor(step);
+              if (pool) await pool.acquire();
+              try {
+                await launchStep(step);
+              } finally {
+                pool?.release();
+              }
+            }),
+        );
+        async function launchStep(step: StepRecord) {
               const attempts = (step.retry ?? 0) + 1;
               // A step re-attaching to its sandbox is still on the attempt
               // that started it; a step re-run after a backoff picks up the
@@ -2951,6 +3285,31 @@ function driveRunInner(
                 !step.sandbox && !!step.attempts && /retrying in \d+s/.test(step.events[step.events.length - 1]?.text ?? "");
               const from = step.sandbox ? Math.max(1, step.attempts ?? 1) : interruptedInBackoff ? Math.min(attempts, step.attempts! + 1) : 1;
               for (let attempt = from; attempt <= attempts; attempt++) {
+                const ceiling = ceilingFor(step);
+                // Nothing left is a refusal, not "no ceiling": a zero passed
+                // down would read as unset, and the one step that must not
+                // run would run with no cap at all. On a retry this is what
+                // stops the second attempt spending a share the first one
+                // already spent.
+                if (ceiling.ceilingUsd === 0) {
+                  step.status = "failed";
+                  step.events.push({
+                    t: new Date().toISOString(),
+                    type: "error",
+                    text:
+                      attempt === from
+                        ? `over budget — nothing left to spend before this step (${ceiling.note}); it was not started`
+                        : `over budget — nothing left to spend for attempt ${attempt} (${ceiling.note}); not retried`,
+                  });
+                  save();
+                  return;
+                }
+                // Re-attaching to a sandbox is the same attempt, so its row
+                // is replaced; anything else is a new row, a loop cycle's
+                // re-run included.
+                const reattach = !!step.sandbox;
+                const attemptStartedAt = new Date().toISOString();
+                const eventsBefore = step.events.length;
                 step.attempts = attempt;
                 step.status = "running";
                 save();
@@ -2965,11 +3324,21 @@ function driveRunInner(
                   effortOverride,
                   run.id,
                   ctxData,
-                  ceilingUsd,
+                  ceiling.ceilingUsd,
+                  ceiling.note,
+                  run.test === true,
+                  // save() merges a stop written by another process into
+                  // this record; the model loop polls this on a clock.
+                  () => {
+                    save();
+                    return run.stopRequested === true;
+                  },
                 );
                 // runStep mutates step.status; read it through a widened local
                 // so TS doesn't keep the "running" narrowing from above.
                 const outcome: string = step.status;
+                recordAttempt(step, attempt, outcome === "completed" ? "completed" : "failed", attemptStartedAt, eventsBefore, reattach);
+                save();
                 if (outcome !== "failed" || attempt === attempts) break;
                 // A stop destroys the step's sandbox, which reads here as a
                 // failed attempt — and a failed attempt used to be retried in
@@ -3014,8 +3383,7 @@ function driveRunInner(
                   break;
                 }
               }
-            }),
-        );
+        }
 
         // on-fail: another agent takes the step over. Sequential and rare:
         // the rescue is a second full attempt at the step's job with the
@@ -3067,7 +3435,7 @@ function driveRunInner(
           rescue.attempts = 1;
           rescue.status = "running";
           save();
-          await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData);
+          await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData, null, undefined, run.test === true);
           // runStep mutates rescue.status; widen past TS's "running" narrowing.
           const rescueOutcome: string = rescue.status;
           if (rescueOutcome === "completed") {
@@ -3197,16 +3565,25 @@ function driveRunInner(
       for (const s of run.steps) if (s.status === "pending") s.status = "skipped";
     } catch (err) {
       run.status = "failed";
+      const text = err instanceof Error ? err.message : String(err);
+      let told = false;
       run.steps.forEach((s) => {
         if (s.status === "running") {
           s.status = "failed";
-          s.events.push({
-            t: new Date().toISOString(),
-            type: "error",
-            text: err instanceof Error ? err.message : String(err),
-          });
+          s.events.push({ t: new Date().toISOString(), type: "error", text });
+          told = true;
         }
       });
+      // Thrown between groups, with nothing running: the error still has to
+      // land on some step's trail, or the run reads as failed for no reason.
+      if (!told) {
+        const witness = run.steps.findLast((s) => s.status !== "pending") ?? run.steps[0];
+        witness?.events.push({
+          t: new Date().toISOString(),
+          type: "error",
+          text: `run failed between steps: ${text}`,
+        });
+      }
     } finally {
       if (!parked) {
         run.finishedAt = new Date().toISOString();
@@ -3216,6 +3593,9 @@ function driveRunInner(
         // learn, and the runs list and the notification stop having to say
         // only "completed".
         run.summary = runSummary(run);
+        // A test run's headline says so wherever the headline goes — the
+        // runs list, the history tool, a later run's recall.
+        if (run.test && run.summary) run.summary = testHeadline(run.summary);
         // Archive before the next run resets outputs/ — a failed run's
         // artifacts are usually the most interesting ones, so this runs on
         // failure too.
@@ -3256,16 +3636,6 @@ function driveRunInner(
   })();
 }
 
-function readFlow(tenant: string, workspace: string, flowName: string) {
-  const dir = path.join(workspaceDir(tenant, workspace), "flows");
-  if (!fs.existsSync(dir)) return null;
-  for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".md"))) {
-    const flow = parseFlow(f, fs.readFileSync(path.join(dir, f), "utf8"));
-    if (flow.name === flowName) return flow;
-  }
-  return null;
-}
-
 // Splice `[[flow:other]]` steps into the parent's step list, renumbering
 // groups so the nested flow's own parallelism survives. Depth-limited, and
 // a cycle raises rather than expanding forever.