@foldrun/core 2.5.0 → 2.7.0

package/src/package.ts

@@ -26,7 +26,7 @@
 
 import fs from "node:fs";
 import path from "node:path";
-import matter from "./matter.ts";
+import matter, { configMatter } from "./matter.ts";
 import {
   assertSafeName,
   isEditablePath,
@@ -592,7 +592,13 @@ interface AgentFront {
 }
 
 function frontOf(content: string): AgentFront {
-  const d = safeFront(content);
+  // An agent's secrets: and tools: may sit in its guardrail: block.
+  let d: Record<string, unknown>;
+  try {
+    d = configMatter(content).data as Record<string, unknown>;
+  } catch {
+    d = {};
+  }
   const apis = Array.isArray(d.apis) ? d.apis.map((a) => (a && typeof a === "object" ? String((a as { name?: unknown }).name ?? "") : "")) : [];
   const mcp = d.mcpServers && typeof d.mcpServers === "object" ? Object.keys(d.mcpServers) : [];
   const scripts = list(d.scripts);

package/src/provider-health.ts

@@ -21,7 +21,7 @@
 import { accountModelBlock } from "./model-credential.ts";
 import fs from "node:fs";
 import path from "node:path";
-import matter from "./matter.ts";
+import { configMatter } from "./matter.ts";
 import { getSecret } from "./secrets.ts";
 import { noteSecretUse, healthKey } from "./secret-health.ts";
 import { accountDir, workspaceDir, listWorkspaces, listAgents, parseProvider, type ProviderSpec } from "./store.ts";
@@ -205,7 +205,7 @@ function cheapestModel(spec: ProviderSpec): string {
 
 function frontmatterOf(file: string): Record<string, unknown> {
   try {
-    return (matter(fs.readFileSync(file, "utf8")).data ?? {}) as Record<string, unknown>;
+    return (configMatter(fs.readFileSync(file, "utf8")).data ?? {}) as Record<string, unknown>;
   } catch {
     return {};
   }

package/src/run-container.ts

@@ -73,6 +73,9 @@ export interface ContainerStepInput {
   budgetNote?: string;
   price?: TokenPrice | null;
   verify?: string;
+  /** The model `verify: judge:` grades with (the workspace's `judge:`). */
+  judgeModel?: string;
+  judgeEffort?: string;
   /** `output: json` — checked inside, where the reply is. */
   output?: "json";
   /** `schema:` and `max_turns:`, resolved host-side — see ExecOptions. */
@@ -556,7 +559,8 @@ export function runtimeCacheMount(
 
 // ---------- the runner image ----------
 
-const DRIVER = `// The runner container's entrypoint. Reads one step, executes it with the
+/** The runner container's entrypoint, as JS text: written into the image as driver.mjs, so it must be plain JS — a TypeScript cast here broke every step on v2026.10.21.5 (tests/run-container.test.ts checks it). */
+export const DRIVER = `// The runner container's entrypoint. Reads one step, executes it with the
 // same core the server uses, speaks JSONL on stdout, exits.
 import fs from "node:fs";
 const input = JSON.parse(fs.readFileSync("/opt/runner/job/input.json", "utf8"));
@@ -722,6 +726,36 @@ try {
   const peers = input.peers ? buildPeerTools({ channel, self: input.peers.self, siblings: input.peers.siblings, emit }) : { server: null };
   const inbox = channel ? inboxReader(channel, emit) : undefined;
 
+  // Keyless cloud identities (FOLDRUN_IDENTITIES): each token file kept
+  // fresh through the proxy's /identity door, which mints only what this
+  // step's lease was granted. Fetched before the step starts, renewed five
+  // minutes before it expires (with jitter); a failure is said and retried.
+  // Plain JS: this text is the driver, not TypeScript.
+  const identityNames = (env.FOLDRUN_IDENTITIES || "").split(",").filter(Boolean);
+  if (identityNames.length) {
+    // No backslashes: this sits inside a template literal, which eats them.
+    const ep = new RegExp("^(https?://[^/]+)/e/([A-Za-z0-9_-]{16,})/?$").exec(env.FOLDRUN_EGRESS || "");
+    const dir = env.FOLDRUN_IDENTITY_DIR || "/tmp/foldrun-identity";
+    fs.mkdirSync(dir, { recursive: true, mode: 0o700 });
+    const keepFresh = async (name) => {
+      let wait = 30000;
+      try {
+        if (!ep) throw new Error("no egress proxy to ask");
+        const res = await fetch(ep[1] + "/identity/" + ep[2] + "/" + name, { signal: AbortSignal.timeout(15000) });
+        const body = await res.json().catch(() => ({}));
+        if (!res.ok || !body.token) throw new Error((body.error && body.error.message) || ("proxy answered " + res.status));
+        const file = dir + "/" + name.toLowerCase() + ".jwt";
+        fs.writeFileSync(file + ".tmp", body.token, { mode: 0o600 });
+        fs.renameSync(file + ".tmp", file);
+        wait = Math.max(30000, body.expiresAt - Date.now() - 300000 + Math.floor(Math.random() * 60000) - 30000);
+      } catch (err) {
+        emit("error", "identity " + name + ": " + (err && err.message ? err.message : String(err)) + " — trying again in 30s");
+      }
+      setTimeout(() => { void keepFresh(name); }, wait).unref();
+    };
+    await Promise.all(identityNames.map(keepFresh));
+  }
+
   mark("tools");
   const outcome = await executeStep({
     agentDir,
@@ -754,6 +788,8 @@ try {
     price: input.price,
     verify: input.verify,
     verifyEnv: toolEnv,
+    ...(typeof input.judgeModel === "string" ? { judgeModel: input.judgeModel } : {}),
+    ...(typeof input.judgeEffort === "string" ? { judgeEffort: input.judgeEffort } : {}),
     output: input.output,
     schema: input.schema,
     maxTurns: input.maxTurns,
@@ -1358,6 +1394,10 @@ export interface RunInContainerArgs {
    *  without it, "stop" could only mean "stop after this step", and a
    *  browser step has fifteen minutes left to spend. */
   runId?: string;
+  /** Whether a person has stopped the run, read from the record. A step that
+   *  waits for room before its sandbox exists asks this, since there is no
+   *  sandbox yet for the stop to destroy. */
+  stopRequested?: () => boolean;
   /** The reservation class — which limits this step's sandbox holds. */
   size?: "small" | "large" | "heavy";
   /** Re-attach to a sandbox a previous driver started, instead of creating

package/src/runner.ts

@@ -10,7 +10,7 @@ import { hookNames } from "./hooks.ts";
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
-import matter from "./matter.ts";
+import matter, { parseLenient, configMatter } from "./matter.ts";
 import { spawn } from "node:child_process";
 import {
   knownPrice, executeStep, extractJson, stepCeiling, stepCeilingFor, type EventExtra } from "./step-exec.ts";
@@ -21,6 +21,7 @@ import { eventUrl } from "./webhook.ts";
 import { runStepInContainer, sizeLimits, killRunSandboxes, type StepTiming } from "./run-container.ts";
 import { hostSafeEnv, platformApiUrl } from "./host-env.ts";
 import { resolveModelCredential, credentialLine, accountModelBlock, isClaudeLoginToken, isModelEnvName, DECLARED_ENV_PREFIX } from "./model-credential.ts";
+import { parseJudge } from "./judge-verdict.ts";
 import { validateSchema, describeSchemaErrors, looksLikeSchema } from "./json-schema.ts";
 import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute, type EgressGrant } from "./egress.ts";
 import { explainRefusal, isRefusalStatus, refusalFromLine, supplyNote, type SupplyState } from "./refusal.ts";
@@ -70,6 +71,7 @@ import {
   checkFormatVersion,
   readRun,
   writeRun,
+  isRescueRecord,
   runSummary,
   verdictOf,
   type FlowStep,
@@ -88,7 +90,7 @@ import {
 import { canReadBack } from "./tool-output.ts";
 import { PromptLayers, promptCacheKey } from "./prompt-layers.ts";
 import { loadCatalog, checkModel, clampEffort, catalogCost, findModel, tokenPriceOf, type Catalog, type TokenPrice } from "./catalog.ts";
-import { resolveSecrets, getSecret, materializeSecrets, secretSites } from "./secrets.ts";
+import { resolveSecrets, getSecret, materializeSecrets, secretSites, isClientCertValue, clientCertOf, withoutHeldCerts, isIdentityValue, identityOf } from "./secrets.ts";
 import { resolveNetwork, networkLine, networkEnforceWanted, proxyEnv, EGRESS_CA_ENV } from "./network.ts";
 import { materializeFileSecrets, cleanupFileSecrets } from "./secret-files.ts";
 import { buildApiTools, secretsUsedByApi } from "./api-tools.ts";
@@ -335,28 +337,9 @@ const toTags = (v: unknown): string[] =>
 // rather than dropping the skill, so a cross-client skill still loads here.
 function safeMatter(raw: string): { data: Record<string, unknown> } {
   // gray-matter caches a *failed* parse as a hollow success; ./matter.ts
-  // parses every call, so the throw below is deterministic and the fallback
-  // fires every time (it used to clear gray-matter's cache here for that).
-  try {
-    return matter(raw) as { data: Record<string, unknown> };
-  } catch {
-    const fm = raw.match(/^---\r?\n([\s\S]*?)\r?\n---/);
-    if (fm) {
-      const fixed = fm[1].replace(
-        /^([A-Za-z0-9_-]+):[ \t]+(?!["'>|])(.*:.*)$/gm,
-        (_m, k: string, v: string) => `${k}: ${JSON.stringify(v)}`,
-      );
-      try {
-        // A function replacer, not a string: a fixed block containing `$&` or
-        // `$$` (a value with those in it) would otherwise be treated as a
-        // replacement pattern and splice the whole frontmatter into itself.
-        return matter(raw.replace(fm[1], () => fixed)) as { data: Record<string, unknown> };
-      } catch {
-        /* fall through */
-      }
-    }
-    return { data: {} };
-  }
+  // parses every call, so the strict read throws every time and the lenient
+  // one (shared with check, which warns about it) fires every time.
+  return { data: parseLenient(raw).data };
 }
 
 // Both layouts: <root>/<name>/SKILL.md (Agent Skills standard) and the flat
@@ -452,7 +435,7 @@ export function readAgentsMd(dir: string): { data: Record<string, unknown>; body
     const file = path.join(dir, name);
     if (!fs.existsSync(file)) continue;
     try {
-      const parsed = matter(fs.readFileSync(file, "utf8"));
+      const parsed = configMatter(fs.readFileSync(file, "utf8"));
       return { data: parsed.data as Record<string, unknown>, body: withoutAgentRules(parsed.content).trim() };
     } catch {
       return null;
@@ -855,7 +838,7 @@ export function agentContext(
   // agentDir is <data>/<tenant>/projects/<workspace>/agents/<agent>, so the
   // workspace a secret should resolve against is two levels up.
   const workspace = path.basename(path.resolve(agentDir, "..", ".."));
-  const { data: front, content: body } = matter(
+  const { data: front, content: body } = configMatter(
     fs.readFileSync(path.join(agentDir, "agent.md"), "utf8"),
   );
   // The prompt's sections, by name and by how often they change. Added in
@@ -1502,7 +1485,8 @@ export function agentContext(
     // the agent, and the date the platform believes it is.
     {
       ...runtime.env,
-      ...secretEnv,
+      // An unexported client certificate is the proxy's, never a script's.
+      ...withoutHeldCerts(secretEnv),
       ...(identity.runId ? { FOLDRUN_RUN_ID: identity.runId } : {}),
       ...(identity.agent ? { FOLDRUN_AGENT: identity.agent } : {}),
       // And the workspace: a commit or a record made by a shared library
@@ -1783,6 +1767,9 @@ export function agentContext(
     // asked for by name fails inside the container with "cannot find" while
     // the host path, which uses this same merged spec, works fine.
     runtime: { log: runtimeLog, error: runtimeError, executor, spec: runtimeSpec, dispose: runtime.dispose },
+    // Whole: a bound client certificate goes from here into the egress
+    // grant (runStep), which is where it is presented. Programs get theirs
+    // filtered (withoutHeldCerts) where their environments are built.
     secretEnv: { ...runtime.env, ...secretEnv },
     formatWarning: checkFormatVersion(workspaceFrontmatter(agentDir, tenant).foldrun_version).warning,
     providerEnv,
@@ -2089,6 +2076,15 @@ async function runStep(
   // AGENTS.md — every model call always; each tool call's input and output
   // unless it says `metadata`.
   const capture: "metadata" | "full" = workspaceFrontmatter(agentDir, tenant).capture === "metadata" ? "metadata" : "full";
+  // `judge:` (AGENTS.md): the model and effort every `verify: judge:` here
+  // grades with — field by field, the workspace's else the account's.
+  const { judgeModel, judgeEffort } = (() => {
+    const ws = parseJudge(readAgentsMd(workspaceRootOf(agentDir))?.data?.judge);
+    const account = parseJudge(readAgentsMd(accountDir(tenant))?.data?.judge);
+    const model = ws.model ?? account.model;
+    const effort = resolveEffort(ws.effort ?? account.effort ?? "");
+    return { judgeModel: model ? resolveModel(model) : undefined, judgeEffort: effort ?? undefined };
+  })();
   // A step that pours out content stops being recorded past this — its
   // calls keep their names, sizes and times (FOL-54 step 4).
   let capturedChars = 0;
@@ -2694,11 +2690,27 @@ async function runStep(
         readAgentsMd(accountDir(tenant))?.data?.network_enforce,
         readAgentsMd(workspaceRootOf(agentDir))?.data?.network_enforce,
       );
+      // Keyless cloud identities the step declared: the proxy may mint tokens
+      // for these audiences only, for this workload (oidc-issuer.ts).
+      // A test run withholds them as it withholds every credential
+      // (test-mode.ts): a token is a credential, however it is delivered.
+      const declaredIdentities = Object.entries(liveSecrets).filter(([, v]) => typeof v === "string" && isIdentityValue(v));
+      const identitiesHeld = testRun
+        ? withholdSecrets(Object.fromEntries(declaredIdentities), {
+            outward: scriptSpecs.some((sc) => sc.outward),
+            allow: scriptSpecs.some((sc) => sc.testMode === "allow"),
+          }).withheld
+        : [];
+      const identities = Object.fromEntries(
+        declaredIdentities.filter(([name]) => !identitiesHeld.includes(name)).map(([name, v]) => [name, { audience: identityOf(v).audience }]),
+      );
+      if (Object.keys(identities).length) grant.identities = identities;
       const lease = await platform.egress.lease({
         tenant,
         runId: runId ?? "adhoc",
         grant,
         test: testRun,
+        who: { workspace: workspaceName, agent: step.agent },
         ...(stepIndex !== undefined && stepIndex >= 0 ? { step: stepIndex } : {}),
         // tools: [peers] — the roster the proxy serves, and the inboxes a
         // colleague's message may be addressed to. Only when granted: a
@@ -2771,6 +2783,34 @@ async function runStep(
           push("info", `test: ${name} withheld from the sandbox — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from the sandbox` } });
         }
       }
+      // A keyless identity holds nothing to hand over: the sandbox gets the
+      // path of a token file the driver keeps fresh through the proxy's
+      // /identity door. Without a lease there is no door, and so no token.
+      const identityDeclared = declared.filter((name) => typeof liveSecrets[name] === "string" && isIdentityValue(liveSecrets[name]!));
+      for (const name of identityDeclared) delete sandboxSecrets[name];
+      for (const name of identitiesHeld) {
+        push("info", `test: ${name} withheld — no identity token on a test run`, { effect: { kind: "withheld", summary: `${name} withheld from the sandbox` } });
+      }
+      const identityNames = identityDeclared.filter((name) => !identitiesHeld.includes(name));
+      if (identityNames.length) {
+        if (lease) {
+          const dir = "/tmp/foldrun-identity";
+          sandboxSecrets.FOLDRUN_IDENTITIES = identityNames.join(",");
+          sandboxSecrets.FOLDRUN_IDENTITY_DIR = dir;
+          for (const name of identityNames) sandboxSecrets[`${name}_FILE`] = `${dir}/${name.toLowerCase()}.jwt`;
+          push("info", `identity: ${identityNames.join(", ")} — a short-lived token in $NAME_FILE, renewed by the platform while the step runs`);
+        } else {
+          push("error", `identity: ${identityNames.join(", ")} needs a sandboxed step on the platform — no token here`);
+        }
+      }
+      // A client certificate is the proxy's to present, to the exact sites it
+      // is bound to; never the sandbox's, unless it was saved with `export`.
+      for (const name of declared) {
+        const v = liveSecrets[name];
+        if (typeof v !== "string" || !isClientCertValue(v) || bound.includes(name) || clientCertOf(v).export) continue;
+        delete sandboxSecrets[name];
+        push("info", `secrets: ${name} is a client certificate — the egress proxy presents it to the sites it is bound to (\`foldrun secrets sites ${name} <host>\`); unbound, it stays out of the sandbox`);
+      }
       // On a test run the http tools always go through the proxy as
       // placeholders, even when a script made the step materialise: the
       // proxy is where the outward-write policy lives, and a header
@@ -2857,6 +2897,8 @@ async function runStep(
           budgetNote: stepBudgetNote,
           price: tokenPrice,
           verify: step.verify,
+          ...(judgeModel ? { judgeModel } : {}),
+          ...(judgeEffort ? { judgeEffort } : {}),
           output: step.output,
           schema: resolvedSchema ?? undefined,
           maxTurns: step.maxTurns,
@@ -2913,6 +2955,9 @@ async function runStep(
         emit: pushWatching,
         // Stamps the sandbox, so stopping this run can destroy it.
         runId,
+        // A step still waiting for room has no sandbox to destroy; the
+        // executor asks this before it creates one (pod-room.ts).
+        ...(stopRequested ? { stopRequested } : {}),
         size,
         // The pod's network, for the executor's own wall (the k8s tier's
         // network policy): a filtered pod reaches the proxy and DNS only.
@@ -3177,7 +3222,20 @@ async function runStep(
       for (const name of held.withheld) {
         push("info", `test: ${name} withheld from scripts — TEST_MODE_WITHHELD`, { effect: { kind: "withheld", summary: `${name} withheld from scripts` } });
       }
-      const stepSecrets = held.env;
+      // A client certificate not saved with `export` has no proxy here to
+      // present it: it stays out of everything this step runs — tools,
+      // consults, verify commands — not only out of the files.
+      const stepSecrets = Object.fromEntries(
+        Object.entries(held.env).filter(([name, v]) => {
+          if (typeof v === "string" && isIdentityValue(v)) {
+            push("error", `identity: ${name} needs a sandboxed step on the platform — no token here`);
+            return false;
+          }
+          if (typeof v !== "string" || !isClientCertValue(v) || clientCertOf(v).export) return true;
+          push("info", `secrets: ${name} is a client certificate — no egress proxy here to present it; save it with export to use it as files`);
+          return false;
+        }),
+      );
       // The model credential for this machine's Claude Code — the agent's
       // own provider when it names one. The host allowlist drops it, so
       // without this every in-process step ran on the machine's login.
@@ -3252,6 +3310,8 @@ async function runStep(
         budgetNote: stepBudgetNote,
         price: tokenPrice,
         verify: step.verify,
+        ...(judgeModel ? { judgeModel } : {}),
+        ...(judgeEffort ? { judgeEffort } : {}),
         output: step.output,
         schema: resolvedSchema ?? undefined,
         maxTurns: step.maxTurns,
@@ -3455,7 +3515,7 @@ export function copyTreeBytes(from: string, to: string) {
 function openApiSources(agentDir: string, tenant: string): string[] {
   const out = new Set<string>();
   try {
-    const { data } = matter(fs.readFileSync(path.join(agentDir, "agent.md"), "utf8"));
+    const { data } = configMatter(fs.readFileSync(path.join(agentDir, "agent.md"), "utf8"));
     const front = data as { apis?: unknown; tools?: unknown };
     for (const api of parseApis(front.apis)) if (api.openapi?.startsWith("https://")) out.add(api.openapi);
     const workspace = path.basename(path.resolve(agentDir, "..", ".."));
@@ -4039,7 +4099,14 @@ export function driveRun(
   opts: { parkOnApproval?: boolean; effortOverride?: string | null } = {},
 ): Promise<void> {
   drivingRuns.add(run.id);
+  // The run's egress leases are renewed while it is driven, by whichever
+  // door drove it — the queue, or a reconcile that picked it back up — so an
+  // identity token is renewed for a live step however it was started.
+  const renew = platform.egress.renew;
+  const beat = renew ? setInterval(() => void renew(run.id).catch(() => undefined), 5 * 60_000) : null;
+  beat?.unref?.();
   return driveRunInner(tenant, workspace, run, modelOverride, tags, opts).finally(() => {
+    if (beat) clearInterval(beat);
     drivingRuns.delete(run.id);
   });
 }
@@ -4080,10 +4147,30 @@ function driveRunInner(
    *
    * The flag is one-way: nothing here clears it, so the read-modify-write can
    * only ever turn a stop on, never off.
+   *
+   * And once it is on, this copy may not undo what stopRun wrote: the run
+   * stays failed, and steps that had not started stay skipped. Before
+   * 21 Oct 2026 only the flag came across, so the next save put the record
+   * back to `running` with its steps pending; a second stop then answered
+   * "failed" instead of "already failed", and the group kept starting pods
+   * (medium-desk run-mvho48wp-99e1). A step running in THIS process is left
+   * as it is: its sandbox is gone, it ends on its own, and records how.
    */
   const save = () => {
-    if (!run.stopRequested && readRun(tenant, workspace, run.id)?.stopRequested) {
-      run.stopRequested = true;
+    const disk = run.stopRequested ? null : readRun(tenant, workspace, run.id);
+    if (disk?.stopRequested) run.stopRequested = true;
+    if (run.stopRequested) {
+      if (run.status !== "completed" && run.status !== "failed") {
+        run.status = "failed";
+        run.finishedAt ??= disk?.finishedAt ?? new Date().toISOString();
+      }
+      run.parkedAt = null;
+      for (const s of run.steps) {
+        if (s.status === "pending" || s.status === "awaiting-approval") {
+          s.status = "skipped";
+          s.skipReason ??= "run stopped";
+        }
+      }
     }
     writeRun(tenant, workspace, run);
   };
@@ -4301,13 +4388,20 @@ function driveRunInner(
         for (let i = gi - 1; i >= 0; i--) if (groupData[i] !== undefined) return groupData[i];
         return undefined;
       };
-      const dataOf = (group: StepRecord[]): unknown => {
+      // A rescue's result already stands on the step it took over, so the
+      // three handoffs below read the flow's steps only: counting the rescue
+      // too handed the next group the same output twice, or a stale one
+      // from an evaluator loop's earlier cycle.
+      const flowStepsOf = (group: StepRecord[]) => group.filter((s) => !isRescueRecord(s));
+      const dataOf = (all: StepRecord[]): unknown => {
+        const group = flowStepsOf(all);
         const carrying = group.filter((s) => s.status === "completed" && s.data !== undefined);
         if (carrying.length === 0) return undefined;
         if (carrying.length === 1 && !carrying[0].item) return carrying[0].data;
         return carrying.map((s) => ({ agent: s.agent, ...(s.item ? { item: s.item } : {}), data: s.data }));
       };
-      const resultsOf = (group: StepRecord[]): string | null => {
+      const resultsOf = (all: StepRecord[]): string | null => {
+        const group = flowStepsOf(all);
         const results = group
           .filter((s) => s.status === "completed" && s.result)
           .map((s) =>
@@ -4321,8 +4415,8 @@ function driveRunInner(
       };
       // What each step concluded — its final turn — for the handoffs that
       // should read a conclusion rather than a transcript (`each: lines`).
-      const conclusionsOf = (group: StepRecord[]): string | null => {
-        const out = group
+      const conclusionsOf = (all: StepRecord[]): string | null => {
+        const out = flowStepsOf(all)
           .filter((s) => s.status === "completed" && (s.conclusion || s.result))
           .map((s) => s.conclusion || s.result!);
         return out.length ? out.join("\n") : null;
@@ -4709,7 +4803,7 @@ function driveRunInner(
           save();
           if (freshGroup.every((s) => s.status !== "pending")) {
             // everything in this group was rejected
-            if (freshGroup.some((s) => s.status === "failed" && !s.optional && !s.carriedFrom)) {
+            if (freshGroup.some((s) => s.status === "failed" && !s.optional && !s.carriedFrom && !isRescueRecord(s))) {
               run.status = "failed";
               break;
             }
@@ -4723,7 +4817,11 @@ function driveRunInner(
         // itself mid-turn at its share, so a fan-out of twenty cannot end
         // twenty steps over — the shares sum to what was left. Between
         // groups the check above still refuses to start the next one.
-        const launching = freshGroup.filter((s) => s.status === "pending");
+        // A rescue record is the on-fail of a step in this group, not a step
+        // of the flow: one left pending by a drive that died is picked up by
+        // rescueStep below, never launched alongside the group (21 Oct 2026:
+        // a resumed run launched 77 leftover fixers at once).
+        const launching = freshGroup.filter((s) => s.status === "pending" && !isRescueRecord(s));
         // tools: [peers] — what each step of the group is told about the
         // others, and the bus they share when they run in this process.
         const peerBus = localPeerBus();
@@ -4788,7 +4886,7 @@ function driveRunInner(
                 // lead's reply or for a slot — must not start it: the
                 // between-groups check runs only once this group is over,
                 // and save() is what carries the flag in from the record.
-                if (lead || pool) {
+                {
                   save();
                   if (run.stopRequested) {
                     step.status = "skipped";
@@ -4968,6 +5066,13 @@ function driveRunInner(
         // runs out of cycles below, which used to fail the run without ever
         // reaching the step's on-fail (medium-desk's claim-trimmer was
         // written for exactly that case and could never run).
+        // A rescue hears a stop the way a group step does: its model loop and
+        // its executor's wait for room both poll this (no sandbox exists yet
+        // for a stop to destroy while it waits).
+        const stopPoll = () => {
+          save();
+          return run.stopRequested === true;
+        };
         const rescueStep = async (step: StepRecord): Promise<void> => {
           if (!step.onFail) return;
           const rescuerDir = path.join(pDir, "agents", step.onFail);
@@ -4979,6 +5084,38 @@ function driveRunInner(
             });
             return;
           }
+          // One rescue per failure, across drives. A worker that died
+          // mid-rescue left an unfinished record behind it; the drive that
+          // resumes picks that one up instead of splicing another — a crash
+          // loop used to add one per restart (blog-desk run-mvh1zra7-ae4k,
+          // ~80). The leftover carries the same instruction, built from the
+          // same failure, and a running one may still hold a sandbox to
+          // re-attach to, so it is THE rescue; any further unfinished copies
+          // are superseded. A FINISHED rescue is history, never reused: it
+          // answered an earlier failure — an evaluator loop's previous cycle
+          // — not this one.
+          const leftovers: StepRecord[] = [];
+          for (let j = run.steps.indexOf(step) + 1; j < run.steps.length && isRescueRecord(run.steps[j]); j++) {
+            const r = run.steps[j];
+            if (r.agent === step.onFail && (r.rescueOf === undefined || r.rescueOf === step.agent) && (r.status === "pending" || r.status === "running")) {
+              leftovers.push(r);
+            }
+          }
+          const resumed = leftovers[0];
+          for (const r of leftovers.slice(1)) {
+            r.status = "skipped";
+            r.skipReason = "superseded on resume — one rescue per failure";
+          }
+          if (resumed) {
+            step.events.push({ t: new Date().toISOString(), type: "info", text: `resuming the rescue by [[${step.onFail}]] that an earlier drive started` });
+            resumed.rescueOf ??= step.agent;
+            resumed.attempts = Math.max(1, resumed.attempts ?? 0);
+            resumed.status = "running";
+            save();
+            await runStep(rescuerDir, tenant, resumed, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData, null, undefined, run.test === true, stopPoll, flowTimezone, run.steps.indexOf(resumed));
+            settleRescue(step, resumed);
+            return;
+          }
           step.events.push({
             t: new Date().toISOString(),
             type: "info",
@@ -4995,6 +5132,7 @@ function driveRunInner(
           const owesMarker = step.until ? `\nThe step's loop ended without its marker: your reply must end with a line saying ${step.until}.` : "";
           const rescue: StepRecord = {
             agent: step.onFail,
+            rescueOf: step.agent,
             instruction:
               `${step.instruction}\n\n<rescue>\nYou are taking this step over: [[${step.agent}]] attempted it and failed. ` +
               `Its failure, for whatever it is worth:\n${failure}${approved}${owesMarker}\n</rescue>`,
@@ -5025,7 +5163,11 @@ function driveRunInner(
           rescue.attempts = 1;
           rescue.status = "running";
           save();
-          await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData, null, undefined, run.test === true, undefined, flowTimezone, run.steps.indexOf(rescue));
+          await runStep(rescuerDir, tenant, rescue, ctx, save, modelOverride, tags, effortOverride, run.id, ctxData, null, undefined, run.test === true, stopPoll, flowTimezone, run.steps.indexOf(rescue));
+          settleRescue(step, rescue);
+        };
+        // What a rescue's end means for the step it took over.
+        const settleRescue = (step: StepRecord, rescue: StepRecord): void => {
           // runStep mutates rescue.status; widen past TS's "running" narrowing.
           const rescueOutcome: string = rescue.status;
           if (rescueOutcome === "completed") {
@@ -5057,7 +5199,10 @@ function driveRunInner(
         // here: the person chose to start after it. Counting it stopped
         // fix-desk's "re-run from the verifier" (run-muwfi5jn-4217) in under
         // a second, before the one step it was asked to run.
-        const requiredFailed = freshGroup.some((s) => s.status === "failed" && !s.optional && !s.carriedFrom);
+        // A rescue record's own failure is already on the step it took over
+        // ("also failed"); counted here it would fail a group whose step a
+        // later rescue completed.
+        const requiredFailed = freshGroup.some((s) => s.status === "failed" && !s.optional && !s.carriedFrom && !isRescueRecord(s));
         if (requiredFailed) {
           run.status = "failed";
           break;
@@ -5133,6 +5278,10 @@ function driveRunInner(
             });
             for (const s of [...ordered[gi - 1], ...freshGroup]) {
               if (s.status === "skipped" && s.each) continue; // an expanded template stays expanded
+              // A rescue answered its own failure and stays as it ended: reset
+              // to pending it read as an interrupted rescue, and the next
+              // cycle's failure was handed the old one's instructions.
+              if (isRescueRecord(s)) continue;
               s.status = "pending";
               s.result = null;
             }