@foldrun/core 0.6.0 → 1.0.0

package/src/runner.ts

@@ -3,7 +3,9 @@
 // fail without failing the flow. A direct agent run is a one-step ad-hoc
 // flow, so there is a single execution path.
 
+import { hookNames } from "./hooks.ts";
 import fs from "node:fs";
+import os from "node:os";
 import path from "node:path";
 import matter from "gray-matter";
 import { spawn } from "node:child_process";
@@ -14,8 +16,8 @@ import { resolveAgentPath } from "./confine.ts";
 import { browserPodLine, podLossDecision, RERAN_ON_FULL } from "./browser-pod.ts";
 import { eventUrl } from "./webhook.ts";
 import { runStepInContainer, sizeLimits, killRunSandboxes, type StepTiming } from "./run-container.ts";
-import { hostSafeEnv } from "./host-env.ts";
-import { resolveModelCredential, credentialLine } from "./model-credential.ts";
+import { hostSafeEnv, platformApiUrl } from "./host-env.ts";
+import { resolveModelCredential, credentialLine, accountModelBlock, isClaudeLoginToken } from "./model-credential.ts";
 import { validateSchema, describeSchemaErrors, looksLikeSchema } from "./json-schema.ts";
 import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute, type EgressGrant } from "./egress.ts";
 import { explainRefusal, isRefusalStatus, refusalFromLine, supplyNote, type SupplyState } from "./refusal.ts";
@@ -88,8 +90,10 @@ import { startTranslator, translatorSpecFor, type TranslatorSpec } from "./trans
 import { providerPreset } from "./providers.ts";
 import { buildScriptTools, parseScripts, type ExecutionContext, type ScriptSpec } from "./script-tools.ts";
 import { libraryDir, libraryTools, libraryMemoryIndex } from "./library.ts";
-import { fingerprint, mergeRuntimes, parseRuntime, prepareRuntime, type PreparedRuntime, type RuntimeSpec } from "./runtime.ts";
-import { materializeFiles, harvestFiles } from "./storage.ts";
+import { fingerprint, mergeRuntimes, parseRuntime, prepareRuntime, stepInstallEnv, type PreparedRuntime, type RuntimeSpec } from "./runtime.ts";
+import { buildInstallTool, INSTALL_TOOL } from "./install-tool.ts";
+import { dataRoot } from "./paths.ts";
+import { materializeFiles, harvestFiles, storageBaseline } from "./storage.ts";
 import { chooseExecutor, ensureImage } from "./container.ts";
 import { stampBundle } from "./okf.ts";
 import type { McpServerConfig } from "@anthropic-ai/claude-agent-sdk";
@@ -707,7 +711,7 @@ export function resolveDocLinks(text: string, workspaceRoot: string): string {
   });
 }
 
-function agentContext(
+export function agentContext(
   agentDir: string,
   tenant: string,
   tags: string[] = [],
@@ -1124,8 +1128,11 @@ function agentContext(
   //
   // The URL sits in git; the token is a ${SECRET} resolved server-side. An
   // agent's own block wins over the workspace's, like everything else here.
-  const providerBlock =
+  // Then the account's own model key (Settings → Model), when it set one:
+  // an account that brings its key need not write a block anywhere.
+  const fileBlock =
     (front.provider as Record<string, unknown> | undefined) ?? workspaceFrontmatter(agentDir, tenant).provider;
+  const providerBlock = fileBlock ?? accountModelBlock((name) => getSecret(tenant, name)?.value ?? null);
   let providerEnv: Record<string, string> = {};
   let providerLabel: string | null = null;
   // Secret values pulled into the provider block, name → value. Kept apart
@@ -1278,6 +1285,9 @@ function agentContext(
     // one — the `websearch` library tool reads it. Not a secret; rides with
     // the clock so it reaches scripts, the sandbox and verify alike.
     ...(process.env.FOLDRUN_SEARCH_URL ? { FOLDRUN_SEARCH_URL: process.env.FOLDRUN_SEARCH_URL } : {}),
+    // Where this install's own API answers (host-env.ts platformApiUrl), so a
+    // script calls `$FOLDRUN_API_URL/workspaces/…` instead of naming a host.
+    ...(platformApiUrl() ? { FOLDRUN_API_URL: platformApiUrl()! } : {}),
     // Whose index our web search asks when the agent named a search API.
     // Unset means the account's own engine. The name, never the key.
     ...(searchChoice.shape === "direct" && searchChoice.provider
@@ -1332,6 +1342,7 @@ function agentContext(
     ...(browseSettings.webgpu ? { FOLDRUN_BROWSER_WEBGPU: "1" } : {}),
     ...(browseSettings.ignore_https_errors ? { FOLDRUN_BROWSER_IGNORE_HTTPS_ERRORS: "1" } : {}),
     ...(browseSettings.state_key ? { FOLDRUN_BROWSER_STATE_KEY: browseSettings.state_key } : {}),
+    ...(browseSettings.state ? { FOLDRUN_BROWSER_STATE: browseSettings.state } : {}),
     ...(browseSettings.video ? { FOLDRUN_BROWSER_VIDEO: "1" } : {}),
     ...(browseSettings.live_view === false ? { FOLDRUN_BROWSER_LIVE_VIEW: "0" } : {}),
   };
@@ -1365,7 +1376,20 @@ function agentContext(
   );
   if (scriptTools.promptLines.length) {
     parts.push(
-      `# Tools from scripts\n\nCall these directly — they run workspace scripts for you:\n\n${scriptTools.promptLines.join("\n")}`,
+      `# Tools from scripts\n\nCall these directly — they run workspace scripts for you. Each is called by its full name, ` +
+        `\`mcp__foldrun_scripts__<name>\` (\`web\` is \`mcp__foldrun_scripts__web\`); a bare name is refused:\n\n${scriptTools.promptLines.join("\n")}`,
+    );
+  }
+  // The web tool's own description lists every action, so an agent whose
+  // search or fetch a model provider answers called web action=search,
+  // was refused, and only then used WebSearch — eight refusals in one
+  // reddit-desk validator step on 6 Oct 2026. Say it before the first call.
+  if (builtins.length) {
+    const named = builtins.map((b) => b.split("="));
+    parts.push(
+      `# Web search and fetch on this run\n\n` +
+        named.map(([action, tool]) => `- To ${action}, call **${tool}** directly. \`web\` with action=${action} is switched off on this run.`).join("\n") +
+        `\n\nThe web tool's other actions work as usual.`,
     );
   }
   if (declared.length) {
@@ -1379,7 +1403,13 @@ function agentContext(
     "Write deliverables for people to workspace/storage/ (kept and downloadable); write working text for later steps to outputs/; " +
       "what the next run needs goes to workspace/state/. Paths are relative to your agent directory, or start with workspace/ " +
       "for the workspace root (workspace/storage/ and the older ../../storage/ are the same place). There is no /tmp for you: a " +
-      "directory a tool made outside the workspace, such as a checkout, is reached only through that tool.",
+      "directory a tool made outside the workspace, such as a checkout, is reached only through that tool. " +
+      // 45 refusals in 34 runs (6–7 Oct 2026), nearly all an agent rewriting
+      // a plan or draft an earlier run left: the Write tool refuses a file
+      // that exists and has not been Read in this step, before any hook of
+      // ours can see the call. Said once here instead of learned per step.
+      "To replace a file that already exists, Read it first — Write refuses a file you have not Read in this step (a new file needs no Read). " +
+      "Built-in tools are capitalised: Read, Write, Edit, Glob, Grep, Bash.",
   );
 
   // The run's one-line summary is the first meaningful line of the last step
@@ -1449,6 +1479,12 @@ function agentContext(
     // names it (FOLDRUN_WEB_BUILTIN) rather than asking our engine.
     if (toolName === "web" && !ref.linked) {
       allowed.push(...providerWebTools);
+      // web writes what it found to files and says "Read that file" — a
+      // screenshot, a crawl, a page saved whole. An agent granted web alone
+      // was told to Read and refused it (blog-desk claim-checker, 7 Oct
+      // 2026: "Read is disabled for this session"). Reading is confined to
+      // the workspace like every file tool; disallowedTools still removes it.
+      if (!allowed.includes("Read")) allowed.push("Read");
       continue;
     }
 
@@ -1470,6 +1506,8 @@ function agentContext(
       else if (toolName === "ask") wantAsk = true;
       else wantDesks = true;
       if (available[toolName]) shadowed.push(toolName);
+    } else if (toolName === "install") {
+      // Granted to every agent anyway (below); naming it is harmless.
     } else if (BUILTIN_TOOLS.has(toolName)) {
       allowed.push(toolName);
       if (available[toolName]) shadowed.push(toolName);
@@ -1531,6 +1569,11 @@ function agentContext(
   // what it exposes, which is the unit a person actually reasons about anyway.
   for (const name of mcpNames) allowed.push(`mcp__${name}`, `mcp__${name}__*`);
 
+  // install — every agent may install what it finds it needs mid-step (apt,
+  // pip, npm), declared or not (install-tool.ts). Granted by default; taken
+  // away by disallowedTools: [install], and absent in plan mode (read-only).
+  allowed.push(INSTALL_TOOL);
+
   // Claude Code's spellings, so an agent written for it mostly runs here.
   // `permissionMode: plan` is its read-only mode; ours is the `read` group.
   if (front.permissionMode === "plan") {
@@ -1538,6 +1581,7 @@ function agentContext(
     allowed.push(...TOOL_MAP.read, ...apiTools.toolNames, ...scriptTools.toolNames);
   }
   const denied: string[] = refNames(front.disallowedTools);
+  if (denied.includes("install")) denied.push(INSTALL_TOOL);
   const finalAllowed = allowed.filter((t) => !denied.includes(t));
 
   return {
@@ -1756,6 +1800,42 @@ async function terminalAsk(question: string, options?: string[]): Promise<string
   }
 }
 
+/**
+ * What the person at the gate said, framed for the step's prompt — or "".
+ *
+ * The note is the freshest instruction the step has — later than the flow
+ * file, aimed at this exact run. When the gate asked a question (ask:), the
+ * note IS the answer and is framed as one.
+ *
+ * Both halves go through resolveDocLinks for the same reason the
+ * instruction does: this is authored text on its way to becoming a prompt,
+ * and the rule is that ALL of it expands. A person answering "which list?"
+ * with [[dead-ends]] was sending the model two literal brackets — the one
+ * reference syntax the workspace teaches, and the one place it silently did
+ * nothing. The previous step's results are NOT resolved, deliberately: that
+ * is model output, and a model that happens to emit [[brackets]] must not
+ * thereby name a file.
+ *
+ * An ask: gate approved with no note still says so. It used to add nothing,
+ * so the step never learned its question had been answered: gbp-desk's
+ * rivals sender, told "a plain approval means all offices", saw no answer
+ * at all and stopped BLOCKED (run-muu6r18y-417e, 2026-10-05).
+ */
+export function operatorBlock(step: Pick<StepRecord, "ask" | "approvalNote" | "approvedAt">, workspaceRoot: string): string {
+  const ask = step.ask ? resolveDocLinks(step.ask, workspaceRoot) : "";
+  if (step.approvalNote) {
+    const note = resolveDocLinks(step.approvalNote, workspaceRoot);
+    return step.ask
+      ? `\n\n<operator_answer>\nThis step asked a human: ${ask}\nTheir answer:\n${note}\n</operator_answer>`
+      : `\n\n<operator_guidance>\nThe human who approved this step added:\n${note}\n</operator_guidance>`;
+  }
+  if (step.ask && step.approvedAt) {
+    return `\n\n<operator_answer>\nThis step asked a human: ${ask}\nTheir answer: they approved it with no note — a plain yes.\n</operator_answer>`;
+  }
+  return "";
+}
+
+
 async function runStep(
   agentDir: string,
   tenant: string,
@@ -1807,6 +1887,7 @@ async function runStep(
   // cover a script's own stdout.
   let redactions: [string, string][] = [];
   let fileDir: string | null = null;
+  let installDir: string | null = null;
   let disposeRuntime: (() => void) | undefined;
   // Can this step's tools act outside the workspace? Set once its context is
   // read; the verdict at the end of this function reads it.
@@ -1963,6 +2044,29 @@ async function runStep(
       const line = credentialLine(credential);
       push(line.type, line.text);
     }
+    // A hosted platform runs only its operator's accounts on its own model
+    // credential; every other account brings its own API key. Refused here,
+    // before a sandbox is rented, with what to do about it — not run on the
+    // operator's key, and not failed later by a provider.
+    if (!platform.platformKeyAllowed(tenant)) {
+      const own = Object.values(providerSecrets).concat(Object.values(providerEnv));
+      if (!providerLabel) {
+        push(
+          "error",
+          `no model key: this account runs on its own API key, and none is set. Add one in Settings → Model ` +
+            `(or \`foldrun model set <provider> --key …\`), or write a provider: block in AGENTS.md`,
+        );
+        step.status = "failed";
+        save();
+        return;
+      }
+      if (own.some((v) => isClaudeLoginToken(v))) {
+        push("error", "model key: a Claude login token (sk-ant-oat…) cannot be used here — use an API key from your provider");
+        step.status = "failed";
+        save();
+        return;
+      }
+    }
     for (const w of providerWarnings) push("error", w);
     // A level that wrote a language nobody can read was skipped, not obeyed.
     for (const l of language.lines) push("info", l);
@@ -2143,28 +2247,7 @@ async function runStep(
       // happening to contain brackets.
       prompt += `\n\n<event>\nThis step waited for an external event. What arrived:\n${step.eventPayload}\n</event>`;
     }
-    if (step.approvalNote) {
-      // The person at the gate said something while letting the run through.
-      // That is the freshest instruction the step has — later than the flow
-      // file, aimed at this exact run. When the gate asked a question
-      // (ask:), the note IS the answer and is framed as one.
-      //
-      // Both halves go through resolveDocLinks for the same reason the
-      // instruction above does: this is authored text on its way to becoming
-      // a prompt, and the rule is that ALL of it expands. A person answering
-      // "which list?" with [[dead-ends]] was sending the model two literal
-      // brackets — the one reference syntax the workspace teaches, and the
-      // one place it silently did nothing.
-      //
-      // What is NOT resolved, deliberately: the previous step's results
-      // above. That is model output, not something a person wrote, and a
-      // model that happens to emit [[brackets]] must not thereby name a file.
-      const ask = step.ask ? resolveDocLinks(step.ask, workspaceRoot) : "";
-      const note = resolveDocLinks(step.approvalNote, workspaceRoot);
-      prompt += step.ask
-        ? `\n\n<operator_answer>\nThis step asked a human: ${ask}\nTheir answer:\n${note}\n</operator_answer>`
-        : `\n\n<operator_guidance>\nThe human who approved this step added:\n${note}\n</operator_guidance>`;
-    }
+    prompt += operatorBlock(step, workspaceRoot);
     if (step.delegate?.length) {
       // The chooser's contract. The allowed set comes from the flow file;
       // the runner parses exactly this shape back out of the result, so the
@@ -2210,6 +2293,9 @@ async function runStep(
     // watched on both paths, because a refusal reads the same from a pod
     // and from this process.
     let lastRefusal = "";
+    // The step's model connection was cut (isModelConnectionLost): the
+    // platform's side going away under the step, not the step failing.
+    let lastDrop = "";
     // And WHY it refused, in words, when anything on the way saw the
     // headers that say so: the translator here, the egress proxy over
     // there. Kept apart from `lastRefusal` — that one decides whether to
@@ -2217,6 +2303,7 @@ async function runStep(
     let refusalWhy = "";
     const pushWatching: typeof push = (type, text, extra) => {
       if (type === "error" && isProviderRefusal(text)) lastRefusal = text;
+      if (type === "error" && isModelConnectionLost(text)) lastDrop = text;
       const why = refusalFromLine(text);
       if (why) refusalWhy = why;
       push(type, text, extra);
@@ -2270,6 +2357,17 @@ async function runStep(
     const limitOwners = toolOwners(apiTools.owners, scriptSpecs);
     const limitOpts = Object.keys(stepLimits).length ? { limits: stepLimits, toolOwners: limitOwners } : {};
     if (limitOpts.limits) push("info", `limits: ${Object.entries(stepLimits).map(([k, n]) => `${k} ${n}`).join(", ")} — per step; a call past one is refused`);
+    // `hooks:` — the account's, the workspace's, then the agent's own, each
+    // once; resolved and run inside the step (hooks.ts, step-exec.ts).
+    const stepHookNames = [...new Set([
+      ...hookNames(readAgentsMd(accountDir(tenant))?.data?.hooks),
+      ...hookNames(readAgentsMd(workspaceRootOf(agentDir))?.data?.hooks),
+      ...hookNames(front.hooks),
+    ])];
+    const hookOpts = stepHookNames.length
+      ? { hookNames: stepHookNames, hookBase: { run_id: runId ?? null, step: stepIndex ?? null, agent: step.agent, workspace: path.basename(workspaceRootOf(agentDir)) } }
+      : {};
+    if (stepHookNames.length) push("info", `hooks: ${stepHookNames.join(", ")}`);
 
     const isolation = process.env.FOLDRUN_RUN_ISOLATION;
     if (isolatedRun()) {
@@ -2436,6 +2534,7 @@ async function runStep(
           schema: resolvedSchema ?? undefined,
           maxTurns: step.maxTurns,
           ...limitOpts,
+          ...hookOpts,
           translator: keyName === MODEL_KEY_NAME ? primaryTranslator : secondTranslator,
           // Outward steps keep a browser of their own: an executor that
           // could browse from the slim image through the account's pod
@@ -2511,6 +2610,7 @@ async function runStep(
         lastArgs = a;
         return runIsolated(a);
       };
+      let attemptFrom = step.events.length;
       let outcome = await runTracked(firstArgs);
       step.sandbox = null; // whatever happens next starts its own
       // A 401 from a token that rotated mid-step is not a broken key: the
@@ -2548,6 +2648,48 @@ async function runStep(
         outcome = await runTracked(again);
         outcome = withEarlierTiming(outcome, previous);
       }
+      // The model connection was cut under the step — the egress proxy or the
+      // worker went away (6 Oct 2026: a worker restarted every 4.5 minutes and
+      // every desk running failed on "Connection refused"). That is the
+      // platform's fault, so the step runs again, whatever its `retry:` —
+      // unless it can act outside the workspace and its tools already ran:
+      // the post may be up, and a second attempt would post it again.
+      for (let n = 1; n <= DROP_RETRIES && outcome.status === "failed" && lastDrop; n++) {
+        if (!dropRetrySafe(canActOutward, step.events.slice(attemptFrom))) {
+          push("info", `the model connection dropped after this step's tools ran — not retried: they can act outside the workspace, and another attempt could repeat what they did`);
+          break;
+        }
+        const wait = Math.round(retryBaseMs() * 2 * n);
+        push("info", `the model connection dropped (${lastDrop.slice(0, 80)}) — the platform's side, not the step's; running the step again in ${Math.round(wait / 1000)}s (${n} of ${DROP_RETRIES})`);
+        await sleep(wait);
+        lastDrop = "";
+        lastRefusal = "";
+        attemptFrom = step.events.length;
+        const previous = outcome.timing;
+        const again = isolatedArgs(platformModelEnv());
+        await lease?.commit();
+        outcome = withEarlierTiming(await runTracked(again), previous);
+      }
+      // The model wrote its tool call out as text — `<invoke name="…">` in
+      // its last message — and stopped. Nothing ran, so nothing was done:
+      // seo-digest's editor three times (27 Sep, 4 and 5 Oct 2026), its own
+      // retry included, because the retry asked the same way. Run it once
+      // more with that said plainly, on the same safety rule as a dropped
+      // connection: never again after an outward tool already ran.
+      for (let n = 1; n <= TEXT_CALL_RETRIES && wroteToolCallAsText(attemptText(outcome.result, step.events.slice(attemptFrom))); n++) {
+        if (!dropRetrySafe(canActOutward, step.events.slice(attemptFrom))) {
+          push("info", "the model wrote a tool call as text instead of making it — not run again: this step's tools can act outside the workspace and already ran");
+          break;
+        }
+        push("info", `the model wrote a tool call as text instead of making it, so nothing ran — running the step again with a note (${n} of ${TEXT_CALL_RETRIES})`);
+        attemptFrom = step.events.length;
+        lastRefusal = "";
+        const previous = outcome.timing;
+        const base = isolatedArgs(platformModelEnv());
+        const again = { ...base, input: { ...base.input, prompt: `${base.input.prompt}\n\n${TEXT_CALL_NOTE}` } };
+        await lease?.commit();
+        outcome = withEarlierTiming(await runTracked(again), previous);
+      }
       // The second supply, tried exactly once, and only when the primary
       // refused over money/auth/limits rather than the work failing. The
       // retry is a fresh sandbox — the failed one is gone, and driveRun's
@@ -2706,6 +2848,20 @@ async function runStep(
       // finally — once, however many attempts follow.
       const mat = materializeFileSecrets(agentDir, stepSecrets);
       fileDir = mat.dir;
+      // install: pip and npm land in a directory of this step's own, gone in
+      // the finally; apt has no helper on a bare host and says so.
+      installDir = allowed.includes(INSTALL_TOOL) ? fs.mkdtempSync(path.join(os.tmpdir(), "foldrun-step-")) : null;
+      const installEnv = installDir ? stepInstallEnv(installDir, { ...process.env, ...mat.env }) : {};
+      const installTools = installDir
+        ? buildInstallTool({
+            dir: installDir,
+            socket: null,
+            cacheRoot: path.join(dataRoot(), tenant, ".runtimes"),
+            python: mat.env.VIRTUAL_ENV ? path.join(mat.env.VIRTUAL_ENV, "bin", "python") : "python3",
+            env: { ...hostSafeEnv(), ...mat.env, ...installEnv },
+            emit: (type, text) => push(type, text),
+          })
+        : null;
       const attemptInProcess = (modelEnv: Record<string, string | undefined>) => executeStep({
         agentDir,
         workspaceRoot,
@@ -2725,6 +2881,7 @@ async function runStep(
           ...(historyTools.server ? { foldrun_history: historyTools.server } : {}),
           ...(deskTools.server ? { foldrun_desks: deskTools.server } : {}),
           ...(askTools ? { foldrun_ask: askTools.server } : {}),
+          ...(installTools ? { foldrun_install: installTools.server } : {}),
           ...mcpServers,
         },
         // Declared secrets reach the agent's scripts as env vars; the model
@@ -2732,7 +2889,7 @@ async function runStep(
         // the allowlisted host environment, not process.env: the SDK's
         // Bash tool runs with this, and on a platform process.env holds
         // every tenant's root key and the datastore credentials.
-        env: { ...hostSafeEnv(), ...clockEnv, ...(testRun ? testModeEnv() : {}), ...mat.env, ...modelEnv },
+        env: { ...hostSafeEnv(), ...clockEnv, ...(testRun ? testModeEnv() : {}), ...mat.env, ...installEnv, ...modelEnv },
         timeoutSec: step.timeout,
         budgetUsd: stepBudgetUsd,
         budgetNote: stepBudgetNote,
@@ -2742,6 +2899,7 @@ async function runStep(
         schema: resolvedSchema ?? undefined,
         maxTurns: step.maxTurns,
         ...limitOpts,
+        ...hookOpts,
         stopRequested,
         // What the scripts saw, the verify sees: a flow can then check that
         // the proof a step left names THIS run, not one that came before —
@@ -2840,6 +2998,7 @@ async function runStep(
   } finally {
     try {
       cleanupFileSecrets(fileDir);
+      if (installDir) fs.rmSync(installDir, { recursive: true, force: true });
     } catch {
       // best-effort — a leftover scratch file is swept with the run's outputs
     }
@@ -2996,8 +3155,12 @@ function platformFallbackEnv(): Record<string, string> | null {
 const CREDENTIAL_ROTATION_WAIT_MS = 90_000;
 const CREDENTIAL_POLL_MS = 2_000;
 
-function isAuthRefusal(text: string): boolean {
-  return /API Error: 401|access token has (been revoked|expired)/i.test(text);
+/** A refusal of the credential itself, in any of the wordings Claude Code
+ *  has used. "Failed to authenticate: OAuth token revoked" (9 Oct 2026, a
+ *  step started one second before the 09:07:52Z refresh) matched none of the
+ *  older two, so the step failed instead of waiting for the new token. */
+export function isAuthRefusal(text: string): boolean {
+  return /API Error: 401|access token has (been revoked|expired)|OAuth token (has )?(been )?(revoked|expired)|Failed to authenticate/i.test(text);
 }
 
 /**
@@ -3048,6 +3211,49 @@ function isProviderRefusal(text: string): boolean {
   );
 }
 
+/** A step's own model connection cut: the egress proxy or the worker it
+ *  runs through went away. Only the run's final error, never a tool's — a
+ *  WebFetch that could not connect is the step's business, not ours. */
+export function isModelConnectionLost(text: string): boolean {
+  return /^Claude Code returned an error result: API Error: (Connection (refused|dropped|error)|.*\b(ECONNREFUSED|ECONNRESET|socket hang up)\b)/i.test(text);
+}
+
+/** A tool call written out as text rather than made: the model's last
+ *  message holds `<invoke name=…>` / `<function_calls>` markup. Only the tail
+ *  counts — a reply that quotes such markup mid-way and goes on is fine. */
+export function wroteToolCallAsText(text: string): boolean {
+  const tail = text.slice(-1500);
+  const open = tail.search(/<(?:antml:)?(?:invoke|function_calls)\b[^>]*>/);
+  if (open < 0) return false;
+  // Quoted mid-reply and then the reply carries on: not a stalled call.
+  const after = tail.slice(open).replace(/<\/(?:antml:)?(?:invoke|function_calls)>/g, "\u0000");
+  const lastClose = after.lastIndexOf("\u0000");
+  return lastClose < 0 || after.slice(lastClose + 1).trim().length < 200;
+}
+
+/** What an attempt said last: its result, or — when it ended without one —
+ *  the error that carried the model's final text. */
+function attemptText(result: string | null | undefined, events: { type: string; text?: string }[]): string {
+  if (result && result.trim()) return result;
+  const lastError = [...events].reverse().find((e) => e.type === "error");
+  return lastError?.text ?? "";
+}
+
+/** Runs again after a tool call written as text. */
+export const TEXT_CALL_RETRIES = 1;
+export const TEXT_CALL_NOTE =
+  "Note from the platform: your last attempt ended by writing a tool call out as text (an <invoke …> block), so it was not run and nothing happened. " +
+  "Make every tool call as a real tool call — never type its markup into your message.";
+
+/** How many times a step whose model connection was cut runs again. */
+export const DROP_RETRIES = 2;
+
+/** May a step whose connection dropped run again? Yes, unless its tools can
+ *  act outward and one ran in this attempt (actedThenFailedCheck's rule). */
+export function dropRetrySafe(canActOutward: boolean, attemptEvents: { type: string }[]): boolean {
+  return !(canActOutward && attemptEvents.some((e) => e.type === "tool"));
+}
+
 /** How many times a busy provider is given another moment, and the longest
  *  any one wait may be. Three is the industry's habit and the point where
  *  waiting longer stops being kinder than failing. */
@@ -3637,6 +3843,32 @@ function driveRunInner(
       //
       // Runs on resume too — a parked run's container is long gone, and the
       // one it comes back in needs the same files the first one had.
+      //
+      // A run re-driven after its worker died is the exception to "the store
+      // is newer than the disk": a park harvests before it parks, a crash does
+      // not, so what this run's finished steps wrote to storage/ is only on
+      // disk, and materializing first wrote the store's older copy over it.
+      // On 6 Oct 2026 a worker restart put reddit-desk's 2 Oct post back over
+      // that morning's draft, and a rerun from the validator would have
+      // posted it a second time. So a run that has been here before (its
+      // baseline is on disk) stores what it changed first; files it did not
+      // touch still come fresh from the store.
+      const baselineFile = path.join(runRoot, "runs", run.id, "storage-baseline.json");
+      if (fs.existsSync(baselineFile)) {
+        try {
+          const changedFrom = JSON.parse(fs.readFileSync(baselineFile, "utf8")) as Record<string, string>;
+          const { saved } = await harvestFiles(tenant, workspace, `run:${run.id}`, { changedFrom });
+          if (saved.length) {
+            run.steps[0]?.events.push({
+              t: new Date().toISOString(),
+              type: "info",
+              text: `files: kept ${saved.join(", ")} written before the run was picked up again`,
+            });
+          }
+        } catch {
+          // the materialize below still runs; at worst this is the old behaviour
+        }
+      }
       try {
         const brought = await materializeFiles(tenant, workspace);
         if (brought.length) {
@@ -3646,6 +3878,8 @@ function driveRunInner(
             text: `files: ${brought.length} brought into the workspace`,
           });
         }
+        fs.mkdirSync(path.dirname(baselineFile), { recursive: true });
+        fs.writeFileSync(baselineFile, JSON.stringify(storageBaseline(tenant, workspace)));
       } catch (err) {
         // A file store that is down must not take the run with it: most flows
         // never touch storage/, and the ones that do will fail their own verify.
@@ -4111,7 +4345,7 @@ function driveRunInner(
           save();
           if (freshGroup.every((s) => s.status !== "pending")) {
             // everything in this group was rejected
-            if (freshGroup.some((s) => s.status === "failed" && !s.optional)) {
+            if (freshGroup.some((s) => s.status === "failed" && !s.optional && !s.carriedFrom)) {
               run.status = "failed";
               break;
             }
@@ -4353,8 +4587,13 @@ function driveRunInner(
             effort: step.effort,
             timeout: step.timeout,
             // The rescuer inherits the contract, not just the job: a step
-            // that promised data still owes data when someone else does it.
+            // that promised data still owes data when someone else does it,
+            // and a step that had to pass a check still has to pass it — a
+            // rescue whose result stood unchecked was a way round the very
+            // check that failed (found 7 Oct 2026, adding rescuers to
+            // medium-desk).
             output: step.output,
+            ...(step.verify ? { verify: step.verify } : {}),
             attempts: 0,
             status: "pending",
             events: [],
@@ -4389,7 +4628,11 @@ function driveRunInner(
           save();
         }
 
-        const requiredFailed = freshGroup.some((s) => s.status === "failed" && !s.optional);
+        // A step carried into a re-run failed in the run it came from, not
+        // here: the person chose to start after it. Counting it stopped
+        // fix-desk's "re-run from the verifier" (run-muwfi5jn-4217) in under
+        // a second, before the one step it was asked to run.
+        const requiredFailed = freshGroup.some((s) => s.status === "failed" && !s.optional && !s.carriedFrom);
         if (requiredFailed) {
           run.status = "failed";
           break;