@foldrun/core 0.6.0 → 0.7.0

package/src/provider-health.ts

@@ -18,6 +18,7 @@
 // It reads the same provider block a run reads, resolved the same way, so a
 // pass here means a run would reach the same place with the same key.
 
+import { accountModelBlock } from "./model-credential.ts";
 import fs from "node:fs";
 import path from "node:path";
 import matter from "gray-matter";
@@ -236,6 +237,8 @@ export function providersInUse(tenant: string): ProviderInUse[] {
   };
 
   add(frontmatterOf(path.join(accountDir(tenant), "AGENTS.md")).provider, null, "account");
+  // The account's own key from Settings → Model, which needs no block.
+  add(accountModelBlock((name) => getSecret(tenant, name)?.value ?? null), null, "account (Settings → Model)");
   for (const ws of safeList(() => listWorkspaces(tenant))) {
     const dir = workspaceDir(tenant, ws.name);
     add(frontmatterOf(path.join(dir, "AGENTS.md")).provider, ws.name, ws.name);

package/src/run-container.ts

@@ -28,7 +28,7 @@ import { isPlatformPath, type ApiSpec, type Effort } from "./store.ts";
 import { divertedSummary, isDivertedPath, overlayTestWrites } from "./test-mode.ts";
 import type { EventExtra } from "./step-exec.ts";
 import { isFileValue, fileContent } from "./secrets.ts";
-import { safeTenantSegment, type RuntimeSpec } from "./runtime.ts";
+import { safeTenantSegment, systemInstallSh, SYSTEM_FROM_INPUT_SH, INSTALL_SOCKET, type RuntimeSpec } from "./runtime.ts";
 import type { ScriptSpec } from "./script-tools.ts";
 import type { ConsultSpec } from "./agent-tools.ts";
 import type { SubagentSpec } from "./subagents.ts";
@@ -293,18 +293,37 @@ export function mergeAppends(was: Buffer | string | null, host: Buffer, next: Bu
   let common = 0;
   const max = Math.min(h.length, n.length);
   while (common < max && h.charCodeAt(common) === n.charCodeAt(common)) common++;
-  for (let end = common; end >= 0; end--) {
-    if (end > 0 && h[end - 1] !== "\n") continue;
-    const prefix = h.slice(0, end);
-    const matches =
-      typeof was === "string" ? sha(Buffer.from(prefix, "utf8")) === was : Buffer.from(prefix, "utf8").equals(was);
-    if (!matches) continue;
-    const hostAdded = h.slice(end);
-    const stepAdded = n.slice(end);
-    const joined = hostAdded && !hostAdded.endsWith("\n") ? hostAdded + "\n" : hostAdded;
-    return Buffer.from(prefix + joined + stepAdded, "utf8");
+  // One pass over the shared prefix. Rebuilding and hashing the prefix at
+  // every line boundary, from the longest down, was quadratic: on 6 Oct 2026
+  // a conflicted file of a few MB held the worker's event loop for minutes,
+  // the liveness probe killed it every 4.5 minutes, and every step on the box
+  // lost its model connection.
+  let end = -1;
+  if (typeof was === "string") {
+    const run = crypto.createHash("sha256");
+    if (run.copy().digest("hex") === was) end = 0;
+    let from = 0;
+    for (let nl = h.indexOf("\n"); nl !== -1 && nl < common; nl = h.indexOf("\n", nl + 1)) {
+      run.update(h.slice(from, nl + 1), "utf8");
+      from = nl + 1;
+      if (run.copy().digest("hex") === was) end = from;
+    }
+  } else {
+    // The base's bytes are known, so it can only end at their length.
+    const base = was.toString("utf8");
+    if (
+      base.length <= common &&
+      (base.length === 0 || base.endsWith("\n")) &&
+      h.startsWith(base) &&
+      Buffer.from(base, "utf8").equals(was)
+    )
+      end = base.length;
   }
-  return null;
+  if (end < 0) return null;
+  const hostAdded = h.slice(end);
+  const stepAdded = n.slice(end);
+  const joined = hostAdded && !hostAdded.endsWith("\n") ? hostAdded + "\n" : hostAdded;
+  return Buffer.from(h.slice(0, end) + joined + stepAdded, "utf8");
 }
 
 export function applyContainerChanges(
@@ -408,6 +427,13 @@ export function applyContainerChanges(
  */
 export const RUNTIME_CACHE = "/home/agent/.foldrun/runner/.runtimes";
 
+/** Starts the install helper as root, in the background, before the drop to
+ *  the agent user: the root half of the `install` tool (install-helper.ts).
+ *  The agent reaches it only through the socket; it installs only validated
+ *  Debian package names, and dies with the container. */
+export const INSTALL_HELPER_SH = `FOLDRUN_INSTALL_SOCKET=${INSTALL_SOCKET}; export FOLDRUN_INSTALL_SOCKET
+node /opt/runner/node_modules/@foldrun/core/dist/src/install-helper.js "$FOLDRUN_INSTALL_SOCKET" ${RUNTIME_CACHE} >/dev/null 2>&1 &`;
+
 /**
  * Where the platform's shared runtime layer is mounted, read-only, in a run
  * pod — and read-write, one entry at a time, in the pod that builds an entry
@@ -548,6 +574,25 @@ try {
   // meet the packages its runtime declared, not the image's bare python.
   Object.assign(env, runtime.env);
 
+  // install: what the agent finds it needs mid-step. apt goes through the
+  // root start-up's helper (install-helper.ts); pip and npm land in a
+  // directory of this container's own, on the paths from the first turn.
+  const { buildInstallTool, INSTALL_TOOL } = await import("@foldrun/core/install-tool");
+  const { stepInstallEnv, INSTALL_SOCKET } = await import("@foldrun/core/runtime");
+  const installDir = "/home/agent/.foldrun/step";
+  const installing = (input.allowed ?? []).includes(INSTALL_TOOL);
+  if (installing) Object.assign(env, stepInstallEnv(installDir, env));
+  const install = installing
+    ? buildInstallTool({
+        dir: installDir,
+        socket: env.FOLDRUN_INSTALL_SOCKET || INSTALL_SOCKET,
+        cacheRoot: "${RUNTIME_CACHE}",
+        python: env.VIRTUAL_ENV ? env.VIRTUAL_ENV + "/bin/python" : "python3",
+        env,
+        emit,
+      })
+    : { server: null };
+
   // API and script tools, rebuilt in here from their specs. Secrets were
   // substituted into API headers before the input crossed the boundary, so
   // buildApiTools gets an already-resolved environment. verify: gets the
@@ -598,6 +643,7 @@ try {
       ...(history.server ? { foldrun_history: history.server } : {}),
       ...(desks.server ? { foldrun_desks: desks.server } : {}),
       ...(ask.server ? { foldrun_ask: ask.server } : {}),
+      ...(install.server ? { foldrun_install: install.server } : {}),
       ...input.mcpServers,
     },
     ...(inbox ? { inbox } : {}),
@@ -893,7 +939,14 @@ RUN node -e ' \\
 FROM browsers AS full
 ${CORE_INSTALL}`;
 
+// runtime: system: — apt as root, before the drop (runtime.ts systemInstallSh).
+// set -e is off around it: a failed install is reported by the driver, which
+// reads the status file and fails the step with apt's own words.
 const ENTRY = `#!/bin/sh
+set +e
+${SYSTEM_FROM_INPUT_SH}
+${systemInstallSh(RUNTIME_CACHE)}
+${INSTALL_HELPER_SH}
 set -e
 chown -R agent:agent /workspace /library /opt/runner/job
 # The runtime cache, when one is mounted. NOT recursive: a warm cache is tens

package/src/runner.ts

@@ -4,6 +4,7 @@
 // flow, so there is a single execution path.
 
 import fs from "node:fs";
+import os from "node:os";
 import path from "node:path";
 import matter from "gray-matter";
 import { spawn } from "node:child_process";
@@ -14,8 +15,8 @@ import { resolveAgentPath } from "./confine.ts";
 import { browserPodLine, podLossDecision, RERAN_ON_FULL } from "./browser-pod.ts";
 import { eventUrl } from "./webhook.ts";
 import { runStepInContainer, sizeLimits, killRunSandboxes, type StepTiming } from "./run-container.ts";
-import { hostSafeEnv } from "./host-env.ts";
-import { resolveModelCredential, credentialLine } from "./model-credential.ts";
+import { hostSafeEnv, platformApiUrl } from "./host-env.ts";
+import { resolveModelCredential, credentialLine, accountModelBlock, isClaudeLoginToken } from "./model-credential.ts";
 import { validateSchema, describeSchemaErrors, looksLikeSchema } from "./json-schema.ts";
 import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute, type EgressGrant } from "./egress.ts";
 import { explainRefusal, isRefusalStatus, refusalFromLine, supplyNote, type SupplyState } from "./refusal.ts";
@@ -88,8 +89,10 @@ import { startTranslator, translatorSpecFor, type TranslatorSpec } from "./trans
 import { providerPreset } from "./providers.ts";
 import { buildScriptTools, parseScripts, type ExecutionContext, type ScriptSpec } from "./script-tools.ts";
 import { libraryDir, libraryTools, libraryMemoryIndex } from "./library.ts";
-import { fingerprint, mergeRuntimes, parseRuntime, prepareRuntime, type PreparedRuntime, type RuntimeSpec } from "./runtime.ts";
-import { materializeFiles, harvestFiles } from "./storage.ts";
+import { fingerprint, mergeRuntimes, parseRuntime, prepareRuntime, stepInstallEnv, type PreparedRuntime, type RuntimeSpec } from "./runtime.ts";
+import { buildInstallTool, INSTALL_TOOL } from "./install-tool.ts";
+import { dataRoot } from "./paths.ts";
+import { materializeFiles, harvestFiles, storageBaseline } from "./storage.ts";
 import { chooseExecutor, ensureImage } from "./container.ts";
 import { stampBundle } from "./okf.ts";
 import type { McpServerConfig } from "@anthropic-ai/claude-agent-sdk";
@@ -707,7 +710,7 @@ export function resolveDocLinks(text: string, workspaceRoot: string): string {
   });
 }
 
-function agentContext(
+export function agentContext(
   agentDir: string,
   tenant: string,
   tags: string[] = [],
@@ -1124,8 +1127,11 @@ function agentContext(
   //
   // The URL sits in git; the token is a ${SECRET} resolved server-side. An
   // agent's own block wins over the workspace's, like everything else here.
-  const providerBlock =
+  // Then the account's own model key (Settings → Model), when it set one:
+  // an account that brings its key need not write a block anywhere.
+  const fileBlock =
     (front.provider as Record<string, unknown> | undefined) ?? workspaceFrontmatter(agentDir, tenant).provider;
+  const providerBlock = fileBlock ?? accountModelBlock((name) => getSecret(tenant, name)?.value ?? null);
   let providerEnv: Record<string, string> = {};
   let providerLabel: string | null = null;
   // Secret values pulled into the provider block, name → value. Kept apart
@@ -1278,6 +1284,9 @@ function agentContext(
     // one — the `websearch` library tool reads it. Not a secret; rides with
     // the clock so it reaches scripts, the sandbox and verify alike.
     ...(process.env.FOLDRUN_SEARCH_URL ? { FOLDRUN_SEARCH_URL: process.env.FOLDRUN_SEARCH_URL } : {}),
+    // Where this install's own API answers (host-env.ts platformApiUrl), so a
+    // script calls `$FOLDRUN_API_URL/workspaces/…` instead of naming a host.
+    ...(platformApiUrl() ? { FOLDRUN_API_URL: platformApiUrl()! } : {}),
     // Whose index our web search asks when the agent named a search API.
     // Unset means the account's own engine. The name, never the key.
     ...(searchChoice.shape === "direct" && searchChoice.provider
@@ -1365,7 +1374,20 @@ function agentContext(
   );
   if (scriptTools.promptLines.length) {
     parts.push(
-      `# Tools from scripts\n\nCall these directly — they run workspace scripts for you:\n\n${scriptTools.promptLines.join("\n")}`,
+      `# Tools from scripts\n\nCall these directly — they run workspace scripts for you. Each is called by its full name, ` +
+        `\`mcp__foldrun_scripts__<name>\` (\`web\` is \`mcp__foldrun_scripts__web\`); a bare name is refused:\n\n${scriptTools.promptLines.join("\n")}`,
+    );
+  }
+  // The web tool's own description lists every action, so an agent whose
+  // search or fetch a model provider answers called web action=search,
+  // was refused, and only then used WebSearch — eight refusals in one
+  // reddit-desk validator step on 6 Oct 2026. Say it before the first call.
+  if (builtins.length) {
+    const named = builtins.map((b) => b.split("="));
+    parts.push(
+      `# Web search and fetch on this run\n\n` +
+        named.map(([action, tool]) => `- To ${action}, call **${tool}** directly. \`web\` with action=${action} is switched off on this run.`).join("\n") +
+        `\n\nThe web tool's other actions work as usual.`,
     );
   }
   if (declared.length) {
@@ -1379,7 +1401,13 @@ function agentContext(
     "Write deliverables for people to workspace/storage/ (kept and downloadable); write working text for later steps to outputs/; " +
       "what the next run needs goes to workspace/state/. Paths are relative to your agent directory, or start with workspace/ " +
       "for the workspace root (workspace/storage/ and the older ../../storage/ are the same place). There is no /tmp for you: a " +
-      "directory a tool made outside the workspace, such as a checkout, is reached only through that tool.",
+      "directory a tool made outside the workspace, such as a checkout, is reached only through that tool. " +
+      // 45 refusals in 34 runs (6–7 Oct 2026), nearly all an agent rewriting
+      // a plan or draft an earlier run left: the Write tool refuses a file
+      // that exists and has not been Read in this step, before any hook of
+      // ours can see the call. Said once here instead of learned per step.
+      "To replace a file that already exists, Read it first — Write refuses a file you have not Read in this step (a new file needs no Read). " +
+      "Built-in tools are capitalised: Read, Write, Edit, Glob, Grep, Bash.",
   );
 
   // The run's one-line summary is the first meaningful line of the last step
@@ -1449,6 +1477,12 @@ function agentContext(
     // names it (FOLDRUN_WEB_BUILTIN) rather than asking our engine.
     if (toolName === "web" && !ref.linked) {
       allowed.push(...providerWebTools);
+      // web writes what it found to files and says "Read that file" — a
+      // screenshot, a crawl, a page saved whole. An agent granted web alone
+      // was told to Read and refused it (blog-desk claim-checker, 7 Oct
+      // 2026: "Read is disabled for this session"). Reading is confined to
+      // the workspace like every file tool; disallowedTools still removes it.
+      if (!allowed.includes("Read")) allowed.push("Read");
       continue;
     }
 
@@ -1470,6 +1504,8 @@ function agentContext(
       else if (toolName === "ask") wantAsk = true;
       else wantDesks = true;
       if (available[toolName]) shadowed.push(toolName);
+    } else if (toolName === "install") {
+      // Granted to every agent anyway (below); naming it is harmless.
     } else if (BUILTIN_TOOLS.has(toolName)) {
       allowed.push(toolName);
       if (available[toolName]) shadowed.push(toolName);
@@ -1531,6 +1567,11 @@ function agentContext(
   // what it exposes, which is the unit a person actually reasons about anyway.
   for (const name of mcpNames) allowed.push(`mcp__${name}`, `mcp__${name}__*`);
 
+  // install — every agent may install what it finds it needs mid-step (apt,
+  // pip, npm), declared or not (install-tool.ts). Granted by default; taken
+  // away by disallowedTools: [install], and absent in plan mode (read-only).
+  allowed.push(INSTALL_TOOL);
+
   // Claude Code's spellings, so an agent written for it mostly runs here.
   // `permissionMode: plan` is its read-only mode; ours is the `read` group.
   if (front.permissionMode === "plan") {
@@ -1538,6 +1579,7 @@ function agentContext(
     allowed.push(...TOOL_MAP.read, ...apiTools.toolNames, ...scriptTools.toolNames);
   }
   const denied: string[] = refNames(front.disallowedTools);
+  if (denied.includes("install")) denied.push(INSTALL_TOOL);
   const finalAllowed = allowed.filter((t) => !denied.includes(t));
 
   return {
@@ -1756,6 +1798,42 @@ async function terminalAsk(question: string, options?: string[]): Promise<string
   }
 }
 
+/**
+ * What the person at the gate said, framed for the step's prompt — or "".
+ *
+ * The note is the freshest instruction the step has — later than the flow
+ * file, aimed at this exact run. When the gate asked a question (ask:), the
+ * note IS the answer and is framed as one.
+ *
+ * Both halves go through resolveDocLinks for the same reason the
+ * instruction does: this is authored text on its way to becoming a prompt,
+ * and the rule is that ALL of it expands. A person answering "which list?"
+ * with [[dead-ends]] was sending the model two literal brackets — the one
+ * reference syntax the workspace teaches, and the one place it silently did
+ * nothing. The previous step's results are NOT resolved, deliberately: that
+ * is model output, and a model that happens to emit [[brackets]] must not
+ * thereby name a file.
+ *
+ * An ask: gate approved with no note still says so. It used to add nothing,
+ * so the step never learned its question had been answered: gbp-desk's
+ * rivals sender, told "a plain approval means all offices", saw no answer
+ * at all and stopped BLOCKED (run-muu6r18y-417e, 2026-10-05).
+ */
+export function operatorBlock(step: Pick<StepRecord, "ask" | "approvalNote" | "approvedAt">, workspaceRoot: string): string {
+  const ask = step.ask ? resolveDocLinks(step.ask, workspaceRoot) : "";
+  if (step.approvalNote) {
+    const note = resolveDocLinks(step.approvalNote, workspaceRoot);
+    return step.ask
+      ? `\n\n<operator_answer>\nThis step asked a human: ${ask}\nTheir answer:\n${note}\n</operator_answer>`
+      : `\n\n<operator_guidance>\nThe human who approved this step added:\n${note}\n</operator_guidance>`;
+  }
+  if (step.ask && step.approvedAt) {
+    return `\n\n<operator_answer>\nThis step asked a human: ${ask}\nTheir answer: they approved it with no note — a plain yes.\n</operator_answer>`;
+  }
+  return "";
+}
+
+
 async function runStep(
   agentDir: string,
   tenant: string,
@@ -1807,6 +1885,7 @@ async function runStep(
   // cover a script's own stdout.
   let redactions: [string, string][] = [];
   let fileDir: string | null = null;
+  let installDir: string | null = null;
   let disposeRuntime: (() => void) | undefined;
   // Can this step's tools act outside the workspace? Set once its context is
   // read; the verdict at the end of this function reads it.
@@ -1963,6 +2042,29 @@ async function runStep(
       const line = credentialLine(credential);
       push(line.type, line.text);
     }
+    // A hosted platform runs only its operator's accounts on its own model
+    // credential; every other account brings its own API key. Refused here,
+    // before a sandbox is rented, with what to do about it — not run on the
+    // operator's key, and not failed later by a provider.
+    if (!platform.platformKeyAllowed(tenant)) {
+      const own = Object.values(providerSecrets).concat(Object.values(providerEnv));
+      if (!providerLabel) {
+        push(
+          "error",
+          `no model key: this account runs on its own API key, and none is set. Add one in Settings → Model ` +
+            `(or \`foldrun model set <provider> --key …\`), or write a provider: block in AGENTS.md`,
+        );
+        step.status = "failed";
+        save();
+        return;
+      }
+      if (own.some((v) => isClaudeLoginToken(v))) {
+        push("error", "model key: a Claude login token (sk-ant-oat…) cannot be used here — use an API key from your provider");
+        step.status = "failed";
+        save();
+        return;
+      }
+    }
     for (const w of providerWarnings) push("error", w);
     // A level that wrote a language nobody can read was skipped, not obeyed.
     for (const l of language.lines) push("info", l);
@@ -2143,28 +2245,7 @@ async function runStep(
       // happening to contain brackets.
       prompt += `\n\n<event>\nThis step waited for an external event. What arrived:\n${step.eventPayload}\n</event>`;
     }
-    if (step.approvalNote) {
-      // The person at the gate said something while letting the run through.
-      // That is the freshest instruction the step has — later than the flow
-      // file, aimed at this exact run. When the gate asked a question
-      // (ask:), the note IS the answer and is framed as one.
-      //
-      // Both halves go through resolveDocLinks for the same reason the
-      // instruction above does: this is authored text on its way to becoming
-      // a prompt, and the rule is that ALL of it expands. A person answering
-      // "which list?" with [[dead-ends]] was sending the model two literal
-      // brackets — the one reference syntax the workspace teaches, and the
-      // one place it silently did nothing.
-      //
-      // What is NOT resolved, deliberately: the previous step's results
-      // above. That is model output, not something a person wrote, and a
-      // model that happens to emit [[brackets]] must not thereby name a file.
-      const ask = step.ask ? resolveDocLinks(step.ask, workspaceRoot) : "";
-      const note = resolveDocLinks(step.approvalNote, workspaceRoot);
-      prompt += step.ask
-        ? `\n\n<operator_answer>\nThis step asked a human: ${ask}\nTheir answer:\n${note}\n</operator_answer>`
-        : `\n\n<operator_guidance>\nThe human who approved this step added:\n${note}\n</operator_guidance>`;
-    }
+    prompt += operatorBlock(step, workspaceRoot);
     if (step.delegate?.length) {
       // The chooser's contract. The allowed set comes from the flow file;
       // the runner parses exactly this shape back out of the result, so the
@@ -2210,6 +2291,9 @@ async function runStep(
     // watched on both paths, because a refusal reads the same from a pod
     // and from this process.
     let lastRefusal = "";
+    // The step's model connection was cut (isModelConnectionLost): the
+    // platform's side going away under the step, not the step failing.
+    let lastDrop = "";
     // And WHY it refused, in words, when anything on the way saw the
     // headers that say so: the translator here, the egress proxy over
     // there. Kept apart from `lastRefusal` — that one decides whether to
@@ -2217,6 +2301,7 @@ async function runStep(
     let refusalWhy = "";
     const pushWatching: typeof push = (type, text, extra) => {
       if (type === "error" && isProviderRefusal(text)) lastRefusal = text;
+      if (type === "error" && isModelConnectionLost(text)) lastDrop = text;
       const why = refusalFromLine(text);
       if (why) refusalWhy = why;
       push(type, text, extra);
@@ -2511,6 +2596,7 @@ async function runStep(
         lastArgs = a;
         return runIsolated(a);
       };
+      let attemptFrom = step.events.length;
       let outcome = await runTracked(firstArgs);
       step.sandbox = null; // whatever happens next starts its own
       // A 401 from a token that rotated mid-step is not a broken key: the
@@ -2548,6 +2634,48 @@ async function runStep(
         outcome = await runTracked(again);
         outcome = withEarlierTiming(outcome, previous);
       }
+      // The model connection was cut under the step — the egress proxy or the
+      // worker went away (6 Oct 2026: a worker restarted every 4.5 minutes and
+      // every desk running failed on "Connection refused"). That is the
+      // platform's fault, so the step runs again, whatever its `retry:` —
+      // unless it can act outside the workspace and its tools already ran:
+      // the post may be up, and a second attempt would post it again.
+      for (let n = 1; n <= DROP_RETRIES && outcome.status === "failed" && lastDrop; n++) {
+        if (!dropRetrySafe(canActOutward, step.events.slice(attemptFrom))) {
+          push("info", `the model connection dropped after this step's tools ran — not retried: they can act outside the workspace, and another attempt could repeat what they did`);
+          break;
+        }
+        const wait = Math.round(retryBaseMs() * 2 * n);
+        push("info", `the model connection dropped (${lastDrop.slice(0, 80)}) — the platform's side, not the step's; running the step again in ${Math.round(wait / 1000)}s (${n} of ${DROP_RETRIES})`);
+        await sleep(wait);
+        lastDrop = "";
+        lastRefusal = "";
+        attemptFrom = step.events.length;
+        const previous = outcome.timing;
+        const again = isolatedArgs(platformModelEnv());
+        await lease?.commit();
+        outcome = withEarlierTiming(await runTracked(again), previous);
+      }
+      // The model wrote its tool call out as text — `<invoke name="…">` in
+      // its last message — and stopped. Nothing ran, so nothing was done:
+      // seo-digest's editor three times (27 Sep, 4 and 5 Oct 2026), its own
+      // retry included, because the retry asked the same way. Run it once
+      // more with that said plainly, on the same safety rule as a dropped
+      // connection: never again after an outward tool already ran.
+      for (let n = 1; n <= TEXT_CALL_RETRIES && wroteToolCallAsText(attemptText(outcome.result, step.events.slice(attemptFrom))); n++) {
+        if (!dropRetrySafe(canActOutward, step.events.slice(attemptFrom))) {
+          push("info", "the model wrote a tool call as text instead of making it — not run again: this step's tools can act outside the workspace and already ran");
+          break;
+        }
+        push("info", `the model wrote a tool call as text instead of making it, so nothing ran — running the step again with a note (${n} of ${TEXT_CALL_RETRIES})`);
+        attemptFrom = step.events.length;
+        lastRefusal = "";
+        const previous = outcome.timing;
+        const base = isolatedArgs(platformModelEnv());
+        const again = { ...base, input: { ...base.input, prompt: `${base.input.prompt}\n\n${TEXT_CALL_NOTE}` } };
+        await lease?.commit();
+        outcome = withEarlierTiming(await runTracked(again), previous);
+      }
       // The second supply, tried exactly once, and only when the primary
       // refused over money/auth/limits rather than the work failing. The
       // retry is a fresh sandbox — the failed one is gone, and driveRun's
@@ -2706,6 +2834,20 @@ async function runStep(
       // finally — once, however many attempts follow.
       const mat = materializeFileSecrets(agentDir, stepSecrets);
       fileDir = mat.dir;
+      // install: pip and npm land in a directory of this step's own, gone in
+      // the finally; apt has no helper on a bare host and says so.
+      installDir = allowed.includes(INSTALL_TOOL) ? fs.mkdtempSync(path.join(os.tmpdir(), "foldrun-step-")) : null;
+      const installEnv = installDir ? stepInstallEnv(installDir, { ...process.env, ...mat.env }) : {};
+      const installTools = installDir
+        ? buildInstallTool({
+            dir: installDir,
+            socket: null,
+            cacheRoot: path.join(dataRoot(), tenant, ".runtimes"),
+            python: mat.env.VIRTUAL_ENV ? path.join(mat.env.VIRTUAL_ENV, "bin", "python") : "python3",
+            env: { ...hostSafeEnv(), ...mat.env, ...installEnv },
+            emit: (type, text) => push(type, text),
+          })
+        : null;
       const attemptInProcess = (modelEnv: Record<string, string | undefined>) => executeStep({
         agentDir,
         workspaceRoot,
@@ -2725,6 +2867,7 @@ async function runStep(
           ...(historyTools.server ? { foldrun_history: historyTools.server } : {}),
           ...(deskTools.server ? { foldrun_desks: deskTools.server } : {}),
           ...(askTools ? { foldrun_ask: askTools.server } : {}),
+          ...(installTools ? { foldrun_install: installTools.server } : {}),
           ...mcpServers,
         },
         // Declared secrets reach the agent's scripts as env vars; the model
@@ -2732,7 +2875,7 @@ async function runStep(
         // the allowlisted host environment, not process.env: the SDK's
         // Bash tool runs with this, and on a platform process.env holds
         // every tenant's root key and the datastore credentials.
-        env: { ...hostSafeEnv(), ...clockEnv, ...(testRun ? testModeEnv() : {}), ...mat.env, ...modelEnv },
+        env: { ...hostSafeEnv(), ...clockEnv, ...(testRun ? testModeEnv() : {}), ...mat.env, ...installEnv, ...modelEnv },
         timeoutSec: step.timeout,
         budgetUsd: stepBudgetUsd,
         budgetNote: stepBudgetNote,
@@ -2840,6 +2983,7 @@ async function runStep(
   } finally {
     try {
       cleanupFileSecrets(fileDir);
+      if (installDir) fs.rmSync(installDir, { recursive: true, force: true });
     } catch {
       // best-effort — a leftover scratch file is swept with the run's outputs
     }
@@ -2996,8 +3140,12 @@ function platformFallbackEnv(): Record<string, string> | null {
 const CREDENTIAL_ROTATION_WAIT_MS = 90_000;
 const CREDENTIAL_POLL_MS = 2_000;
 
-function isAuthRefusal(text: string): boolean {
-  return /API Error: 401|access token has (been revoked|expired)/i.test(text);
+/** A refusal of the credential itself, in any of the wordings Claude Code
+ *  has used. "Failed to authenticate: OAuth token revoked" (9 Oct 2026, a
+ *  step started one second before the 09:07:52Z refresh) matched none of the
+ *  older two, so the step failed instead of waiting for the new token. */
+export function isAuthRefusal(text: string): boolean {
+  return /API Error: 401|access token has (been revoked|expired)|OAuth token (has )?(been )?(revoked|expired)|Failed to authenticate/i.test(text);
 }
 
 /**
@@ -3048,6 +3196,49 @@ function isProviderRefusal(text: string): boolean {
   );
 }
 
+/** A step's own model connection cut: the egress proxy or the worker it
+ *  runs through went away. Only the run's final error, never a tool's — a
+ *  WebFetch that could not connect is the step's business, not ours. */
+export function isModelConnectionLost(text: string): boolean {
+  return /^Claude Code returned an error result: API Error: (Connection (refused|dropped|error)|.*\b(ECONNREFUSED|ECONNRESET|socket hang up)\b)/i.test(text);
+}
+
+/** A tool call written out as text rather than made: the model's last
+ *  message holds `<invoke name=…>` / `<function_calls>` markup. Only the tail
+ *  counts — a reply that quotes such markup mid-way and goes on is fine. */
+export function wroteToolCallAsText(text: string): boolean {
+  const tail = text.slice(-1500);
+  const open = tail.search(/<(?:antml:)?(?:invoke|function_calls)\b[^>]*>/);
+  if (open < 0) return false;
+  // Quoted mid-reply and then the reply carries on: not a stalled call.
+  const after = tail.slice(open).replace(/<\/(?:antml:)?(?:invoke|function_calls)>/g, "\u0000");
+  const lastClose = after.lastIndexOf("\u0000");
+  return lastClose < 0 || after.slice(lastClose + 1).trim().length < 200;
+}
+
+/** What an attempt said last: its result, or — when it ended without one —
+ *  the error that carried the model's final text. */
+function attemptText(result: string | null | undefined, events: { type: string; text?: string }[]): string {
+  if (result && result.trim()) return result;
+  const lastError = [...events].reverse().find((e) => e.type === "error");
+  return lastError?.text ?? "";
+}
+
+/** Runs again after a tool call written as text. */
+export const TEXT_CALL_RETRIES = 1;
+export const TEXT_CALL_NOTE =
+  "Note from the platform: your last attempt ended by writing a tool call out as text (an <invoke …> block), so it was not run and nothing happened. " +
+  "Make every tool call as a real tool call — never type its markup into your message.";
+
+/** How many times a step whose model connection was cut runs again. */
+export const DROP_RETRIES = 2;
+
+/** May a step whose connection dropped run again? Yes, unless its tools can
+ *  act outward and one ran in this attempt (actedThenFailedCheck's rule). */
+export function dropRetrySafe(canActOutward: boolean, attemptEvents: { type: string }[]): boolean {
+  return !(canActOutward && attemptEvents.some((e) => e.type === "tool"));
+}
+
 /** How many times a busy provider is given another moment, and the longest
  *  any one wait may be. Three is the industry's habit and the point where
  *  waiting longer stops being kinder than failing. */
@@ -3637,6 +3828,32 @@ function driveRunInner(
       //
       // Runs on resume too — a parked run's container is long gone, and the
       // one it comes back in needs the same files the first one had.
+      //
+      // A run re-driven after its worker died is the exception to "the store
+      // is newer than the disk": a park harvests before it parks, a crash does
+      // not, so what this run's finished steps wrote to storage/ is only on
+      // disk, and materializing first wrote the store's older copy over it.
+      // On 6 Oct 2026 a worker restart put reddit-desk's 2 Oct post back over
+      // that morning's draft, and a rerun from the validator would have
+      // posted it a second time. So a run that has been here before (its
+      // baseline is on disk) stores what it changed first; files it did not
+      // touch still come fresh from the store.
+      const baselineFile = path.join(runRoot, "runs", run.id, "storage-baseline.json");
+      if (fs.existsSync(baselineFile)) {
+        try {
+          const changedFrom = JSON.parse(fs.readFileSync(baselineFile, "utf8")) as Record<string, string>;
+          const { saved } = await harvestFiles(tenant, workspace, `run:${run.id}`, { changedFrom });
+          if (saved.length) {
+            run.steps[0]?.events.push({
+              t: new Date().toISOString(),
+              type: "info",
+              text: `files: kept ${saved.join(", ")} written before the run was picked up again`,
+            });
+          }
+        } catch {
+          // the materialize below still runs; at worst this is the old behaviour
+        }
+      }
       try {
         const brought = await materializeFiles(tenant, workspace);
         if (brought.length) {
@@ -3646,6 +3863,8 @@ function driveRunInner(
             text: `files: ${brought.length} brought into the workspace`,
           });
         }
+        fs.mkdirSync(path.dirname(baselineFile), { recursive: true });
+        fs.writeFileSync(baselineFile, JSON.stringify(storageBaseline(tenant, workspace)));
       } catch (err) {
         // A file store that is down must not take the run with it: most flows
         // never touch storage/, and the ones that do will fail their own verify.
@@ -4111,7 +4330,7 @@ function driveRunInner(
           save();
           if (freshGroup.every((s) => s.status !== "pending")) {
             // everything in this group was rejected
-            if (freshGroup.some((s) => s.status === "failed" && !s.optional)) {
+            if (freshGroup.some((s) => s.status === "failed" && !s.optional && !s.carriedFrom)) {
               run.status = "failed";
               break;
             }
@@ -4353,8 +4572,13 @@ function driveRunInner(
             effort: step.effort,
             timeout: step.timeout,
             // The rescuer inherits the contract, not just the job: a step
-            // that promised data still owes data when someone else does it.
+            // that promised data still owes data when someone else does it,
+            // and a step that had to pass a check still has to pass it — a
+            // rescue whose result stood unchecked was a way round the very
+            // check that failed (found 7 Oct 2026, adding rescuers to
+            // medium-desk).
             output: step.output,
+            ...(step.verify ? { verify: step.verify } : {}),
             attempts: 0,
             status: "pending",
             events: [],
@@ -4389,7 +4613,11 @@ function driveRunInner(
           save();
         }
 
-        const requiredFailed = freshGroup.some((s) => s.status === "failed" && !s.optional);
+        // A step carried into a re-run failed in the run it came from, not
+        // here: the person chose to start after it. Counting it stopped
+        // fix-desk's "re-run from the verifier" (run-muwfi5jn-4217) in under
+        // a second, before the one step it was asked to run.
+        const requiredFailed = freshGroup.some((s) => s.status === "failed" && !s.optional && !s.carriedFrom);
         if (requiredFailed) {
           run.status = "failed";
           break;