@foldrun/core 0.6.0 → 0.7.0

package/dist/src/runner.js

@@ -3,6 +3,7 @@
 // fail without failing the flow. A direct agent run is a one-step ad-hoc
 // flow, so there is a single execution path.
 import fs from "node:fs";
+import os from "node:os";
 import path from "node:path";
 import matter from "gray-matter";
 import { knownPrice, executeStep, extractJson, stepCeiling, stepCeilingFor } from "./step-exec.js";
@@ -11,8 +12,8 @@ import { resolveAgentPath } from "./confine.js";
 import { browserPodLine, podLossDecision, RERAN_ON_FULL } from "./browser-pod.js";
 import { eventUrl } from "./webhook.js";
 import { runStepInContainer, sizeLimits, killRunSandboxes } from "./run-container.js";
-import { hostSafeEnv } from "./host-env.js";
-import { resolveModelCredential, credentialLine } from "./model-credential.js";
+import { hostSafeEnv, platformApiUrl } from "./host-env.js";
+import { resolveModelCredential, credentialLine, accountModelBlock, isClaudeLoginToken } from "./model-credential.js";
 import { validateSchema, describeSchemaErrors, looksLikeSchema } from "./json-schema.js";
 import { EGRESS_ENV, MODEL_KEY_NAME, addGrant, hostOf, placeholderNames, proxyModelEnv, unsubstitute } from "./egress.js";
 import { explainRefusal, isRefusalStatus, refusalFromLine, supplyNote } from "./refusal.js";
@@ -42,8 +43,10 @@ import { startTranslator, translatorSpecFor } from "./translator.js";
 import { providerPreset } from "./providers.js";
 import { buildScriptTools, parseScripts } from "./script-tools.js";
 import { libraryDir, libraryTools, libraryMemoryIndex } from "./library.js";
-import { fingerprint, mergeRuntimes, parseRuntime, prepareRuntime } from "./runtime.js";
-import { materializeFiles, harvestFiles } from "./storage.js";
+import { fingerprint, mergeRuntimes, parseRuntime, prepareRuntime, stepInstallEnv } from "./runtime.js";
+import { buildInstallTool, INSTALL_TOOL } from "./install-tool.js";
+import { dataRoot } from "./paths.js";
+import { materializeFiles, harvestFiles, storageBaseline } from "./storage.js";
 import { chooseExecutor, ensureImage } from "./container.js";
 import { stampBundle } from "./okf.js";
 import { readBrowseSettings, readSearchSettings, resolveSearch, searchSettingsEnv, webConfig, browseSessionProblems } from "./providers.js";
@@ -614,7 +617,7 @@ export function resolveDocLinks(text, workspaceRoot) {
         return hit ? `\`${hit}\`` : whole;
     });
 }
-function agentContext(agentDir, tenant, tags = [], 
+export function agentContext(agentDir, tenant, tags = [], 
 /** Who is running, so scripts can stamp what they write with its provenance. */
 identity = {}, 
 /** The flow's own `timezone:`, when this step belongs to one. Between the
@@ -1000,7 +1003,10 @@ grantsOnly = false) {
     //
     // The URL sits in git; the token is a ${SECRET} resolved server-side. An
     // agent's own block wins over the workspace's, like everything else here.
-    const providerBlock = front.provider ?? workspaceFrontmatter(agentDir, tenant).provider;
+    // Then the account's own model key (Settings → Model), when it set one:
+    // an account that brings its key need not write a block anywhere.
+    const fileBlock = front.provider ?? workspaceFrontmatter(agentDir, tenant).provider;
+    const providerBlock = fileBlock ?? accountModelBlock((name) => getSecret(tenant, name)?.value ?? null);
     let providerEnv = {};
     let providerLabel = null;
     // Secret values pulled into the provider block, name → value. Kept apart
@@ -1151,6 +1157,9 @@ grantsOnly = false) {
         // one — the `websearch` library tool reads it. Not a secret; rides with
         // the clock so it reaches scripts, the sandbox and verify alike.
         ...(process.env.FOLDRUN_SEARCH_URL ? { FOLDRUN_SEARCH_URL: process.env.FOLDRUN_SEARCH_URL } : {}),
+        // Where this install's own API answers (host-env.ts platformApiUrl), so a
+        // script calls `$FOLDRUN_API_URL/workspaces/…` instead of naming a host.
+        ...(platformApiUrl() ? { FOLDRUN_API_URL: platformApiUrl() } : {}),
         // Whose index our web search asks when the agent named a search API.
         // Unset means the account's own engine. The name, never the key.
         ...(searchChoice.shape === "direct" && searchChoice.provider
@@ -1228,7 +1237,18 @@ grantsOnly = false) {
     parts.push(`# Clock\n\nToday is ${clockEnv.FOLDRUN_DATE} (${timezone}). Use this date for anything you ` +
         `write or check — it is the workspace's calendar, not the container's clock.`);
     if (scriptTools.promptLines.length) {
-        parts.push(`# Tools from scripts\n\nCall these directly — they run workspace scripts for you:\n\n${scriptTools.promptLines.join("\n")}`);
+        parts.push(`# Tools from scripts\n\nCall these directly — they run workspace scripts for you. Each is called by its full name, ` +
+            `\`mcp__foldrun_scripts__<name>\` (\`web\` is \`mcp__foldrun_scripts__web\`); a bare name is refused:\n\n${scriptTools.promptLines.join("\n")}`);
+    }
+    // The web tool's own description lists every action, so an agent whose
+    // search or fetch a model provider answers called web action=search,
+    // was refused, and only then used WebSearch — eight refusals in one
+    // reddit-desk validator step on 6 Oct 2026. Say it before the first call.
+    if (builtins.length) {
+        const named = builtins.map((b) => b.split("="));
+        parts.push(`# Web search and fetch on this run\n\n` +
+            named.map(([action, tool]) => `- To ${action}, call **${tool}** directly. \`web\` with action=${action} is switched off on this run.`).join("\n") +
+            `\n\nThe web tool's other actions work as usual.`);
     }
     if (declared.length) {
         parts.push(`# Secrets\n\nAvailable to your scripts as environment variables: ${declared.join(", ")}. ` +
@@ -1237,7 +1257,13 @@ grantsOnly = false) {
     parts.push("Write deliverables for people to workspace/storage/ (kept and downloadable); write working text for later steps to outputs/; " +
         "what the next run needs goes to workspace/state/. Paths are relative to your agent directory, or start with workspace/ " +
         "for the workspace root (workspace/storage/ and the older ../../storage/ are the same place). There is no /tmp for you: a " +
-        "directory a tool made outside the workspace, such as a checkout, is reached only through that tool.");
+        "directory a tool made outside the workspace, such as a checkout, is reached only through that tool. " +
+        // 45 refusals in 34 runs (6–7 Oct 2026), nearly all an agent rewriting
+        // a plan or draft an earlier run left: the Write tool refuses a file
+        // that exists and has not been Read in this step, before any hook of
+        // ours can see the call. Said once here instead of learned per step.
+        "To replace a file that already exists, Read it first — Write refuses a file you have not Read in this step (a new file needs no Read). " +
+        "Built-in tools are capitalised: Read, Write, Edit, Glob, Grep, Bash.");
     // The run's one-line summary is the first meaningful line of the last step
     // that produced a result — read, never declared (runSummary in store.ts).
     // That convention was load-bearing and unstated: it decides the runs-list
@@ -1299,6 +1325,13 @@ grantsOnly = false) {
         // names it (FOLDRUN_WEB_BUILTIN) rather than asking our engine.
         if (toolName === "web" && !ref.linked) {
             allowed.push(...providerWebTools);
+            // web writes what it found to files and says "Read that file" — a
+            // screenshot, a crawl, a page saved whole. An agent granted web alone
+            // was told to Read and refused it (blog-desk claim-checker, 7 Oct
+            // 2026: "Read is disabled for this session"). Reading is confined to
+            // the workspace like every file tool; disallowedTools still removes it.
+            if (!allowed.includes("Read"))
+                allowed.push("Read");
             continue;
         }
         // Resolution order: built-in group, exact SDK name, then your own tools.
@@ -1328,6 +1361,9 @@ grantsOnly = false) {
             if (available[toolName])
                 shadowed.push(toolName);
         }
+        else if (toolName === "install") {
+            // Granted to every agent anyway (below); naming it is harmless.
+        }
         else if (BUILTIN_TOOLS.has(toolName)) {
             allowed.push(toolName);
             if (available[toolName])
@@ -1391,6 +1427,10 @@ grantsOnly = false) {
     // what it exposes, which is the unit a person actually reasons about anyway.
     for (const name of mcpNames)
         allowed.push(`mcp__${name}`, `mcp__${name}__*`);
+    // install — every agent may install what it finds it needs mid-step (apt,
+    // pip, npm), declared or not (install-tool.ts). Granted by default; taken
+    // away by disallowedTools: [install], and absent in plan mode (read-only).
+    allowed.push(INSTALL_TOOL);
     // Claude Code's spellings, so an agent written for it mostly runs here.
     // `permissionMode: plan` is its read-only mode; ours is the `read` group.
     if (front.permissionMode === "plan") {
@@ -1398,6 +1438,8 @@ grantsOnly = false) {
         allowed.push(...TOOL_MAP.read, ...apiTools.toolNames, ...scriptTools.toolNames);
     }
     const denied = refNames(front.disallowedTools);
+    if (denied.includes("install"))
+        denied.push(INSTALL_TOOL);
     const finalAllowed = allowed.filter((t) => !denied.includes(t));
     return {
         front,
@@ -1600,6 +1642,40 @@ async function terminalAsk(question, options) {
         rl.close();
     }
 }
+/**
+ * What the person at the gate said, framed for the step's prompt — or "".
+ *
+ * The note is the freshest instruction the step has — later than the flow
+ * file, aimed at this exact run. When the gate asked a question (ask:), the
+ * note IS the answer and is framed as one.
+ *
+ * Both halves go through resolveDocLinks for the same reason the
+ * instruction does: this is authored text on its way to becoming a prompt,
+ * and the rule is that ALL of it expands. A person answering "which list?"
+ * with [[dead-ends]] was sending the model two literal brackets — the one
+ * reference syntax the workspace teaches, and the one place it silently did
+ * nothing. The previous step's results are NOT resolved, deliberately: that
+ * is model output, and a model that happens to emit [[brackets]] must not
+ * thereby name a file.
+ *
+ * An ask: gate approved with no note still says so. It used to add nothing,
+ * so the step never learned its question had been answered: gbp-desk's
+ * rivals sender, told "a plain approval means all offices", saw no answer
+ * at all and stopped BLOCKED (run-muu6r18y-417e, 2026-10-05).
+ */
+export function operatorBlock(step, workspaceRoot) {
+    const ask = step.ask ? resolveDocLinks(step.ask, workspaceRoot) : "";
+    if (step.approvalNote) {
+        const note = resolveDocLinks(step.approvalNote, workspaceRoot);
+        return step.ask
+            ? `\n\n<operator_answer>\nThis step asked a human: ${ask}\nTheir answer:\n${note}\n</operator_answer>`
+            : `\n\n<operator_guidance>\nThe human who approved this step added:\n${note}\n</operator_guidance>`;
+    }
+    if (step.ask && step.approvedAt) {
+        return `\n\n<operator_answer>\nThis step asked a human: ${ask}\nTheir answer: they approved it with no note — a plain yes.\n</operator_answer>`;
+    }
+    return "";
+}
 async function runStep(agentDir, tenant, step, context, save, modelOverride, tags = [], effortOverride, runId, 
 /** What the previous group returned as DATA (an `output: json` step's
  *  value), beside the prose in `context`. Undefined when nothing did. */
@@ -1641,6 +1717,7 @@ stepIndex) {
     // cover a script's own stdout.
     let redactions = [];
     let fileDir = null;
+    let installDir = null;
     let disposeRuntime;
     // Can this step's tools act outside the workspace? Set once its context is
     // read; the verdict at the end of this function reads it.
@@ -1784,6 +1861,26 @@ stepIndex) {
             const line = credentialLine(credential);
             push(line.type, line.text);
         }
+        // A hosted platform runs only its operator's accounts on its own model
+        // credential; every other account brings its own API key. Refused here,
+        // before a sandbox is rented, with what to do about it — not run on the
+        // operator's key, and not failed later by a provider.
+        if (!platform.platformKeyAllowed(tenant)) {
+            const own = Object.values(providerSecrets).concat(Object.values(providerEnv));
+            if (!providerLabel) {
+                push("error", `no model key: this account runs on its own API key, and none is set. Add one in Settings → Model ` +
+                    `(or \`foldrun model set <provider> --key …\`), or write a provider: block in AGENTS.md`);
+                step.status = "failed";
+                save();
+                return;
+            }
+            if (own.some((v) => isClaudeLoginToken(v))) {
+                push("error", "model key: a Claude login token (sk-ant-oat…) cannot be used here — use an API key from your provider");
+                step.status = "failed";
+                save();
+                return;
+            }
+        }
         for (const w of providerWarnings)
             push("error", w);
         // A level that wrote a language nobody can read was skipped, not obeyed.
@@ -1958,28 +2055,7 @@ stepIndex) {
             // happening to contain brackets.
             prompt += `\n\n<event>\nThis step waited for an external event. What arrived:\n${step.eventPayload}\n</event>`;
         }
-        if (step.approvalNote) {
-            // The person at the gate said something while letting the run through.
-            // That is the freshest instruction the step has — later than the flow
-            // file, aimed at this exact run. When the gate asked a question
-            // (ask:), the note IS the answer and is framed as one.
-            //
-            // Both halves go through resolveDocLinks for the same reason the
-            // instruction above does: this is authored text on its way to becoming
-            // a prompt, and the rule is that ALL of it expands. A person answering
-            // "which list?" with [[dead-ends]] was sending the model two literal
-            // brackets — the one reference syntax the workspace teaches, and the
-            // one place it silently did nothing.
-            //
-            // What is NOT resolved, deliberately: the previous step's results
-            // above. That is model output, not something a person wrote, and a
-            // model that happens to emit [[brackets]] must not thereby name a file.
-            const ask = step.ask ? resolveDocLinks(step.ask, workspaceRoot) : "";
-            const note = resolveDocLinks(step.approvalNote, workspaceRoot);
-            prompt += step.ask
-                ? `\n\n<operator_answer>\nThis step asked a human: ${ask}\nTheir answer:\n${note}\n</operator_answer>`
-                : `\n\n<operator_guidance>\nThe human who approved this step added:\n${note}\n</operator_guidance>`;
-        }
+        prompt += operatorBlock(step, workspaceRoot);
         if (step.delegate?.length) {
             // The chooser's contract. The allowed set comes from the flow file;
             // the runner parses exactly this shape back out of the result, so the
@@ -2023,6 +2099,9 @@ stepIndex) {
         // watched on both paths, because a refusal reads the same from a pod
         // and from this process.
         let lastRefusal = "";
+        // The step's model connection was cut (isModelConnectionLost): the
+        // platform's side going away under the step, not the step failing.
+        let lastDrop = "";
         // And WHY it refused, in words, when anything on the way saw the
         // headers that say so: the translator here, the egress proxy over
         // there. Kept apart from `lastRefusal` — that one decides whether to
@@ -2031,6 +2110,8 @@ stepIndex) {
         const pushWatching = (type, text, extra) => {
             if (type === "error" && isProviderRefusal(text))
                 lastRefusal = text;
+            if (type === "error" && isModelConnectionLost(text))
+                lastDrop = text;
             const why = refusalFromLine(text);
             if (why)
                 refusalWhy = why;
@@ -2317,6 +2398,7 @@ stepIndex) {
                 lastArgs = a;
                 return runIsolated(a);
             };
+            let attemptFrom = step.events.length;
             let outcome = await runTracked(firstArgs);
             step.sandbox = null; // whatever happens next starts its own
             // A 401 from a token that rotated mid-step is not a broken key: the
@@ -2355,6 +2437,48 @@ stepIndex) {
                 outcome = await runTracked(again);
                 outcome = withEarlierTiming(outcome, previous);
             }
+            // The model connection was cut under the step — the egress proxy or the
+            // worker went away (6 Oct 2026: a worker restarted every 4.5 minutes and
+            // every desk running failed on "Connection refused"). That is the
+            // platform's fault, so the step runs again, whatever its `retry:` —
+            // unless it can act outside the workspace and its tools already ran:
+            // the post may be up, and a second attempt would post it again.
+            for (let n = 1; n <= DROP_RETRIES && outcome.status === "failed" && lastDrop; n++) {
+                if (!dropRetrySafe(canActOutward, step.events.slice(attemptFrom))) {
+                    push("info", `the model connection dropped after this step's tools ran — not retried: they can act outside the workspace, and another attempt could repeat what they did`);
+                    break;
+                }
+                const wait = Math.round(retryBaseMs() * 2 * n);
+                push("info", `the model connection dropped (${lastDrop.slice(0, 80)}) — the platform's side, not the step's; running the step again in ${Math.round(wait / 1000)}s (${n} of ${DROP_RETRIES})`);
+                await sleep(wait);
+                lastDrop = "";
+                lastRefusal = "";
+                attemptFrom = step.events.length;
+                const previous = outcome.timing;
+                const again = isolatedArgs(platformModelEnv());
+                await lease?.commit();
+                outcome = withEarlierTiming(await runTracked(again), previous);
+            }
+            // The model wrote its tool call out as text — `<invoke name="…">` in
+            // its last message — and stopped. Nothing ran, so nothing was done:
+            // seo-digest's editor three times (27 Sep, 4 and 5 Oct 2026), its own
+            // retry included, because the retry asked the same way. Run it once
+            // more with that said plainly, on the same safety rule as a dropped
+            // connection: never again after an outward tool already ran.
+            for (let n = 1; n <= TEXT_CALL_RETRIES && wroteToolCallAsText(attemptText(outcome.result, step.events.slice(attemptFrom))); n++) {
+                if (!dropRetrySafe(canActOutward, step.events.slice(attemptFrom))) {
+                    push("info", "the model wrote a tool call as text instead of making it — not run again: this step's tools can act outside the workspace and already ran");
+                    break;
+                }
+                push("info", `the model wrote a tool call as text instead of making it, so nothing ran — running the step again with a note (${n} of ${TEXT_CALL_RETRIES})`);
+                attemptFrom = step.events.length;
+                lastRefusal = "";
+                const previous = outcome.timing;
+                const base = isolatedArgs(platformModelEnv());
+                const again = { ...base, input: { ...base.input, prompt: `${base.input.prompt}\n\n${TEXT_CALL_NOTE}` } };
+                await lease?.commit();
+                outcome = withEarlierTiming(await runTracked(again), previous);
+            }
             // The second supply, tried exactly once, and only when the primary
             // refused over money/auth/limits rather than the work failing. The
             // retry is a fresh sandbox — the failed one is gone, and driveRun's
@@ -2508,6 +2632,20 @@ stepIndex) {
             // finally — once, however many attempts follow.
             const mat = materializeFileSecrets(agentDir, stepSecrets);
             fileDir = mat.dir;
+            // install: pip and npm land in a directory of this step's own, gone in
+            // the finally; apt has no helper on a bare host and says so.
+            installDir = allowed.includes(INSTALL_TOOL) ? fs.mkdtempSync(path.join(os.tmpdir(), "foldrun-step-")) : null;
+            const installEnv = installDir ? stepInstallEnv(installDir, { ...process.env, ...mat.env }) : {};
+            const installTools = installDir
+                ? buildInstallTool({
+                    dir: installDir,
+                    socket: null,
+                    cacheRoot: path.join(dataRoot(), tenant, ".runtimes"),
+                    python: mat.env.VIRTUAL_ENV ? path.join(mat.env.VIRTUAL_ENV, "bin", "python") : "python3",
+                    env: { ...hostSafeEnv(), ...mat.env, ...installEnv },
+                    emit: (type, text) => push(type, text),
+                })
+                : null;
             const attemptInProcess = (modelEnv) => executeStep({
                 agentDir,
                 workspaceRoot,
@@ -2527,6 +2665,7 @@ stepIndex) {
                     ...(historyTools.server ? { foldrun_history: historyTools.server } : {}),
                     ...(deskTools.server ? { foldrun_desks: deskTools.server } : {}),
                     ...(askTools ? { foldrun_ask: askTools.server } : {}),
+                    ...(installTools ? { foldrun_install: installTools.server } : {}),
                     ...mcpServers,
                 },
                 // Declared secrets reach the agent's scripts as env vars; the model
@@ -2534,7 +2673,7 @@ stepIndex) {
                 // the allowlisted host environment, not process.env: the SDK's
                 // Bash tool runs with this, and on a platform process.env holds
                 // every tenant's root key and the datastore credentials.
-                env: { ...hostSafeEnv(), ...clockEnv, ...(testRun ? testModeEnv() : {}), ...mat.env, ...modelEnv },
+                env: { ...hostSafeEnv(), ...clockEnv, ...(testRun ? testModeEnv() : {}), ...mat.env, ...installEnv, ...modelEnv },
                 timeoutSec: step.timeout,
                 budgetUsd: stepBudgetUsd,
                 budgetNote: stepBudgetNote,
@@ -2647,6 +2786,8 @@ stepIndex) {
     finally {
         try {
             cleanupFileSecrets(fileDir);
+            if (installDir)
+                fs.rmSync(installDir, { recursive: true, force: true });
         }
         catch {
             // best-effort — a leftover scratch file is swept with the run's outputs
@@ -2804,8 +2945,12 @@ function platformFallbackEnv() {
  */
 const CREDENTIAL_ROTATION_WAIT_MS = 90_000;
 const CREDENTIAL_POLL_MS = 2_000;
-function isAuthRefusal(text) {
-    return /API Error: 401|access token has (been revoked|expired)/i.test(text);
+/** A refusal of the credential itself, in any of the wordings Claude Code
+ *  has used. "Failed to authenticate: OAuth token revoked" (9 Oct 2026, a
+ *  step started one second before the 09:07:52Z refresh) matched none of the
+ *  older two, so the step failed instead of waiting for the new token. */
+export function isAuthRefusal(text) {
+    return /API Error: 401|access token has (been revoked|expired)|OAuth token (has )?(been )?(revoked|expired)|Failed to authenticate/i.test(text);
 }
 /**
  * Wait, briefly, for the mounted credential to become one we have not
@@ -2848,6 +2993,44 @@ function isProviderRefusal(text) {
     // precisely what the fallback provider is for.
     return /API Error: (401|402|403|429|5\d\d)|credits?\b|quota|billing|insufficient|overloaded|(weekly|daily|monthly|usage) limit|rate.?limit(ed)?\b.*resets?\b|limit\b.*\bresets?\b/i.test(text);
 }
+/** A step's own model connection cut: the egress proxy or the worker it
+ *  runs through went away. Only the run's final error, never a tool's — a
+ *  WebFetch that could not connect is the step's business, not ours. */
+export function isModelConnectionLost(text) {
+    return /^Claude Code returned an error result: API Error: (Connection (refused|dropped|error)|.*\b(ECONNREFUSED|ECONNRESET|socket hang up)\b)/i.test(text);
+}
+/** A tool call written out as text rather than made: the model's last
+ *  message holds `<invoke name=…>` / `<function_calls>` markup. Only the tail
+ *  counts — a reply that quotes such markup mid-way and goes on is fine. */
+export function wroteToolCallAsText(text) {
+    const tail = text.slice(-1500);
+    const open = tail.search(/<(?:antml:)?(?:invoke|function_calls)\b[^>]*>/);
+    if (open < 0)
+        return false;
+    // Quoted mid-reply and then the reply carries on: not a stalled call.
+    const after = tail.slice(open).replace(/<\/(?:antml:)?(?:invoke|function_calls)>/g, "\u0000");
+    const lastClose = after.lastIndexOf("\u0000");
+    return lastClose < 0 || after.slice(lastClose + 1).trim().length < 200;
+}
+/** What an attempt said last: its result, or — when it ended without one —
+ *  the error that carried the model's final text. */
+function attemptText(result, events) {
+    if (result && result.trim())
+        return result;
+    const lastError = [...events].reverse().find((e) => e.type === "error");
+    return lastError?.text ?? "";
+}
+/** Runs again after a tool call written as text. */
+export const TEXT_CALL_RETRIES = 1;
+export const TEXT_CALL_NOTE = "Note from the platform: your last attempt ended by writing a tool call out as text (an <invoke …> block), so it was not run and nothing happened. " +
+    "Make every tool call as a real tool call — never type its markup into your message.";
+/** How many times a step whose model connection was cut runs again. */
+export const DROP_RETRIES = 2;
+/** May a step whose connection dropped run again? Yes, unless its tools can
+ *  act outward and one ran in this attempt (actedThenFailedCheck's rule). */
+export function dropRetrySafe(canActOutward, attemptEvents) {
+    return !(canActOutward && attemptEvents.some((e) => e.type === "tool"));
+}
 /** How many times a busy provider is given another moment, and the longest
  *  any one wait may be. Three is the industry's habit and the point where
  *  waiting longer stops being kinder than failing. */
@@ -3412,6 +3595,33 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
             //
             // Runs on resume too — a parked run's container is long gone, and the
             // one it comes back in needs the same files the first one had.
+            //
+            // A run re-driven after its worker died is the exception to "the store
+            // is newer than the disk": a park harvests before it parks, a crash does
+            // not, so what this run's finished steps wrote to storage/ is only on
+            // disk, and materializing first wrote the store's older copy over it.
+            // On 6 Oct 2026 a worker restart put reddit-desk's 2 Oct post back over
+            // that morning's draft, and a rerun from the validator would have
+            // posted it a second time. So a run that has been here before (its
+            // baseline is on disk) stores what it changed first; files it did not
+            // touch still come fresh from the store.
+            const baselineFile = path.join(runRoot, "runs", run.id, "storage-baseline.json");
+            if (fs.existsSync(baselineFile)) {
+                try {
+                    const changedFrom = JSON.parse(fs.readFileSync(baselineFile, "utf8"));
+                    const { saved } = await harvestFiles(tenant, workspace, `run:${run.id}`, { changedFrom });
+                    if (saved.length) {
+                        run.steps[0]?.events.push({
+                            t: new Date().toISOString(),
+                            type: "info",
+                            text: `files: kept ${saved.join(", ")} written before the run was picked up again`,
+                        });
+                    }
+                }
+                catch {
+                    // the materialize below still runs; at worst this is the old behaviour
+                }
+            }
             try {
                 const brought = await materializeFiles(tenant, workspace);
                 if (brought.length) {
@@ -3421,6 +3631,8 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                         text: `files: ${brought.length} brought into the workspace`,
                     });
                 }
+                fs.mkdirSync(path.dirname(baselineFile), { recursive: true });
+                fs.writeFileSync(baselineFile, JSON.stringify(storageBaseline(tenant, workspace)));
             }
             catch (err) {
                 // A file store that is down must not take the run with it: most flows
@@ -3877,7 +4089,7 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                     save();
                     if (freshGroup.every((s) => s.status !== "pending")) {
                         // everything in this group was rejected
-                        if (freshGroup.some((s) => s.status === "failed" && !s.optional)) {
+                        if (freshGroup.some((s) => s.status === "failed" && !s.optional && !s.carriedFrom)) {
                             run.status = "failed";
                             break;
                         }
@@ -4099,8 +4311,13 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                         effort: step.effort,
                         timeout: step.timeout,
                         // The rescuer inherits the contract, not just the job: a step
-                        // that promised data still owes data when someone else does it.
+                        // that promised data still owes data when someone else does it,
+                        // and a step that had to pass a check still has to pass it — a
+                        // rescue whose result stood unchecked was a way round the very
+                        // check that failed (found 7 Oct 2026, adding rescuers to
+                        // medium-desk).
                         output: step.output,
+                        ...(step.verify ? { verify: step.verify } : {}),
                         attempts: 0,
                         status: "pending",
                         events: [],
@@ -4136,7 +4353,11 @@ function driveRunInner(tenant, workspace, run, modelOverride, tags, opts) {
                     }
                     save();
                 }
-                const requiredFailed = freshGroup.some((s) => s.status === "failed" && !s.optional);
+                // A step carried into a re-run failed in the run it came from, not
+                // here: the person chose to start after it. Counting it stopped
+                // fix-desk's "re-run from the verifier" (run-muwfi5jn-4217) in under
+                // a second, before the one step it was asked to run.
+                const requiredFailed = freshGroup.some((s) => s.status === "failed" && !s.optional && !s.carriedFrom);
                 if (requiredFailed) {
                     run.status = "failed";
                     break;

package/dist/src/runtime.d.ts

@@ -3,6 +3,12 @@ export interface RuntimeSpec {
     packages: string[];
     node?: string | boolean;
     npm: string[];
+    /** Debian packages the step needs from the operating system — a library's
+     *  headers for a pip package that compiles (`libcairo2-dev`), a C compiler
+     *  (`build-essential`), a program (`ffmpeg`). Installed with apt by the run
+     *  pod's root start-up, before anything model-directed runs, from a .deb
+     *  cache on the runtime volume (SYSTEM_INSTALL_SH); verified here. */
+    system?: string[];
     /** Declared entries that are not valid requirements. Kept so the run can
      *  SAY they were dropped: silently discarding `pandas>=2` and then failing
      *  the script with "no module named pandas" is the least debuggable outcome
@@ -13,6 +19,14 @@ export interface RuntimeSpec {
      *  from the same argv as its packages — and said so in the build log. */
     ignored?: string[];
 }
+export declare const PIP_REQUIREMENT: RegExp;
+/** npm's own shape: an optional @scope, then a name, then an optional @range. */
+export declare const NPM_REQUIREMENT: RegExp;
+/** A Debian package name as policy defines it: lowercase letters, digits,
+ *  `+`, `-` and `.`, starting with a letter or digit, two or more characters.
+ *  Like the two above, a guard first: it can never begin with a dash, so it
+ *  can never become an apt option, and it holds no shell characters. */
+export declare const DEB_PACKAGE: RegExp;
 export declare function parseRuntime(raw: unknown): RuntimeSpec | null;
 /**
  * Several declarations, one environment.
@@ -90,4 +104,67 @@ export declare function checkEntry(root: string, spec: RuntimeSpec): string | nu
  * moment a cache grows — so it costs nothing on the hot path.
  */
 export declare function pruneRuntimes(cacheRoot: string, keep: string): string[];
+/**
+ * An installer's failure, said so that the first line is the cause.
+ *
+ * Until 11 Oct 2026 a failure was the installer's last 500 characters, and
+ * the run log shows a message's first line: a pycairo build that died for
+ * want of a C compiler read, in full, `uv pip install failed: files.` — the
+ * tail end of a setuptools deprecation notice. The cause was on the last
+ * line of 4 KB nobody saw. So: find the cause and the package it hit, say
+ * the fix when it is one we know, then the last lines of the log for the rest.
+ */
+export declare function summarizeInstallFailure(what: string, out: string): string;
+/** Sets $FOLDRUN_SYSTEM_PACKAGES from the step's input.json (its merged
+ *  runtime spec) — the run pod's and the docker entry's first line. */
+export declare const SYSTEM_FROM_INPUT_SH = "FOLDRUN_SYSTEM_PACKAGES=\"$(node -e 'try{const r=JSON.parse(require(\"fs\").readFileSync(\"/opt/runner/job/input.json\",\"utf8\")).runtime;process.stdout.write(((r&&r.system)||[]).join(\" \"))}catch(e){}')\"";
+/** Where a run container's root start-up records the system install. */
+export declare const SYSTEM_STATUS_FILE = "/opt/runner/job/system-install.json";
+export declare const SYSTEM_LOG_FILE = "/opt/runner/job/system-install.log";
+/**
+ * The root start-up's half of `runtime: system:` — one shell snippet, used by
+ * the k8s run pod, the docker entry and the deploy-time warm pod alike.
+ * Reads the package names from $FOLDRUN_SYSTEM_PACKAGES (space-separated,
+ * each re-checked against DEB_PACKAGE's shape here too, so nothing else can
+ * ride in), installs them with apt as root BEFORE the drop to the agent user,
+ * and writes SYSTEM_STATUS_FILE for prepareRuntime to read.
+ *
+ * The first step to need a package set installs it with apt (the .debs kept
+ * in `<cache>/.apt/archives`) and then saves the files those packages added,
+ * with their dpkg records, as a snapshot: `<cache>/.apt/snapshots/<key>.tar`.
+ * Every later step restores the snapshot instead — measured 11 Oct 2026 in a
+ * gVisor run pod on the NFS cache: build-essential, pkg-config, python3-dev,
+ * libcairo2-dev and libpango1.0-dev (154 packages, 432 MB) took 23-30 s
+ * through apt and 0.76 s from the snapshot, gcc and pkg-config working after.
+ * The key is the package set AND a checksum of the image's own dpkg status,
+ * so a new runner image never restores files built against the old one. A
+ * snapshot is written under a temporary name and renamed, so two pods
+ * building it at once cannot leave a torn one; a restore that fails falls
+ * back to apt, and prepareRuntime verifies the result either way. Unused for
+ * 30 days, one is pruned. Package lists stay per pod: two pods updating one
+ * lists directory on a shared volume would corrupt it.
+ */
+export declare function systemInstallSh(cacheDir: string, logDir?: string): string;
+/**
+ * Are the declared system packages on this machine? The install happened in
+ * the container's root start-up (systemInstallSh); this only reads the
+ * result. Without dpkg (a laptop, macOS) there is nothing to check against
+ * and nothing this process could install: say so and carry on.
+ */
+export declare function checkSystemPackages(pkgs: string[]): {
+    log: string[];
+    error: string | null;
+};
 export declare function prepareRuntime(tenant: string, spec: RuntimeSpec | null): PreparedRuntime;
+/** Where the root start-up's install helper listens, inside a run container. */
+export declare const INSTALL_SOCKET = "/opt/runner/job/install.sock";
+/**
+ * What a step installs for itself mid-run (the `install` tool) lands under
+ * one per-step directory: pip packages in `<dir>/py` (`uv pip install
+ * --target`), npm packages in `<dir>/npm` (`npm install --prefix`). Both are
+ * on the step's paths from its first turn, so a tool installed by one call is
+ * found by the next — `ffmpeg`, `import manim`, `require("sharp")`. Inside a
+ * run container the directory is the container's own filesystem and goes
+ * when the container does.
+ */
+export declare function stepInstallEnv(dir: string, base: Record<string, string | undefined>): Record<string, string>;