@foldrun/core 0.6.0 → 0.7.0

package/src/runtime.ts

@@ -33,6 +33,12 @@ export interface RuntimeSpec {
   packages: string[]; // pip
   node?: string | boolean;
   npm: string[];
+  /** Debian packages the step needs from the operating system — a library's
+   *  headers for a pip package that compiles (`libcairo2-dev`), a C compiler
+   *  (`build-essential`), a program (`ffmpeg`). Installed with apt by the run
+   *  pod's root start-up, before anything model-directed runs, from a .deb
+   *  cache on the runtime volume (SYSTEM_INSTALL_SH); verified here. */
+  system?: string[];
   /** Declared entries that are not valid requirements. Kept so the run can
    *  SAY they were dropped: silently discarding `pandas>=2` and then failing
    *  the script with "no module named pandas" is the least debuggable outcome
@@ -61,11 +67,17 @@ export interface RuntimeSpec {
  */
 const PIP_NAME = String.raw`[A-Za-z0-9][A-Za-z0-9._-]*(\[[A-Za-z0-9._,-]+\])?`;
 const PIP_SPECIFIER = String.raw`(==|!=|<=|>=|~=|<|>)[A-Za-z0-9._*+!-]+`;
-const PIP_REQUIREMENT = new RegExp(`^${PIP_NAME}(${PIP_SPECIFIER}(,${PIP_SPECIFIER})*)?$`);
+export const PIP_REQUIREMENT = new RegExp(`^${PIP_NAME}(${PIP_SPECIFIER}(,${PIP_SPECIFIER})*)?$`);
 /** npm's own shape: an optional @scope, then a name, then an optional @range. */
-const NPM_REQUIREMENT =
+export const NPM_REQUIREMENT =
   /^(@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*(@[A-Za-z0-9._^~*>=< |-]+)?$/;
 
+/** A Debian package name as policy defines it: lowercase letters, digits,
+ *  `+`, `-` and `.`, starting with a letter or digit, two or more characters.
+ *  Like the two above, a guard first: it can never begin with a dash, so it
+ *  can never become an apt option, and it holds no shell characters. */
+export const DEB_PACKAGE = /^[a-z0-9][a-z0-9+.-]{1,62}$/;
+
 export function parseRuntime(raw: unknown): RuntimeSpec | null {
   if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
   const e = raw as Record<string, unknown>;
@@ -85,9 +97,11 @@ export function parseRuntime(raw: unknown): RuntimeSpec | null {
     node: typeof e.node === "string" || typeof e.node === "boolean" ? e.node : undefined,
     npm: list(e.npm, NPM_REQUIREMENT),
   };
+  const system = list(e.system ?? e.apt, DEB_PACKAGE);
+  if (system.length) spec.system = system;
   if (rejected.length) spec.rejected = rejected;
   if (Array.isArray(e.ignored) && e.ignored.length) spec.ignored = e.ignored.map(String);
-  return wantsPython(spec) || wantsNode(spec) ? spec : null;
+  return wantsPython(spec) || wantsNode(spec) || wantsSystem(spec) ? spec : null;
 }
 
 /**
@@ -117,6 +131,8 @@ export function mergeRuntimes(...specs: (RuntimeSpec | null | undefined)[]): Run
     node: pick("node"),
     npm: uniq(present.flatMap((s) => s.npm)),
   };
+  const system = uniq(present.flatMap((s) => s.system ?? []));
+  if (system.length) merged.system = system;
   const rejected = uniq(present.flatMap((s) => s.rejected ?? []));
   if (rejected.length) merged.rejected = rejected;
   const ignored = uniq(present.flatMap((s) => s.ignored ?? []));
@@ -130,6 +146,11 @@ export function fingerprint(spec: RuntimeSpec): string {
     packages: [...spec.packages].sort(),
     node: spec.node ?? null,
     npm: [...spec.npm].sort(),
+    // Only when declared: adding the key unconditionally would give every
+    // environment already cached a new fingerprint and rebuild them all.
+    // Part of it when declared, because a venv whose packages compiled
+    // against these libraries is not the venv built without them.
+    ...(spec.system?.length ? { system: [...spec.system].sort() } : {}),
   });
   return crypto.createHash("sha256").update(canonical).digest("hex").slice(0, 16);
 }
@@ -244,6 +265,7 @@ function wire(root: string, spec: RuntimeSpec, note: string): PreparedRuntime {
 
 // `python: false` says "not this one" — it used to read as a declaration
 // because it was not undefined, and built a venv nobody asked for.
+const wantsSystem = (s: RuntimeSpec) => (s.system?.length ?? 0) > 0;
 const wantsPython = (s: RuntimeSpec) => (s.python !== undefined && s.python !== false) || s.packages.length > 0;
 const wantsNode = (s: RuntimeSpec) => (s.node !== undefined && s.node !== false) || s.npm.length > 0;
 
@@ -556,8 +578,193 @@ function findUv(): string | null {
 }
 
 // Build (or reuse) the environment for one runtime declaration.
+/**
+ * An installer's failure, said so that the first line is the cause.
+ *
+ * Until 11 Oct 2026 a failure was the installer's last 500 characters, and
+ * the run log shows a message's first line: a pycairo build that died for
+ * want of a C compiler read, in full, `uv pip install failed: files.` — the
+ * tail end of a setuptools deprecation notice. The cause was on the last
+ * line of 4 KB nobody saw. So: find the cause and the package it hit, say
+ * the fix when it is one we know, then the last lines of the log for the rest.
+ */
+export function summarizeInstallFailure(what: string, out: string): string {
+  const text = out ?? "";
+  const lines = text.split(/\r?\n/).map((l) => l.replace(/\s+$/, "")).filter(Boolean);
+  const pkg =
+    /Failed to build `([^`]+)`/.exec(text)?.[1] ??
+    /Failed building wheel for ([\w.-]+)/.exec(text)?.[1] ??
+    /Building wheel for ([\w.-]+) \(.*\) did not run successfully/.exec(text)?.[1] ??
+    /Failed to (?:download|prepare|fetch) `([^`]+)`/.exec(text)?.[1] ??
+    null;
+  const subject = pkg ? `\`${pkg}\`` : "a package";
+  let cause: string | null = null;
+  let hint = "";
+  const header = /fatal error: ([\w./+-]+\.h): No such file/.exec(text);
+  const pc = /Package '?([\w.+-]+)'?,? (?:required by [^,]+, )?(?:was )?not found|Dependency "([\w.+-]+)" not found/.exec(text);
+  if (/No such file or directory: '[^']*gcc'|unable to execute '[^']*gcc'|\bgcc: (?:command )?not found|command '[^']*gcc' failed|error: command '[^']*\b(?:cc|gcc|c\+\+|g\+\+)' failed|C compiler cannot create executables|no acceptable C compiler/i.test(text)) {
+    cause = `${subject} has to be compiled and this runner has no C compiler`;
+    hint = " — add `system: [build-essential]` (and the -dev package of any library it builds against) to the runtime, or pin a version that ships a prebuilt wheel";
+  } else if (/pkg-config(?:: command)? not found|Did not find pkg-config|pkg-config.*No such file/i.test(text)) {
+    cause = `${subject} looks for its libraries with pkg-config, which this runner lacks`;
+    hint = " — add `system: [pkg-config]` and the library's -dev package to the runtime";
+  } else if (header && /^Python\.h$/.test(header[1])) {
+    cause = `${subject} builds against Python's own C headers, which this runner lacks`;
+    hint = " — add `system: [python3-dev, build-essential]` to the runtime";
+  } else if (pc || header) {
+    const lib = pc?.[1] ?? pc?.[2] ?? header![1];
+    cause = `${subject} needs the system library ${lib} (its headers) to build`;
+    hint = ` — add its Debian -dev package to \`runtime: system:\`${pc ? ` (for ${lib}, usually \`lib${lib.replace(/^lib/, "")}-dev\`)` : ""}, plus build-essential and pkg-config`;
+  } else if (/E: Unable to locate package (\S+)/.test(text)) {
+    cause = `apt has no package \`${/E: Unable to locate package (\S+)/.exec(text)![1]}\``;
+    hint = " — check the name on packages.debian.org for trixie";
+  } else if (/No solution found when resolving dependencies/.test(text)) {
+    const i = lines.findIndex((l) => /No solution found/.test(l));
+    cause = `the requirements cannot all be satisfied: ${lines.slice(i + 1, i + 3).map((l) => l.replace(/^[\s╰─▶×]+/, "")).join(" ")}`.trim();
+  } else if (/No matching distribution found for (\S+)|Could not find a version that satisfies the requirement (\S+)|Because (\S+) was not found in the package registry/.test(text)) {
+    const m = /No matching distribution found for (\S+)|Could not find a version that satisfies the requirement (\S+)|Because (\S+) was not found in the package registry/.exec(text)!;
+    cause = `no published version of \`${m[1] ?? m[2] ?? m[3]}\` matches`;
+  } else if (/npm (?:ERR!|error) code (\S+)/.test(text)) {
+    const code = /npm (?:ERR!|error) code (\S+)/.exec(text)![1];
+    const detail = lines.find((l) => /npm (?:ERR!|error) (?:404|notarget|E\w+)\b/.test(l) && !/ code /.test(l));
+    cause = `npm ${code}${detail ? `: ${detail.replace(/^npm (?:ERR!|error)\s*/, "")}` : ""}`;
+  }
+  if (!cause) {
+    const last = [...lines].reverse().find((l) => /error/i.test(l)) ?? lines[lines.length - 1] ?? "no output";
+    cause = (pkg ? `${subject}: ` : "") + last.replace(/^\s*(?:error:|ERROR:|×)\s*/, "");
+  }
+  const excerpt = lines.slice(-25).join("\n");
+  return `${what} failed — ${cause}${hint}\n${excerpt.length > 2000 ? excerpt.slice(-2000) : excerpt}`;
+}
+
+/** Sets $FOLDRUN_SYSTEM_PACKAGES from the step's input.json (its merged
+ *  runtime spec) — the run pod's and the docker entry's first line. */
+export const SYSTEM_FROM_INPUT_SH = `FOLDRUN_SYSTEM_PACKAGES="$(node -e 'try{const r=JSON.parse(require("fs").readFileSync("/opt/runner/job/input.json","utf8")).runtime;process.stdout.write(((r&&r.system)||[]).join(" "))}catch(e){}')"`;
+
+/** Where a run container's root start-up records the system install. */
+export const SYSTEM_STATUS_FILE = "/opt/runner/job/system-install.json";
+export const SYSTEM_LOG_FILE = "/opt/runner/job/system-install.log";
+
+/**
+ * The root start-up's half of `runtime: system:` — one shell snippet, used by
+ * the k8s run pod, the docker entry and the deploy-time warm pod alike.
+ * Reads the package names from $FOLDRUN_SYSTEM_PACKAGES (space-separated,
+ * each re-checked against DEB_PACKAGE's shape here too, so nothing else can
+ * ride in), installs them with apt as root BEFORE the drop to the agent user,
+ * and writes SYSTEM_STATUS_FILE for prepareRuntime to read.
+ *
+ * The first step to need a package set installs it with apt (the .debs kept
+ * in `<cache>/.apt/archives`) and then saves the files those packages added,
+ * with their dpkg records, as a snapshot: `<cache>/.apt/snapshots/<key>.tar`.
+ * Every later step restores the snapshot instead — measured 11 Oct 2026 in a
+ * gVisor run pod on the NFS cache: build-essential, pkg-config, python3-dev,
+ * libcairo2-dev and libpango1.0-dev (154 packages, 432 MB) took 23-30 s
+ * through apt and 0.76 s from the snapshot, gcc and pkg-config working after.
+ * The key is the package set AND a checksum of the image's own dpkg status,
+ * so a new runner image never restores files built against the old one. A
+ * snapshot is written under a temporary name and renamed, so two pods
+ * building it at once cannot leave a torn one; a restore that fails falls
+ * back to apt, and prepareRuntime verifies the result either way. Unused for
+ * 30 days, one is pruned. Package lists stay per pod: two pods updating one
+ * lists directory on a shared volume would corrupt it.
+ */
+export function systemInstallSh(cacheDir: string, logDir = "/opt/runner/job"): string {
+  return `if [ -n "$FOLDRUN_SYSTEM_PACKAGES" ]; then
+  SYS=""; for p in $FOLDRUN_SYSTEM_PACKAGES; do case "$p" in -*|*[!a-z0-9+.-]*) ;; *) SYS="$SYS $p";; esac; done
+  D=\${FOLDRUN_DPKG_DIR:-/var/lib/dpkg}; R=\${FOLDRUN_SYS_ROOT:-/}
+  C=${cacheDir}/.apt; A=$C/archives; S=$C/snapshots
+  mkdir -p "$A/partial" "$S" 2>/dev/null || { A=/var/cache/apt/archives; S=""; }
+  KEY=v3-$( { printf '%s\\n' $SYS | sort -u; cksum < "$D/status"; } | cksum | tr ' ' '-')
+  T0=$(date +%s); MODE=apt; RC=1
+  if [ -n "$S" ] && [ -f "$S/$KEY.tar" ] && tar -xf "$S/$KEY.tar" -C "$R" > ${logDir}/system-install.log 2>&1; then
+    MODE=snapshot; RC=0; touch "$S/$KEY.tar" 2>/dev/null
+    # apt runs ldconfig through dpkg's trigger; a restore must too, or a
+    # library the snapshot brought is missing from /etc/ld.so.cache.
+    [ "$R" = / ] && ldconfig 2>/dev/null
+  else
+    ls "$D"/info/*.list 2>/dev/null | sort > /tmp/.sys-before
+    touch /tmp/.sys-mark; sleep 1
+    { apt-get -o Acquire::Retries=3 -qq update && DEBIAN_FRONTEND=noninteractive apt-get -o Dir::Cache::Archives="$A" -o Acquire::Retries=3 -o Dpkg::Use-Pty=0 install -y -qq --no-install-recommends $SYS; } > ${logDir}/system-install.log 2>&1
+    RC=$?
+    if [ $RC -eq 0 ] && [ -n "$S" ]; then
+      ls "$D"/info/*.list 2>/dev/null | sort > /tmp/.sys-after
+      comm -13 /tmp/.sys-before /tmp/.sys-after > /tmp/.sys-new
+      if [ -s /tmp/.sys-new ]; then
+        { cat $(cat /tmp/.sys-new) | sort -u | while read -r f; do if [ -f "$f" ] || [ -L "$f" ]; then echo "$f"; fi; done
+          for l in $(cat /tmp/.sys-new); do ls "\${l%.list}".* 2>/dev/null; done
+          echo "$D/status"
+          # What the packages' install scripts made, which dpkg's lists do not
+          # name: update-alternatives links (libblas.so.3 → /etc/alternatives),
+          # generated caches and configs. By ctime, not mtime: those links carry
+          # the package's own (2025) mtime, and only ctime cannot be backdated.
+          find \${FOLDRUN_SYS_SCAN:-/etc /usr /var/lib} -xdev \\( -path /var/lib/apt -o -path /var/lib/dpkg/info \\) -prune -o \\( -type f -o -type l \\) -cnewer /tmp/.sys-mark -print 2>/dev/null
+        } | sort -u > /tmp/.sys-files
+        tar -cf "$S/.$KEY.$$.tar" -T /tmp/.sys-files 2>/dev/null && mv -f "$S/.$KEY.$$.tar" "$S/$KEY.tar" 2>/dev/null
+        rm -f "$S/.$KEY.$$.tar" 2>/dev/null
+      fi
+      find "$S" -name '*.tar' -mtime +30 -delete 2>/dev/null
+    fi
+  fi
+  printf '{"ok": %s, "exit": %s, "seconds": %s, "mode": "%s", "packages": "%s"}\\n' "$([ $RC -eq 0 ] && echo true || echo false)" "$RC" "$(( $(date +%s) - T0 ))" "$MODE" "$SYS" > ${logDir}/system-install.json
+  rm -rf /var/lib/apt/lists/* 2>/dev/null || true
+fi`;
+}
+
+/**
+ * Are the declared system packages on this machine? The install happened in
+ * the container's root start-up (systemInstallSh); this only reads the
+ * result. Without dpkg (a laptop, macOS) there is nothing to check against
+ * and nothing this process could install: say so and carry on.
+ */
+export function checkSystemPackages(pkgs: string[]): { log: string[]; error: string | null } {
+  if (!pkgs.length) return { log: [], error: null };
+  const statusFile = process.env.FOLDRUN_SYSTEM_STATUS ?? SYSTEM_STATUS_FILE;
+  const logFile = process.env.FOLDRUN_SYSTEM_LOG ?? SYSTEM_LOG_FILE;
+  let status: { ok?: boolean; seconds?: number; mode?: string } | null = null;
+  try {
+    status = JSON.parse(fs.readFileSync(statusFile, "utf8"));
+  } catch {
+    status = null;
+  }
+  const q = spawnSync("dpkg-query", ["-W", "-f=${Package} ${db:Status-Status}\n", ...pkgs], { encoding: "utf8" });
+  if (q.error) {
+    return { log: [`system packages ${pkgs.join(", ")} install only in a run container; on this machine they are assumed present`], error: null };
+  }
+  const installed = new Set(
+    (q.stdout ?? "")
+      .split("\n")
+      .map((l) => l.trim().split(/\s+/))
+      .filter((f) => f[1] === "installed")
+      .map((f) => f[0].replace(/:.*$/, "")),
+  );
+  const missing = pkgs.filter((p) => !installed.has(p));
+  if (!missing.length) {
+    return { log: [`system packages ready: ${pkgs.join(", ")}${status?.seconds !== undefined ? ` (${status.mode === "snapshot" ? "snapshot" : "apt"}, ${status.seconds}s)` : ""}`], error: null };
+  }
+  let aptOut = "";
+  try {
+    aptOut = fs.readFileSync(logFile, "utf8");
+  } catch {
+    aptOut = "";
+  }
+  const reason = !status
+    ? "this runner did not install them — system packages are installed by a run container's start-up (k8s or docker isolation), not on a bare host"
+    : status.ok
+      ? "apt reported success, yet they are not installed"
+      : summarizeInstallFailure("apt-get install", aptOut).replace(/^apt-get install failed — /, "apt: ");
+  return { log: [], error: `system packages not installed: ${missing.join(", ")} — ${reason}` };
+}
+
 export function prepareRuntime(tenant: string, spec: RuntimeSpec | null): PreparedRuntime {
   if (!spec) return EMPTY;
+  const system = checkSystemPackages(spec.system ?? []);
+  if (system.error) return { ...EMPTY, log: system.log, error: system.error };
+  if (!wantsPython(spec) && !wantsNode(spec)) return { ...EMPTY, log: system.log };
+  const built = prepareLanguageRuntime(tenant, spec);
+  return system.log.length ? { ...built, log: [...system.log, ...built.log] } : built;
+}
+
+function prepareLanguageRuntime(tenant: string, spec: RuntimeSpec): PreparedRuntime {
 
   const fp = fingerprint(spec);
   const cacheRoot = path.join(dataRoot(), tenant, ".runtimes");
@@ -733,7 +940,7 @@ export function prepareRuntime(tenant: string, spec: RuntimeSpec | null): Prepar
         ? install(uv, ["pip", "install", "-q", "--python", venvPython, ...spec.packages], root, 300_000, uvEnv)
         : install(path.join(root, "venv", "bin", "pip"), ["install", "--disable-pip-version-check", "-q", ...spec.packages], root);
       if (!installed.ok) {
-        return fail(`${uv ? "uv pip" : "pip"} install failed: ${installed.out.slice(-500)}`);
+        return fail(summarizeInstallFailure(`${uv ? "uv pip" : "pip"} install`, installed.out));
       }
       built.packages = spec.packages;
       log.push(`runtime ${fp}: installed ${spec.packages.join(", ")}`);
@@ -766,7 +973,7 @@ export function prepareRuntime(tenant: string, spec: RuntimeSpec | null): Prepar
         },
       );
       if (!installed.ok) {
-        return fail(`npm install failed: ${installed.out.slice(-500)}`);
+        return fail(summarizeInstallFailure("npm install", installed.out));
       }
       built.npm = spec.npm;
       log.push(`runtime ${fp}: installed ${spec.npm.join(", ")}`);
@@ -834,3 +1041,26 @@ function touch(file: string): void {
     // read-only or gone; neither is a reason to fail the step
   }
 }
+
+// ---------- installing mid-step ----------
+
+/** Where the root start-up's install helper listens, inside a run container. */
+export const INSTALL_SOCKET = "/opt/runner/job/install.sock";
+
+/**
+ * What a step installs for itself mid-run (the `install` tool) lands under
+ * one per-step directory: pip packages in `<dir>/py` (`uv pip install
+ * --target`), npm packages in `<dir>/npm` (`npm install --prefix`). Both are
+ * on the step's paths from its first turn, so a tool installed by one call is
+ * found by the next — `ffmpeg`, `import manim`, `require("sharp")`. Inside a
+ * run container the directory is the container's own filesystem and goes
+ * when the container does.
+ */
+export function stepInstallEnv(dir: string, base: Record<string, string | undefined>): Record<string, string> {
+  return {
+    // After whatever is already there: a declared environment keeps winning.
+    PATH: [base.PATH ?? "", path.join(dir, "py", "bin"), path.join(dir, "npm", "node_modules", ".bin")].filter(Boolean).join(path.delimiter),
+    PYTHONPATH: [base.PYTHONPATH ?? "", path.join(dir, "py")].filter(Boolean).join(path.delimiter),
+    NODE_PATH: [base.NODE_PATH ?? "", path.join(dir, "npm", "node_modules")].filter(Boolean).join(path.delimiter),
+  };
+}

package/src/script-tools.ts

@@ -38,6 +38,8 @@ export interface ScriptSpec {
    *  script inline. Exactly one of the two is the program. */
   run: string;
   description: string;
+  /** The tool.md body, for the model: how to call it and what it means. */
+  guide?: string;
   args: Record<string, string>; // arg name → description
   interpreter?: string; // optional override, e.g. "python3", "bash"
   /** Seconds this script may run — for the crawl that legitimately takes
@@ -91,6 +93,7 @@ export function parseScripts(raw: unknown): ScriptSpec[] {
       run: run ?? "",
       ...(code ? { code, codeExt: typeof e.codeExt === "string" ? e.codeExt : ".mjs" } : {}),
       description: typeof e.description === "string" ? e.description : "",
+      ...(typeof e.guide === "string" && e.guide.trim() ? { guide: e.guide } : {}),
       args,
       interpreter: typeof e.interpreter === "string" ? e.interpreter : undefined,
       secrets: e.secrets === "proxied" ? "proxied" : undefined,
@@ -427,7 +430,8 @@ export function buildScriptTools(
     return tool(
       spec.name,
       `${spec.description || `Run ${spec.run}.`} Runs the workspace script ${spec.run}; returns its output. ` +
-        `Credentials it needs are already in its environment.`,
+        `Credentials it needs are already in its environment.` +
+        (spec.guide ? `\n\nHow to use it (from its tool.md):\n${spec.guide}` : ""),
       shape,
       async (args) => {
         const started = Date.now();

package/src/secrets.ts

@@ -505,6 +505,15 @@ export interface OAuth2Config {
 
 const OAUTH2_PREFIX = "@oauth2 ";
 
+/** Tries at a token endpoint before a refresh is called failed: the first,
+ *  and two more after a short wait, for a network error, a 5xx or a 429. */
+export const OAUTH_REFRESH_TRIES = 3;
+/** Wait before try n (1-based retry): 1s, 3s — FOLDRUN_OAUTH_RETRY_MS scales it for tests. */
+export function oauthRetryDelayMs(n: number): number {
+  const base = Number(process.env.FOLDRUN_OAUTH_RETRY_MS ?? 1000);
+  return base * (n === 1 ? 1 : 3);
+}
+
 export function setOAuth2Secret(
   tenant: string,
   name: string,
@@ -550,12 +559,31 @@ async function exchange(config: OAuth2Config, cacheKey: string): Promise<string>
       client_secret: config.client_secret,
       ...(config.extra ?? {}),
     });
-    const res = await fetchUntrusted(config.token_url, {
-      method: "POST",
-      headers: { "content-type": "application/x-www-form-urlencoded" },
-      body: body.toString(),
-      signal: AbortSignal.timeout(15_000),
-    }, { sameOrigin: true });
+    // A network blip is retried; a provider's answer is not. "fetch failed"
+    // (a reset, a DNS hiccup, a timeout) failed whole steps on the first try
+    // — gbp-desk's Thursday posts, 7 Oct 2026. A 4xx (revoked, wrong client)
+    // will say the same thing again, so it goes straight to the error below
+    // and the reconnect mail; a 5xx or 429 is the provider busy, retried.
+    let res: Response | undefined;
+    let lastNetwork: unknown;
+    for (let attempt = 0; attempt < OAUTH_REFRESH_TRIES; attempt++) {
+      if (attempt) await new Promise((r) => setTimeout(r, oauthRetryDelayMs(attempt)));
+      try {
+        res = await fetchUntrusted(config.token_url, {
+          method: "POST",
+          headers: { "content-type": "application/x-www-form-urlencoded" },
+          body: body.toString(),
+          signal: AbortSignal.timeout(15_000),
+        }, { sameOrigin: true });
+      } catch (err) {
+        lastNetwork = err;
+        res = undefined;
+        continue;
+      }
+      if (res.status === 429 || res.status >= 500) continue;
+      break;
+    }
+    if (!res) throw lastNetwork instanceof Error ? lastNetwork : new Error(String(lastNetwork));
     const payload = (await res.json().catch(() => ({}))) as {
       access_token?: string;
       expires_in?: number;

package/src/starter.ts

@@ -47,6 +47,10 @@ foldrun_version: "0.1"
 # that declares the same key replaces it *whole* — base_url and token never
 # merge across scopes, so a provider block belongs entirely at one level.
 #
+# Your own model key needs no block: Settings → Model provider, or
+# \`foldrun account model set anthropic --key …\` (an API key, not a login).
+# A provider: block here or lower wins over it.
+#
 # provider:
 #   base_url: https://openrouter.ai/api
 #   token: \${OPENROUTER_API_KEY}
@@ -342,13 +346,14 @@ All of it, \`/api/workspaces/<workspace>\` unless noted:
 | \`/runs/<id>/message\` | POST | tell a running step something; \`step\` picks one of a parallel group |
 | \`/flows\` · \`/flows/<f>\` | GET POST · POST DELETE PATCH | list, create, edit |
 | \`/flows/<f>/run\` | POST | start a flow |
-| \`/agents\` · \`/agents/<a>/run\` | GET POST · POST | list agents, run one |
+| \`/agents\` · \`/agents/<a>/run\` · \`/agents/import\` | GET POST · POST · GET POST | list agents, run one, copy one in from another workspace |
 | \`/evals\` · \`/evals/<e>/run\` | GET POST · POST | list evals, run one |
 | \`/tools/<t>/test\` | POST | exercise one tool alone |
 | \`/storage\` · \`/storage/download\` · \`/storage/upload-url\` · \`/storage/preview\` | GET POST PUT DELETE · GET · POST · GET | workspace files, any type; \`preview\` returns one parsed for reading |
 | \`/assets\` | POST | upload an asset |
 | \`/hooks/<f>/rotate\` | POST | new webhook token for a flow |
 | \`/notify/deliveries\` · \`/notify/deliveries/<id>/redeliver\` | GET · POST | every notify: webhook sent and each attempt; send one again |
+| \`/export\` · \`/import\` | GET · POST | the workspace, a flow (\`?kind=flow&name=\`) or an agent as a .zip; take one in (\`?dryRun=1\` first, \`?overwrite=1\` to replace) |
 | \`/restore\` | POST | the workspace's source back to a point in its history — \`dryRun\` first, then \`confirm\` |
 | \`/vocabulary\` | GET | what this workspace's documents may say |
 
@@ -458,10 +463,11 @@ subagents: [researcher]     # colleagues it may delegate a job to (own context,
 secrets: [CRM_TOKEN]        # keys the agent uses itself; a tool's own come with the tool
 limits: {web.search: 40, crm: 20, calls: 300}  # per step; a call past a limit is refused, never run
 timezone: Australia/Sydney  # the calendar it works to — IANA name or UTC+10
-provider: {}                # BYOK: your own model credential
+provider: {}                # BYOK for this agent; account-wide: \`foldrun account model set\`
 permissionMode: plan        # plan first, act once approved
 runtime:                    # only if scripts need packages
   packages: [requests]
+  system: [ffmpeg]          # Debian packages, apt-installed before the step
 ---
 
 You research a topic and report what you found...
@@ -475,8 +481,12 @@ flow's, else the workspace's AGENTS.md, else the account's, else UTC. It sets
 \`TZ\` and the date the agent is told, so a Sydney desk stamps the Sydney date.
 
 \`runtime:\` installs pip/npm packages in the sandbox each agent runs in. Pin
-versions the normal way (\`pandas>=2\`, \`lodash@^4\`). Python and Node are
-supported; anything else should ship as a committed binary.
+versions the normal way (\`pandas>=2\`, \`lodash@^4\`). \`system:\` adds Debian
+packages \u{2014} a program (\`ffmpeg\`), a library's headers for a pip package that
+compiles (\`libcairo2-dev\`, \`python3-dev\`), \`build-essential\` \u{2014} installed with
+apt before the step starts and cached, so later runs only unpack them. An agent that
+finds it needs something mid-run installs it itself with the \`install\` tool
+(apt, pip or npm), which every agent has; declaring is only a speed-up.
 
 Link documents to each other with \`[[wikilinks]]\` — the name, not the path.
 `,
@@ -499,3 +509,31 @@ expect:
     },
   ];
 }
+
+/**
+ * A new workspace with nothing in it to delete: its AGENTS.md, the
+ * .gitignore that keeps its secret key out of git, and the CLAUDE.md a coding
+ * tool reads. No example agents or flow.
+ *
+ * The starter's researcher and writer were a demo, and a person making a
+ * workspace for real work met them as clutter — "why does a new workspace
+ * have agents?" (owner, 2026-10-05). The agents worth having are usually ones
+ * already written in another workspace, which `agents/import` copies in; the
+ * example is still one flag away (`foldrun new <name> --starter`).
+ */
+export function blankWorkspaceFiles(workspace: string): StarterFile[] {
+  const keep = new Set(["AGENTS.md", ".gitignore", "CLAUDE.md"]);
+  return starterFiles(workspace)
+    .filter((f) => keep.has(f.path))
+    .map((f) =>
+      f.path === "AGENTS.md"
+        ? {
+            ...f,
+            content: f.content.replace(
+              "description: A starter workspace — edit the agents, then run the flow.",
+              "description: A new workspace — add an agent or import one from another workspace, then write a flow.",
+            ),
+          }
+        : f,
+    );
+}

package/src/step-exec.ts

@@ -329,6 +329,13 @@ async function executeStepInner(opts: ExecOptions, runQuery: QueryFn): Promise<E
       // declares no tools gets none, instead of seeing the full Claude
       // Code toolset and burning turns on denied calls.
       tools: opts.allowed,
+      // Stated, never left to the SDK. Unset, it chose its own mode for the
+      // login it ran under, and in auto mode its classifier refused a granted
+      // script before canUseTool saw it: "denied by the Claude Code auto mode
+      // classifier. Reason: [Irreversible Local Destruction]" for backlink-
+      // desk's outreach_reset, which only marks old bodies STALE (2026-10-04
+      // and 05). The decision is ours: canUseTool and the PreToolUse hook.
+      permissionMode: "default",
       // Deliberately NOT allowedTools: listing them there auto-approves and
       // skips canUseTool entirely, which is where confinement happens. The
       // toolset is still restricted by `tools` above.
@@ -632,11 +639,23 @@ async function executeStepInner(opts: ExecOptions, runQuery: QueryFn): Promise<E
             ...(block.is_error ? { err: true } : {}),
             ...(open.subagent ? { subagent: open.subagent } : {}),
           });
+          // Why it failed, on the trail. The span above carried only `err`,
+          // so a refused call left the agent's own guess as the only account
+          // of it: backlink-desk's outreach_reset "denied by the permission
+          // classifier" (run-muufboma-2t6l, 2026-10-05) could not be checked.
+          if (block.is_error) {
+            const why = toolResultText(block.content).replace(/\s+/g, " ").trim().slice(0, 300);
+            if (why) emit("error", `${open.name}: ${why}`, { call: block.tool_use_id, ...(open.subagent ? { subagent: open.subagent } : {}) });
+          }
           readPod(block.tool_use_id);
         }
       }
     } else if (message.type === "result") {
       status = message.subtype === "success" ? "completed" : "failed";
+      // Calls the SDK itself refused, which canUseTool never saw.
+      for (const d of ("permission_denials" in message && Array.isArray(message.permission_denials) ? message.permission_denials : [])) {
+        emit("error", `refused before it ran: ${String((d as { tool_name?: unknown }).tool_name ?? "a tool")}`);
+      }
       if (message.subtype === "error_max_turns") {
         emit("error", `stopped after ${opts.maxTurns ?? "its"} turns (max_turns: in the flow file) — the step did not finish`);
       }
@@ -877,8 +896,8 @@ export async function checkVerify(
      *  and `matches:` test this, because it is also what the run's headline
      *  is read from: a reporter that narrated between tool calls ("Now let
      *  me write the report…") failed `matches: ^BAD\b` with a correct
-     *  headline while the check read every turn joined. `judge:` still grades
-     *  the whole result. */
+     *  headline while the check read every turn joined. `judge:` grades this
+     *  too, as "the reply", with the whole result beside it as the work. */
     conclusion?: string | null;
     data?: unknown;
     /** The step's own model environment, for `judge:` — it grades on the
@@ -941,7 +960,13 @@ export async function checkVerify(
       return { ok, headline: ok ? "present and non-empty" : `${value} is missing or empty`, detail: "" };
     }
     case "judge": {
-      const verdict = await judgeReply(value, output, ctx.modelEnv ?? {}, ctx.signal);
+      // The reply is the final turn, as everywhere else; every turn goes
+      // beside it as the work, so a claim about what the step did can still
+      // be checked. Both as one <reply> failed `judge: the reply opens with
+      // GOOD` on a reply that did (fix-desk, run-muwevg72-7mza): the joined
+      // text opened with the narration before it.
+      const reply = ctx.conclusion?.trim() ? ctx.conclusion : output;
+      const verdict = await judgeReply(value, reply, output, ctx.modelEnv ?? {}, ctx.signal);
       const ok = /^\s*PASS\b/i.test(verdict);
       return { ok, headline: ok ? "PASS" : "FAIL", detail: verdict.slice(0, 300) };
     }
@@ -954,9 +979,24 @@ export async function checkVerify(
  * Same shape as the eval judge, and answered with one word first so the
  * verdict is a prefix test rather than a reading.
  */
+export function judgePrompt(rubric: string, reply: string, work: string): string {
+  const shown = reply.slice(0, 20_000);
+  const context = work.trim() && work.trim() !== reply.trim()
+    ? `Everything the step wrote while working, for context. "The reply" means only the reply block below, ` +
+      `never this:\n<work>\n${work.slice(-20_000)}\n</work>\n\n`
+    : "";
+  return (
+    `You are grading a reply against one requirement. Answer with PASS or FAIL as the ` +
+    `first word, then one sentence of reason.\n\nRequirement: ${rubric}\n\n` +
+    context +
+    `<reply>\n${shown}\n</reply>`
+  );
+}
+
 async function judgeReply(
   rubric: string,
-  output: string,
+  reply: string,
+  work: string,
   env: Record<string, string | undefined>,
   signal?: AbortSignal,
 ): Promise<string> {
@@ -966,10 +1006,7 @@ async function judgeReply(
   signal?.addEventListener("abort", () => abortController.abort(), { once: true });
   try {
     const q = query({
-      prompt:
-        `You are grading a reply against one requirement. Answer with PASS or FAIL as the ` +
-        `first word, then one sentence of reason.\n\nRequirement: ${rubric}\n\n` +
-        `<reply>\n${output.slice(0, 40_000)}\n</reply>`,
+      prompt: judgePrompt(rubric, reply, work),
       options: {
         model: "haiku",
         systemPrompt: "You grade text against a stated requirement. Be strict and literal.",
@@ -1060,3 +1097,14 @@ function runVerify(
 export function sdkEnv(base: Record<string, string | undefined> | undefined, extra: Record<string, string> = {}): Record<string, string | undefined> {
   return { ...(base ?? process.env), ...extra, ENABLE_CLAUDEAI_MCP_SERVERS: "false" };
 }
+
+/** The text of a tool result, whichever shape it came in: a string, or a
+ *  list of content blocks of which only the text ones say anything. */
+export function toolResultText(content: unknown): string {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return "";
+  return content
+    .filter((b): b is { type: "text"; text: string } => !!b && (b as { type?: unknown }).type === "text" && typeof (b as { text?: unknown }).text === "string")
+    .map((b) => b.text)
+    .join(" ");
+}

package/src/storage.ts

@@ -839,6 +839,14 @@ export async function materializeFiles(tenant: string, workspace: string): Promi
   return brought;
 }
 
+/** path → sha of every stored file: what `storage/` holds once
+ *  materializeFiles has run. A run keeps it to tell its own writes from the
+ *  files it was handed. */
+export function storageBaseline(tenant: string, workspace: string): Record<string, string> {
+  if (!fileStoreEnabled()) return {};
+  return Object.fromEntries(readIndex(tenant, workspace).files.map((f) => [f.path, f.sha]));
+}
+
 /**
  * Take what the run left in `storage/` back into the store, stamped with the
  * run that wrote it.
@@ -851,6 +859,7 @@ export async function harvestFiles(
   tenant: string,
   workspace: string,
   by: string,
+  opts: { changedFrom?: Record<string, string> } = {},
 ): Promise<{ saved: string[]; errors: string[] }> {
   if (!fileStoreEnabled()) return { saved: [], errors: [] };
   adoptLegacyFilesDir(workspaceDir(tenant, workspace));
@@ -885,6 +894,10 @@ export async function harvestFiles(
         // Same size is not proof, so confirm by hash before skipping a write.
         if (sha256(fs.readFileSync(abs)) === previous.sha) continue;
       }
+      // Only what the run itself changed: a file still holding the bytes it
+      // was handed is not this run's work, and the index may have moved on
+      // since (an upload while the run was going) — that must win.
+      if (opts.changedFrom && opts.changedFrom[norm] === sha256(fs.readFileSync(abs))) continue;
       await putFileQuietly(tenant, workspace, norm, fs.readFileSync(abs), by);
       saved.push(norm);
     } catch (err) {