@foldrun/core 0.5.0 → 0.7.0

package/dist/src/runtime.d.ts

@@ -3,6 +3,12 @@ export interface RuntimeSpec {
     packages: string[];
     node?: string | boolean;
     npm: string[];
+    /** Debian packages the step needs from the operating system — a library's
+     *  headers for a pip package that compiles (`libcairo2-dev`), a C compiler
+     *  (`build-essential`), a program (`ffmpeg`). Installed with apt by the run
+     *  pod's root start-up, before anything model-directed runs, from a .deb
+     *  cache on the runtime volume (SYSTEM_INSTALL_SH); verified here. */
+    system?: string[];
     /** Declared entries that are not valid requirements. Kept so the run can
      *  SAY they were dropped: silently discarding `pandas>=2` and then failing
      *  the script with "no module named pandas" is the least debuggable outcome
@@ -13,6 +19,14 @@ export interface RuntimeSpec {
      *  from the same argv as its packages — and said so in the build log. */
     ignored?: string[];
 }
+export declare const PIP_REQUIREMENT: RegExp;
+/** npm's own shape: an optional @scope, then a name, then an optional @range. */
+export declare const NPM_REQUIREMENT: RegExp;
+/** A Debian package name as policy defines it: lowercase letters, digits,
+ *  `+`, `-` and `.`, starting with a letter or digit, two or more characters.
+ *  Like the two above, a guard first: it can never begin with a dash, so it
+ *  can never become an apt option, and it holds no shell characters. */
+export declare const DEB_PACKAGE: RegExp;
 export declare function parseRuntime(raw: unknown): RuntimeSpec | null;
 /**
  * Several declarations, one environment.
@@ -90,4 +104,67 @@ export declare function checkEntry(root: string, spec: RuntimeSpec): string | nu
  * moment a cache grows — so it costs nothing on the hot path.
  */
 export declare function pruneRuntimes(cacheRoot: string, keep: string): string[];
+/**
+ * An installer's failure, said so that the first line is the cause.
+ *
+ * Until 11 Oct 2026 a failure was the installer's last 500 characters, and
+ * the run log shows a message's first line: a pycairo build that died for
+ * want of a C compiler read, in full, `uv pip install failed: files.` — the
+ * tail end of a setuptools deprecation notice. The cause was on the last
+ * line of 4 KB nobody saw. So: find the cause and the package it hit, say
+ * the fix when it is one we know, then the last lines of the log for the rest.
+ */
+export declare function summarizeInstallFailure(what: string, out: string): string;
+/** Sets $FOLDRUN_SYSTEM_PACKAGES from the step's input.json (its merged
+ *  runtime spec) — the run pod's and the docker entry's first line. */
+export declare const SYSTEM_FROM_INPUT_SH = "FOLDRUN_SYSTEM_PACKAGES=\"$(node -e 'try{const r=JSON.parse(require(\"fs\").readFileSync(\"/opt/runner/job/input.json\",\"utf8\")).runtime;process.stdout.write(((r&&r.system)||[]).join(\" \"))}catch(e){}')\"";
+/** Where a run container's root start-up records the system install. */
+export declare const SYSTEM_STATUS_FILE = "/opt/runner/job/system-install.json";
+export declare const SYSTEM_LOG_FILE = "/opt/runner/job/system-install.log";
+/**
+ * The root start-up's half of `runtime: system:` — one shell snippet, used by
+ * the k8s run pod, the docker entry and the deploy-time warm pod alike.
+ * Reads the package names from $FOLDRUN_SYSTEM_PACKAGES (space-separated,
+ * each re-checked against DEB_PACKAGE's shape here too, so nothing else can
+ * ride in), installs them with apt as root BEFORE the drop to the agent user,
+ * and writes SYSTEM_STATUS_FILE for prepareRuntime to read.
+ *
+ * The first step to need a package set installs it with apt (the .debs kept
+ * in `<cache>/.apt/archives`) and then saves the files those packages added,
+ * with their dpkg records, as a snapshot: `<cache>/.apt/snapshots/<key>.tar`.
+ * Every later step restores the snapshot instead — measured 11 Oct 2026 in a
+ * gVisor run pod on the NFS cache: build-essential, pkg-config, python3-dev,
+ * libcairo2-dev and libpango1.0-dev (154 packages, 432 MB) took 23-30 s
+ * through apt and 0.76 s from the snapshot, gcc and pkg-config working after.
+ * The key is the package set AND a checksum of the image's own dpkg status,
+ * so a new runner image never restores files built against the old one. A
+ * snapshot is written under a temporary name and renamed, so two pods
+ * building it at once cannot leave a torn one; a restore that fails falls
+ * back to apt, and prepareRuntime verifies the result either way. Unused for
+ * 30 days, one is pruned. Package lists stay per pod: two pods updating one
+ * lists directory on a shared volume would corrupt it.
+ */
+export declare function systemInstallSh(cacheDir: string, logDir?: string): string;
+/**
+ * Are the declared system packages on this machine? The install happened in
+ * the container's root start-up (systemInstallSh); this only reads the
+ * result. Without dpkg (a laptop, macOS) there is nothing to check against
+ * and nothing this process could install: say so and carry on.
+ */
+export declare function checkSystemPackages(pkgs: string[]): {
+    log: string[];
+    error: string | null;
+};
 export declare function prepareRuntime(tenant: string, spec: RuntimeSpec | null): PreparedRuntime;
+/** Where the root start-up's install helper listens, inside a run container. */
+export declare const INSTALL_SOCKET = "/opt/runner/job/install.sock";
+/**
+ * What a step installs for itself mid-run (the `install` tool) lands under
+ * one per-step directory: pip packages in `<dir>/py` (`uv pip install
+ * --target`), npm packages in `<dir>/npm` (`npm install --prefix`). Both are
+ * on the step's paths from its first turn, so a tool installed by one call is
+ * found by the next — `ffmpeg`, `import manim`, `require("sharp")`. Inside a
+ * run container the directory is the container's own filesystem and goes
+ * when the container does.
+ */
+export declare function stepInstallEnv(dir: string, base: Record<string, string | undefined>): Record<string, string>;

package/dist/src/runtime.js

@@ -43,9 +43,14 @@ import { spawnSync } from "node:child_process";
  */
 const PIP_NAME = String.raw `[A-Za-z0-9][A-Za-z0-9._-]*(\[[A-Za-z0-9._,-]+\])?`;
 const PIP_SPECIFIER = String.raw `(==|!=|<=|>=|~=|<|>)[A-Za-z0-9._*+!-]+`;
-const PIP_REQUIREMENT = new RegExp(`^${PIP_NAME}(${PIP_SPECIFIER}(,${PIP_SPECIFIER})*)?$`);
+export const PIP_REQUIREMENT = new RegExp(`^${PIP_NAME}(${PIP_SPECIFIER}(,${PIP_SPECIFIER})*)?$`);
 /** npm's own shape: an optional @scope, then a name, then an optional @range. */
-const NPM_REQUIREMENT = /^(@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*(@[A-Za-z0-9._^~*>=< |-]+)?$/;
+export const NPM_REQUIREMENT = /^(@[a-z0-9][a-z0-9._-]*\/)?[a-z0-9][a-z0-9._-]*(@[A-Za-z0-9._^~*>=< |-]+)?$/;
+/** A Debian package name as policy defines it: lowercase letters, digits,
+ *  `+`, `-` and `.`, starting with a letter or digit, two or more characters.
+ *  Like the two above, a guard first: it can never begin with a dash, so it
+ *  can never become an apt option, and it holds no shell characters. */
+export const DEB_PACKAGE = /^[a-z0-9][a-z0-9+.-]{1,62}$/;
 export function parseRuntime(raw) {
     if (!raw || typeof raw !== "object" || Array.isArray(raw))
         return null;
@@ -67,11 +72,14 @@ export function parseRuntime(raw) {
         node: typeof e.node === "string" || typeof e.node === "boolean" ? e.node : undefined,
         npm: list(e.npm, NPM_REQUIREMENT),
     };
+    const system = list(e.system ?? e.apt, DEB_PACKAGE);
+    if (system.length)
+        spec.system = system;
     if (rejected.length)
         spec.rejected = rejected;
     if (Array.isArray(e.ignored) && e.ignored.length)
         spec.ignored = e.ignored.map(String);
-    return wantsPython(spec) || wantsNode(spec) ? spec : null;
+    return wantsPython(spec) || wantsNode(spec) || wantsSystem(spec) ? spec : null;
 }
 /**
  * Several declarations, one environment.
@@ -103,6 +111,9 @@ export function mergeRuntimes(...specs) {
         node: pick("node"),
         npm: uniq(present.flatMap((s) => s.npm)),
     };
+    const system = uniq(present.flatMap((s) => s.system ?? []));
+    if (system.length)
+        merged.system = system;
     const rejected = uniq(present.flatMap((s) => s.rejected ?? []));
     if (rejected.length)
         merged.rejected = rejected;
@@ -117,6 +128,11 @@ export function fingerprint(spec) {
         packages: [...spec.packages].sort(),
         node: spec.node ?? null,
         npm: [...spec.npm].sort(),
+        // Only when declared: adding the key unconditionally would give every
+        // environment already cached a new fingerprint and rebuild them all.
+        // Part of it when declared, because a venv whose packages compiled
+        // against these libraries is not the venv built without them.
+        ...(spec.system?.length ? { system: [...spec.system].sort() } : {}),
     });
     return crypto.createHash("sha256").update(canonical).digest("hex").slice(0, 16);
 }
@@ -210,6 +226,7 @@ function wire(root, spec, note) {
 }
 // `python: false` says "not this one" — it used to read as a declaration
 // because it was not undefined, and built a venv nobody asked for.
+const wantsSystem = (s) => (s.system?.length ?? 0) > 0;
 const wantsPython = (s) => (s.python !== undefined && s.python !== false) || s.packages.length > 0;
 const wantsNode = (s) => (s.node !== undefined && s.node !== false) || s.npm.length > 0;
 /** How long a build may hold the claim, and therefore how long another step
@@ -524,9 +541,197 @@ function findUv() {
     return uvKnown;
 }
 // Build (or reuse) the environment for one runtime declaration.
+/**
+ * An installer's failure, said so that the first line is the cause.
+ *
+ * Until 11 Oct 2026 a failure was the installer's last 500 characters, and
+ * the run log shows a message's first line: a pycairo build that died for
+ * want of a C compiler read, in full, `uv pip install failed: files.` — the
+ * tail end of a setuptools deprecation notice. The cause was on the last
+ * line of 4 KB nobody saw. So: find the cause and the package it hit, say
+ * the fix when it is one we know, then the last lines of the log for the rest.
+ */
+export function summarizeInstallFailure(what, out) {
+    const text = out ?? "";
+    const lines = text.split(/\r?\n/).map((l) => l.replace(/\s+$/, "")).filter(Boolean);
+    const pkg = /Failed to build `([^`]+)`/.exec(text)?.[1] ??
+        /Failed building wheel for ([\w.-]+)/.exec(text)?.[1] ??
+        /Building wheel for ([\w.-]+) \(.*\) did not run successfully/.exec(text)?.[1] ??
+        /Failed to (?:download|prepare|fetch) `([^`]+)`/.exec(text)?.[1] ??
+        null;
+    const subject = pkg ? `\`${pkg}\`` : "a package";
+    let cause = null;
+    let hint = "";
+    const header = /fatal error: ([\w./+-]+\.h): No such file/.exec(text);
+    const pc = /Package '?([\w.+-]+)'?,? (?:required by [^,]+, )?(?:was )?not found|Dependency "([\w.+-]+)" not found/.exec(text);
+    if (/No such file or directory: '[^']*gcc'|unable to execute '[^']*gcc'|\bgcc: (?:command )?not found|command '[^']*gcc' failed|error: command '[^']*\b(?:cc|gcc|c\+\+|g\+\+)' failed|C compiler cannot create executables|no acceptable C compiler/i.test(text)) {
+        cause = `${subject} has to be compiled and this runner has no C compiler`;
+        hint = " — add `system: [build-essential]` (and the -dev package of any library it builds against) to the runtime, or pin a version that ships a prebuilt wheel";
+    }
+    else if (/pkg-config(?:: command)? not found|Did not find pkg-config|pkg-config.*No such file/i.test(text)) {
+        cause = `${subject} looks for its libraries with pkg-config, which this runner lacks`;
+        hint = " — add `system: [pkg-config]` and the library's -dev package to the runtime";
+    }
+    else if (header && /^Python\.h$/.test(header[1])) {
+        cause = `${subject} builds against Python's own C headers, which this runner lacks`;
+        hint = " — add `system: [python3-dev, build-essential]` to the runtime";
+    }
+    else if (pc || header) {
+        const lib = pc?.[1] ?? pc?.[2] ?? header[1];
+        cause = `${subject} needs the system library ${lib} (its headers) to build`;
+        hint = ` — add its Debian -dev package to \`runtime: system:\`${pc ? ` (for ${lib}, usually \`lib${lib.replace(/^lib/, "")}-dev\`)` : ""}, plus build-essential and pkg-config`;
+    }
+    else if (/E: Unable to locate package (\S+)/.test(text)) {
+        cause = `apt has no package \`${/E: Unable to locate package (\S+)/.exec(text)[1]}\``;
+        hint = " — check the name on packages.debian.org for trixie";
+    }
+    else if (/No solution found when resolving dependencies/.test(text)) {
+        const i = lines.findIndex((l) => /No solution found/.test(l));
+        cause = `the requirements cannot all be satisfied: ${lines.slice(i + 1, i + 3).map((l) => l.replace(/^[\s╰─▶×]+/, "")).join(" ")}`.trim();
+    }
+    else if (/No matching distribution found for (\S+)|Could not find a version that satisfies the requirement (\S+)|Because (\S+) was not found in the package registry/.test(text)) {
+        const m = /No matching distribution found for (\S+)|Could not find a version that satisfies the requirement (\S+)|Because (\S+) was not found in the package registry/.exec(text);
+        cause = `no published version of \`${m[1] ?? m[2] ?? m[3]}\` matches`;
+    }
+    else if (/npm (?:ERR!|error) code (\S+)/.test(text)) {
+        const code = /npm (?:ERR!|error) code (\S+)/.exec(text)[1];
+        const detail = lines.find((l) => /npm (?:ERR!|error) (?:404|notarget|E\w+)\b/.test(l) && !/ code /.test(l));
+        cause = `npm ${code}${detail ? `: ${detail.replace(/^npm (?:ERR!|error)\s*/, "")}` : ""}`;
+    }
+    if (!cause) {
+        const last = [...lines].reverse().find((l) => /error/i.test(l)) ?? lines[lines.length - 1] ?? "no output";
+        cause = (pkg ? `${subject}: ` : "") + last.replace(/^\s*(?:error:|ERROR:|×)\s*/, "");
+    }
+    const excerpt = lines.slice(-25).join("\n");
+    return `${what} failed — ${cause}${hint}\n${excerpt.length > 2000 ? excerpt.slice(-2000) : excerpt}`;
+}
+/** Sets $FOLDRUN_SYSTEM_PACKAGES from the step's input.json (its merged
+ *  runtime spec) — the run pod's and the docker entry's first line. */
+export const SYSTEM_FROM_INPUT_SH = `FOLDRUN_SYSTEM_PACKAGES="$(node -e 'try{const r=JSON.parse(require("fs").readFileSync("/opt/runner/job/input.json","utf8")).runtime;process.stdout.write(((r&&r.system)||[]).join(" "))}catch(e){}')"`;
+/** Where a run container's root start-up records the system install. */
+export const SYSTEM_STATUS_FILE = "/opt/runner/job/system-install.json";
+export const SYSTEM_LOG_FILE = "/opt/runner/job/system-install.log";
+/**
+ * The root start-up's half of `runtime: system:` — one shell snippet, used by
+ * the k8s run pod, the docker entry and the deploy-time warm pod alike.
+ * Reads the package names from $FOLDRUN_SYSTEM_PACKAGES (space-separated,
+ * each re-checked against DEB_PACKAGE's shape here too, so nothing else can
+ * ride in), installs them with apt as root BEFORE the drop to the agent user,
+ * and writes SYSTEM_STATUS_FILE for prepareRuntime to read.
+ *
+ * The first step to need a package set installs it with apt (the .debs kept
+ * in `<cache>/.apt/archives`) and then saves the files those packages added,
+ * with their dpkg records, as a snapshot: `<cache>/.apt/snapshots/<key>.tar`.
+ * Every later step restores the snapshot instead — measured 11 Oct 2026 in a
+ * gVisor run pod on the NFS cache: build-essential, pkg-config, python3-dev,
+ * libcairo2-dev and libpango1.0-dev (154 packages, 432 MB) took 23-30 s
+ * through apt and 0.76 s from the snapshot, gcc and pkg-config working after.
+ * The key is the package set AND a checksum of the image's own dpkg status,
+ * so a new runner image never restores files built against the old one. A
+ * snapshot is written under a temporary name and renamed, so two pods
+ * building it at once cannot leave a torn one; a restore that fails falls
+ * back to apt, and prepareRuntime verifies the result either way. Unused for
+ * 30 days, one is pruned. Package lists stay per pod: two pods updating one
+ * lists directory on a shared volume would corrupt it.
+ */
+export function systemInstallSh(cacheDir, logDir = "/opt/runner/job") {
+    return `if [ -n "$FOLDRUN_SYSTEM_PACKAGES" ]; then
+  SYS=""; for p in $FOLDRUN_SYSTEM_PACKAGES; do case "$p" in -*|*[!a-z0-9+.-]*) ;; *) SYS="$SYS $p";; esac; done
+  D=\${FOLDRUN_DPKG_DIR:-/var/lib/dpkg}; R=\${FOLDRUN_SYS_ROOT:-/}
+  C=${cacheDir}/.apt; A=$C/archives; S=$C/snapshots
+  mkdir -p "$A/partial" "$S" 2>/dev/null || { A=/var/cache/apt/archives; S=""; }
+  KEY=v3-$( { printf '%s\\n' $SYS | sort -u; cksum < "$D/status"; } | cksum | tr ' ' '-')
+  T0=$(date +%s); MODE=apt; RC=1
+  if [ -n "$S" ] && [ -f "$S/$KEY.tar" ] && tar -xf "$S/$KEY.tar" -C "$R" > ${logDir}/system-install.log 2>&1; then
+    MODE=snapshot; RC=0; touch "$S/$KEY.tar" 2>/dev/null
+    # apt runs ldconfig through dpkg's trigger; a restore must too, or a
+    # library the snapshot brought is missing from /etc/ld.so.cache.
+    [ "$R" = / ] && ldconfig 2>/dev/null
+  else
+    ls "$D"/info/*.list 2>/dev/null | sort > /tmp/.sys-before
+    touch /tmp/.sys-mark; sleep 1
+    { apt-get -o Acquire::Retries=3 -qq update && DEBIAN_FRONTEND=noninteractive apt-get -o Dir::Cache::Archives="$A" -o Acquire::Retries=3 -o Dpkg::Use-Pty=0 install -y -qq --no-install-recommends $SYS; } > ${logDir}/system-install.log 2>&1
+    RC=$?
+    if [ $RC -eq 0 ] && [ -n "$S" ]; then
+      ls "$D"/info/*.list 2>/dev/null | sort > /tmp/.sys-after
+      comm -13 /tmp/.sys-before /tmp/.sys-after > /tmp/.sys-new
+      if [ -s /tmp/.sys-new ]; then
+        { cat $(cat /tmp/.sys-new) | sort -u | while read -r f; do if [ -f "$f" ] || [ -L "$f" ]; then echo "$f"; fi; done
+          for l in $(cat /tmp/.sys-new); do ls "\${l%.list}".* 2>/dev/null; done
+          echo "$D/status"
+          # What the packages' install scripts made, which dpkg's lists do not
+          # name: update-alternatives links (libblas.so.3 → /etc/alternatives),
+          # generated caches and configs. By ctime, not mtime: those links carry
+          # the package's own (2025) mtime, and only ctime cannot be backdated.
+          find \${FOLDRUN_SYS_SCAN:-/etc /usr /var/lib} -xdev \\( -path /var/lib/apt -o -path /var/lib/dpkg/info \\) -prune -o \\( -type f -o -type l \\) -cnewer /tmp/.sys-mark -print 2>/dev/null
+        } | sort -u > /tmp/.sys-files
+        tar -cf "$S/.$KEY.$$.tar" -T /tmp/.sys-files 2>/dev/null && mv -f "$S/.$KEY.$$.tar" "$S/$KEY.tar" 2>/dev/null
+        rm -f "$S/.$KEY.$$.tar" 2>/dev/null
+      fi
+      find "$S" -name '*.tar' -mtime +30 -delete 2>/dev/null
+    fi
+  fi
+  printf '{"ok": %s, "exit": %s, "seconds": %s, "mode": "%s", "packages": "%s"}\\n' "$([ $RC -eq 0 ] && echo true || echo false)" "$RC" "$(( $(date +%s) - T0 ))" "$MODE" "$SYS" > ${logDir}/system-install.json
+  rm -rf /var/lib/apt/lists/* 2>/dev/null || true
+fi`;
+}
+/**
+ * Are the declared system packages on this machine? The install happened in
+ * the container's root start-up (systemInstallSh); this only reads the
+ * result. Without dpkg (a laptop, macOS) there is nothing to check against
+ * and nothing this process could install: say so and carry on.
+ */
+export function checkSystemPackages(pkgs) {
+    if (!pkgs.length)
+        return { log: [], error: null };
+    const statusFile = process.env.FOLDRUN_SYSTEM_STATUS ?? SYSTEM_STATUS_FILE;
+    const logFile = process.env.FOLDRUN_SYSTEM_LOG ?? SYSTEM_LOG_FILE;
+    let status = null;
+    try {
+        status = JSON.parse(fs.readFileSync(statusFile, "utf8"));
+    }
+    catch {
+        status = null;
+    }
+    const q = spawnSync("dpkg-query", ["-W", "-f=${Package} ${db:Status-Status}\n", ...pkgs], { encoding: "utf8" });
+    if (q.error) {
+        return { log: [`system packages ${pkgs.join(", ")} install only in a run container; on this machine they are assumed present`], error: null };
+    }
+    const installed = new Set((q.stdout ?? "")
+        .split("\n")
+        .map((l) => l.trim().split(/\s+/))
+        .filter((f) => f[1] === "installed")
+        .map((f) => f[0].replace(/:.*$/, "")));
+    const missing = pkgs.filter((p) => !installed.has(p));
+    if (!missing.length) {
+        return { log: [`system packages ready: ${pkgs.join(", ")}${status?.seconds !== undefined ? ` (${status.mode === "snapshot" ? "snapshot" : "apt"}, ${status.seconds}s)` : ""}`], error: null };
+    }
+    let aptOut = "";
+    try {
+        aptOut = fs.readFileSync(logFile, "utf8");
+    }
+    catch {
+        aptOut = "";
+    }
+    const reason = !status
+        ? "this runner did not install them — system packages are installed by a run container's start-up (k8s or docker isolation), not on a bare host"
+        : status.ok
+            ? "apt reported success, yet they are not installed"
+            : summarizeInstallFailure("apt-get install", aptOut).replace(/^apt-get install failed — /, "apt: ");
+    return { log: [], error: `system packages not installed: ${missing.join(", ")} — ${reason}` };
+}
 export function prepareRuntime(tenant, spec) {
     if (!spec)
         return EMPTY;
+    const system = checkSystemPackages(spec.system ?? []);
+    if (system.error)
+        return { ...EMPTY, log: system.log, error: system.error };
+    if (!wantsPython(spec) && !wantsNode(spec))
+        return { ...EMPTY, log: system.log };
+    const built = prepareLanguageRuntime(tenant, spec);
+    return system.log.length ? { ...built, log: [...system.log, ...built.log] } : built;
+}
+function prepareLanguageRuntime(tenant, spec) {
     const fp = fingerprint(spec);
     const cacheRoot = path.join(dataRoot(), tenant, ".runtimes");
     const shared = path.join(cacheRoot, fp);
@@ -695,7 +900,7 @@ export function prepareRuntime(tenant, spec) {
                     ? install(uv, ["pip", "install", "-q", "--python", venvPython, ...spec.packages], root, 300_000, uvEnv)
                     : install(path.join(root, "venv", "bin", "pip"), ["install", "--disable-pip-version-check", "-q", ...spec.packages], root);
                 if (!installed.ok) {
-                    return fail(`${uv ? "uv pip" : "pip"} install failed: ${installed.out.slice(-500)}`);
+                    return fail(summarizeInstallFailure(`${uv ? "uv pip" : "pip"} install`, installed.out));
                 }
                 built.packages = spec.packages;
                 log.push(`runtime ${fp}: installed ${spec.packages.join(", ")}`);
@@ -718,7 +923,7 @@ export function prepareRuntime(tenant, spec) {
                     npm_config_cache: path.join(root, ".npm-cache"),
                 });
                 if (!installed.ok) {
-                    return fail(`npm install failed: ${installed.out.slice(-500)}`);
+                    return fail(summarizeInstallFailure("npm install", installed.out));
                 }
                 built.npm = spec.npm;
                 log.push(`runtime ${fp}: installed ${spec.npm.join(", ")}`);
@@ -790,3 +995,23 @@ function touch(file) {
         // read-only or gone; neither is a reason to fail the step
     }
 }
+// ---------- installing mid-step ----------
+/** Where the root start-up's install helper listens, inside a run container. */
+export const INSTALL_SOCKET = "/opt/runner/job/install.sock";
+/**
+ * What a step installs for itself mid-run (the `install` tool) lands under
+ * one per-step directory: pip packages in `<dir>/py` (`uv pip install
+ * --target`), npm packages in `<dir>/npm` (`npm install --prefix`). Both are
+ * on the step's paths from its first turn, so a tool installed by one call is
+ * found by the next — `ffmpeg`, `import manim`, `require("sharp")`. Inside a
+ * run container the directory is the container's own filesystem and goes
+ * when the container does.
+ */
+export function stepInstallEnv(dir, base) {
+    return {
+        // After whatever is already there: a declared environment keeps winning.
+        PATH: [base.PATH ?? "", path.join(dir, "py", "bin"), path.join(dir, "npm", "node_modules", ".bin")].filter(Boolean).join(path.delimiter),
+        PYTHONPATH: [base.PYTHONPATH ?? "", path.join(dir, "py")].filter(Boolean).join(path.delimiter),
+        NODE_PATH: [base.NODE_PATH ?? "", path.join(dir, "npm", "node_modules")].filter(Boolean).join(path.delimiter),
+    };
+}

package/dist/src/script-tools.d.ts

@@ -6,6 +6,8 @@ export interface ScriptSpec {
      *  script inline. Exactly one of the two is the program. */
     run: string;
     description: string;
+    /** The tool.md body, for the model: how to call it and what it means. */
+    guide?: string;
     args: Record<string, string>;
     interpreter?: string;
     /** Seconds this script may run — for the crawl that legitimately takes
@@ -45,6 +47,20 @@ export declare function runScript(cwd: string, spec: ScriptSpec, values: Record<
     code: number | null;
     out: string;
 }>;
+/**
+ * A host path as the script's container sees it: under the agent's own
+ * directory it is /workspace, under a mounted folder it is that mount, and
+ * anything else (an argument that only looks like a path) is left alone.
+ * Containment, not a text prefix: `<ws>/scripts-old/x.py` is not under
+ * `<ws>/scripts`. The longest mount wins, so a nested one is not shadowed.
+ */
+export declare function containerPath(p: string, agentDir: string, mounts: Record<string, string>): string;
+/** Where a `workspace/…` script argument is staged in a script container,
+ *  or null when it is not one (or climbs out of the workspace). */
+export declare function stageWorkspaceArg(arg: string, workspaceRoot: string): {
+    host: string;
+    container: string;
+} | null;
 export interface ExecutionContext {
     /** "docker" nests each script in its own container; "sandbox" spawns it
      *  directly inside the isolation the step already has. */

package/dist/src/script-tools.js

@@ -55,6 +55,7 @@ export function parseScripts(raw) {
             run: run ?? "",
             ...(code ? { code, codeExt: typeof e.codeExt === "string" ? e.codeExt : ".mjs" } : {}),
             description: typeof e.description === "string" ? e.description : "",
+            ...(typeof e.guide === "string" && e.guide.trim() ? { guide: e.guide } : {}),
             args,
             interpreter: typeof e.interpreter === "string" ? e.interpreter : undefined,
             secrets: e.secrets === "proxied" ? "proxied" : undefined,
@@ -183,14 +184,18 @@ export function runScript(cwd, spec, values, env, libraryScripts, interpreters,
         // Container path: same command, executed inside an isolated container
         // with only the agent's directory (plus declared shared dirs) mounted.
         if (exec && exec.executor === "docker") {
+            // `workspace/…` is the workspace root on the host (the step's link)
+            // and in a run container; here there is no link, so each such
+            // argument is staged: copied in, pointed at, and copied back after.
+            const workspaceRoot = path.resolve(cwd, "..", "..");
+            const staged = [];
             const inContainer = (p) => {
-                if (p.startsWith(path.resolve(cwd)))
-                    return p.replace(path.resolve(cwd), "/workspace");
-                for (const [host, mount] of Object.entries(exec.mounts)) {
-                    if (p.startsWith(host))
-                        return p.replace(host, mount);
-                }
-                return p;
+                const w = stageWorkspaceArg(p, workspaceRoot);
+                if (!w)
+                    return containerPath(p, cwd, exec.mounts);
+                if (!staged.some((s) => s.host === w.host))
+                    staged.push(w);
+                return w.container;
             };
             const argv = [
                 // The interpreter inside the image, not the host's venv path.
@@ -203,6 +208,7 @@ export function runScript(cwd, spec, values, env, libraryScripts, interpreters,
             runInContainer({
                 agentDir: path.resolve(cwd),
                 readOnly: exec.mounts,
+                staged,
                 image: exec.image,
                 argv,
                 env,
@@ -238,6 +244,41 @@ export function runScript(cwd, spec, values, env, libraryScripts, interpreters,
         }));
     });
 }
+/**
+ * A host path as the script's container sees it: under the agent's own
+ * directory it is /workspace, under a mounted folder it is that mount, and
+ * anything else (an argument that only looks like a path) is left alone.
+ * Containment, not a text prefix: `<ws>/scripts-old/x.py` is not under
+ * `<ws>/scripts`. The longest mount wins, so a nested one is not shadowed.
+ */
+export function containerPath(p, agentDir, mounts) {
+    if (!path.isAbsolute(p))
+        return p;
+    const under = (root) => {
+        const rel = path.relative(root, p);
+        return rel === "" || (!rel.startsWith("..") && !path.isAbsolute(rel)) ? rel : null;
+    };
+    const own = under(path.resolve(agentDir));
+    if (own !== null)
+        return path.posix.join("/workspace", own.split(path.sep).join("/"));
+    const hit = Object.entries(mounts)
+        .map(([host, mount]) => ({ mount, rel: under(path.resolve(host)), len: host.length }))
+        .filter((m) => m.rel !== null)
+        .sort((a, b) => b.len - a.len)[0];
+    return hit ? path.posix.join(hit.mount, hit.rel.split(path.sep).join("/")) : p;
+}
+/** Where a `workspace/…` script argument is staged in a script container,
+ *  or null when it is not one (or climbs out of the workspace). */
+export function stageWorkspaceArg(arg, workspaceRoot) {
+    if (arg !== "workspace" && !arg.startsWith("workspace/"))
+        return null;
+    const rel = arg.slice("workspace".length).replace(/^\/+/, "");
+    const host = path.resolve(workspaceRoot, rel);
+    const within = path.relative(workspaceRoot, host);
+    if (within.startsWith("..") || path.isAbsolute(within))
+        return null;
+    return { host, container: path.posix.join("/workspace-root", within.split(path.sep).join("/")) };
+}
 // `run:` accepts:
 //   scripts/x.py              this agent's own script
 //   workspace/tools/<t>/x.py  a folder tool's own code, in this workspace
@@ -306,7 +347,8 @@ export function buildScriptTools(agentDir, scripts, env, libraryScripts = "", in
     const tools = scripts.map((spec) => {
         const shape = scriptArgShape(spec.args);
         return tool(spec.name, `${spec.description || `Run ${spec.run}.`} Runs the workspace script ${spec.run}; returns its output. ` +
-            `Credentials it needs are already in its environment.`, shape, async (args) => {
+            `Credentials it needs are already in its environment.` +
+            (spec.guide ? `\n\nHow to use it (from its tool.md):\n${spec.guide}` : ""), shape, async (args) => {
             const started = Date.now();
             const values = Object.fromEntries(Object.entries(args ?? {}).map(([k, v]) => [k, v === undefined ? "" : String(v)]));
             const { code, out } = await runScript(agentDir, spec, values, env, libraryScripts, interpreters, exec);

package/dist/src/secrets.d.ts

@@ -123,6 +123,11 @@ export interface OAuth2Config {
     /** Extra form fields some providers want (audience, scope, …). */
     extra?: Record<string, string>;
 }
+/** Tries at a token endpoint before a refresh is called failed: the first,
+ *  and two more after a short wait, for a network error, a 5xx or a 429. */
+export declare const OAUTH_REFRESH_TRIES = 3;
+/** Wait before try n (1-based retry): 1s, 3s — FOLDRUN_OAUTH_RETRY_MS scales it for tests. */
+export declare function oauthRetryDelayMs(n: number): number;
 export declare function setOAuth2Secret(tenant: string, name: string, config: OAuth2Config, workspace?: string): void;
 export declare function isOAuth2Value(value: string): boolean;
 /**

package/dist/src/secrets.js

@@ -402,6 +402,14 @@ export function rotateMasterKey(tenant, from, to) {
     return { rewritten, unreadable };
 }
 const OAUTH2_PREFIX = "@oauth2 ";
+/** Tries at a token endpoint before a refresh is called failed: the first,
+ *  and two more after a short wait, for a network error, a 5xx or a 429. */
+export const OAUTH_REFRESH_TRIES = 3;
+/** Wait before try n (1-based retry): 1s, 3s — FOLDRUN_OAUTH_RETRY_MS scales it for tests. */
+export function oauthRetryDelayMs(n) {
+    const base = Number(process.env.FOLDRUN_OAUTH_RETRY_MS ?? 1000);
+    return base * (n === 1 ? 1 : 3);
+}
 export function setOAuth2Secret(tenant, name, config, workspace) {
     for (const field of ["token_url", "client_id", "client_secret"]) {
         if (!config[field]?.trim())
@@ -440,12 +448,35 @@ async function exchange(config, cacheKey) {
             client_secret: config.client_secret,
             ...(config.extra ?? {}),
         });
-        const res = await fetchUntrusted(config.token_url, {
-            method: "POST",
-            headers: { "content-type": "application/x-www-form-urlencoded" },
-            body: body.toString(),
-            signal: AbortSignal.timeout(15_000),
-        }, { sameOrigin: true });
+        // A network blip is retried; a provider's answer is not. "fetch failed"
+        // (a reset, a DNS hiccup, a timeout) failed whole steps on the first try
+        // — gbp-desk's Thursday posts, 7 Oct 2026. A 4xx (revoked, wrong client)
+        // will say the same thing again, so it goes straight to the error below
+        // and the reconnect mail; a 5xx or 429 is the provider busy, retried.
+        let res;
+        let lastNetwork;
+        for (let attempt = 0; attempt < OAUTH_REFRESH_TRIES; attempt++) {
+            if (attempt)
+                await new Promise((r) => setTimeout(r, oauthRetryDelayMs(attempt)));
+            try {
+                res = await fetchUntrusted(config.token_url, {
+                    method: "POST",
+                    headers: { "content-type": "application/x-www-form-urlencoded" },
+                    body: body.toString(),
+                    signal: AbortSignal.timeout(15_000),
+                }, { sameOrigin: true });
+            }
+            catch (err) {
+                lastNetwork = err;
+                res = undefined;
+                continue;
+            }
+            if (res.status === 429 || res.status >= 500)
+                continue;
+            break;
+        }
+        if (!res)
+            throw lastNetwork instanceof Error ? lastNetwork : new Error(String(lastNetwork));
         const payload = (await res.json().catch(() => ({})));
         if (!res.ok || !payload.access_token) {
             throw new Error(`token refresh failed (${res.status}): ${payload.error_description ?? payload.error ?? "no access_token in the reply"}`);

package/dist/src/starter.d.ts

@@ -28,3 +28,15 @@ export declare function accountFiles(account: string): StarterFile[];
  */
 export declare function accountGitignore(): StarterFile;
 export declare function starterFiles(workspace: string): StarterFile[];
+/**
+ * A new workspace with nothing in it to delete: its AGENTS.md, the
+ * .gitignore that keeps its secret key out of git, and the CLAUDE.md a coding
+ * tool reads. No example agents or flow.
+ *
+ * The starter's researcher and writer were a demo, and a person making a
+ * workspace for real work met them as clutter — "why does a new workspace
+ * have agents?" (owner, 2026-10-05). The agents worth having are usually ones
+ * already written in another workspace, which `agents/import` copies in; the
+ * example is still one flag away (`foldrun new <name> --starter`).
+ */
+export declare function blankWorkspaceFiles(workspace: string): StarterFile[];