@foldrun/core 2.12.1 → 2.13.1

package/src/run-container.ts

@@ -33,6 +33,7 @@ import type { TokenPrice } from "./catalog.ts";
 import { isFileValue, fileContent } from "./secrets.ts";
 import { safeTenantSegment, systemInstallSh, SYSTEM_FROM_INPUT_SH, INSTALL_SOCKET, type RuntimeSpec } from "./runtime.ts";
 import { egressCaSh, type NetworkLevel } from "./network.ts";
+import { anyMemoryMi, quantityMi, type ResourceOverride } from "./resources.ts";
 import type { ScriptSpec } from "./script-tools.ts";
 import type { ConsultSpec } from "./agent-tools.ts";
 import type { DelegateSpec, SubagentSpec } from "./subagents.ts";
@@ -181,6 +182,8 @@ export function parseDriverMark(line: string): { name: StartupPhase; at: number
 export interface StepActuals {
   busyCpuSecs: number | null;
   peakMemBytes: number | null;
+  /** Seconds the CPU cap held the sandbox back. Absent before 26 Oct 2026. */
+  throttledSecs?: number | null;
   rxBytes: number | null;
   txBytes: number | null;
 }
@@ -619,6 +622,11 @@ function readActuals() {
   const cpuStat = read("/sys/fs/cgroup/cpu.stat");
   const usage = cpuStat?.match(/^usage_usec (\\d+)/m);
   const cpuacctNs = usage ? NaN : Number(read("/sys/fs/cgroup/cpuacct/cpuacct.usage")?.trim());
+  // Time the CPU cap held the sandbox back: v2's throttled_usec in the same
+  // cpu.stat, v1's throttled_time (nanoseconds) under gVisor. Large against
+  // busy time means the step wanted more cores than its class gives.
+  const throttledUs = cpuStat?.match(/^throttled_usec (\\d+)/m);
+  const throttledNs = throttledUs ? null : read("/sys/fs/cgroup/cpu/cpu.stat")?.match(/^throttled_time (\\d+)/m);
   const peak =
     read("/sys/fs/cgroup/memory.peak") ??
     read("/sys/fs/cgroup/memory/memory.max_usage_in_bytes") ??
@@ -638,6 +646,7 @@ function readActuals() {
         ? cpuacctNs / 1e9
         : null,
     peakMemBytes: peak ? Number(peak.trim()) || null : null,
+    throttledSecs: throttledUs ? Number(throttledUs[1]) / 1e6 : throttledNs ? Number(throttledNs[1]) / 1e9 : null,
     rxBytes: sawNet ? rx : null,
     txBytes: sawNet ? tx : null,
   };
@@ -1262,7 +1271,8 @@ export function buildFailureSummary(log: string, opts: { tail?: number; maxLines
   const key: string[] = [];
   for (const raw of log.split("\n")) {
     const line = raw.replace(/^#\d+ \d+(\.\d+)? /, "").trim();
-    if (!/^runner image:|^ERROR\b|^error:/i.test(line) && !/did not complete successfully/.test(line)) continue;
+    // apt's own errors start "E: "; the legacy builder ends with "returned a non-zero code".
+    if (!/^runner image:|^ERROR\b|^error:|^E: /i.test(line) && !/did not complete successfully|returned a non-zero code|no space left on device/i.test(line)) continue;
     const cut = line.length > 300 ? line.slice(0, 300) + " …" : line;
     if (seen.has(cut)) continue;
     seen.add(cut);
@@ -1270,6 +1280,11 @@ export function buildFailureSummary(log: string, opts: { tail?: number; maxLines
   }
   // The ones that say what is missing first: a reader stops at the top.
   key.sort((a, b) => Number(/MISSING ENGINES/.test(b)) - Number(/MISSING ENGINES/.test(a)));
+  // A full Docker disk is the commonest cause on one machine, and the fix is
+  // not in foldrun: say it first, in plain words.
+  if (/enough free space|no space left on device/i.test(log)) {
+    key.unshift("the Docker disk is full: free space (`docker system df`, then `docker builder prune` / `docker image prune`) and run it again");
+  }
   const head = key.length
     ? key.slice(0, maxLines).map((l) => `  ${l}`).join("\n") + (key.length > maxLines ? `\n  … ${key.length - maxLines} more` : "")
     : "  (no engine or ERROR line in the log)";
@@ -1372,6 +1387,8 @@ export function parseDriverLine(
             ? {
                 busyCpuSecs: typeof parsed.res.busyCpuSecs === "number" ? parsed.res.busyCpuSecs : null,
                 peakMemBytes: typeof parsed.res.peakMemBytes === "number" ? parsed.res.peakMemBytes : null,
+                // A driver from before 26 Oct 2026 sends none: unknown, not zero.
+                throttledSecs: typeof parsed.res.throttledSecs === "number" ? parsed.res.throttledSecs : null,
                 rxBytes: typeof parsed.res.rxBytes === "number" ? parsed.res.rxBytes : null,
                 txBytes: typeof parsed.res.txBytes === "number" ? parsed.res.txBytes : null,
               }
@@ -1408,6 +1425,65 @@ function ensureRunNetwork(): string {
   return RUN_NETWORK;
 }
 
+// The cores the container engine can hand out, asked once. `--cpus` above it
+// is refused outright ("range of CPUs is from 0.01 to N"), so a heavy step
+// on a two-core server would fail before it began. The engine may be a VM
+// (Docker Desktop) with fewer cores than this machine, which is why it is
+// asked rather than read from os; podman answers a different template, and
+// an engine that does not answer falls back to this machine's count.
+let engineCpuCount: number | null = null;
+function engineCpus(): number {
+  if (engineCpuCount !== null) return engineCpuCount;
+  const out = spawnSync(cli(), ["info", "--format", "{{.NCPU}}"], { encoding: "utf8", timeout: 10_000 });
+  const n = Number((out.stdout ?? "").trim());
+  engineCpuCount = Number.isInteger(n) && n > 0 ? n : os.availableParallelism();
+  return engineCpuCount;
+}
+
+/**
+ * A memory figure Docker will accept: the mirror of k8sMemory.
+ *
+ * Docker reads b/k/m/g (binary) and refuses Kubernetes' Mi and Gi outright
+ * ("invalid suffix: 'mi'", Docker 29). The class defaults for small and
+ * heavy are Kubernetes quantities (1Gi, 8Gi), and `resources: memory` is
+ * normalised to Mi, so before 26 Oct 2026 every small and heavy step failed
+ * to start on the docker executor; only large's `2g` worked. Binary suffixes
+ * become whole bytes (1.5Gi has no Docker spelling), Docker's own pass
+ * through, and anything else is handed over for Docker's error to name.
+ */
+export function dockerMemory(value: string): string {
+  const s = String(value).trim();
+  // Lowercase only: case decides, as in k8sMemory — `1G` is Kubernetes'
+  // decimal gigabyte, and Docker would read it as binary.
+  if (/^\d+(\.\d+)?[bkmg]?$/.test(s)) return s;
+  const m = /^(\d+(?:\.\d+)?)(Ki|Mi|Gi|Ti|K|M|G|T)$/.exec(s);
+  if (!m) return s;
+  const pow: Record<string, number> = { Ki: 1, Mi: 2, Gi: 3, Ti: 4 };
+  const dec: Record<string, number> = { K: 1e3, M: 1e6, G: 1e9, T: 1e12 };
+  const bytes = m[2] in pow ? Number(m[1]) * 1024 ** pow[m[2]] : Number(m[1]) * dec[m[2]];
+  return String(Math.round(bytes));
+}
+
+/** The `--cpus` a step's container gets: its class's billed cores, no more
+ *  than the engine has. Null when the figure is not a number, so the flag is
+ *  left off rather than failing the step on an install's typo. */
+export function dockerCpus(cpus: string, available: number = engineCpus()): string | null {
+  const n = Number(cpus);
+  if (!Number.isFinite(n) || n <= 0) return null;
+  return String(Math.min(n, available));
+}
+
+/** The cores a step's sandbox actually holds, for its record (the bill and
+ *  the compute in use): the asked-for figure, except on the docker
+ *  executor, which lowers --cpus to what the engine has (dockerCpus). Until
+ *  26 Oct 2026 the record said heavy's 4 on a two-core machine that gave 2. */
+export function heldCores(cpus: string, isolation: string | undefined, available?: number): number {
+  const asked = Number(cpus) || 2;
+  if (isolation !== "container") return asked;
+  const given = Number(available === undefined ? dockerCpus(cpus) : dockerCpus(cpus, available));
+  return Number.isFinite(given) && given > 0 ? given : asked;
+}
+
 export interface RunInContainerArgs {
   workspaceRoot: string;
   libraryRoot: string;
@@ -1435,6 +1511,9 @@ export interface RunInContainerArgs {
   stopRequested?: () => boolean;
   /** The reservation class — which limits this step's sandbox holds. */
   size?: "small" | "large" | "heavy";
+  /** The class's figures with the agent's `resources:` over them
+   *  (resolveResources). Absent: the class's own. */
+  limits?: StepLimits;
   /** Re-attach to a sandbox a previous driver started, instead of creating
    *  one: its handle, and how many output lines that driver had applied.
    *  The executor replays nothing before `consumed`, and reads the step's
@@ -1549,8 +1628,8 @@ export function sizeLimits(size?: "small" | "large" | "heavy"): { cpus: string;
     // For steps that render a browser or hold real data. The memory figure is
     // a hard ceiling (memory is not compressible — over it, the kernel OOM-kills
     // the pod, which on a single-box install can take neighbours with it). The
-    // cpus figure is a billing weight, not a ceiling: on k8s the pod bursts CPU
-    // freely, and this is the rate its seconds are priced at.
+    // cpus figure is both the rate its seconds are priced at and, since
+    // 26 Oct 2026, the cores the sandbox is capped at.
     return {
       cpus: process.env.FOLDRUN_RUNNER_CPUS_HEAVY ?? "4",
       memory: process.env.FOLDRUN_RUNNER_MEMORY_HEAVY ?? "8Gi",
@@ -1562,6 +1641,66 @@ export function sizeLimits(size?: "small" | "large" | "heavy"): { cpus: string;
   };
 }
 
+/** What one step's sandbox holds: cores and memory (each a hard limit and a
+ *  billing figure) and, where the agent set it, disk. */
+export interface StepLimits {
+  cpus: string;
+  memory: string;
+  /** The agent's `resources: disk`, or null for the class's (the k8s tier's
+   *  diskFor; Docker has no per-container disk cap). */
+  disk: string | null;
+}
+
+/**
+ * The class's figures with an agent's `resources:` over them, key by key.
+ *
+ * `atLeastClass` is for a retry that moved the step up a class after the
+ * cluster ended it for want of memory or disk: each override is raised to
+ * the bigger class's figure, so `memory: 3Gi` OOM-killed at large retries at
+ * heavy's 8Gi rather than the same 3Gi that already failed.
+ */
+export function resolveResources(
+  size?: "small" | "large" | "heavy",
+  override?: ResourceOverride | null,
+  opts: { atLeastClass?: boolean } = {},
+): StepLimits {
+  const base = sizeLimits(size);
+  const bigger = (mine: number, theirs: number) => (opts.atLeastClass ? Math.max(mine, theirs) : mine);
+  const baseMi = anyMemoryMi(base.memory);
+  let cpus = base.cpus;
+  if (override?.cpu !== undefined) {
+    const b = Number(base.cpus);
+    cpus = String(Number.isFinite(b) ? bigger(override.cpu, b) : override.cpu);
+  }
+  let memory = base.memory;
+  if (override?.memory !== undefined) {
+    const mi = quantityMi(override.memory);
+    memory = Number.isFinite(baseMi) && opts.atLeastClass && baseMi > mi ? base.memory : override.memory;
+  }
+  // Disk the same way: a step evicted for disk at its own `disk: 2Gi` would
+  // fail again at 2Gi, whatever class the retry moved it to.
+  let disk = override?.disk ?? null;
+  if (disk && opts.atLeastClass) {
+    const classMi = anyMemoryMi(classDisk(size));
+    if (Number.isFinite(classMi) && classMi > quantityMi(disk)) disk = null;
+  }
+  return { cpus, memory, disk };
+}
+
+/** The node disk a class may write (the k8s `ephemeral-storage` limit;
+ *  Docker has no per-container cap). Env-tunable like the cores and memory. */
+export function classDisk(size?: "small" | "large" | "heavy"): string {
+  if (size === "small") return process.env.FOLDRUN_RUNNER_DISK_SMALL ?? "2Gi";
+  if (size === "heavy") return process.env.FOLDRUN_RUNNER_DISK_HEAVY ?? "20Gi";
+  return process.env.FOLDRUN_RUNNER_DISK ?? "8Gi";
+}
+
+/** A memory figure in GiB, for the step's reservation record. */
+export function memoryGiB(memory: string): number {
+  const mi = anyMemoryMi(memory);
+  return Number.isFinite(mi) ? mi / 1024 : 2;
+}
+
 /** The label both executors stamp, so one name means one thing. */
 export const RUN_LABEL = "foldrun-run-id";
 
@@ -1624,6 +1763,8 @@ export async function runStepInContainer(args: RunInContainerArgs): Promise<Cont
       { mode: 0o600 },
     );
 
+    const lim = args.limits ?? resolveResources(args.size);
+    const cpus = dockerCpus(lim.cpus);
     const flags = [
       "--security-opt", "no-new-privileges",
       ...(args.runId ? ["--label", `${RUN_LABEL}=${args.runId}`] : []),
@@ -1633,10 +1774,11 @@ export async function runStepInContainer(args: RunInContainerArgs): Promise<Cont
       "--cap-add", "CHOWN", "--cap-add", "SETUID", "--cap-add", "SETGID",
       "--pids-limit", "512",
       // Memory is a hard ceiling (not compressible — an OOM protects the host).
-      // CPU is deliberately NOT capped, matching the k8s path: a cap only
-      // throttles work the customer pays for, and the tier's cpu figure is a
-      // billing weight, applied in usage.ts, not a docker --cpus limit here.
-      "--memory", sizeLimits(args.size).memory,
+      // CPU is capped at the class's billed cores, matching the k8s path
+      // (since 26 Oct 2026): a step runs on what it pays for and cannot take
+      // the whole machine from the steps beside it.
+      "--memory", dockerMemory(lim.memory),
+      ...(cpus ? ["--cpus", cpus] : []),
       "--env-file", envFile,
     ];
     // gVisor (or kata) where the host has it: one env var, because the flag

package/src/runner.ts

@@ -19,7 +19,8 @@ import { cascadeLimits, toolOwners } from "./limits.ts";
 import { resolveAgentPath } from "./confine.ts";
 import { browserPodLine, podLossDecision, RERAN_ON_FULL } from "./browser-pod.ts";
 import { eventUrl } from "./webhook.ts";
-import { runStepInContainer, sizeLimits, killRunSandboxes, type StepTiming } from "./run-container.ts";
+import { runStepInContainer, resolveResources, memoryGiB, heldCores, killRunSandboxes, type StepTiming } from "./run-container.ts";
+import { readResources, hasOverride } from "./resources.ts";
 import { hostSafeEnv, platformApiUrl } from "./host-env.ts";
 import { resolveModelCredential, credentialLine, accountModelBlock, isClaudeLoginToken, isModelEnvName, DECLARED_ENV_PREFIX } from "./model-credential.ts";
 import { parseJudge } from "./judge-verdict.ts";
@@ -1296,9 +1297,10 @@ export function agentContext(
   // is large — the install's configured limits — so existing agents change
   // nothing. `size: small` is the opt-in for agents that wait on APIs more
   // than they work; `size: heavy` for a rendered browser or real data in
-  // memory. Memory is a hard ceiling per class; CPU bursts uncapped and bills
-  // at the class rate, so heavy both may use more and costs more. The actuals
-  // line in any trace is the evidence for right-sizing.
+  // memory. Memory and cores are hard ceilings per class and bill at the
+  // class rate (cores capped since 26 Oct 2026), so heavy both may use more
+  // and costs more. The actuals line in any trace — busy, throttled, peak —
+  // is the evidence for right-sizing.
   const size: "small" | "large" | "heavy" =
     front.size === "small" ? "small" : front.size === "heavy" ? "heavy" : "large";
 
@@ -2311,6 +2313,16 @@ async function runStep(
     // A retry that moved the step up a class (an evicted or OOM-killed
     // attempt) wins over the agent's own `size:` for the attempts after.
     const size = step.sizeUp ?? agentSize;
+    // `resources:` over the class's figures, key by key. A value check would
+    // refuse is dropped here too (the class's figure holds) and said so, so
+    // `foldrun check` and the sandbox never disagree. After a retry moved up
+    // a class, an override never holds the step below the new class.
+    const asked = readResources(front.resources);
+    for (const p of asked.problems) push("error", `resources: ${p}`);
+    const limits = resolveResources(size, asked.value, { atLeastClass: !!step.sizeUp });
+    // What the sandbox holds while it exists — `step.holding`, what the
+    // account's compute in use sums (Settings, `foldrun usage compute`).
+    const held = { cpus: heldCores(limits.cpus, process.env.FOLDRUN_RUN_ISOLATION), memGiB: memoryGiB(limits.memory) };
 
     // oauth2 secrets become live access tokens here — the one async moment
     // in secret resolution, at the last host-side point before anything is
@@ -2345,7 +2357,10 @@ async function runStep(
     const noKey = !providerLabel && !platform.platformKeyAllowed(tenant);
     const credential = providerLabel || noKey ? null : resolveModelCredential(isolatedRun() ? platformModelEnv() : process.env);
     step.credential = credential ? credential.label : noKey ? "none — this account has no model key" : `provider: ${providerLabel}`;
-    if (credential) {
+    // Core's own Docker container has no login to fall back to (below):
+    // there, "none set" is refused with what to do, not described.
+    const noCredentialInContainer = credential?.kind === "none" && process.env.FOLDRUN_RUN_ISOLATION === "container";
+    if (credential && !noCredentialInContainer) {
       const line = credentialLine(credential);
       push(line.type, line.text);
     }
@@ -2372,6 +2387,24 @@ async function runStep(
         return;
       }
     }
+    // A step in core's own Docker container (self-hosting) has no machine
+    // login to fall back to: with no credential at all it could only fail at
+    // the model, with Claude Code's own "Not logged in · Please run /login".
+    // Refused here, before a sandbox is rented, saying what to set (a
+    // self-hosted install with no key, 26 Oct 2026). A local `foldrun run`
+    // keeps the machine's login; a registered executor brings its own.
+    if (noCredentialInContainer) {
+      push(
+        "error",
+        "no model credential: this install has none and the account has no key of its own. Set ANTHROPIC_API_KEY " +
+          "(or CLAUDE_CODE_OAUTH_TOKEN from `claude setup-token`) in the platform's environment — for Docker Compose, " +
+          "self-hosting/.env, then `docker compose up -d` — or add the account's key in Settings → Model " +
+          "(`foldrun account model set <provider> --key …`)",
+      );
+      step.status = "failed";
+      save();
+      return;
+    }
     for (const w of providerWarnings) push("error", w);
     // A level that wrote a language nobody can read was skipped, not obeyed.
     for (const l of language.lines) push("info", l);
@@ -3080,6 +3113,7 @@ async function runStep(
         // executor asks this before it creates one (pod-room.ts).
         ...(stopRequested ? { stopRequested } : {}),
         size,
+        limits,
         // The pod's network, for the executor's own wall (the k8s tier's
         // network policy): a filtered pod reaches the proxy and DNS only.
         network: { level: network.level, enforced: forward?.enforce === true },
@@ -3096,6 +3130,8 @@ async function runStep(
         // saves the record, so this only updates the fields.
         checkpoint: (ref: string, consumed: number) => {
           step.sandbox = { kind: isolation!, ref, consumed, since: step.sandbox?.since ?? new Date().toISOString() };
+          // The sandbox exists: past the room gate, it now holds its figures.
+          if (consumed === 0) step.holding = held;
           if (consumed === 0) save();
         },
       });
@@ -3111,9 +3147,23 @@ async function runStep(
       // The arguments of the attempt whose outcome is current — what a
       // re-run on the full image repeats, on the same model supply.
       let lastArgs: typeof firstArgs = firstArgs;
-      const runTracked = (a: typeof firstArgs) => {
+      const runTracked = async (a: typeof firstArgs) => {
         lastArgs = a;
-        return runIsolated(a);
+        // An executor that cannot report its sandbox (Docker: no room gate,
+        // it starts at once) holds from the call; one that can says so at
+        // its first checkpoint, after any wait for room.
+        if (!platform.sandboxResumable(isolation!)) {
+          step.holding = held;
+          save();
+        }
+        try {
+          return await runIsolated(a);
+        } finally {
+          // Saved now: the clean-up after an attempt (the log drain, the
+          // lease, publishing) can take a while, and the sandbox is gone.
+          step.holding = null;
+          save();
+        }
       };
       let attemptFrom = step.events.length;
       let outcome = await runTracked(firstArgs);
@@ -3306,6 +3356,7 @@ async function runStep(
         push(
           "info",
           `resources: cpu busy ${outcome.res.busyCpuSecs?.toFixed(1) ?? "?"}s · ` +
+            (outcome.res.throttledSecs ? `throttled ${outcome.res.throttledSecs.toFixed(1)}s · ` : "") +
             `peak mem ${mb(outcome.res.peakMemBytes)}MB · ` +
             `net ${mb(outcome.res.rxBytes)}MB in / ${mb(outcome.res.txBytes)}MB out`,
         );
@@ -3326,13 +3377,8 @@ async function runStep(
       // is per core-second and per GiB-second, and what was held must come
       // from the record, never from what the config says at billing time.
       step.size = size;
-      const lim = sizeLimits(size);
-      const gib = (() => {
-        const m = lim.memory.match(/^(\d+(?:\.\d+)?)\s*(Gi?|Mi?|g|m)?/i);
-        const n = m ? Number(m[1]) : 2;
-        return (m?.[2] ?? "Gi").toLowerCase().startsWith("m") ? n / 1024 : n;
-      })();
-      step.reserved = { cpus: Number(lim.cpus) || 2, memGiB: gib };
+      if (hasOverride(asked.value)) step.resources = asked.value;
+      step.reserved = held;
     } else {
       // Same withholding as the sandbox path — the in-process step is what
       // `foldrun run --test` drives on a laptop, where there is no proxy

package/src/starter.ts

@@ -490,6 +490,7 @@ owner: ana@example.com      # who answers for it; else the workspace's, else the
 model: default              # fast | default | max
 effort: low                 # how hard to think about it
 size: large                 # small | large | heavy — the sandbox it rents
+resources: { memory: 4Gi }  # optional: cpu, memory, disk in numbers, over the size class
 tools: [web, read, ask, peers, my-folder-tool]  # built-ins and your own tools/, one list; \`ask\` lets it ask you mid-step, \`peers\` lets it message the steps running beside it
 disallowedTools: [Bash]     # subtract from what it would otherwise have
 skills: [house-style]       # from skills/

package/src/store.ts

@@ -3533,6 +3533,14 @@ export interface StepRecord {
    *  from before sizes existed — billing falls back to the flat rate. */
   size?: "small" | "large" | "heavy";
   reserved?: { cpus: number; memGiB: number } | null;
+  /** The agent's `resources:` as applied (resources.ts), when it set any:
+   *  why `reserved` differs from the class's figures. Absent before 26 Oct
+   *  2026 and on steps of an agent that wrote none. */
+  resources?: { cpu?: number; memory?: string; disk?: string };
+  /** What the step's sandbox holds while it exists — set once it is past
+   *  the room gate, cleared when the attempt ends. What an account's
+   *  compute in use sums; `reserved` is the bill's record of the same. */
+  holding?: { cpus: number; memGiB: number } | null;
   /** Token counts behind costUsd, kept so usage reports can say "how many"
    *  and not only "how much" — the price of a token changes, the count is
    *  the fact. Absent on steps recorded before this existed. */
@@ -3557,6 +3565,9 @@ export interface StepRecord {
   actual?: {
     busyCpuSecs: number | null;
     peakMemBytes: number | null;
+    /** Seconds the CPU cap (the class's cores) held the sandbox back, from
+     *  cpu.stat. Absent before 26 Oct 2026. */
+    throttledSecs?: number | null;
     rxBytes: number | null;
     txBytes: number | null;
   } | null;