@foldrun/core 2.10.1 → 2.11.0

package/dist/src/triggers.js

@@ -54,6 +54,17 @@ export function chainedFlows(tenant, workspace, finished) {
         f.name !== finished.flow &&
         (f.on === "any" || f.on === endingOf(finished)));
 }
+/** What a chained flow's `filter:` reads: the run that just settled. */
+export function chainFields(finished) {
+    return {
+        run: finished.id,
+        flow: finished.flow,
+        status: finished.status,
+        ending: endingOf(finished),
+        verdict: runVerdict(finished) ?? null,
+        summary: finished.summary ?? null,
+    };
+}
 /**
  * Start every flow chained on a run that just settled. The finished run's
  * one-line summary and its final result are the task; the run id rides
@@ -73,7 +84,7 @@ export async function fireChainedFlows(tenant, workspace, finished) {
         (result ? `\n${result}` : "");
     for (const flow of flows) {
         try {
-            const gate = await platform.admitTrigger(tenant, workspace, flow, { body, tag: "previous_run" });
+            const gate = await platform.admitTrigger(tenant, workspace, flow, { body, tag: "previous_run", fields: chainFields(finished) });
             if (!gate.admit) {
                 console.log(`[foldrun] trigger: flow — ${tenant}/${workspace}/${flow.name} not started: ${gate.detail}`);
                 continue;
@@ -138,7 +149,7 @@ export async function fireStorageTriggers(tenant, workspace, paths, by) {
             const body = `by: ${by}\n${hits.map((h) => `- storage/${h}`).join("\n")}`;
             // A deploy that writes forty files under one prefix is one event, not
             // forty runs — which is exactly what `debounce:` is for.
-            const gate = await platform.admitTrigger(tenant, workspace, flow, { body, tag: "storage_event" });
+            const gate = await platform.admitTrigger(tenant, workspace, flow, { body, tag: "storage_event", fields: { by, paths: hits } });
             if (!gate.admit) {
                 console.log(`[foldrun] trigger: storage — ${tenant}/${workspace}/${flow.name} not started: ${gate.detail}`);
                 continue;

package/dist/src/untrusted-fetch.d.ts

@@ -1,6 +1,20 @@
 /** How many redirects one request may follow. fetch's own limit is 20;
  *  nothing a tenant points the platform at needs more than a couple. */
 export declare const MAX_REDIRECTS = 5;
+/** What one answer may be, read, unless the caller says otherwise. */
+export declare const MAX_BYTES: number;
+export declare class ResponseTooLarge extends Error {
+    constructor(url: string, maxBytes: number);
+}
+/**
+ * `res`, its body read to at most `maxBytes`. The status and headers stand;
+ * the body errors with ResponseTooLarge at the limit — `text()`, `json()`
+ * and a reader all reject, never resolve short. A declared length past the
+ * limit errors at the first read, before a byte is fetched; an undeclared
+ * one is counted as it streams (after decompression, so a small gzip that
+ * unpacks huge is caught too). Infinity: no limit.
+ */
+export declare function capBody(res: Response, maxBytes: number, url: string): Response;
 export declare class RedirectRefused extends Error {
     constructor(message: string);
 }
@@ -8,4 +22,6 @@ export declare function fetchUntrusted(url: string, init?: RequestInit, opts?: {
     /** The request carries a credential (in its body or its headers):
      *  follow a redirect only within the origin it was sent to. */
     sameOrigin?: boolean;
+    /** The most the answer may be, read; MAX_BYTES unless said. */
+    maxBytes?: number;
 }): Promise<Response>;

package/dist/src/untrusted-fetch.js

@@ -14,6 +14,12 @@
 // the first one was. A hop to another origin keeps only the headers that
 // cannot carry a credential, and a caller whose request IS a credential
 // (a token exchange, a provider key in a header) follows none at all.
+//
+// And every answer is read to a limit (maxBytes, 1 MiB unless the caller
+// says otherwise): a tenant's URL can answer forever, and the worker or web
+// tier holding it all in memory is the platform's problem, not the tenant's.
+// Past the limit the body errors — a caller never gets the first megabyte
+// passed off as the whole answer.
 import { platform } from "./platform.js";
 /** How many redirects one request may follow. fetch's own limit is 20;
  *  nothing a tenant points the platform at needs more than a couple. */
@@ -23,6 +29,57 @@ const REDIRECT = new Set([301, 302, 303, 307, 308]);
  *  credential. Everything else — Authorization, a cookie, an x-api-key, a
  *  webhook's signature — stays with the origin it was meant for. */
 const CROSS_ORIGIN_SAFE = new Set(["accept", "accept-language", "content-type", "user-agent"]);
+/** What one answer may be, read, unless the caller says otherwise. */
+export const MAX_BYTES = 1024 * 1024;
+export class ResponseTooLarge extends Error {
+    constructor(url, maxBytes) {
+        super(`${url} answered more than ${sizeLabel(maxBytes)} — not read`);
+        this.name = "ResponseTooLarge";
+    }
+}
+const sizeLabel = (n) => (n % (1024 * 1024) === 0 ? `${n / (1024 * 1024)} MiB` : `${n} bytes`);
+/** Statuses whose Response may carry no body (the constructor throws on one). */
+const NULL_BODY = new Set([101, 204, 205, 304]);
+/**
+ * `res`, its body read to at most `maxBytes`. The status and headers stand;
+ * the body errors with ResponseTooLarge at the limit — `text()`, `json()`
+ * and a reader all reject, never resolve short. A declared length past the
+ * limit errors at the first read, before a byte is fetched; an undeclared
+ * one is counted as it streams (after decompression, so a small gzip that
+ * unpacks huge is caught too). Infinity: no limit.
+ */
+export function capBody(res, maxBytes, url) {
+    if (!Number.isFinite(maxBytes) || !res.body || NULL_BODY.has(res.status))
+        return res;
+    const declared = Number(res.headers.get("content-length") ?? NaN);
+    const tooLong = Number.isFinite(declared) && declared > maxBytes;
+    const reader = res.body.getReader();
+    let seen = 0;
+    const body = new ReadableStream({
+        async pull(controller) {
+            if (tooLong) {
+                reader.cancel().catch(() => { });
+                return controller.error(new ResponseTooLarge(url, maxBytes));
+            }
+            const { done, value } = await reader.read();
+            if (done)
+                return controller.close();
+            seen += value.byteLength;
+            if (seen > maxBytes) {
+                reader.cancel().catch(() => { });
+                return controller.error(new ResponseTooLarge(url, maxBytes));
+            }
+            controller.enqueue(value);
+        },
+        cancel(reason) {
+            return reader.cancel(reason);
+        },
+    });
+    const out = new Response(body, { status: res.status, statusText: res.statusText, headers: res.headers });
+    // Where the answer came from, as the original said.
+    Object.defineProperty(out, "url", { value: res.url });
+    return out;
+}
 export class RedirectRefused extends Error {
     constructor(message) {
         super(message);
@@ -30,13 +87,14 @@ export class RedirectRefused extends Error {
     }
 }
 export async function fetchUntrusted(url, init = {}, opts = {}) {
+    const maxBytes = opts.maxBytes ?? MAX_BYTES;
     let current = url;
     let req = { ...init };
     for (let hop = 0;; hop++) {
-        const res = await platform.fetchUntrusted(current, { ...req, redirect: "manual" });
+        const res = await platform.fetchUntrusted(current, { ...req, redirect: "manual" }, { maxBytes });
         const location = REDIRECT.has(res.status) ? res.headers.get("location") : null;
         if (!location)
-            return res;
+            return capBody(res, maxBytes, current);
         // The body of a redirect is never read; free the socket.
         await res.body?.cancel().catch(() => { });
         if (hop >= MAX_REDIRECTS)

package/index.ts

@@ -18,7 +18,10 @@ export * from "./src/starter.ts";
 export * from "./src/runner.ts";
 export * from "./src/okf.ts";
 export * from "./src/evals.ts";
+export * from "./src/grades.ts";
+export * from "./src/eval-compare.ts";
 export * from "./src/flow-lint.ts";
+export * from "./src/trigger-filter.ts";
 export * from "./src/catalog.ts";
 export * from "./src/model-probe.ts";
 export * from "./src/providers.ts";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@foldrun/core",
-  "version": "2.10.1",
+  "version": "2.11.0",
   "description": "Agents are just folders. The format, the runner, the checks.",
   "license": "SEE LICENSE IN LICENSE",
   "type": "module",
@@ -55,7 +55,7 @@
   "dependencies": {
     "@anthropic-ai/claude-agent-sdk": "^0.3.296",
     "gray-matter": "^4.0.3",
-    "zod": "4.4.3"
+    "zod": "4.6.7"
   },
   "devDependencies": {
     "@types/node": "^26",

package/src/catalog.ts

@@ -121,6 +121,10 @@ export function readCatalog(baseUrl: string): Catalog | null {
   }
 }
 
+/** The most a gateway's model list may be, read: OpenRouter's is about
+ *  800 KB and grows, near fetchUntrusted's 1 MiB default. */
+export const CATALOG_MAX_BYTES = 8 * 1024 * 1024;
+
 /**
  * The cache, refreshed over the network when it is older than the TTL.
  * A failed fetch keeps the stale copy — offline is a reason to be out of
@@ -134,7 +138,7 @@ export async function loadCatalog(baseUrl: string): Promise<Catalog | null> {
   try {
     const res = await fetchUntrusted(`${trimSlashes(baseUrl)}/v1/models`, {
       signal: AbortSignal.timeout(10_000),
-    });
+    }, { maxBytes: CATALOG_MAX_BYTES });
     if (!res.ok) throw new Error(`${res.status}`);
     const models = parseModels(await res.json());
     if (!models.length) throw new Error("empty model list");

package/src/deploy.ts

@@ -636,6 +636,16 @@ export function deployIssues(files: DeployFile[]): DeployIssue[] {
     }
   }
 
+  // An eval whose settings cannot be read: a `sample:` past 100, a grader
+  // (`trigger: runs`) with a task nothing receives or a file:/run: check
+  // that would act on the platform's host. Refused where they are written.
+  for (const f of files) {
+    if (!/^evals\/[^/]+\.md$/.test(f.path)) continue;
+    let info;
+    try { info = parseEval(f.path, f.content); } catch { continue; }
+    for (const p of info.problems) at(f.path, p);
+  }
+
   // OKF conformance, which needs the files as a tree. Staged in a temp
   // directory and thrown away — nothing touches the live workspace until every
   // check above has passed.

package/src/eval-compare.ts

@@ -0,0 +1,243 @@
+// Model comparisons: one eval's cases, run under two to four model settings,
+// side by side (feature 2, 24 Oct 2026).
+//
+// "Could this step run on the fast model?" was a guess: switch it, watch a
+// week of desks, switch it back. A comparison answers it from the cases you
+// already have — each configuration runs every case, `repeat` times, and the
+// result says per configuration how many passed, what it cost (the runs and
+// their judges) and how long a run took, and per case where they differ.
+//
+// The rules, settled in a Codex debate (24 Oct 2026):
+//   · The model is forced on the steps (every step of a flow eval, or only
+//     `agent`'s), so it wins over the agent's own `model:` and the flow's.
+//   · Every run is a test run, whatever the eval or the flow says about
+//     `live:` — a comparison is never the reason something was sent.
+//   · The judge is the eval's own, the same for every configuration: what
+//     changes is the subject, never the grader.
+//   · Bounded twice: at most MAX_TRIALS case runs, and `maxUsd` — once the
+//     runs and judges have spent it, the rest are recorded as not run.
+//   · A run's time is from a worker picking it up to its end; queueing is
+//     left out, so a busy account does not read as a slow model.
+//   · The repeats interleave (each configuration once, then again), so a
+//     provider that slows down mid-way slows every configuration alike.
+//   · Graders and saved inputs are not comparable: they have no cases to run.
+//   · Nothing here writes the eval's latest result or its history.
+
+import { runEval, type EvalInfo, type EvalResult } from "./evals.ts";
+import { resolveEffort, type Effort } from "./store.ts";
+import { loadFlow } from "./runner.ts";
+
+export const MAX_CONFIGS = 4;
+export const MAX_REPEAT = 5;
+export const MAX_TRIALS = 60;
+export const DEFAULT_MAX_USD = 5;
+export const MAX_MAX_USD = 100;
+
+export interface CompareConfig {
+  /** A tier (fast, default, max) or a model id. */
+  model: string;
+  effort?: Effort | null;
+  /** Flow evals: only this agent's steps take the model. */
+  agent?: string | null;
+}
+
+export interface ComparePlan {
+  configs: CompareConfig[];
+  repeat: number;
+  maxUsd: number;
+  /** cases × configs × repeat. */
+  trials: number;
+}
+
+export interface CompareCaseRow {
+  case: string;
+  /** Per configuration, in order: passed of trials. */
+  passed: number[];
+  trials: number[];
+}
+
+export interface CompareConfigResult extends CompareConfig {
+  label: string;
+  trials: number;
+  passed: number;
+  /** Trials that were not run (the cap) or could not run (an error before any check). */
+  notRun: number;
+  /** passed / decided trials; null when none ran. */
+  rate: number | null;
+  /** The runs and their judges, USD. */
+  costUsd: number;
+  judgeCostUsd: number;
+  /** Per case run that finished, the mean, USD and seconds. */
+  avgCostUsd: number | null;
+  avgSecs: number | null;
+  /** Every run this configuration started, for opening one. */
+  runIds: string[];
+}
+
+export interface EvalComparison {
+  eval: string;
+  target: { flow: string | null; agent: string | null };
+  judge: { model: string; effort: Effort | null };
+  startedAt: string;
+  finishedAt: string;
+  repeat: number;
+  maxUsd: number;
+  spentUsd: number;
+  /** Why it stopped early, when it did. */
+  stopped: string | null;
+  configs: CompareConfigResult[];
+  cases: CompareCaseRow[];
+}
+
+function refuse(message: string, status: number): Error {
+  return Object.assign(new Error(message), { status });
+}
+
+const MODEL = /^[A-Za-z0-9][\w.:/-]{0,99}$/;
+
+export const configLabel = (c: CompareConfig) =>
+  `${c.model}${c.effort ? ` · ${c.effort}` : ""}${c.agent ? ` · ${c.agent} only` : ""}`;
+
+/**
+ * Read and bound a request. Throws with a status: 400 for a shape, 413 when
+ * it is too big to run. The eval must have cases and not be a grader.
+ */
+export function planComparison(info: EvalInfo, body: unknown): ComparePlan {
+  if (info.trigger === "runs") throw refuse(`${info.name} grades real runs — it has no cases to compare models on`, 400);
+  if (info.inputs) throw refuse(`${info.name} is a file of saved inputs, not an eval`, 400);
+  if (info.cases.length === 0) throw refuse(`${info.name} has no cases`, 400);
+  // A comparison promises test runs only; a run: check is a command on the
+  // platform's host, run again for every case of every setting and repeat.
+  const runs = info.cases.find((c) => c.expect.some((a) => a.type === "run"));
+  if (runs) throw refuse(`${info.name} has a run: check (case "${runs.name}") — a comparison repeats every check many times and run: executes a command, so it cannot be compared; use contains, matches or judge`, 400);
+  const b = (body && typeof body === "object" ? body : {}) as Record<string, unknown>;
+  if (!Array.isArray(b.configs) || b.configs.length < 2) throw refuse("expected { configs: [{ model, effort?, agent? }, …] } — two to four settings to compare", 400);
+  if (b.configs.length > MAX_CONFIGS) throw refuse(`at most ${MAX_CONFIGS} settings in one comparison`, 400);
+  const configs: CompareConfig[] = b.configs.map((raw, i) => {
+    const r = (raw && typeof raw === "object" ? raw : {}) as Record<string, unknown>;
+    if (typeof r.model !== "string" || !MODEL.test(r.model.trim())) throw refuse(`configs[${i}].model is a tier (fast, default, max) or a model id`, 400);
+    let effort: Effort | null = null;
+    if (r.effort !== undefined && r.effort !== null && r.effort !== "") {
+      effort = resolveEffort(r.effort);
+      if (!effort) throw refuse(`configs[${i}].effort is low, medium, high, xhigh or max`, 400);
+    }
+    let agent: string | null = null;
+    if (r.agent !== undefined && r.agent !== null && r.agent !== "") {
+      if (!info.flow) throw refuse(`configs[${i}].agent picks one agent's steps of a flow eval — ${info.name} tests agent ${info.agent} already`, 400);
+      if (typeof r.agent !== "string" || !/^[a-z0-9][a-z0-9-]{0,60}$/.test(r.agent)) throw refuse(`configs[${i}].agent is an agent's name`, 400);
+      agent = r.agent;
+    }
+    return { model: r.model.trim(), effort, agent };
+  });
+  const keys = configs.map(configLabel);
+  if (new Set(keys).size !== keys.length) throw refuse("two settings are the same — each one should differ", 400);
+  const repeat = b.repeat === undefined ? 1 : Number(b.repeat);
+  if (!Number.isInteger(repeat) || repeat < 1 || repeat > MAX_REPEAT) throw refuse(`repeat is a whole number from 1 to ${MAX_REPEAT}`, 400);
+  const maxUsd = b.maxUsd === undefined ? DEFAULT_MAX_USD : Number(b.maxUsd);
+  if (!Number.isFinite(maxUsd) || maxUsd <= 0 || maxUsd > MAX_MAX_USD) throw refuse(`maxUsd is a spend limit in USD, above 0 and at most ${MAX_MAX_USD}`, 400);
+  const trials = info.cases.length * configs.length * repeat;
+  if (trials > MAX_TRIALS) {
+    throw refuse(`${info.cases.length} cases × ${configs.length} settings × ${repeat} = ${trials} runs — at most ${MAX_TRIALS}; fewer settings or repeats`, 413);
+  }
+  return { configs, repeat, maxUsd, trials };
+}
+
+/**
+ * Run the comparison. `onProgress` hears each finished pass of one
+ * configuration (done of total case runs). `run` is the eval runner —
+ * injectable, so a test needs no model.
+ */
+/** A flow eval's `agent` must have steps in the flow, or the comparison
+ *  would run the original models under the requested labels. */
+function assertAgentsInFlow(tenant: string, workspace: string, info: EvalInfo, plan: ComparePlan) {
+  const named = plan.configs.map((c) => c.agent).filter((a): a is string => !!a);
+  if (!info.flow || !named.length) return;
+  const flow = loadFlow(tenant, workspace, info.flow);
+  const agents = new Set((flow?.steps ?? []).map((s) => s.agent));
+  const missing = named.find((a) => !agents.has(a));
+  if (missing) throw refuse(`flow ${info.flow} has no step by ${missing} — pick an agent it runs`, 400);
+}
+
+export async function compareEval(
+  tenant: string,
+  workspace: string,
+  info: EvalInfo,
+  plan: ComparePlan,
+  opts: { onProgress?: (done: number, total: number) => void | Promise<void>; run?: typeof runEval } = {},
+): Promise<EvalComparison> {
+  if (!opts.run) assertAgentsInFlow(tenant, workspace, info, plan);
+  const run = opts.run ?? runEval;
+  const startedAt = new Date().toISOString();
+  const results: EvalResult[][] = plan.configs.map(() => []);
+  let spent = 0;
+  let stopped: string | null = null;
+  let done = 0;
+  for (let r = 0; r < plan.repeat; r++) {
+    for (let i = 0; i < plan.configs.length; i++) {
+      const cfg = plan.configs[i];
+      const result = await run(tenant, workspace, info, {
+        model: cfg.model,
+        effort: cfg.effort ?? null,
+        agent: cfg.agent ?? null,
+        forceTest: true,
+        beforeCase: () => {
+          if (spent >= plan.maxUsd) {
+            stopped ??= `the spend limit of $${plan.maxUsd.toFixed(2)} was reached`;
+            return stopped;
+          }
+          return null;
+        },
+        // Counted as each case ends, so the limit holds inside a setting too.
+        afterCase: (c) => {
+          spent += c.costUsd + (c.judgeCostUsd ?? 0);
+        },
+      });
+      results[i].push(result);
+      done += info.cases.length;
+      await opts.onProgress?.(done, plan.trials);
+    }
+  }
+
+  const round = (n: number, d = 4) => Math.round(n * 10 ** d) / 10 ** d;
+  const configs: CompareConfigResult[] = plan.configs.map((cfg, i) => {
+    const cases = results[i].flatMap((r) => r.cases);
+    const ran = cases.filter((c) => c.runId);
+    const notRun = cases.filter((c) => !c.passed && c.error && !c.assertions.length).length;
+    const passed = cases.filter((c) => c.passed).length;
+    const decided = cases.length - notRun;
+    const cost = cases.reduce((n, c) => n + c.costUsd, 0);
+    const judge = cases.reduce((n, c) => n + (c.judgeCostUsd ?? 0), 0);
+    const timed = ran.filter((c) => typeof c.secs === "number");
+    return {
+      ...cfg,
+      label: configLabel(cfg),
+      trials: cases.length,
+      passed,
+      notRun,
+      rate: decided ? round(passed / decided, 3) : null,
+      costUsd: round(cost + judge),
+      judgeCostUsd: round(judge),
+      avgCostUsd: ran.length ? round((cost + judge) / ran.length) : null,
+      avgSecs: timed.length ? Math.round((timed.reduce((n, c) => n + (c.secs ?? 0), 0) / timed.length) * 10) / 10 : null,
+      runIds: ran.map((c) => c.runId!),
+    };
+  });
+  const cases: CompareCaseRow[] = info.cases.map((c) => ({
+    case: c.name,
+    passed: results.map((rs) => rs.flatMap((r) => r.cases).filter((x) => x.name === c.name && x.passed).length),
+    trials: results.map((rs) => rs.flatMap((r) => r.cases).filter((x) => x.name === c.name).length),
+  }));
+  return {
+    eval: info.name,
+    target: { flow: info.flow, agent: info.agent },
+    judge: { model: info.model, effort: info.effort },
+    startedAt,
+    finishedAt: new Date().toISOString(),
+    repeat: plan.repeat,
+    maxUsd: plan.maxUsd,
+    spentUsd: round(spent),
+    stopped,
+    configs,
+    cases,
+  };
+}