@foldrun/core 2.10.1 → 2.11.0

package/dist/index.d.ts

@@ -6,7 +6,10 @@ export * from "./src/starter.ts";
 export * from "./src/runner.ts";
 export * from "./src/okf.ts";
 export * from "./src/evals.ts";
+export * from "./src/grades.ts";
+export * from "./src/eval-compare.ts";
 export * from "./src/flow-lint.ts";
+export * from "./src/trigger-filter.ts";
 export * from "./src/catalog.ts";
 export * from "./src/model-probe.ts";
 export * from "./src/providers.ts";

package/dist/index.js

@@ -17,7 +17,10 @@ export * from "./src/starter.js";
 export * from "./src/runner.js";
 export * from "./src/okf.js";
 export * from "./src/evals.js";
+export * from "./src/grades.js";
+export * from "./src/eval-compare.js";
 export * from "./src/flow-lint.js";
+export * from "./src/trigger-filter.js";
 export * from "./src/catalog.js";
 export * from "./src/model-probe.js";
 export * from "./src/providers.js";

package/dist/src/catalog.d.ts

@@ -35,6 +35,9 @@ export declare function parseModels(payload: unknown): CatalogModel[];
 /** The cache as it stands, however old — null only when nothing was ever
  *  fetched. Reading never touches the network. */
 export declare function readCatalog(baseUrl: string): Catalog | null;
+/** The most a gateway's model list may be, read: OpenRouter's is about
+ *  800 KB and grows, near fetchUntrusted's 1 MiB default. */
+export declare const CATALOG_MAX_BYTES: number;
 /**
  * The cache, refreshed over the network when it is older than the TTL.
  * A failed fetch keeps the stale copy — offline is a reason to be out of

package/dist/src/catalog.js

@@ -83,6 +83,9 @@ export function readCatalog(baseUrl) {
         return null;
     }
 }
+/** The most a gateway's model list may be, read: OpenRouter's is about
+ *  800 KB and grows, near fetchUntrusted's 1 MiB default. */
+export const CATALOG_MAX_BYTES = 8 * 1024 * 1024;
 /**
  * The cache, refreshed over the network when it is older than the TTL.
  * A failed fetch keeps the stale copy — offline is a reason to be out of
@@ -96,7 +99,7 @@ export async function loadCatalog(baseUrl) {
     try {
         const res = await fetchUntrusted(`${trimSlashes(baseUrl)}/v1/models`, {
             signal: AbortSignal.timeout(10_000),
-        });
+        }, { maxBytes: CATALOG_MAX_BYTES });
         if (!res.ok)
             throw new Error(`${res.status}`);
         const models = parseModels(await res.json());

package/dist/src/deploy.js

@@ -611,6 +611,22 @@ export function deployIssues(files) {
             }
         }
     }
+    // An eval whose settings cannot be read: a `sample:` past 100, a grader
+    // (`trigger: runs`) with a task nothing receives or a file:/run: check
+    // that would act on the platform's host. Refused where they are written.
+    for (const f of files) {
+        if (!/^evals\/[^/]+\.md$/.test(f.path))
+            continue;
+        let info;
+        try {
+            info = parseEval(f.path, f.content);
+        }
+        catch {
+            continue;
+        }
+        for (const p of info.problems)
+            at(f.path, p);
+    }
     // OKF conformance, which needs the files as a tree. Staged in a temp
     // directory and thrown away — nothing touches the live workspace until every
     // check above has passed.

package/dist/src/eval-compare.d.ts

@@ -0,0 +1,74 @@
+import { runEval, type EvalInfo } from "./evals.ts";
+import { type Effort } from "./store.ts";
+export declare const MAX_CONFIGS = 4;
+export declare const MAX_REPEAT = 5;
+export declare const MAX_TRIALS = 60;
+export declare const DEFAULT_MAX_USD = 5;
+export declare const MAX_MAX_USD = 100;
+export interface CompareConfig {
+    /** A tier (fast, default, max) or a model id. */
+    model: string;
+    effort?: Effort | null;
+    /** Flow evals: only this agent's steps take the model. */
+    agent?: string | null;
+}
+export interface ComparePlan {
+    configs: CompareConfig[];
+    repeat: number;
+    maxUsd: number;
+    /** cases × configs × repeat. */
+    trials: number;
+}
+export interface CompareCaseRow {
+    case: string;
+    /** Per configuration, in order: passed of trials. */
+    passed: number[];
+    trials: number[];
+}
+export interface CompareConfigResult extends CompareConfig {
+    label: string;
+    trials: number;
+    passed: number;
+    /** Trials that were not run (the cap) or could not run (an error before any check). */
+    notRun: number;
+    /** passed / decided trials; null when none ran. */
+    rate: number | null;
+    /** The runs and their judges, USD. */
+    costUsd: number;
+    judgeCostUsd: number;
+    /** Per case run that finished, the mean, USD and seconds. */
+    avgCostUsd: number | null;
+    avgSecs: number | null;
+    /** Every run this configuration started, for opening one. */
+    runIds: string[];
+}
+export interface EvalComparison {
+    eval: string;
+    target: {
+        flow: string | null;
+        agent: string | null;
+    };
+    judge: {
+        model: string;
+        effort: Effort | null;
+    };
+    startedAt: string;
+    finishedAt: string;
+    repeat: number;
+    maxUsd: number;
+    spentUsd: number;
+    /** Why it stopped early, when it did. */
+    stopped: string | null;
+    configs: CompareConfigResult[];
+    cases: CompareCaseRow[];
+}
+export declare const configLabel: (c: CompareConfig) => string;
+/**
+ * Read and bound a request. Throws with a status: 400 for a shape, 413 when
+ * it is too big to run. The eval must have cases and not be a grader.
+ */
+export declare function planComparison(info: EvalInfo, body: unknown): ComparePlan;
+export declare function compareEval(tenant: string, workspace: string, info: EvalInfo, plan: ComparePlan, opts?: {
+    onProgress?: (done: number, total: number) => void | Promise<void>;
+    run?: typeof runEval;
+}): Promise<EvalComparison>;

package/dist/src/eval-compare.js

@@ -0,0 +1,187 @@
+// Model comparisons: one eval's cases, run under two to four model settings,
+// side by side (feature 2, 24 Oct 2026).
+//
+// "Could this step run on the fast model?" was a guess: switch it, watch a
+// week of desks, switch it back. A comparison answers it from the cases you
+// already have — each configuration runs every case, `repeat` times, and the
+// result says per configuration how many passed, what it cost (the runs and
+// their judges) and how long a run took, and per case where they differ.
+//
+// The rules, settled in a Codex debate (24 Oct 2026):
+//   · The model is forced on the steps (every step of a flow eval, or only
+//     `agent`'s), so it wins over the agent's own `model:` and the flow's.
+//   · Every run is a test run, whatever the eval or the flow says about
+//     `live:` — a comparison is never the reason something was sent.
+//   · The judge is the eval's own, the same for every configuration: what
+//     changes is the subject, never the grader.
+//   · Bounded twice: at most MAX_TRIALS case runs, and `maxUsd` — once the
+//     runs and judges have spent it, the rest are recorded as not run.
+//   · A run's time is from a worker picking it up to its end; queueing is
+//     left out, so a busy account does not read as a slow model.
+//   · The repeats interleave (each configuration once, then again), so a
+//     provider that slows down mid-way slows every configuration alike.
+//   · Graders and saved inputs are not comparable: they have no cases to run.
+//   · Nothing here writes the eval's latest result or its history.
+import { runEval } from "./evals.js";
+import { resolveEffort } from "./store.js";
+import { loadFlow } from "./runner.js";
+export const MAX_CONFIGS = 4;
+export const MAX_REPEAT = 5;
+export const MAX_TRIALS = 60;
+export const DEFAULT_MAX_USD = 5;
+export const MAX_MAX_USD = 100;
+function refuse(message, status) {
+    return Object.assign(new Error(message), { status });
+}
+const MODEL = /^[A-Za-z0-9][\w.:/-]{0,99}$/;
+export const configLabel = (c) => `${c.model}${c.effort ? ` · ${c.effort}` : ""}${c.agent ? ` · ${c.agent} only` : ""}`;
+/**
+ * Read and bound a request. Throws with a status: 400 for a shape, 413 when
+ * it is too big to run. The eval must have cases and not be a grader.
+ */
+export function planComparison(info, body) {
+    if (info.trigger === "runs")
+        throw refuse(`${info.name} grades real runs — it has no cases to compare models on`, 400);
+    if (info.inputs)
+        throw refuse(`${info.name} is a file of saved inputs, not an eval`, 400);
+    if (info.cases.length === 0)
+        throw refuse(`${info.name} has no cases`, 400);
+    // A comparison promises test runs only; a run: check is a command on the
+    // platform's host, run again for every case of every setting and repeat.
+    const runs = info.cases.find((c) => c.expect.some((a) => a.type === "run"));
+    if (runs)
+        throw refuse(`${info.name} has a run: check (case "${runs.name}") — a comparison repeats every check many times and run: executes a command, so it cannot be compared; use contains, matches or judge`, 400);
+    const b = (body && typeof body === "object" ? body : {});
+    if (!Array.isArray(b.configs) || b.configs.length < 2)
+        throw refuse("expected { configs: [{ model, effort?, agent? }, …] } — two to four settings to compare", 400);
+    if (b.configs.length > MAX_CONFIGS)
+        throw refuse(`at most ${MAX_CONFIGS} settings in one comparison`, 400);
+    const configs = b.configs.map((raw, i) => {
+        const r = (raw && typeof raw === "object" ? raw : {});
+        if (typeof r.model !== "string" || !MODEL.test(r.model.trim()))
+            throw refuse(`configs[${i}].model is a tier (fast, default, max) or a model id`, 400);
+        let effort = null;
+        if (r.effort !== undefined && r.effort !== null && r.effort !== "") {
+            effort = resolveEffort(r.effort);
+            if (!effort)
+                throw refuse(`configs[${i}].effort is low, medium, high, xhigh or max`, 400);
+        }
+        let agent = null;
+        if (r.agent !== undefined && r.agent !== null && r.agent !== "") {
+            if (!info.flow)
+                throw refuse(`configs[${i}].agent picks one agent's steps of a flow eval — ${info.name} tests agent ${info.agent} already`, 400);
+            if (typeof r.agent !== "string" || !/^[a-z0-9][a-z0-9-]{0,60}$/.test(r.agent))
+                throw refuse(`configs[${i}].agent is an agent's name`, 400);
+            agent = r.agent;
+        }
+        return { model: r.model.trim(), effort, agent };
+    });
+    const keys = configs.map(configLabel);
+    if (new Set(keys).size !== keys.length)
+        throw refuse("two settings are the same — each one should differ", 400);
+    const repeat = b.repeat === undefined ? 1 : Number(b.repeat);
+    if (!Number.isInteger(repeat) || repeat < 1 || repeat > MAX_REPEAT)
+        throw refuse(`repeat is a whole number from 1 to ${MAX_REPEAT}`, 400);
+    const maxUsd = b.maxUsd === undefined ? DEFAULT_MAX_USD : Number(b.maxUsd);
+    if (!Number.isFinite(maxUsd) || maxUsd <= 0 || maxUsd > MAX_MAX_USD)
+        throw refuse(`maxUsd is a spend limit in USD, above 0 and at most ${MAX_MAX_USD}`, 400);
+    const trials = info.cases.length * configs.length * repeat;
+    if (trials > MAX_TRIALS) {
+        throw refuse(`${info.cases.length} cases × ${configs.length} settings × ${repeat} = ${trials} runs — at most ${MAX_TRIALS}; fewer settings or repeats`, 413);
+    }
+    return { configs, repeat, maxUsd, trials };
+}
+/**
+ * Run the comparison. `onProgress` hears each finished pass of one
+ * configuration (done of total case runs). `run` is the eval runner —
+ * injectable, so a test needs no model.
+ */
+/** A flow eval's `agent` must have steps in the flow, or the comparison
+ *  would run the original models under the requested labels. */
+function assertAgentsInFlow(tenant, workspace, info, plan) {
+    const named = plan.configs.map((c) => c.agent).filter((a) => !!a);
+    if (!info.flow || !named.length)
+        return;
+    const flow = loadFlow(tenant, workspace, info.flow);
+    const agents = new Set((flow?.steps ?? []).map((s) => s.agent));
+    const missing = named.find((a) => !agents.has(a));
+    if (missing)
+        throw refuse(`flow ${info.flow} has no step by ${missing} — pick an agent it runs`, 400);
+}
+export async function compareEval(tenant, workspace, info, plan, opts = {}) {
+    if (!opts.run)
+        assertAgentsInFlow(tenant, workspace, info, plan);
+    const run = opts.run ?? runEval;
+    const startedAt = new Date().toISOString();
+    const results = plan.configs.map(() => []);
+    let spent = 0;
+    let stopped = null;
+    let done = 0;
+    for (let r = 0; r < plan.repeat; r++) {
+        for (let i = 0; i < plan.configs.length; i++) {
+            const cfg = plan.configs[i];
+            const result = await run(tenant, workspace, info, {
+                model: cfg.model,
+                effort: cfg.effort ?? null,
+                agent: cfg.agent ?? null,
+                forceTest: true,
+                beforeCase: () => {
+                    if (spent >= plan.maxUsd) {
+                        stopped ??= `the spend limit of $${plan.maxUsd.toFixed(2)} was reached`;
+                        return stopped;
+                    }
+                    return null;
+                },
+                // Counted as each case ends, so the limit holds inside a setting too.
+                afterCase: (c) => {
+                    spent += c.costUsd + (c.judgeCostUsd ?? 0);
+                },
+            });
+            results[i].push(result);
+            done += info.cases.length;
+            await opts.onProgress?.(done, plan.trials);
+        }
+    }
+    const round = (n, d = 4) => Math.round(n * 10 ** d) / 10 ** d;
+    const configs = plan.configs.map((cfg, i) => {
+        const cases = results[i].flatMap((r) => r.cases);
+        const ran = cases.filter((c) => c.runId);
+        const notRun = cases.filter((c) => !c.passed && c.error && !c.assertions.length).length;
+        const passed = cases.filter((c) => c.passed).length;
+        const decided = cases.length - notRun;
+        const cost = cases.reduce((n, c) => n + c.costUsd, 0);
+        const judge = cases.reduce((n, c) => n + (c.judgeCostUsd ?? 0), 0);
+        const timed = ran.filter((c) => typeof c.secs === "number");
+        return {
+            ...cfg,
+            label: configLabel(cfg),
+            trials: cases.length,
+            passed,
+            notRun,
+            rate: decided ? round(passed / decided, 3) : null,
+            costUsd: round(cost + judge),
+            judgeCostUsd: round(judge),
+            avgCostUsd: ran.length ? round((cost + judge) / ran.length) : null,
+            avgSecs: timed.length ? Math.round((timed.reduce((n, c) => n + (c.secs ?? 0), 0) / timed.length) * 10) / 10 : null,
+            runIds: ran.map((c) => c.runId),
+        };
+    });
+    const cases = info.cases.map((c) => ({
+        case: c.name,
+        passed: results.map((rs) => rs.flatMap((r) => r.cases).filter((x) => x.name === c.name && x.passed).length),
+        trials: results.map((rs) => rs.flatMap((r) => r.cases).filter((x) => x.name === c.name).length),
+    }));
+    return {
+        eval: info.name,
+        target: { flow: info.flow, agent: info.agent },
+        judge: { model: info.model, effort: info.effort },
+        startedAt,
+        finishedAt: new Date().toISOString(),
+        repeat: plan.repeat,
+        maxUsd: plan.maxUsd,
+        spentUsd: round(spent),
+        stopped,
+        configs,
+        cases,
+    };
+}

package/dist/src/evals.d.ts

@@ -1,3 +1,4 @@
+import type { TranslatorSpec } from "./translator.ts";
 import { type Effort } from "./store.ts";
 export type Assertion = {
     type: "contains";
@@ -36,8 +37,20 @@ export interface EvalInfo {
     effort: Effort | null;
     /** When it runs. `deploy` (default): after every push, with the rest.
      *  `manual`: only when a person presses Run or the CLI asks — for a flow
-     *  eval that costs a whole run and touches real systems every time. */
-    trigger: "deploy" | "manual";
+     *  eval that costs a whole run and touches real systems every time.
+     *  `runs`: never run as cases — it grades real finished runs of its agent
+     *  or flow (grades.ts), a `sample` of them, as they finish. */
+    trigger: "deploy" | "manual" | "runs";
+    /** trigger: runs — the percent of finished runs graded, 0–100. */
+    sample: number;
+    /** trigger: runs — propose a fix (improve.ts) once this many graded runs
+     *  failed and no person has looked yet. Null: never on its own. */
+    improve: number | null;
+    /** What in the file cannot be read as written, one sentence each — the
+     *  deploy's checks and `foldrun check` report them. */
+    problems: string[];
+    /** A hash of the file as it was read (listEvals), when it was. */
+    revision?: string;
     /** An eval's runs are test runs — nothing outward, state untouched —
      *  unless the file says `live: true`. A flow eval that must really send
      *  to prove itself opts in; nothing opts in by accident. */
@@ -55,6 +68,8 @@ export declare function parseEval(file: string, raw: string, defaults?: {
     model?: string;
     effort?: string;
 }): EvalInfo;
+/** Judges one graded run may cost — the grade pass counts runs, not calls. */
+export declare const MAX_GRADER_JUDGES = 5;
 export declare function listEvals(tenant: string, workspace: string): EvalInfo[];
 /** `judge:` from AGENTS.md, field by field: the workspace's model (or
  *  effort), else the account's — so a workspace that sets only an effort
@@ -92,6 +107,8 @@ export interface AssertionResult {
     assertion: Assertion;
     passed: boolean;
     detail: string;
+    /** What a judge cost, USD; absent for the free checks. */
+    costUsd?: number;
 }
 export interface CaseResult {
     name: string;
@@ -101,7 +118,34 @@ export interface CaseResult {
     /** Set when the agent itself failed, before any assertion could run. */
     error: string | null;
     assertions: AssertionResult[];
+    /** What the case's run cost — the subject, not the judge. */
     costUsd: number;
+    /** What its judges cost, USD. Absent on results from before 24 Oct 2026. */
+    judgeCostUsd?: number;
+    /** How long the run took from a worker picking it up to its end — its
+     *  time in the queue left out, so a busy account does not read as a slow
+     *  model. Null when it never ran. */
+    secs?: number | null;
+}
+/** How a model comparison (eval-compare.ts) runs one eval: every step it
+ *  forces onto a model, always as a test run, and whether to go on. */
+export interface EvalRunOptions {
+    /** Forced onto the subject's steps — every step of a flow eval, or only
+     *  `agent`'s when it is set — as a step-level setting, so it wins over
+     *  the agent's own `model:` and the flow's defaults. `effort` likewise
+     *  when given; none keeps each step's own effort, so a model-only
+     *  comparison changes the model and nothing else. */
+    model?: string;
+    effort?: Effort | null;
+    agent?: string | null;
+    /** A test run whatever the eval or flow says about `live:`. */
+    forceTest?: boolean;
+    /** Asked before each case; a reason stops the eval there, the rest of its
+     *  cases recorded as not run with it. */
+    beforeCase?: () => string | null;
+    /** Told each case as it ends — what a comparison counts its spend by, so
+     *  its limit holds between the cases of one setting, not only after. */
+    afterCase?: (c: CaseResult) => void;
 }
 export interface EvalResult {
     eval: string;
@@ -117,6 +161,17 @@ export interface EvalResult {
      *  answers. */
     error?: string;
 }
+/** How long one judge may take before it is cut off: a verdict on one
+ *  sentence is seconds; a stalled model must not hold a grade pass. */
+export declare const JUDGE_TIMEOUT_MS = 120000;
+/** One judge call: its words and what it cost (the SDK's result line). */
+export declare function judgeCall(rubric: string, output: string, model: string, effort: Effort | null, env: Record<string, string | undefined>, 
+/** What the output was an answer to — a live run's task, so the judge
+ *  grades the reply against what was asked rather than in a vacuum. */
+context?: string): Promise<{
+    text: string;
+    costUsd: number;
+}>;
 /**
  * The judge's environment: this process's, with the platform's model
  * credential read live. The process's own CLAUDE_CODE_OAUTH_TOKEN is the copy
@@ -126,7 +181,11 @@ export interface EvalResult {
  * agent steps (which read the live file) kept working (2026-10-21).
  */
 export declare function judgeProcessEnv(): Record<string, string | undefined>;
-export declare function runEval(tenant: string, workspace: string, info: EvalInfo): Promise<EvalResult>;
+export declare function checkAssertion(assertion: Assertion, output: string, agentDir: string, model: string, effort: Effort | null, judgeEnv?: () => {
+    env: Record<string, string | undefined>;
+    translator?: TranslatorSpec | null;
+}): Promise<AssertionResult>;
+export declare function runEval(tenant: string, workspace: string, info: EvalInfo, opts?: EvalRunOptions): Promise<EvalResult>;
 /**
  * The latest result, and one line of history.
  *