@foldrun/core 2.10.1 → 2.11.1

package/dist/src/evals.js

@@ -34,6 +34,7 @@ import { accountModelEnv, withModelEnv } from "./model-env.js";
 import { sdkEnv } from "./step-exec.js";
 import { platformModelEnv } from "./runner.js";
 import fs from "node:fs";
+import crypto from "node:crypto";
 import path from "node:path";
 import { deployedCommit } from "./deploy.js";
 import { latestRevisionId } from "./history.js";
@@ -137,6 +138,45 @@ export function parseEval(file, raw, defaults = {}) {
     }
     if (current)
         cases.push(current);
+    const trigger = data.trigger === "manual" ? "manual" : data.trigger === "runs" ? "runs" : "deploy";
+    const problems = [];
+    const sample = parseSample(data.sample, problems);
+    const improve = parseImprove(data.improve, problems);
+    if (trigger !== "runs") {
+        if (data.sample !== undefined)
+            problems.push("sample: only applies to an eval with trigger: runs — this one runs its cases");
+        if (data.improve !== undefined)
+            problems.push("improve: only applies to an eval with trigger: runs");
+    }
+    else {
+        // A grader reads what a run said; it never starts one, so a task has
+        // nothing to go to, and file:/run: would execute or read on the
+        // platform's host rather than in the run's sandbox.
+        for (const c of cases) {
+            if (c.task)
+                problems.push(`## ${c.name}: task: does nothing in a trigger: runs eval — it grades real runs, so each check is only an expect: list`);
+            for (const a of c.expect) {
+                if (a.type === "file" || a.type === "run")
+                    problems.push(`## ${c.name}: ${a.type}: cannot grade a finished run — use contains, not-contains or judge`);
+                // Graded on the platform's scheduler, over whatever a run said: a
+                // regular expression that backtracks would hold the scheduler with it.
+                if (a.type === "matches")
+                    problems.push(`## ${c.name}: matches: is not used to grade live runs — a regular expression over any run's output can stall the platform; use contains, not-contains or judge`);
+            }
+            if (c.expect.length === 0)
+                problems.push(`## ${c.name}: has no expect: lines, so it grades nothing`);
+        }
+        if (cases.length === 0)
+            problems.push("a grader needs at least one check — a `## <name>` section with an expect: list; with none it would pass every run");
+        // The pass budgets graded runs; this bounds the judges each one costs.
+        const judges = cases.reduce((n, c) => n + c.expect.filter((x) => x.type === "judge").length, 0);
+        if (judges > MAX_GRADER_JUDGES)
+            problems.push(`${judges} judge: lines — a grader asks at most ${MAX_GRADER_JUDGES} judges per run; fold related claims into one`);
+        if (data.live === true)
+            problems.push("live: does nothing in a trigger: runs eval — it never starts a run");
+        if (data.inputs === true)
+            problems.push("inputs: and trigger: runs cannot go together");
+    }
     return {
         name: data.name ?? file.replace(/\.md$/, ""),
         file,
@@ -144,12 +184,41 @@ export function parseEval(file, raw, defaults = {}) {
         flow: typeof data.flow === "string" ? data.flow : null,
         model: resolveModel(data.model ?? defaults.model ?? "fast"),
         effort: resolveEffort(data.effort ?? defaults.effort ?? "low"),
-        trigger: data.trigger === "manual" ? "manual" : "deploy",
+        trigger,
+        sample,
+        improve: trigger === "runs" ? improve : null,
+        problems,
         live: data.live === true,
         inputs: data.inputs === true,
         cases,
     };
 }
+/** Judges one graded run may cost — the grade pass counts runs, not calls. */
+export const MAX_GRADER_JUDGES = 5;
+/** `sample: 25`, `sample: "25%"` — percent of finished runs graded. */
+function parseSample(v, problems) {
+    if (v === undefined || v === null)
+        return 25;
+    const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.trim().replace(/%$/, "")) : NaN;
+    if (!Number.isFinite(n) || n < 0 || n > 100) {
+        problems.push(`sample: ${String(v)} — a percent from 0 to 100 (25 grades about one run in four; 0 grades none on its own)`);
+        return 25;
+    }
+    return Math.round(n * 100) / 100;
+}
+/** `improve: true` (5), `improve: 3`, `improve: false`. */
+function parseImprove(v, problems) {
+    if (v === undefined || v === null || v === false)
+        return null;
+    if (v === true)
+        return 5;
+    const n = typeof v === "number" ? v : NaN;
+    if (!Number.isInteger(n) || n < 1 || n > 50) {
+        problems.push(`improve: ${String(v)} — true, or how many failed grades open a proposal (1 to 50)`);
+        return null;
+    }
+    return n;
+}
 export function listEvals(tenant, workspace) {
     const dir = path.join(workspaceDir(tenant, workspace), "evals");
     if (!fs.existsSync(dir))
@@ -158,7 +227,13 @@ export function listEvals(tenant, workspace) {
         .readdirSync(dir)
         .filter((f) => f.endsWith(".md"))
         .sort()
-        .map((f) => parseEval(f, fs.readFileSync(path.join(dir, f), "utf8"), judgeDefaults(tenant, workspace)));
+        .map((f) => {
+        const raw = fs.readFileSync(path.join(dir, f), "utf8");
+        // The version is taken from the same read the eval is parsed from, so
+        // a grade names the rubric that produced it even if the file changes
+        // while its judges answer.
+        return { ...parseEval(f, raw, judgeDefaults(tenant, workspace)), revision: crypto.createHash("sha256").update(raw).digest("hex").slice(0, 12) };
+    });
 }
 /** `judge:` from AGENTS.md, field by field: the workspace's model (or
  *  effort), else the account's — so a workspace that sets only an effort
@@ -386,32 +461,59 @@ function shell(command, cwd) {
 // LAST, after the reasoning: asked for it first, the judge committed before
 // it had looked, then argued the other way — "FAIL … which meets the
 // criterion" (vaza evals, 21 Oct 2026).
-async function judge(rubric, output, model, effort, env) {
+/** How long one judge may take before it is cut off: a verdict on one
+ *  sentence is seconds; a stalled model must not hold a grade pass. */
+export const JUDGE_TIMEOUT_MS = 120_000;
+/** One judge call: its words and what it cost (the SDK's result line). */
+export async function judgeCall(rubric, output, model, effort, env, 
+/** What the output was an answer to — a live run's task, so the judge
+ *  grades the reply against what was asked rather than in a vacuum. */
+context) {
     const prompt = `You are grading one piece of output against one criterion. ${JUDGE_VERDICT_INSTRUCTION}\n\n` +
-        `Criterion: ${rubric}\n\n<output>\n${output.slice(0, 20000)}\n</output>`;
+        `Criterion: ${rubric}\n\n` +
+        (context ? `<task>\n${context.slice(0, 4000)}\n</task>\n\n` : "") +
+        `<output>\n${output.slice(0, 20000)}\n</output>`;
     let text = "";
-    const q = query({
-        prompt,
-        options: {
-            model,
-            ...(effort ? { effort } : {}),
-            tools: [],
-            allowedTools: [],
-            settingSources: [],
-            strictMcpConfig: true,
-            env,
-            maxTurns: 1,
-        },
-    });
-    for await (const message of q) {
-        if (message.type === "assistant") {
-            for (const block of message.message.content) {
-                if (block.type === "text")
-                    text += block.text;
+    let costUsd = 0;
+    const abort = new AbortController();
+    const timer = setTimeout(() => abort.abort(), JUDGE_TIMEOUT_MS);
+    timer.unref?.();
+    try {
+        const q = query({
+            prompt,
+            options: {
+                model,
+                ...(effort ? { effort } : {}),
+                tools: [],
+                allowedTools: [],
+                settingSources: [],
+                strictMcpConfig: true,
+                env,
+                maxTurns: 1,
+                abortController: abort,
+            },
+        });
+        for await (const message of q) {
+            if (message.type === "assistant") {
+                for (const block of message.message.content) {
+                    if (block.type === "text")
+                        text += block.text;
+                }
+            }
+            else if (message.type === "result") {
+                costUsd = Number(message.total_cost_usd) || 0;
             }
         }
     }
-    return text.trim();
+    catch (err) {
+        if (abort.signal.aborted)
+            throw new Error(`the judge took longer than ${JUDGE_TIMEOUT_MS / 1000}s and was stopped`);
+        throw err;
+    }
+    finally {
+        clearTimeout(timer);
+    }
+    return { text: text.trim(), costUsd };
 }
 /**
  * The judge's environment: this process's, with the platform's model
@@ -425,7 +527,7 @@ export function judgeProcessEnv() {
     const live = Object.fromEntries(Object.entries(platformModelEnv()).filter(([, v]) => v !== undefined));
     return sdkEnv({ ...process.env, ...live });
 }
-async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => ({ env: judgeProcessEnv() })) {
+export async function checkAssertion(assertion, output, agentDir, model, effort, judgeEnv = () => ({ env: judgeProcessEnv() })) {
     const hay = output.toLowerCase();
     switch (assertion.type) {
         case "contains": {
@@ -467,13 +569,16 @@ async function checkAssertion(assertion, output, agentDir, model, effort, judgeE
             catch (err) {
                 return { assertion, passed: false, detail: `judge not run — ${err instanceof Error ? err.message : String(err)}` };
             }
-            const text = await withModelEnv(m, (env) => judge(assertion.value, output, model, effort, env));
-            const { passed, detail } = judgeVerdict(text);
-            return { assertion, passed, detail };
+            const said = await withModelEnv(m, (env) => judgeCall(assertion.value, output, model, effort, env));
+            const { passed, detail } = judgeVerdict(said.text);
+            return { assertion, passed, detail, costUsd: said.costUsd };
         }
     }
 }
-export async function runEval(tenant, workspace, info) {
+export async function runEval(tenant, workspace, info, opts = {}) {
+    if (info.trigger === "runs") {
+        throw refuse(`${info.name} grades real runs (trigger: runs) — it has no cases to run; grade a run with POST …/evals/${info.name}/grade { run }`, 400);
+    }
     const startedAt = new Date().toISOString();
     const results = [];
     // Saved inputs carry no assertions: running them would start and bill a
@@ -485,6 +590,16 @@ export async function runEval(tenant, workspace, info) {
         let output = "";
         let error = null;
         let costUsd = 0;
+        let secs = null;
+        const stop = opts.beforeCase?.() ?? null;
+        if (stop) {
+            results.push({ name: testCase.name, task: testCase.task, runId: null, error: `not run — ${stop}`, passed: false, assertions: [], costUsd: 0, judgeCostUsd: 0, secs: null });
+            continue;
+        }
+        // A step forced onto the comparison's model: step-level, so it wins.
+        const force = (st) => opts.model && (!opts.agent || st.agent === opts.agent)
+            ? { ...st, model: opts.model, ...(opts.effort ? { effort: opts.effort } : {}) }
+            : st;
         try {
             let run;
             if (info.flow) {
@@ -492,27 +607,33 @@ export async function runEval(tenant, workspace, info) {
                 if (!flow || flow.steps.length === 0)
                     throw new Error(`flow ${info.flow} not found or empty`);
                 // The case's task reaches the first step; output passing carries it on.
-                const steps = flow.steps.map((s, i) => i === 0
+                const steps = flow.steps.map((s, i) => force(i === 0
                     ? { ...s, instruction: `${s.instruction}\n\n<run_task>\n${testCase.task}\n</run_task>` }
-                    : s);
+                    : s));
                 // Test unless the eval OR the flow says live: an eval that runs a
-                // flow written `live: true` was written by someone who knew.
-                run = await beginEvalRun(tenant, workspace, steps, `eval:${info.name}`, flow.model, flow.effort, !(info.live || flow.live));
+                // flow written `live: true` was written by someone who knew. A
+                // comparison is a test whatever they say.
+                run = await beginEvalRun(tenant, workspace, steps, `eval:${info.name}`, flow.model, flow.effort, opts.forceTest ? true : !(info.live || flow.live));
             }
             else {
                 if (!target)
                     throw new Error("this eval names neither an agent nor a flow");
-                run = await beginEvalRun(tenant, workspace, [{ agent: target, instruction: testCase.task, group: 1, optional: false }], `eval:${info.name}`, undefined, undefined, !info.live);
+                run = await beginEvalRun(tenant, workspace, [force({ agent: target, instruction: testCase.task, group: 1, optional: false })], `eval:${info.name}`, undefined, undefined, opts.forceTest ? true : !info.live);
             }
             runId = run.id;
             const finished = await waitForRun(tenant, workspace, run.id);
             if (!finished)
                 throw new Error("run disappeared");
+            // What the steps before a gate spent is spent: counted before the
+            // gate is refused, so a comparison's limit sees it.
+            costUsd = finished.steps.reduce((sum, s) => sum + (s.costUsd ?? 0), 0);
             if (finished.status === "awaiting-approval") {
                 throw new Error("this run pauses for approval — an eval cannot answer a human gate");
             }
             output = finished.steps.map((s) => s.result ?? "").join("\n\n");
-            costUsd = finished.steps.reduce((sum, s) => sum + (s.costUsd ?? 0), 0);
+            const from = Date.parse(finished.drivenAt ?? finished.startedAt);
+            const to = finished.finishedAt ? Date.parse(finished.finishedAt) : NaN;
+            secs = Number.isFinite(from) && Number.isFinite(to) ? Math.max(0, Math.round((to - from) / 100) / 10) : null;
             if (finished.status === "failed" && !output.trim()) {
                 // Quote the run's own last error. "Produced nothing" on a fresh
                 // workspace is almost always "secret X is not set" or a tool that
@@ -556,7 +677,10 @@ export async function runEval(tenant, workspace, info) {
             passed: !error && assertions.length > 0 && assertions.every((r) => r.passed),
             assertions,
             costUsd,
+            judgeCostUsd: Math.round(assertions.reduce((n, a) => n + (a.costUsd ?? 0), 0) * 1e6) / 1e6,
+            secs,
         });
+        opts.afterCase?.(results[results.length - 1]);
     }
     return {
         eval: info.name,
@@ -661,8 +785,9 @@ commit = null) {
     if (!stamp)
         return { ran: 0, skipped: "no revision to stamp the result with" };
     // Only the evals that asked to run on deploy — a `trigger: manual` eval is
-    // one a person runs on purpose, because it costs a whole flow run.
-    const evals = listEvals(tenant, workspace).filter((e) => e.trigger !== "manual" && !e.inputs);
+    // one a person runs on purpose, because it costs a whole flow run, and a
+    // `trigger: runs` one grades real runs and has no cases to run.
+    const evals = listEvals(tenant, workspace).filter((e) => e.trigger === "deploy" && !e.inputs);
     if (evals.length === 0)
         return { ran: 0, skipped: "no evals" };
     const dir = path.join(workspaceDir(tenant, workspace), "evals", ".results");

package/dist/src/flow-lint.js

@@ -12,6 +12,7 @@
 // blocked — flows still run, warnings are advisory.
 import { whenRowsPath } from "./store.js";
 import { timezoneProblem } from "./clock.js";
+import { FILTERABLE_TRIGGERS } from "./trigger-filter.js";
 /** What a `[[flow:x]]` step's options are called in a flow file, for the
  *  ones expandSubflows does not carry into the nested steps. */
 const SUBFLOW_DROPPED = [
@@ -209,6 +210,9 @@ export function lintFlow(flow, known) {
     if (flow.signature && !flow.signingSecret) {
         warnings.push({ step: null, message: `signature: ${flow.signature} needs \`signing_secret: \${NAME}\``, detail: "Every delivery will be refused until the secret is named and set in Settings → Secrets." });
     }
+    if (flow.filter != null && !FILTERABLE_TRIGGERS.includes(flow.trigger)) {
+        warnings.push({ step: null, message: `filter: only applies to a webhook, email, flow or storage trigger`, detail: `trigger: ${flow.trigger} delivers no payload to match, so the filter does nothing here.` });
+    }
     if (flow.signature && flow.trigger !== "webhook") {
         warnings.push({ step: null, message: "signature: only applies to trigger: webhook", detail: "Inbox and other triggers do not carry a provider signature; the line does nothing here." });
     }

package/dist/src/frontmatter-fields.js

@@ -113,6 +113,7 @@ const FLOW = [
     { key: "url", type: "string", help: "trigger: watch — the page watched" },
     { key: "every", type: ["string", "number"], help: "trigger: watch — how often" },
     { key: "path", type: "string", help: "trigger: storage — the prefix under storage/" },
+    { key: "filter", type: "object", validated: true, help: "webhook, email, flow or storage trigger: the fields a delivery must have to start a run — e.g. {action: [opened], sender: {type: {not_in: [Bot]}}}" },
     { key: "idempotency", type: "string", help: "a header name, or body:<field>, that marks a repeat delivery" },
     { key: "debounce", type: ["string", "number"], help: "wait for a burst to end" },
     { key: "throttle", type: ["string", "number"], help: "at most one run per window" },
@@ -166,7 +167,9 @@ const EVAL = [
     { key: "flow", type: "string", help: "or the flow" },
     { key: "model", type: "string", help: "the judge's model" },
     { key: "effort", type: "string", values: EFFORT_LEVELS, help: "the judge's effort" },
-    { key: "trigger", type: "string", values: ["deploy", "manual"], help: "deploy (default) | manual; anything else is deploy" },
+    { key: "trigger", type: "string", values: ["deploy", "manual", "runs"], help: "deploy (default) | manual | runs — runs: grade real finished runs of the agent or flow instead of running cases" },
+    { key: "sample", type: ["number", "string"], validated: true, help: "trigger: runs — the share of finished runs graded, 0 to 100 percent (default 25)" },
+    { key: "improve", type: ["boolean", "number"], validated: true, help: "trigger: runs — true (or a count, default 5): propose a fix once that many graded runs failed" },
     { key: "live", type: "boolean", help: "true: real calls" },
     { key: "inputs", type: "boolean", help: "true: cases carry inputs" },
 ];

package/dist/src/grades.d.ts

@@ -0,0 +1,118 @@
+import { type RunRecord } from "./store.ts";
+import { judgeCall, type EvalInfo, type Assertion } from "./evals.ts";
+/** How far back the pass looks for runs to grade. */
+export declare const GRADE_WINDOW_MS: number;
+/** How long grades are kept, and how many per eval. */
+export declare const GRADE_KEEP_MS: number;
+export declare const GRADE_KEEP_LINES = 5000;
+export interface GradeAssertion {
+    type: Assertion["type"];
+    value: string;
+    passed: boolean;
+    detail: string;
+}
+export interface GradeCheck {
+    /** The `## heading` of the check. */
+    name: string;
+    passed: boolean;
+    /** Set when the check could not be decided — the judge failed to answer.
+     *  An error is not a failure: nobody's work was found wanting. */
+    error?: string;
+    assertions: GradeAssertion[];
+}
+export interface Grade {
+    eval: string;
+    runId: string;
+    /** The run's flow (`adhoc:<agent>` for a direct agent run). */
+    flow: string;
+    at: string;
+    /** pass: every check passed. fail: at least one failed. error: none
+     *  failed but at least one could not be decided. */
+    status: "pass" | "fail" | "error";
+    checks: GradeCheck[];
+    /** What the judges cost, USD — the run's own cost is on the run. */
+    costUsd: number;
+    /** The judge's model. */
+    model: string;
+    /** Which version of the grader graded it (a hash of the eval file), and
+     *  which version of the workspace the run ran (RunRecord.revision). */
+    evalRevision: string;
+    runRevision: string | null;
+    runFinishedAt: string | null;
+    /** `sample` when the pass picked it; a person's email (or key label)
+     *  when someone asked for it. */
+    by: string;
+}
+/** The graders of a workspace: its `trigger: runs` evals that can be read. */
+export declare function listGraders(tenant: string, workspace: string): EvalInfo[];
+/** The version of a grader, as its file reads now. */
+export declare function evalRevision(tenant: string, workspace: string, info: EvalInfo): string;
+/** Is this run in this grader's sample? Stable: a hash, not a coin. */
+export declare function sampled(runId: string, evalName: string, percent: number): boolean;
+/** The text a grader grades: every step's reply for a flow, the agent's
+ *  replies for an agent. Null when the run has nothing of it. */
+export declare function gradedOutput(info: EvalInfo, run: RunRecord): string | null;
+/** Grade one finished run. Writes nothing; `recordGrade` keeps it. */
+export declare function gradeRun(tenant: string, workspace: string, info: EvalInfo, run: RunRecord, opts?: {
+    by?: string;
+    now?: number;
+    judge?: typeof judgeCall;
+}): Promise<Grade>;
+/** Keep a grade: one line, appended. */
+export declare function recordGrade(tenant: string, workspace: string, grade: Grade): void;
+/** A grader's grades, newest first — one per run, the newest line for it. */
+export declare function readGrades(tenant: string, workspace: string, name: string, opts?: {
+    sinceMs?: number;
+}): Grade[];
+/** The grades of one run, across every grader of the workspace. */
+export declare function gradesOfRun(tenant: string, workspace: string, runId: string): Grade[];
+export interface GradeSummary {
+    graded: number;
+    passed: number;
+    failed: number;
+    errors: number;
+    /** passed / (passed + failed); null when nothing was decided. */
+    rate: number | null;
+    costUsd: number;
+    /** One row per UTC day that has grades, oldest first. */
+    byDay: {
+        day: string;
+        graded: number;
+        passed: number;
+        failed: number;
+    }[];
+}
+export declare function summariseGrades(grades: Grade[]): GradeSummary;
+/** When the pass first saw this grader — runs that finished before it are
+ *  never sampled, so a new grader starts from now rather than back-filling. */
+export declare function graderSince(tenant: string, workspace: string, name: string, now?: number): number;
+/** When a grader last had a run graded by the pass (0: never) — the pass
+ *  serves the longest-waiting grader first, so fairness holds across passes. */
+export declare function graderLastServed(tenant: string, workspace: string, name: string): number;
+/** How long an undecided grade (the judge did not answer — a spent usage
+ *  cap, a provider outage) waits before the pass tries the run again, and
+ *  how many tries it gets. */
+export declare const GRADE_RETRY_MS = 3600000;
+export declare const GRADE_TRIES = 3;
+/** The runs the pass should grade for one grader, oldest first. */
+export declare function gradeCandidates(tenant: string, workspace: string, info: EvalInfo, now?: number): string[];
+/** Drop grades past the keeping window or the line cap. Rewritten whole
+ *  (temp, then rename); only the pass prunes. */
+export declare function pruneGrades(tenant: string, workspace: string, name: string, now?: number): number;
+/**
+ * Grade what is due in one workspace: for each grader, the sampled runs that
+ * finished since it was first seen and are not graded yet. At most `limit`
+ * judges' worth of runs; `keepGoing` is asked before each run, so a pass
+ * that lost its lease stops between runs, never mid-grade.
+ */
+export declare function gradeWorkspace(tenant: string, workspace: string, opts?: {
+    now?: number;
+    limit?: number;
+    keepGoing?: () => Promise<boolean> | boolean;
+    onGrade?: (g: Grade, info: EvalInfo) => void | Promise<void>;
+    judge?: typeof judgeCall;
+}): Promise<{
+    graded: number;
+    failed: number;
+    errors: string[];
+}>;