@foldrun/core 2.10.1 → 2.11.1

package/src/evals.ts

@@ -36,6 +36,7 @@ import type { TranslatorSpec } from "./translator.ts";
 import { sdkEnv } from "./step-exec.ts";
 import { platformModelEnv } from "./runner.ts";
 import fs from "node:fs";
+import crypto from "node:crypto";
 import path from "node:path";
 import { deployedCommit } from "./deploy.ts";
 import { latestRevisionId } from "./history.ts";
@@ -86,8 +87,20 @@ export interface EvalInfo {
   effort: Effort | null;
   /** When it runs. `deploy` (default): after every push, with the rest.
    *  `manual`: only when a person presses Run or the CLI asks — for a flow
-   *  eval that costs a whole run and touches real systems every time. */
-  trigger: "deploy" | "manual";
+   *  eval that costs a whole run and touches real systems every time.
+   *  `runs`: never run as cases — it grades real finished runs of its agent
+   *  or flow (grades.ts), a `sample` of them, as they finish. */
+  trigger: "deploy" | "manual" | "runs";
+  /** trigger: runs — the percent of finished runs graded, 0–100. */
+  sample: number;
+  /** trigger: runs — propose a fix (improve.ts) once this many graded runs
+   *  failed and no person has looked yet. Null: never on its own. */
+  improve: number | null;
+  /** What in the file cannot be read as written, one sentence each — the
+   *  deploy's checks and `foldrun check` report them. */
+  problems: string[];
+  /** A hash of the file as it was read (listEvals), when it was. */
+  revision?: string;
   /** An eval's runs are test runs — nothing outward, state untouched —
    *  unless the file says `live: true`. A flow eval that must really send
    *  to prove itself opts in; nothing opts in by accident. */
@@ -198,6 +211,35 @@ export function parseEval(file: string, raw: string, defaults: { model?: string;
   }
   if (current) cases.push(current);
 
+  const trigger: EvalInfo["trigger"] = data.trigger === "manual" ? "manual" : data.trigger === "runs" ? "runs" : "deploy";
+  const problems: string[] = [];
+  const sample = parseSample(data.sample, problems);
+  const improve = parseImprove(data.improve, problems);
+  if (trigger !== "runs") {
+    if (data.sample !== undefined) problems.push("sample: only applies to an eval with trigger: runs — this one runs its cases");
+    if (data.improve !== undefined) problems.push("improve: only applies to an eval with trigger: runs");
+  } else {
+    // A grader reads what a run said; it never starts one, so a task has
+    // nothing to go to, and file:/run: would execute or read on the
+    // platform's host rather than in the run's sandbox.
+    for (const c of cases) {
+      if (c.task) problems.push(`## ${c.name}: task: does nothing in a trigger: runs eval — it grades real runs, so each check is only an expect: list`);
+      for (const a of c.expect) {
+        if (a.type === "file" || a.type === "run") problems.push(`## ${c.name}: ${a.type}: cannot grade a finished run — use contains, not-contains or judge`);
+        // Graded on the platform's scheduler, over whatever a run said: a
+        // regular expression that backtracks would hold the scheduler with it.
+        if (a.type === "matches") problems.push(`## ${c.name}: matches: is not used to grade live runs — a regular expression over any run's output can stall the platform; use contains, not-contains or judge`);
+      }
+      if (c.expect.length === 0) problems.push(`## ${c.name}: has no expect: lines, so it grades nothing`);
+    }
+    if (cases.length === 0) problems.push("a grader needs at least one check — a `## <name>` section with an expect: list; with none it would pass every run");
+    // The pass budgets graded runs; this bounds the judges each one costs.
+    const judges = cases.reduce((n, c) => n + c.expect.filter((x) => x.type === "judge").length, 0);
+    if (judges > MAX_GRADER_JUDGES) problems.push(`${judges} judge: lines — a grader asks at most ${MAX_GRADER_JUDGES} judges per run; fold related claims into one`);
+    if (data.live === true) problems.push("live: does nothing in a trigger: runs eval — it never starts a run");
+    if (data.inputs === true) problems.push("inputs: and trigger: runs cannot go together");
+  }
+
   return {
     name: data.name ?? file.replace(/\.md$/, ""),
     file,
@@ -205,13 +247,42 @@ export function parseEval(file: string, raw: string, defaults: { model?: string;
     flow: typeof data.flow === "string" ? data.flow : null,
     model: resolveModel(data.model ?? defaults.model ?? "fast"),
     effort: resolveEffort(data.effort ?? defaults.effort ?? "low"),
-    trigger: data.trigger === "manual" ? "manual" : "deploy",
+    trigger,
+    sample,
+    improve: trigger === "runs" ? improve : null,
+    problems,
     live: data.live === true,
     inputs: data.inputs === true,
     cases,
   };
 }
 
+/** Judges one graded run may cost — the grade pass counts runs, not calls. */
+export const MAX_GRADER_JUDGES = 5;
+
+/** `sample: 25`, `sample: "25%"` — percent of finished runs graded. */
+function parseSample(v: unknown, problems: string[]): number {
+  if (v === undefined || v === null) return 25;
+  const n = typeof v === "number" ? v : typeof v === "string" ? Number(v.trim().replace(/%$/, "")) : NaN;
+  if (!Number.isFinite(n) || n < 0 || n > 100) {
+    problems.push(`sample: ${String(v)} — a percent from 0 to 100 (25 grades about one run in four; 0 grades none on its own)`);
+    return 25;
+  }
+  return Math.round(n * 100) / 100;
+}
+
+/** `improve: true` (5), `improve: 3`, `improve: false`. */
+function parseImprove(v: unknown, problems: string[]): number | null {
+  if (v === undefined || v === null || v === false) return null;
+  if (v === true) return 5;
+  const n = typeof v === "number" ? v : NaN;
+  if (!Number.isInteger(n) || n < 1 || n > 50) {
+    problems.push(`improve: ${String(v)} — true, or how many failed grades open a proposal (1 to 50)`);
+    return null;
+  }
+  return n;
+}
+
 export function listEvals(tenant: string, workspace: string): EvalInfo[] {
   const dir = path.join(workspaceDir(tenant, workspace), "evals");
   if (!fs.existsSync(dir)) return [];
@@ -219,7 +290,13 @@ export function listEvals(tenant: string, workspace: string): EvalInfo[] {
     .readdirSync(dir)
     .filter((f) => f.endsWith(".md"))
     .sort()
-    .map((f) => parseEval(f, fs.readFileSync(path.join(dir, f), "utf8"), judgeDefaults(tenant, workspace)));
+    .map((f) => {
+      const raw = fs.readFileSync(path.join(dir, f), "utf8");
+      // The version is taken from the same read the eval is parsed from, so
+      // a grade names the rubric that produced it even if the file changes
+      // while its judges answer.
+      return { ...parseEval(f, raw, judgeDefaults(tenant, workspace)), revision: crypto.createHash("sha256").update(raw).digest("hex").slice(0, 12) };
+    });
 }
 
 /** `judge:` from AGENTS.md, field by field: the workspace's model (or
@@ -406,6 +483,8 @@ export interface AssertionResult {
   assertion: Assertion;
   passed: boolean;
   detail: string;
+  /** What a judge cost, USD; absent for the free checks. */
+  costUsd?: number;
 }
 
 export interface CaseResult {
@@ -416,7 +495,35 @@ export interface CaseResult {
   /** Set when the agent itself failed, before any assertion could run. */
   error: string | null;
   assertions: AssertionResult[];
+  /** What the case's run cost — the subject, not the judge. */
   costUsd: number;
+  /** What its judges cost, USD. Absent on results from before 24 Oct 2026. */
+  judgeCostUsd?: number;
+  /** How long the run took from a worker picking it up to its end — its
+   *  time in the queue left out, so a busy account does not read as a slow
+   *  model. Null when it never ran. */
+  secs?: number | null;
+}
+
+/** How a model comparison (eval-compare.ts) runs one eval: every step it
+ *  forces onto a model, always as a test run, and whether to go on. */
+export interface EvalRunOptions {
+  /** Forced onto the subject's steps — every step of a flow eval, or only
+   *  `agent`'s when it is set — as a step-level setting, so it wins over
+   *  the agent's own `model:` and the flow's defaults. `effort` likewise
+   *  when given; none keeps each step's own effort, so a model-only
+   *  comparison changes the model and nothing else. */
+  model?: string;
+  effort?: Effort | null;
+  agent?: string | null;
+  /** A test run whatever the eval or flow says about `live:`. */
+  forceTest?: boolean;
+  /** Asked before each case; a reason stops the eval there, the rest of its
+   *  cases recorded as not run with it. */
+  beforeCase?: () => string | null;
+  /** Told each case as it ends — what a comparison counts its spend by, so
+   *  its limit holds between the cases of one setting, not only after. */
+  afterCase?: (c: CaseResult) => void;
 }
 
 export interface EvalResult {
@@ -507,41 +614,66 @@ function shell(command: string, cwd: string): Promise<{ code: number | null; out
 // LAST, after the reasoning: asked for it first, the judge committed before
 // it had looked, then argued the other way — "FAIL … which meets the
 // criterion" (vaza evals, 21 Oct 2026).
-async function judge(
+/** How long one judge may take before it is cut off: a verdict on one
+ *  sentence is seconds; a stalled model must not hold a grade pass. */
+export const JUDGE_TIMEOUT_MS = 120_000;
+
+/** One judge call: its words and what it cost (the SDK's result line). */
+export async function judgeCall(
   rubric: string,
   output: string,
   model: string,
   effort: Effort | null,
   env: Record<string, string | undefined>,
-): Promise<AssertionResult["detail"] & string> {
+  /** What the output was an answer to — a live run's task, so the judge
+   *  grades the reply against what was asked rather than in a vacuum. */
+  context?: string,
+): Promise<{ text: string; costUsd: number }> {
   const prompt =
     `You are grading one piece of output against one criterion. ${JUDGE_VERDICT_INSTRUCTION}\n\n` +
-    `Criterion: ${rubric}\n\n<output>\n${output.slice(0, 20000)}\n</output>`;
+    `Criterion: ${rubric}\n\n` +
+    (context ? `<task>\n${context.slice(0, 4000)}\n</task>\n\n` : "") +
+    `<output>\n${output.slice(0, 20000)}\n</output>`;
 
   let text = "";
-  const q = query({
-    prompt,
-    options: {
-      model,
-      ...(effort ? { effort } : {}),
-      tools: [],
-      allowedTools: [],
-      settingSources: [],
-      strictMcpConfig: true,
-      env,
-      maxTurns: 1,
-    },
-  });
-  for await (const message of q) {
-    if (message.type === "assistant") {
-      for (const block of message.message.content) {
-        if (block.type === "text") text += block.text;
+  let costUsd = 0;
+  const abort = new AbortController();
+  const timer = setTimeout(() => abort.abort(), JUDGE_TIMEOUT_MS);
+  timer.unref?.();
+  try {
+    const q = query({
+      prompt,
+      options: {
+        model,
+        ...(effort ? { effort } : {}),
+        tools: [],
+        allowedTools: [],
+        settingSources: [],
+        strictMcpConfig: true,
+        env,
+        maxTurns: 1,
+        abortController: abort,
+      },
+    });
+    for await (const message of q) {
+      if (message.type === "assistant") {
+        for (const block of message.message.content) {
+          if (block.type === "text") text += block.text;
+        }
+      } else if (message.type === "result") {
+        costUsd = Number((message as { total_cost_usd?: number }).total_cost_usd) || 0;
       }
     }
+  } catch (err) {
+    if (abort.signal.aborted) throw new Error(`the judge took longer than ${JUDGE_TIMEOUT_MS / 1000}s and was stopped`);
+    throw err;
+  } finally {
+    clearTimeout(timer);
   }
-  return text.trim();
+  return { text: text.trim(), costUsd };
 }
 
+
 /**
  * The judge's environment: this process's, with the platform's model
  * credential read live. The process's own CLAUDE_CODE_OAUTH_TOKEN is the copy
@@ -555,7 +687,7 @@ export function judgeProcessEnv(): Record<string, string | undefined> {
   return sdkEnv({ ...process.env, ...live });
 }
 
-async function checkAssertion(
+export async function checkAssertion(
   assertion: Assertion,
   output: string,
   agentDir: string,
@@ -602,9 +734,9 @@ async function checkAssertion(
       } catch (err) {
         return { assertion, passed: false, detail: `judge not run — ${err instanceof Error ? err.message : String(err)}` };
       }
-      const text = await withModelEnv(m, (env) => judge(assertion.value, output, model, effort, env));
-      const { passed, detail } = judgeVerdict(text);
-      return { assertion, passed, detail };
+      const said = await withModelEnv(m, (env) => judgeCall(assertion.value, output, model, effort, env));
+      const { passed, detail } = judgeVerdict(said.text);
+      return { assertion, passed, detail, costUsd: said.costUsd };
     }
   }
 }
@@ -613,7 +745,11 @@ export async function runEval(
   tenant: string,
   workspace: string,
   info: EvalInfo,
+  opts: EvalRunOptions = {},
 ): Promise<EvalResult> {
+  if (info.trigger === "runs") {
+    throw refuse(`${info.name} grades real runs (trigger: runs) — it has no cases to run; grade a run with POST …/evals/${info.name}/grade { run }`, 400);
+  }
   const startedAt = new Date().toISOString();
   const results: CaseResult[] = [];
   // Saved inputs carry no assertions: running them would start and bill a
@@ -626,6 +762,17 @@ export async function runEval(
     let output = "";
     let error: string | null = null;
     let costUsd = 0;
+    let secs: number | null = null;
+    const stop = opts.beforeCase?.() ?? null;
+    if (stop) {
+      results.push({ name: testCase.name, task: testCase.task, runId: null, error: `not run — ${stop}`, passed: false, assertions: [], costUsd: 0, judgeCostUsd: 0, secs: null });
+      continue;
+    }
+    // A step forced onto the comparison's model: step-level, so it wins.
+    const force = <S extends FlowStep>(st: S): S =>
+      opts.model && (!opts.agent || st.agent === opts.agent)
+        ? { ...st, model: opts.model, ...(opts.effort ? { effort: opts.effort } : {}) }
+        : st;
 
     try {
       let run;
@@ -634,33 +781,39 @@ export async function runEval(
         if (!flow || flow.steps.length === 0) throw new Error(`flow ${info.flow} not found or empty`);
         // The case's task reaches the first step; output passing carries it on.
         const steps = flow.steps.map((s, i) =>
-          i === 0
+          force(i === 0
             ? { ...s, instruction: `${s.instruction}\n\n<run_task>\n${testCase.task}\n</run_task>` }
-            : s,
+            : s),
         );
         // Test unless the eval OR the flow says live: an eval that runs a
-        // flow written `live: true` was written by someone who knew.
-        run = await beginEvalRun(tenant, workspace, steps, `eval:${info.name}`, flow.model, flow.effort, !(info.live || flow.live));
+        // flow written `live: true` was written by someone who knew. A
+        // comparison is a test whatever they say.
+        run = await beginEvalRun(tenant, workspace, steps, `eval:${info.name}`, flow.model, flow.effort, opts.forceTest ? true : !(info.live || flow.live));
       } else {
         if (!target) throw new Error("this eval names neither an agent nor a flow");
         run = await beginEvalRun(
           tenant,
           workspace,
-          [{ agent: target, instruction: testCase.task, group: 1, optional: false }],
+          [force({ agent: target, instruction: testCase.task, group: 1, optional: false } as FlowStep)],
           `eval:${info.name}`,
           undefined,
           undefined,
-          !info.live,
+          opts.forceTest ? true : !info.live,
         );
       }
       runId = run.id;
       const finished = await waitForRun(tenant, workspace, run.id);
       if (!finished) throw new Error("run disappeared");
+      // What the steps before a gate spent is spent: counted before the
+      // gate is refused, so a comparison's limit sees it.
+      costUsd = finished.steps.reduce((sum, s) => sum + (s.costUsd ?? 0), 0);
       if (finished.status === "awaiting-approval") {
         throw new Error("this run pauses for approval — an eval cannot answer a human gate");
       }
       output = finished.steps.map((s) => s.result ?? "").join("\n\n");
-      costUsd = finished.steps.reduce((sum, s) => sum + (s.costUsd ?? 0), 0);
+      const from = Date.parse(finished.drivenAt ?? finished.startedAt);
+      const to = finished.finishedAt ? Date.parse(finished.finishedAt) : NaN;
+      secs = Number.isFinite(from) && Number.isFinite(to) ? Math.max(0, Math.round((to - from) / 100) / 10) : null;
       if (finished.status === "failed" && !output.trim()) {
         // Quote the run's own last error. "Produced nothing" on a fresh
         // workspace is almost always "secret X is not set" or a tool that
@@ -702,7 +855,10 @@ export async function runEval(
       passed: !error && assertions.length > 0 && assertions.every((r) => r.passed),
       assertions,
       costUsd,
+      judgeCostUsd: Math.round(assertions.reduce((n, a) => n + (a.costUsd ?? 0), 0) * 1e6) / 1e6,
+      secs,
     });
+    opts.afterCase?.(results[results.length - 1]);
   }
 
   return {
@@ -837,8 +993,9 @@ export async function evaluateDeployed(
   const stamp = commit ?? deployedCommit(tenant, workspace)?.commit ?? latestRevisionId(tenant, workspace);
   if (!stamp) return { ran: 0, skipped: "no revision to stamp the result with" };
   // Only the evals that asked to run on deploy — a `trigger: manual` eval is
-  // one a person runs on purpose, because it costs a whole flow run.
-  const evals = listEvals(tenant, workspace).filter((e) => e.trigger !== "manual" && !e.inputs);
+  // one a person runs on purpose, because it costs a whole flow run, and a
+  // `trigger: runs` one grades real runs and has no cases to run.
+  const evals = listEvals(tenant, workspace).filter((e) => e.trigger === "deploy" && !e.inputs);
   if (evals.length === 0) return { ran: 0, skipped: "no evals" };
   const dir = path.join(workspaceDir(tenant, workspace), "evals", ".results");
   fs.mkdirSync(dir, { recursive: true });

package/src/flow-lint.ts

@@ -14,6 +14,7 @@
 import type { FlowInfo, FlowStep } from "./store.ts";
 import { whenRowsPath } from "./store.ts";
 import { timezoneProblem } from "./clock.ts";
+import { FILTERABLE_TRIGGERS } from "./trigger-filter.ts";
 
 /** What a `[[flow:x]]` step's options are called in a flow file, for the
  *  ones expandSubflows does not carry into the nested steps. */
@@ -251,6 +252,9 @@ export function lintFlow(flow: FlowInfo, known?: KnownNames): FlowWarning[] {
   if (flow.signature && !flow.signingSecret) {
     warnings.push({ step: null, message: `signature: ${flow.signature} needs \`signing_secret: \${NAME}\``, detail: "Every delivery will be refused until the secret is named and set in Settings → Secrets." });
   }
+  if (flow.filter != null && !(FILTERABLE_TRIGGERS as readonly string[]).includes(flow.trigger)) {
+    warnings.push({ step: null, message: `filter: only applies to a webhook, email, flow or storage trigger`, detail: `trigger: ${flow.trigger} delivers no payload to match, so the filter does nothing here.` });
+  }
   if (flow.signature && flow.trigger !== "webhook") {
     warnings.push({ step: null, message: "signature: only applies to trigger: webhook", detail: "Inbox and other triggers do not carry a provider signature; the line does nothing here." });
   }

package/src/frontmatter-fields.ts

@@ -160,6 +160,7 @@ const FLOW: FieldSpec[] = [
   { key: "url", type: "string", help: "trigger: watch — the page watched" },
   { key: "every", type: ["string", "number"], help: "trigger: watch — how often" },
   { key: "path", type: "string", help: "trigger: storage — the prefix under storage/" },
+  { key: "filter", type: "object", validated: true, help: "webhook, email, flow or storage trigger: the fields a delivery must have to start a run — e.g. {action: [opened], sender: {type: {not_in: [Bot]}}}" },
   { key: "idempotency", type: "string", help: "a header name, or body:<field>, that marks a repeat delivery" },
   { key: "debounce", type: ["string", "number"], help: "wait for a burst to end" },
   { key: "throttle", type: ["string", "number"], help: "at most one run per window" },
@@ -216,7 +217,9 @@ const EVAL: FieldSpec[] = [
   { key: "flow", type: "string", help: "or the flow" },
   { key: "model", type: "string", help: "the judge's model" },
   { key: "effort", type: "string", values: EFFORT_LEVELS, help: "the judge's effort" },
-  { key: "trigger", type: "string", values: ["deploy", "manual"], help: "deploy (default) | manual; anything else is deploy" },
+  { key: "trigger", type: "string", values: ["deploy", "manual", "runs"], help: "deploy (default) | manual | runs — runs: grade real finished runs of the agent or flow instead of running cases" },
+  { key: "sample", type: ["number", "string"], validated: true, help: "trigger: runs — the share of finished runs graded, 0 to 100 percent (default 25)" },
+  { key: "improve", type: ["boolean", "number"], validated: true, help: "trigger: runs — true (or a count, default 5): propose a fix once that many graded runs failed" },
   { key: "live", type: "boolean", help: "true: real calls" },
   { key: "inputs", type: "boolean", help: "true: cases carry inputs" },
 ];