@cursor/july 0.1.15 → 0.1.17

package/src/internal/eval-runner.ts

@@ -13,23 +13,61 @@ import { readdir } from "node:fs/promises";
 import { join, relative, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
 import type {
-  EvalAssertionResult,
+  EvalAssertionHandle,
+  EvalAssertions,
+} from "../evals/assertions.js";
+import {
+  computeVerdict,
+  createAssertions,
+  EvalAssertionRecorder,
+  EvalRequireFailure,
+  EvalSkipSignal,
+} from "../evals/assertions.js";
+import { evaluateExpectation } from "../evals/expect.js";
+import type {
+  EvalJudge,
+  EvalJudgeCaller,
+  EvalJudgeCallOptions,
+  EvalJudgeConfig,
+  EvalJudgeGrader,
+} from "../evals/judge.js";
+import {
+  EVAL_JUDGE_GRADERS,
+  EvalJudgeUnavailableError,
+  renderSubmission,
+} from "../evals/judge.js";
+import { EVAL_ROOT_ENV } from "../evals/loaders.js";
+import { combineReporters } from "../evals/reporters.js";
+import { summarizeEvalResults } from "../evals/results.js";
+import type {
   EvalCaseDefinition,
   EvalConfigFile,
   EvalDefinition,
+  EvalReporter,
+  EvalReporterTarget,
+  EvalRunResult,
+  EvalRunSummary,
   EvalSendOptions,
   EvalTestContext,
   EvalTurnResult,
+  EvalVerdict,
   Expectation,
 } from "../evals.js";
 import {
   EVAL_MAX_CONCURRENCY_LIMIT,
   expandEvalDefinition,
+  expandSummarySuiteSelection,
+  isEvalSummaryCase,
   matchesEvalIdFilter,
   normalizeEvalMaxConcurrency,
 } from "../evals.js";
-import type { SessionEvent } from "../types.js";
+import type { ModelSetting, SessionEvent } from "../types.js";
 import { isDefinitionOfKind } from "./brand.js";
+import {
+  callJudgeModel,
+  EVAL_JUDGE_MODEL_ENV,
+  resolveJudgeModel,
+} from "./eval-judge-model.js";
 import { mapPool } from "./map-pool.js";
 import { RunSessionTimeoutError, runSession } from "./run-client.js";
 import { buildTrajectory } from "./trajectory.js";
@@ -43,30 +81,7 @@ export interface DiscoveredEval {
   definition: EvalCaseDefinition;
 }
 
-export interface EvalRunResult {
-  id: string;
-  path: string;
-  description?: string;
-  ok: boolean;
-  assertions: EvalAssertionResult[];
-  sessionId?: string;
-  /** Messages passed to `t.send` (eval inputs). */
-  inputs: string[];
-  finalText?: string;
-  tools?: string[];
-  /** Tool calls with args/results from the trajectory. */
-  toolCalls?: Array<{
-    toolName: string;
-    args?: unknown;
-    output?: unknown;
-    isError: boolean;
-  }>;
-  error?: string;
-  logs: string[];
-  /** Structured scores from {@link import("../evals.js").EvalTestContext.metric}. */
-  metrics: Record<string, string | number>;
-  durationMs: number;
-}
+export type { EvalRunResult } from "../evals.js";
 
 export interface EvalSelectionOptions {
   filterIds?: string[];
@@ -77,6 +92,21 @@ export interface RunEvalsOptions extends EvalSelectionOptions {
   projectRoot: string;
   baseUrl: string;
   timeoutMs?: number;
+  /** Override `evals.config.ts` `maxConcurrency` (CLI `--max-concurrency`). */
+  maxConcurrency?: number;
+  /**
+   * Reporters attached by the caller (CLI `--junit` / `--artifacts`), in
+   * addition to the ones declared in `evals.config.ts` and on each eval.
+   */
+  reporters?: readonly EvalReporter[];
+  /** Ignore reporters declared in config and eval files (`--skip-report`). */
+  skipReport?: boolean;
+  /** Reported to reporters so a destination can label the run. */
+  target?: EvalReporterTarget;
+  /** Treat soft threshold misses as failures when summarizing. */
+  strict?: boolean;
+  /** Injected judge model call; defaults to the Cursor SDK caller. */
+  judgeCaller?: EvalJudgeCaller;
   bearerToken?: string;
   /** Extra request headers (e.g. hosted engineAccess). */
   headers?: Record<string, string>;
@@ -153,6 +183,11 @@ export async function discoverEvals(
   const evals: DiscoveredEval[] = [];
   let config: Partial<EvalConfigFile> = {};
 
+  // Fixture loaders resolve relative paths against the app root, so a dataset
+  // path reads the same however deep the eval file sits and whatever cwd the
+  // CLI ran from.
+  process.env[EVAL_ROOT_ENV] = root;
+
   for (const abs of files) {
     const rel = relative(evalsDir, abs).replace(/\\/g, "/");
     if (rel === "evals.config.ts" || rel === "evals.config.js") {
@@ -166,16 +201,9 @@ export async function discoverEvals(
       continue;
     }
     const mod = await importModule(abs);
-    const def = mod.default;
-    if (!isDefinitionOfKind(def, "eval")) {
-      throw new Error(`${rel}: default export must be defineEval({ ... })`);
-    }
     const fileId = rel.replace(/\.eval\.(ts|js)$/, "").replace(/\/index$/, "");
     try {
-      for (const expanded of expandEvalDefinition(
-        fileId,
-        def as EvalDefinition
-      )) {
+      for (const expanded of expandEvalModule(fileId, mod.default)) {
         evals.push({
           id: expanded.id,
           fileId,
@@ -192,6 +220,45 @@ export async function discoverEvals(
   return { evals, config };
 }
 
+/**
+ * Expand one eval module's default export into runnable datapoints.
+ *
+ * A file normally exports a single `defineEval(...)`. It may instead export an
+ * **array** of them to fan out over a dataset, in which case ids come from the
+ * file name plus a zero-padded index in array order (`sql/0000`, `sql/0001`, …).
+ * Prefer `cases` when the datapoints are hand-written and deserve stable names;
+ * the array form is for generated datasets where the index *is* the name.
+ */
+export function expandEvalModule(
+  fileId: string,
+  defaultExport: unknown
+): Array<{ id: string; definition: EvalCaseDefinition }> {
+  if (Array.isArray(defaultExport)) {
+    if (defaultExport.length === 0) {
+      throw new Error(
+        "default export array is empty; export at least one defineEval({ ... })"
+      );
+    }
+    return defaultExport.flatMap((entry, index) => {
+      if (!isDefinitionOfKind(entry, "eval")) {
+        throw new Error(
+          `default export array entry ${index} must be defineEval({ ... })`
+        );
+      }
+      return expandEvalDefinition(
+        `${fileId}/${String(index).padStart(4, "0")}`,
+        entry as EvalDefinition
+      );
+    });
+  }
+  if (!isDefinitionOfKind(defaultExport, "eval")) {
+    throw new Error(
+      "default export must be defineEval({ ... }) or an array of them"
+    );
+  }
+  return expandEvalDefinition(fileId, defaultExport as EvalDefinition);
+}
+
 /** Apply id/tag filters used by CLI, playground, and `runDiscoveredEvals`. */
 export function filterDiscoveredEvals(
   evals: readonly DiscoveredEval[],
@@ -208,7 +275,10 @@ export function filterDiscoveredEvals(
       (e.definition.tags ?? []).some((t) => options.tags!.includes(t))
     );
   }
-  return selected;
+  // Suite aggregators (`summary` tag) need sibling cases from the same file.
+  return expandSummarySuiteSelection(evals, selected, (e) =>
+    isEvalSummaryCase(e.definition.tags)
+  );
 }
 
 export async function runDiscoveredEvals(
@@ -234,8 +304,18 @@ export async function runDiscoveredEvals(
     return [];
   }
 
-  const maxConcurrency = resolveEvalMaxConcurrency(config);
+  const maxConcurrency =
+    options.maxConcurrency === undefined
+      ? resolveEvalMaxConcurrency(config)
+      : normalizeEvalMaxConcurrency(options.maxConcurrency);
   const defaultTimeoutMs = options.timeoutMs ?? config.timeoutMs ?? 180_000;
+  const reporter = buildReporterHub(options, config, selected);
+  const startedAt = new Date().toISOString();
+
+  await reporter?.runStart(
+    selected,
+    options.target ?? { baseUrl: options.baseUrl, mode: "local" }
+  );
 
   // Iterations of the same authored datapoint share one concurrency slot and
   // run sequentially. maxConcurrency is therefore the max number of distinct
@@ -261,19 +341,103 @@ export async function runDiscoveredEvals(
           headers: options.headers,
           verbose: Boolean(options.verbose),
           signal: options.signal,
+          judge: {
+            configModel: config.judge?.model,
+            caller: options.judgeCaller ?? callJudgeModel,
+          },
           onEvent: (event) => {
             options.onCaseEvent?.(discovered.id, event);
             options.onEvent?.(event);
           },
         });
         options.onCaseDone?.(result);
+        await reporter?.caseComplete(discovered, result);
         results.push(result);
       }
       return results;
     },
     { signal: options.signal }
   );
-  return nested.flat();
+  const results = nested.flat();
+
+  await reporter?.runComplete(
+    summarizeEvalResults(results, {
+      strict: options.strict === true,
+      startedAt,
+      finishedAt: new Date().toISOString(),
+    })
+  );
+  return results;
+}
+
+interface ReporterHub {
+  runStart(
+    evals: readonly DiscoveredEval[],
+    target: EvalReporterTarget
+  ): Promise<void>;
+  caseComplete(
+    discovered: DiscoveredEval,
+    result: EvalRunResult
+  ): Promise<void>;
+  runComplete(summary: EvalRunSummary): Promise<void>;
+}
+
+/**
+ * Fan results out to the reporters that observe them.
+ *
+ * Run-wide reporters (CLI flags plus `evals.config.ts`) see every case; an
+ * eval's own `reporters` only see that eval. The run lifecycle hooks fire on
+ * every reporter in the run, so a per-eval destination still gets its
+ * `onRunStart` / `onRunComplete` pair.
+ */
+function buildReporterHub(
+  options: RunEvalsOptions,
+  config: Partial<EvalConfigFile>,
+  selected: readonly DiscoveredEval[]
+): ReporterHub | undefined {
+  const skip = options.skipReport === true;
+  const runWide = [
+    ...(options.reporters ?? []),
+    ...(skip ? [] : (config.reporters ?? [])),
+  ];
+  const perEval = skip
+    ? []
+    : selected.flatMap((e) => e.definition.reporters ?? []);
+  const everyReporter = [...new Set([...runWide, ...perEval])];
+  if (everyReporter.length === 0) {
+    return undefined;
+  }
+  const onError = (error: unknown): void => {
+    // A broken destination must not fail the eval run itself.
+    const detail = error instanceof Error ? error.message : String(error);
+    process.stderr.write(`eval reporter error: ${detail}\n`);
+  };
+  const all = combineReporters(everyReporter, onError);
+  return {
+    async runStart(evals, target) {
+      await all.onRunStart?.(
+        evals.map((e) => ({
+          id: e.id,
+          fileId: e.fileId,
+          description: e.definition.description,
+          tags: e.definition.tags,
+        })),
+        target
+      );
+    },
+    async caseComplete(discovered, result) {
+      const observers = [
+        ...new Set([
+          ...runWide,
+          ...(skip ? [] : (discovered.definition.reporters ?? [])),
+        ]),
+      ];
+      await combineReporters(observers, onError).onEvalComplete?.(result);
+    },
+    async runComplete(summary) {
+      await all.onRunComplete?.(summary);
+    },
+  };
 }
 
 /**
@@ -331,6 +495,12 @@ function resolveEvalMaxConcurrency(config: Partial<EvalConfigFile>): number {
   return normalizeEvalMaxConcurrency(config.maxConcurrency);
 }
 
+interface RunOneEvalJudgeOptions {
+  /** Judge model from `evals.config.ts`. */
+  configModel?: ModelSetting;
+  caller: EvalJudgeCaller;
+}
+
 async function runOneEval(
   discovered: DiscoveredEval,
   opts: {
@@ -339,50 +509,29 @@ async function runOneEval(
     bearerToken?: string;
     headers?: Record<string, string>;
     verbose: boolean;
+    judge: RunOneEvalJudgeOptions;
     onEvent?: (event: SessionEvent) => void;
     signal?: AbortSignal;
   }
 ): Promise<EvalRunResult> {
   const logs: string[] = [];
   const metrics: Record<string, string | number> = {};
-  const assertions: EvalAssertionResult[] = [];
+  const recorder = new EvalAssertionRecorder();
   const def = discovered.definition;
   const timeoutMs = def.timeoutMs ?? opts.defaultTimeoutMs;
   const startedAt = Date.now();
 
   let sessionId: string | undefined;
   let continuationToken: string | undefined;
+  // The session stream replays from an index and the client stops at the first
+  // terminal event, so each send must resume after the previous turn's last
+  // event or it would re-read that turn instead of the new one.
+  let nextEventIndex = 0;
   const allEvents: SessionEvent[] = [];
   const inputs: string[] = [];
+  const turns: EvalTurnResult[] = [];
   let reply: string | null = null;
 
-  if (opts.signal?.aborted) {
-    return {
-      id: discovered.id,
-      path: discovered.path,
-      ...(def.description === undefined
-        ? {}
-        : { description: def.description }),
-      ok: false,
-      assertions,
-      error: "cancelled",
-      logs,
-      metrics,
-      inputs,
-      toolCalls: [],
-      tools: [],
-      durationMs: Date.now() - startedAt,
-    };
-  }
-
-  const record = (name: string, passed: boolean, detail?: string): void => {
-    assertions.push({
-      name,
-      passed,
-      ...(detail === undefined ? {} : { detail }),
-    });
-  };
-
   const toolsUsed = (): string[] =>
     buildTrajectory(allEvents).turns.flatMap((t) =>
       t.toolCalls.map((c) => c.toolName)
@@ -398,7 +547,69 @@ async function runOneEval(
       }))
     );
 
+  const finish = (extra: {
+    verdict: EvalVerdict;
+    ok: boolean;
+    error?: string;
+    skipReason?: string;
+    finalText?: string;
+  }): EvalRunResult => ({
+    id: discovered.id,
+    path: discovered.path,
+    ...(def.description === undefined ? {} : { description: def.description }),
+    ok: extra.ok,
+    verdict: extra.verdict,
+    skipReason: extra.skipReason,
+    assertions: recorder.results(),
+    error: extra.error,
+    logs,
+    metrics,
+    metadata: def.metadata,
+    tags: def.tags,
+    inputs,
+    toolCalls: toolCallIo(),
+    ...(sessionId === undefined ? {} : { sessionId }),
+    finalText: extra.finalText,
+    tools: toolsUsed(),
+    durationMs: Date.now() - startedAt,
+  });
+
+  if (opts.signal?.aborted) {
+    return finish({ verdict: "failed", ok: false, error: "cancelled" });
+  }
+
+  // A judge that cannot reach a model must not look like a quality regression:
+  // the case is skipped visibly instead.
+  let judgeUnavailable: string | undefined;
+  const timeout = new AbortController();
+  const timer = setTimeout(() => timeout.abort(), clampTimerDelay(timeoutMs));
+  // A pending case timer must not hold the process open on its own.
+  timer.unref?.();
+
+  const runAssertions: EvalAssertions = createAssertions({
+    recorder,
+    events: () => allEvents,
+  });
+
+  const judge = createJudgeSurface({
+    recorder,
+    grader: (name) => EVAL_JUDGE_GRADERS[name],
+    resolveModel: (callModel) =>
+      resolveJudgeModel({
+        call: callModel,
+        evalLevel: def.judge?.model,
+        config: opts.judge.configModel,
+        env: process.env[EVAL_JUDGE_MODEL_ENV],
+      }),
+    caller: opts.judge.caller,
+    defaultValue: () => reply,
+    onUnavailable: (message) => {
+      judgeUnavailable = judgeUnavailable ?? message;
+    },
+  });
+
   const t: EvalTestContext = {
+    ...runAssertions,
     get reply() {
       return reply;
     },
@@ -408,12 +619,19 @@ async function runOneEval(
     get events() {
       return allEvents;
     },
+    get turns() {
+      return turns;
+    },
     get iteration() {
       return def.iteration ?? 1;
     },
     get iterations() {
       return def.iterations ?? 1;
     },
+    get signal() {
+      return timeout.signal;
+    },
+    judge,
     async send(
       message: string,
       options: EvalSendOptions = {}
@@ -424,6 +642,7 @@ async function runOneEval(
         messages: [message],
         sessionId,
         continuationToken,
+        startIndex: nextEventIndex,
         // Keep defineEval traffic out of live defineAB enrollment/metrics.
         purpose: "eval",
         bearerToken: opts.bearerToken,
@@ -437,57 +656,57 @@ async function runOneEval(
       });
       sessionId = result.sessionId;
       continuationToken = result.continuationToken;
+      nextEventIndex = result.nextEventIndex ?? nextEventIndex;
       allEvents.push(...result.events);
       if (result.trajectory.finalText !== "") {
         reply = result.trajectory.finalText;
       }
-      return {
+      const index = turns.length + 1;
+      // The turn is immutable, so its assertions read a frozen event slice
+      // even after later turns append to the run.
+      const turnEvents = [...result.events];
+      const turn: EvalTurnResult = {
+        ...createAssertions({
+          recorder,
+          events: () => turnEvents,
+          scope: `turn[${index}]`,
+        }),
         message: result.trajectory.finalText,
         sessionId: result.sessionId,
-        events: result.events,
+        events: turnEvents,
         toolCalls: result.trajectory.turns.flatMap((turn) =>
           turn.toolCalls.map((c) => c.toolName)
         ),
         ok: result.ok,
+        index,
+        expectOk() {
+          if (!result.ok) {
+            throw new Error(
+              `turn[${index}] failed: ${result.trajectory.turns.find((x) => x.failed)?.error ?? "unknown error"}`
+            );
+          }
+          return turn;
+        },
       };
+      turns.push(turn);
+      return turn;
     },
-    succeeded() {
-      const traj = buildTrajectory(allEvents);
-      record(
-        "succeeded",
-        traj.ok && traj.turns.length > 0,
-        traj.ok ? undefined : "turn failed"
-      );
-    },
-    calledTool(name: string) {
-      const tools = toolsUsed();
-      record(
-        `calledTool(${name})`,
-        tools.includes(name),
-        tools.includes(name) ? undefined : `got [${tools.join(", ")}]`
-      );
-    },
-    notCalledTool(name: string) {
-      const tools = toolsUsed();
-      record(
-        `notCalledTool(${name})`,
-        !tools.includes(name),
-        tools.includes(name) ? "tool was called" : undefined
-      );
-    },
-    messageIncludes(token: string | RegExp) {
-      const text = reply ?? "";
-      const passed =
-        typeof token === "string" ? text.includes(token) : token.test(text);
-      record(
-        `messageIncludes(${String(token)})`,
-        passed,
-        passed ? undefined : `reply=${JSON.stringify(text.slice(0, 120))}`
-      );
+    async require<T>(value: T, assertion: Expectation): Promise<T> {
+      const outcome = evaluateExpectation(value, assertion);
+      recorder.record({
+        name: `require(${assertion.kind})`,
+        passed: outcome.passed,
+        detail: outcome.detail,
+        score: outcome.score,
+        severity: "gate",
+      });
+      if (!outcome.passed) {
+        throw new EvalRequireFailure(`require(${assertion.kind})`);
+      }
+      return value;
     },
-    check(value: unknown, assertion: Expectation) {
-      const { passed, detail } = matchExpectation(value, assertion);
-      record(`check(${assertion.kind})`, passed, detail);
+    skip(reason: string): never {
+      throw new EvalSkipSignal(reason);
     },
     log(message: string) {
       logs.push(message);
@@ -510,129 +729,161 @@ async function runOneEval(
   try {
     await def.test(t);
   } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    if (error instanceof RunSessionTimeoutError && sessionId === undefined) {
-      sessionId = error.sessionId;
+    if (error instanceof EvalSkipSignal) {
+      clearTimeout(timer);
+      return finish({
+        verdict: "skipped",
+        ok: true,
+        skipReason: error.reason,
+        finalText: reply ?? undefined,
+      });
+    }
+    if (!(error instanceof EvalRequireFailure)) {
+      const message = error instanceof Error ? error.message : String(error);
+      if (error instanceof RunSessionTimeoutError && sessionId === undefined) {
+        sessionId = error.sessionId;
+      }
+      await recorder.waitForPending().catch(() => undefined);
+      clearTimeout(timer);
+      return finish({
+        verdict: "failed",
+        ok: false,
+        error: message,
+        finalText: reply ?? undefined,
+      });
     }
-    return {
-      id: discovered.id,
-      path: discovered.path,
-      ...(def.description === undefined
-        ? {}
-        : { description: def.description }),
-      ok: false,
-      assertions,
-      error: message,
-      logs,
-      metrics,
-      inputs,
-      toolCalls: toolCallIo(),
-      ...(sessionId === undefined ? {} : { sessionId }),
-      ...(reply === null ? {} : { finalText: reply }),
-      tools: toolsUsed(),
-      durationMs: Date.now() - startedAt,
-    };
+    // `require` already recorded the failing gate; fall through so the verdict
+    // comes from the assertions rather than a duplicate execution error.
   }
 
+  await recorder.waitForPending();
+  clearTimeout(timer);
+
+  const assertions = recorder.results();
   const traj = buildTrajectory(allEvents);
-  const ok =
+  // With no assertions at all, keep the historical fallback: a case that just
+  // drives the agent passes when the run itself succeeded.
+  const verdict: EvalVerdict =
     assertions.length > 0
-      ? assertions.every((a) => a.passed)
-      : traj.ok && traj.turns.length > 0;
+      ? computeVerdict(assertions)
+      : traj.ok && traj.turns.length > 0
+        ? "passed"
+        : "failed";
+  const finalText = reply ?? traj.finalText;
 
-  return {
-    id: discovered.id,
-    path: discovered.path,
-    ...(def.description === undefined ? {} : { description: def.description }),
-    ok,
-    assertions,
-    logs,
-    metrics,
-    inputs,
-    toolCalls: toolCallIo(),
-    ...(sessionId === undefined ? {} : { sessionId }),
-    finalText: reply ?? traj.finalText,
-    tools: toolsUsed(),
-    durationMs: Date.now() - startedAt,
-  };
+  // An unreachable judge means the *judged* signal is unknown, so the case is
+  // reported as skipped rather than as a quality regression. It must not also
+  // bury a deterministic gate that already failed: skipping a failed case would
+  // turn a red eval green on any machine that happens to lack credentials.
+  if (judgeUnavailable !== undefined && verdict !== "failed") {
+    logs.push(`judge skipped: ${judgeUnavailable}`);
+    return finish({
+      verdict: "skipped",
+      ok: true,
+      skipReason: judgeUnavailable,
+      finalText,
+    });
+  }
+  if (judgeUnavailable !== undefined) {
+    logs.push(
+      `judge unavailable (${judgeUnavailable}), but a gate already failed — reporting the failure`
+    );
+  }
+
+  return finish({
+    verdict,
+    ok: verdict !== "failed",
+    finalText,
+  });
 }
 
-function matchExpectation(
-  value: unknown,
-  assertion: Expectation
-): { passed: boolean; detail?: string } {
-  switch (assertion.kind) {
-    case "includes": {
-      const text = value === null || value === undefined ? "" : String(value);
-      const passed =
-        typeof assertion.value === "string"
-          ? text.includes(assertion.value)
-          : assertion.value.test(text);
-      return {
-        passed,
-        ...(passed
-          ? {}
-          : {
-              detail: `value=${JSON.stringify(text.slice(0, 120))}`,
-            }),
-      };
-    }
-    case "equals": {
-      const passed = deepEqual(value, assertion.value);
-      // Always attach a short detail so values like recall `40.0% (4/10)`
-      // show up on PASS lines / playground, not only on failure.
-      const rendered =
-        typeof value === "string" ? value : JSON.stringify(value);
-      return {
-        passed,
-        detail: passed ? rendered : `got ${rendered}`,
-      };
+/** Clamp a case timeout to a delay `setTimeout` can represent. */
+function clampTimerDelay(timeoutMs: number): number {
+  const MAX_TIMER = 2_147_483_647;
+  return Math.min(Math.max(timeoutMs, 0), MAX_TIMER);
+}
+
+/**
+ * Build `t.judge`. Each grader records its slot immediately and resolves it
+ * during finalization, so several judges overlap instead of serializing.
+ */
+function createJudgeSurface(args: {
+  recorder: EvalAssertionRecorder;
+  grader: (name: keyof typeof EVAL_JUDGE_GRADERS) => EvalJudgeGrader;
+  resolveModel: (call?: ModelSetting) => ModelSetting | undefined;
+  caller: EvalJudgeCaller;
+  defaultValue: () => unknown;
+  onUnavailable: (message: string) => void;
+}): EvalJudge {
+  const grade = (
+    graderName: keyof typeof EVAL_JUDGE_GRADERS,
+    reference: string,
+    options: EvalJudgeCallOptions | undefined
+  ): EvalAssertionHandle => {
+    const grader = args.grader(graderName);
+    const model = args.resolveModel(options?.model);
+    const name = `judge.${graderName}(${truncateForName(reference)})`;
+    if (model === undefined) {
+      // A missing judge model is an authoring error, not a flaky provider.
+      return args.recorder.record({
+        name,
+        passed: false,
+        severity: "gate",
+        detail:
+          "no judge model configured; set judge.model in evals.config.ts, on the eval, or on the call",
+      });
     }
-    case "satisfies": {
-      const passed = assertion.predicate(value);
+    const submission = renderSubmission(
+      options !== undefined && "on" in options
+        ? options.on
+        : args.defaultValue()
+    );
+    const settle = (async () => {
+      const reply = await args.caller({
+        prompt: grader.buildPrompt({ submission, reference }),
+        model,
+      });
+      const result = grader.parse(reply);
       return {
-        passed,
-        ...(passed ? {} : { detail: assertion.label }),
+        passed: true,
+        score: result.score,
+        detail:
+          result.rationale === undefined
+            ? `${result.choice} (${result.score.toFixed(2)})`
+            : `${result.choice} (${result.score.toFixed(2)}) — ${result.rationale}`,
       };
-    }
-  }
+    })().catch((error: unknown) => {
+      if (error instanceof EvalJudgeUnavailableError) {
+        args.onUnavailable(error.message);
+        return { passed: true, detail: `skipped: ${error.message}` };
+      }
+      throw error;
+    });
+    // Soft by default: a judge is a tracked score until given a bar.
+    return args.recorder.recordAsync(
+      { name, passed: true, severity: "soft", detail: "grading…" },
+      settle
+    );
+  };
+
+  return {
+    async model(prompt, options) {
+      const model = args.resolveModel(options?.model);
+      if (model === undefined) {
+        throw new Error(
+          "t.judge.model: no judge model configured; set judge.model in evals.config.ts, on the eval, or on the call"
+        );
+      }
+      return args.caller({ prompt, model });
+    },
+    factuality: (expected, options) => grade("factuality", expected, options),
+    summarizes: (expected, options) => grade("summarizes", expected, options),
+    closedQA: (criteria, options) => grade("closedQA", criteria, options),
+    sql: (expected, options) => grade("sql", expected, options),
+  };
 }
 
-function deepEqual(a: unknown, b: unknown): boolean {
-  if (Object.is(a, b)) {
-    return true;
-  }
-  if (
-    typeof a !== "object" ||
-    typeof b !== "object" ||
-    a === null ||
-    b === null
-  ) {
-    return false;
-  }
-  if (Array.isArray(a) !== Array.isArray(b)) {
-    return false;
-  }
-  if (Array.isArray(a) && Array.isArray(b)) {
-    if (a.length !== b.length) {
-      return false;
-    }
-    return a.every((item, i) => deepEqual(item, b[i]));
-  }
-  const aKeys = Object.keys(a as object);
-  const bKeys = Object.keys(b as object);
-  if (aKeys.length !== bKeys.length) {
-    return false;
-  }
-  for (const key of aKeys) {
-    if (
-      !deepEqual(
-        (a as Record<string, unknown>)[key],
-        (b as Record<string, unknown>)[key]
-      )
-    ) {
-      return false;
-    }
-  }
-  return true;
+function truncateForName(text: string): string {
+  const oneLine = text.replace(/\s+/g, " ").trim();
+  return oneLine.length > 60 ? `${oneLine.slice(0, 59)}…` : oneLine;
 }