@cursor/bdk 0.2.2 → 0.2.4

package/src/internal/eval-runner.ts

@@ -1,823 +0,0 @@
-/**
- * Discover and run filesystem evals under `<project>/evals/`.
- *
- * Case identity: single-`test` files use the path
- * (`evals/weather/nyc.eval.ts` → `weather/nyc`); multi-`cases` files use
- * `<fileId>/<case.id>` (`evals/weather.eval.ts` + `nyc` → `weather/nyc`).
- * Drive with `t.send`, assert with `t.succeeded` / `t.calledTool` /
- * `t.messageIncludes` / `t.check`.
- */
-
-import type { Dirent } from "node:fs";
-import { readdir } from "node:fs/promises";
-import { join, relative, resolve } from "node:path";
-import type { EvalAssertionHandle, EvalAssertions } from "../evals/assertions.js";
-import {
-  computeVerdict,
-  createAssertions,
-  EvalAssertionRecorder,
-  EvalRequireFailure,
-  EvalSkipSignal,
-} from "../evals/assertions.js";
-import { evaluateExpectation } from "../evals/expect.js";
-import type {
-  EvalJudge,
-  EvalJudgeCaller,
-  EvalJudgeCallOptions,
-  EvalJudgeGrader,
-} from "../evals/judge.js";
-import { EVAL_JUDGE_GRADERS, EvalJudgeUnavailableError, renderSubmission } from "../evals/judge.js";
-import { EVAL_ROOT_ENV } from "../evals/loaders.js";
-import { combineReporters } from "../evals/reporters.js";
-import { summarizeEvalResults } from "../evals/results.js";
-import type {
-  EvalCaseDefinition,
-  EvalConfigFile,
-  EvalDefinition,
-  EvalReporter,
-  EvalReporterTarget,
-  EvalRunResult,
-  EvalRunSummary,
-  EvalSendOptions,
-  EvalTestContext,
-  EvalTurnResult,
-  EvalVerdict,
-  Expectation,
-} from "../evals.js";
-import {
-  EVAL_MAX_CONCURRENCY_LIMIT,
-  expandEvalDefinition,
-  expandSummarySuiteSelection,
-  isEvalSummaryCase,
-  matchesEvalIdFilter,
-  normalizeEvalMaxConcurrency,
-} from "../evals.js";
-import type { ModelSetting, SessionEvent } from "../types.js";
-import { importAuthoredModule } from "./authored-loaders.js";
-import { isDefinitionOfKind } from "./brand.js";
-import { callJudgeModel, EVAL_JUDGE_MODEL_ENV, resolveJudgeModel } from "./eval-judge-model.js";
-import { mapPool } from "./map-pool.js";
-import { RunSessionTimeoutError, runSession } from "./run-client.js";
-import { buildTrajectory } from "./trajectory.js";
-
-export interface DiscoveredEval {
-  /** Runnable datapoint id (`weather/nyc` or file id for single-test files). */
-  id: string;
-  /** File id under `evals/` (`weather` for `evals/weather.eval.ts`). */
-  fileId: string;
-  path: string;
-  definition: EvalCaseDefinition;
-}
-
-export type { EvalRunResult } from "../evals.js";
-
-export interface EvalSelectionOptions {
-  filterIds?: string[];
-  tags?: string[];
-}
-
-export interface RunEvalsOptions extends EvalSelectionOptions {
-  projectRoot: string;
-  baseUrl: string;
-  timeoutMs?: number;
-  /** Override `evals.config.ts` `maxConcurrency` (CLI `--max-concurrency`). */
-  maxConcurrency?: number;
-  /**
-   * Reporters attached by the caller (CLI `--junit` / `--artifacts`), in
-   * addition to the ones declared in `evals.config.ts` and on each eval.
-   */
-  reporters?: readonly EvalReporter[];
-  /** Ignore reporters declared in config and eval files (`--skip-report`). */
-  skipReport?: boolean;
-  /** Reported to reporters so a destination can label the run. */
-  target?: EvalReporterTarget;
-  /** Treat soft threshold misses as failures when summarizing. */
-  strict?: boolean;
-  /** Injected judge model call; defaults to the Cursor SDK caller. */
-  judgeCaller?: EvalJudgeCaller;
-  bearerToken?: string;
-  /** Extra request headers (e.g. hosted engineAccess). */
-  headers?: Record<string, string>;
-  verbose?: boolean;
-  onEvent?: (event: SessionEvent) => void;
-  /** Fired when a case begins executing (after concurrency slot acquired). */
-  onCaseStart?: (id: string) => void;
-  /** Fired when a case finishes (pass or fail). */
-  onCaseDone?: (result: EvalRunResult) => void;
-  /** Fired for each session event during a case (for live progress / sessionId). */
-  onCaseEvent?: (id: string, event: SessionEvent) => void;
-  importModule?: (absolutePath: string) => Promise<Record<string, unknown>>;
-  /**
-   * Pre-discovered evals + config. When set, skips filesystem discovery and
-   * selection filters (caller already chose the case list).
-   */
-  discovered?: {
-    evals: readonly DiscoveredEval[];
-    config: Partial<EvalConfigFile>;
-  };
-  /** When aborted, stop starting new cases and cancel in-flight turns. */
-  signal?: AbortSignal;
-}
-
-async function listEvalFiles(dir: string): Promise<string[]> {
-  const out: string[] = [];
-  async function walk(current: string): Promise<void> {
-    let entries: Dirent[];
-    try {
-      entries = await readdir(current, { withFileTypes: true });
-    } catch {
-      return;
-    }
-    for (const entry of entries) {
-      if (entry.name.startsWith(".")) {
-        continue;
-      }
-      const abs = join(current, entry.name);
-      if (entry.isDirectory()) {
-        if (entry.name === "node_modules") {
-          continue;
-        }
-        await walk(abs);
-      } else if (entry.isFile()) {
-        out.push(abs);
-      }
-    }
-  }
-  await walk(dir);
-  return out.sort();
-}
-
-export async function discoverEvals(
-  projectRoot: string,
-  importModule: (absolutePath: string) => Promise<Record<string, unknown>> = importAuthoredModule,
-): Promise<{
-  evals: DiscoveredEval[];
-  config: Partial<EvalConfigFile>;
-}> {
-  const root = resolve(projectRoot);
-  const evalsDir = join(root, "evals");
-  const files = await listEvalFiles(evalsDir);
-  const evals: DiscoveredEval[] = [];
-  let config: Partial<EvalConfigFile> = {};
-
-  // Fixture loaders resolve relative paths against the app root, so a dataset
-  // path reads the same however deep the eval file sits and whatever cwd the
-  // CLI ran from.
-  process.env[EVAL_ROOT_ENV] = root;
-
-  for (const abs of files) {
-    const rel = relative(evalsDir, abs).replace(/\\/g, "/");
-    if (rel === "evals.config.ts" || rel === "evals.config.js") {
-      const mod = await importModule(abs);
-      if (mod.default !== null && typeof mod.default === "object") {
-        config = mod.default as Partial<EvalConfigFile>;
-      }
-      continue;
-    }
-    if (!rel.endsWith(".eval.ts") && !rel.endsWith(".eval.js")) {
-      continue;
-    }
-    const mod = await importModule(abs);
-    const fileId = rel.replace(/\.eval\.(ts|js)$/, "").replace(/\/index$/, "");
-    try {
-      for (const expanded of expandEvalModule(fileId, mod.default)) {
-        evals.push({
-          id: expanded.id,
-          fileId,
-          path: abs,
-          definition: expanded.definition,
-        });
-      }
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      throw new Error(`${rel}: ${message}`);
-    }
-  }
-
-  return { evals, config };
-}
-
-/**
- * Expand one eval module's default export into runnable datapoints.
- *
- * A file normally exports a single `defineEval(...)`. It may instead export an
- * **array** of them to fan out over a dataset, in which case ids come from the
- * file name plus a zero-padded index in array order (`sql/0000`, `sql/0001`, …).
- * Prefer `cases` when the datapoints are hand-written and deserve stable names;
- * the array form is for generated datasets where the index *is* the name.
- */
-export function expandEvalModule(
-  fileId: string,
-  defaultExport: unknown,
-): Array<{ id: string; definition: EvalCaseDefinition }> {
-  if (Array.isArray(defaultExport)) {
-    if (defaultExport.length === 0) {
-      throw new Error("default export array is empty; export at least one defineEval({ ... })");
-    }
-    return defaultExport.flatMap((entry, index) => {
-      if (!isDefinitionOfKind(entry, "eval")) {
-        throw new Error(`default export array entry ${index} must be defineEval({ ... })`);
-      }
-      return expandEvalDefinition(
-        `${fileId}/${String(index).padStart(4, "0")}`,
-        entry as EvalDefinition,
-      );
-    });
-  }
-  if (!isDefinitionOfKind(defaultExport, "eval")) {
-    throw new Error("default export must be defineEval({ ... }) or an array of them");
-  }
-  return expandEvalDefinition(fileId, defaultExport as EvalDefinition);
-}
-
-/** Apply id/tag filters used by CLI, playground, and `runDiscoveredEvals`. */
-export function filterDiscoveredEvals(
-  evals: readonly DiscoveredEval[],
-  options: EvalSelectionOptions = {},
-): DiscoveredEval[] {
-  let selected = evals as DiscoveredEval[];
-  if (options.filterIds !== undefined && options.filterIds.length > 0) {
-    selected = selected.filter(e => options.filterIds!.some(f => matchesEvalIdFilter(e.id, f)));
-  }
-  if (options.tags !== undefined && options.tags.length > 0) {
-    selected = selected.filter(e => (e.definition.tags ?? []).some(t => options.tags!.includes(t)));
-  }
-  // Suite aggregators (`summary` tag) need sibling cases from the same file.
-  return expandSummarySuiteSelection(evals, selected, e => isEvalSummaryCase(e.definition.tags));
-}
-
-export async function runDiscoveredEvals(options: RunEvalsOptions): Promise<EvalRunResult[]> {
-  let selected: readonly DiscoveredEval[];
-  let config: Partial<EvalConfigFile>;
-
-  if (options.discovered !== undefined) {
-    selected = options.discovered.evals;
-    config = options.discovered.config;
-  } else {
-    const importModule = options.importModule ?? importAuthoredModule;
-    const discovered = await discoverEvals(options.projectRoot, importModule);
-    config = discovered.config;
-    selected = filterDiscoveredEvals(discovered.evals, {
-      filterIds: options.filterIds,
-      tags: options.tags,
-    });
-  }
-
-  if (selected.length === 0) {
-    return [];
-  }
-
-  const maxConcurrency =
-    options.maxConcurrency === undefined
-      ? resolveEvalMaxConcurrency(config)
-      : normalizeEvalMaxConcurrency(options.maxConcurrency);
-  const defaultTimeoutMs = options.timeoutMs ?? config.timeoutMs ?? 180_000;
-  const reporter = buildReporterHub(options, config, selected);
-  const startedAt = new Date().toISOString();
-
-  await reporter?.runStart(selected, options.target ?? { baseUrl: options.baseUrl, mode: "local" });
-
-  // Iterations of the same authored datapoint share one concurrency slot and
-  // run sequentially. maxConcurrency is therefore the max number of distinct
-  // datapoints in flight, not datapoints × iterations.
-  const groups = groupEvalsForIterationSchedule(selected);
-  const nested = await mapPool(
-    groups,
-    maxConcurrency,
-    async group => {
-      const results: EvalRunResult[] = [];
-      for (const discovered of group) {
-        if (options.signal?.aborted) {
-          break;
-        }
-        if (options.verbose) {
-          process.stderr.write(`  eval ${discovered.id}…\n`);
-        }
-        options.onCaseStart?.(discovered.id);
-        const result = await runOneEval(discovered, {
-          baseUrl: options.baseUrl,
-          defaultTimeoutMs,
-          bearerToken: options.bearerToken,
-          headers: options.headers,
-          verbose: Boolean(options.verbose),
-          signal: options.signal,
-          judge: {
-            configModel: config.judge?.model,
-            caller: options.judgeCaller ?? callJudgeModel,
-          },
-          onEvent: event => {
-            options.onCaseEvent?.(discovered.id, event);
-            options.onEvent?.(event);
-          },
-        });
-        options.onCaseDone?.(result);
-        await reporter?.caseComplete(discovered, result);
-        results.push(result);
-      }
-      return results;
-    },
-    { signal: options.signal },
-  );
-  const results = nested.flat();
-
-  await reporter?.runComplete(
-    summarizeEvalResults(results, {
-      strict: options.strict === true,
-      startedAt,
-      finishedAt: new Date().toISOString(),
-    }),
-  );
-  return results;
-}
-
-interface ReporterHub {
-  runStart(evals: readonly DiscoveredEval[], target: EvalReporterTarget): Promise<void>;
-  caseComplete(discovered: DiscoveredEval, result: EvalRunResult): Promise<void>;
-  runComplete(summary: EvalRunSummary): Promise<void>;
-}
-
-/**
- * Fan results out to the reporters that observe them.
- *
- * Run-wide reporters (CLI flags plus `evals.config.ts`) see every case; an
- * eval's own `reporters` only see that eval. The run lifecycle hooks fire on
- * every reporter in the run, so a per-eval destination still gets its
- * `onRunStart` / `onRunComplete` pair.
- */
-function buildReporterHub(
-  options: RunEvalsOptions,
-  config: Partial<EvalConfigFile>,
-  selected: readonly DiscoveredEval[],
-): ReporterHub | undefined {
-  const skip = options.skipReport === true;
-  const runWide = [...(options.reporters ?? []), ...(skip ? [] : (config.reporters ?? []))];
-  const perEval = skip ? [] : selected.flatMap(e => e.definition.reporters ?? []);
-  const everyReporter = [...new Set([...runWide, ...perEval])];
-  if (everyReporter.length === 0) {
-    return undefined;
-  }
-  const onError = (error: unknown): void => {
-    // A broken destination must not fail the eval run itself.
-    const detail = error instanceof Error ? error.message : String(error);
-    process.stderr.write(`eval reporter error: ${detail}\n`);
-  };
-  const all = combineReporters(everyReporter, onError);
-  return {
-    async runStart(evals, target) {
-      await all.onRunStart?.(
-        evals.map(e => ({
-          id: e.id,
-          fileId: e.fileId,
-          description: e.definition.description,
-          tags: e.definition.tags,
-        })),
-        target,
-      );
-    },
-    async caseComplete(discovered, result) {
-      const observers = [
-        ...new Set([...runWide, ...(skip ? [] : (discovered.definition.reporters ?? []))]),
-      ];
-      await combineReporters(observers, onError).onEvalComplete?.(result);
-    },
-    async runComplete(summary) {
-      await all.onRunComplete?.(summary);
-    },
-  };
-}
-
-/**
- * Schedule key for {@link groupEvalsForIterationSchedule}: iteration siblings
- * (`weather/nyc/1`…`/3`) share a key; single-iteration ids keep their full id.
- */
-export function iterationScheduleKey(discovered: DiscoveredEval): string {
-  const iterations = discovered.definition.iterations ?? 1;
-  const iteration = discovered.definition.iteration ?? 1;
-  if (iterations <= 1) {
-    return discovered.id;
-  }
-  const suffix = `/${iteration}`;
-  if (discovered.id.endsWith(suffix)) {
-    return discovered.id.slice(0, -suffix.length);
-  }
-  return discovered.id;
-}
-
-/**
- * Group expanded iteration cases so each authored datapoint's iterations run
- * back-to-back under one {@link mapPool} slot (sorted by iteration index).
- */
-export function groupEvalsForIterationSchedule(
-  evals: readonly DiscoveredEval[],
-): DiscoveredEval[][] {
-  const groups = new Map<string, DiscoveredEval[]>();
-  const order: string[] = [];
-  for (const discovered of evals) {
-    const key = iterationScheduleKey(discovered);
-    let group = groups.get(key);
-    if (group === undefined) {
-      group = [];
-      groups.set(key, group);
-      order.push(key);
-    }
-    group.push(discovered);
-  }
-  return order.map(key => {
-    const group = groups.get(key)!;
-    return group.sort((a, b) => (a.definition.iteration ?? 1) - (b.definition.iteration ?? 1));
-  });
-}
-
-function resolveEvalMaxConcurrency(config: Partial<EvalConfigFile>): number {
-  if (config.maxConcurrency === undefined) {
-    throw new Error(
-      `evals/evals.config.ts must define maxConcurrency (1–${EVAL_MAX_CONCURRENCY_LIMIT}; hard limit is due to model provider request limits). Example:\n` +
-        `  import { defineEvalConfig } from "@cursor/bdk/evals";\n` +
-        `  export default defineEvalConfig({ maxConcurrency: 20 });`,
-    );
-  }
-  return normalizeEvalMaxConcurrency(config.maxConcurrency);
-}
-
-interface RunOneEvalJudgeOptions {
-  /** Judge model from `evals.config.ts`. */
-  configModel?: ModelSetting;
-  caller: EvalJudgeCaller;
-}
-
-async function runOneEval(
-  discovered: DiscoveredEval,
-  opts: {
-    baseUrl: string;
-    defaultTimeoutMs: number;
-    bearerToken?: string;
-    headers?: Record<string, string>;
-    verbose: boolean;
-    judge: RunOneEvalJudgeOptions;
-    onEvent?: (event: SessionEvent) => void;
-    signal?: AbortSignal;
-  },
-): Promise<EvalRunResult> {
-  const logs: string[] = [];
-  const metrics: Record<string, string | number> = {};
-  const recorder = new EvalAssertionRecorder();
-  const def = discovered.definition;
-  const timeoutMs = def.timeoutMs ?? opts.defaultTimeoutMs;
-  const startedAt = Date.now();
-
-  let sessionId: string | undefined;
-  let continuationToken: string | undefined;
-  // The session stream replays from an index and the client stops at the first
-  // terminal event, so each send must resume after the previous turn's last
-  // event or it would re-read that turn instead of the new one.
-  let nextEventIndex = 0;
-  const allEvents: SessionEvent[] = [];
-  const inputs: string[] = [];
-  const turns: EvalTurnResult[] = [];
-  let reply: string | null = null;
-
-  const toolsUsed = (): string[] =>
-    buildTrajectory(allEvents).turns.flatMap(t => t.toolCalls.map(c => c.toolName));
-
-  const toolCallIo = () =>
-    buildTrajectory(allEvents).turns.flatMap(turn =>
-      turn.toolCalls.map(c => ({
-        toolName: c.toolName,
-        ...(c.args === undefined ? {} : { args: c.args }),
-        ...(c.output === undefined ? {} : { output: c.output }),
-        isError: c.isError,
-      })),
-    );
-
-  const finish = (extra: {
-    verdict: EvalVerdict;
-    ok: boolean;
-    error?: string;
-    skipReason?: string;
-    finalText?: string;
-  }): EvalRunResult => ({
-    id: discovered.id,
-    path: discovered.path,
-    ...(def.description === undefined ? {} : { description: def.description }),
-    ok: extra.ok,
-    verdict: extra.verdict,
-    skipReason: extra.skipReason,
-    assertions: recorder.results(),
-    error: extra.error,
-    logs,
-    metrics,
-    metadata: def.metadata,
-    tags: def.tags,
-    inputs,
-    toolCalls: toolCallIo(),
-    ...(sessionId === undefined ? {} : { sessionId }),
-    finalText: extra.finalText,
-    tools: toolsUsed(),
-    durationMs: Date.now() - startedAt,
-  });
-
-  if (opts.signal?.aborted) {
-    return finish({ verdict: "failed", ok: false, error: "cancelled" });
-  }
-
-  // A judge that cannot reach a model must not look like a quality regression:
-  // the case is skipped visibly instead.
-  let judgeUnavailable: string | undefined;
-  const timeout = new AbortController();
-  const timer = setTimeout(() => timeout.abort(), clampTimerDelay(timeoutMs));
-  // A pending case timer must not hold the process open on its own.
-  timer.unref?.();
-
-  const runAssertions: EvalAssertions = createAssertions({
-    recorder,
-    events: () => allEvents,
-  });
-
-  const judge = createJudgeSurface({
-    recorder,
-    grader: name => EVAL_JUDGE_GRADERS[name],
-    resolveModel: callModel =>
-      resolveJudgeModel({
-        call: callModel,
-        evalLevel: def.judge?.model,
-        config: opts.judge.configModel,
-        env: process.env[EVAL_JUDGE_MODEL_ENV],
-      }),
-    caller: opts.judge.caller,
-    defaultValue: () => reply,
-    onUnavailable: message => {
-      judgeUnavailable = judgeUnavailable ?? message;
-    },
-  });
-
-  const t: EvalTestContext = {
-    ...runAssertions,
-    get reply() {
-      return reply;
-    },
-    get sessionId() {
-      return sessionId;
-    },
-    get events() {
-      return allEvents;
-    },
-    get turns() {
-      return turns;
-    },
-    get iteration() {
-      return def.iteration ?? 1;
-    },
-    get iterations() {
-      return def.iterations ?? 1;
-    },
-    get signal() {
-      return timeout.signal;
-    },
-    judge,
-    async send(message: string, options: EvalSendOptions = {}): Promise<EvalTurnResult> {
-      inputs.push(message);
-      const result = await runSession({
-        baseUrl: opts.baseUrl,
-        messages: [message],
-        sessionId,
-        continuationToken,
-        startIndex: nextEventIndex,
-        purpose: "eval",
-        bearerToken: opts.bearerToken,
-        headers: opts.headers,
-        workspaceDir: options.workspaceDir,
-        workspaceFiles: options.workspaceFiles,
-        cloud: options.cloud,
-        timeoutMs,
-        signal: opts.signal,
-        onEvent: opts.onEvent,
-      });
-      sessionId = result.sessionId;
-      continuationToken = result.continuationToken;
-      nextEventIndex = result.nextEventIndex ?? nextEventIndex;
-      allEvents.push(...result.events);
-      if (result.trajectory.finalText !== "") {
-        reply = result.trajectory.finalText;
-      }
-      const index = turns.length + 1;
-      // The turn is immutable, so its assertions read a frozen event slice
-      // even after later turns append to the run.
-      const turnEvents = [...result.events];
-      const turn: EvalTurnResult = {
-        ...createAssertions({
-          recorder,
-          events: () => turnEvents,
-          scope: `turn[${index}]`,
-        }),
-        message: result.trajectory.finalText,
-        sessionId: result.sessionId,
-        events: turnEvents,
-        toolCalls: result.trajectory.turns.flatMap(turn => turn.toolCalls.map(c => c.toolName)),
-        ok: result.ok,
-        index,
-        expectOk() {
-          if (!result.ok) {
-            throw new Error(
-              `turn[${index}] failed: ${result.trajectory.turns.find(x => x.failed)?.error ?? "unknown error"}`,
-            );
-          }
-          return turn;
-        },
-      };
-      turns.push(turn);
-      return turn;
-    },
-    async require<T>(value: T, assertion: Expectation): Promise<T> {
-      const outcome = evaluateExpectation(value, assertion);
-      recorder.record({
-        name: `require(${assertion.kind})`,
-        passed: outcome.passed,
-        detail: outcome.detail,
-        score: outcome.score,
-        severity: "gate",
-      });
-      if (!outcome.passed) {
-        throw new EvalRequireFailure(`require(${assertion.kind})`);
-      }
-      return value;
-    },
-    skip(reason: string): never {
-      throw new EvalSkipSignal(reason);
-    },
-    log(message: string) {
-      logs.push(message);
-      if (opts.verbose) {
-        process.stderr.write(`    ${message}\n`);
-      }
-    },
-    metric(name: string, value: string | number) {
-      const key = name.trim();
-      if (key === "") {
-        return;
-      }
-      metrics[key] = value;
-      if (opts.verbose) {
-        process.stderr.write(`    metric ${key}=${String(value)}\n`);
-      }
-    },
-  };
-
-  try {
-    await def.test(t);
-  } catch (error) {
-    if (error instanceof EvalSkipSignal) {
-      clearTimeout(timer);
-      return finish({
-        verdict: "skipped",
-        ok: true,
-        skipReason: error.reason,
-        finalText: reply ?? undefined,
-      });
-    }
-    if (!(error instanceof EvalRequireFailure)) {
-      const message = error instanceof Error ? error.message : String(error);
-      if (error instanceof RunSessionTimeoutError && sessionId === undefined) {
-        sessionId = error.sessionId;
-      }
-      await recorder.waitForPending().catch(() => undefined);
-      clearTimeout(timer);
-      return finish({
-        verdict: "failed",
-        ok: false,
-        error: message,
-        finalText: reply ?? undefined,
-      });
-    }
-    // `require` already recorded the failing gate; fall through so the verdict
-    // comes from the assertions rather than a duplicate execution error.
-  }
-
-  await recorder.waitForPending();
-  clearTimeout(timer);
-
-  const assertions = recorder.results();
-  const traj = buildTrajectory(allEvents);
-  // With no assertions at all, keep the historical fallback: a case that just
-  // drives the agent passes when the run itself succeeded.
-  const verdict: EvalVerdict =
-    assertions.length > 0
-      ? computeVerdict(assertions)
-      : traj.ok && traj.turns.length > 0
-        ? "passed"
-        : "failed";
-  const finalText = reply ?? traj.finalText;
-
-  // An unreachable judge means the *judged* signal is unknown, so the case is
-  // reported as skipped rather than as a quality regression. It must not also
-  // bury a deterministic gate that already failed: skipping a failed case would
-  // turn a red eval green on any machine that happens to lack credentials.
-  if (judgeUnavailable !== undefined && verdict !== "failed") {
-    logs.push(`judge skipped: ${judgeUnavailable}`);
-    return finish({
-      verdict: "skipped",
-      ok: true,
-      skipReason: judgeUnavailable,
-      finalText,
-    });
-  }
-  if (judgeUnavailable !== undefined) {
-    logs.push(
-      `judge unavailable (${judgeUnavailable}), but a gate already failed — reporting the failure`,
-    );
-  }
-
-  return finish({
-    verdict,
-    ok: verdict !== "failed",
-    finalText,
-  });
-}
-
-/** Clamp a case timeout to a delay `setTimeout` can represent. */
-function clampTimerDelay(timeoutMs: number): number {
-  const MAX_TIMER = 2_147_483_647;
-  return Math.min(Math.max(timeoutMs, 0), MAX_TIMER);
-}
-
-/**
- * Build `t.judge`. Each grader records its slot immediately and resolves it
- * during finalization, so several judges overlap instead of serializing.
- */
-function createJudgeSurface(args: {
-  recorder: EvalAssertionRecorder;
-  grader: (name: keyof typeof EVAL_JUDGE_GRADERS) => EvalJudgeGrader;
-  resolveModel: (call?: ModelSetting) => ModelSetting | undefined;
-  caller: EvalJudgeCaller;
-  defaultValue: () => unknown;
-  onUnavailable: (message: string) => void;
-}): EvalJudge {
-  const grade = (
-    graderName: keyof typeof EVAL_JUDGE_GRADERS,
-    reference: string,
-    options: EvalJudgeCallOptions | undefined,
-  ): EvalAssertionHandle => {
-    const grader = args.grader(graderName);
-    const model = args.resolveModel(options?.model);
-    const name = `judge.${graderName}(${truncateForName(reference)})`;
-    if (model === undefined) {
-      // A missing judge model is an authoring error, not a flaky provider.
-      return args.recorder.record({
-        name,
-        passed: false,
-        severity: "gate",
-        detail:
-          "no judge model configured; set judge.model in evals.config.ts, on the eval, or on the call",
-      });
-    }
-    const submission = renderSubmission(
-      options !== undefined && "on" in options ? options.on : args.defaultValue(),
-    );
-    const settle = (async () => {
-      const reply = await args.caller({
-        prompt: grader.buildPrompt({ submission, reference }),
-        model,
-      });
-      const result = grader.parse(reply);
-      return {
-        passed: true,
-        score: result.score,
-        detail:
-          result.rationale === undefined
-            ? `${result.choice} (${result.score.toFixed(2)})`
-            : `${result.choice} (${result.score.toFixed(2)}) — ${result.rationale}`,
-      };
-    })().catch((error: unknown) => {
-      if (error instanceof EvalJudgeUnavailableError) {
-        args.onUnavailable(error.message);
-        return { passed: true, detail: `skipped: ${error.message}` };
-      }
-      throw error;
-    });
-    // Soft by default: a judge is a tracked score until given a bar.
-    return args.recorder.recordAsync(
-      { name, passed: true, severity: "soft", detail: "grading…" },
-      settle,
-    );
-  };
-
-  return {
-    async model(prompt, options) {
-      const model = args.resolveModel(options?.model);
-      if (model === undefined) {
-        throw new Error(
-          "t.judge.model: no judge model configured; set judge.model in evals.config.ts, on the eval, or on the call",
-        );
-      }
-      return args.caller({ prompt, model });
-    },
-    factuality: (expected, options) => grade("factuality", expected, options),
-    summarizes: (expected, options) => grade("summarizes", expected, options),
-    closedQA: (criteria, options) => grade("closedQA", criteria, options),
-    sql: (expected, options) => grade("sql", expected, options),
-  };
-}
-
-function truncateForName(text: string): string {
-  const oneLine = text.replace(/\s+/g, " ").trim();
-  return oneLine.length > 60 ? `${oneLine.slice(0, 59)}…` : oneLine;
-}