@cursor/july 0.1.20 → 0.1.21

package/src/internal/eval-runner.run.test.ts

@@ -0,0 +1,928 @@
+/**
+ * End-to-end coverage for the eval runner: discovery, the severity model,
+ * judges, verdicts, and reporters. `runSession` is mocked so a case can drive
+ * a scripted agent without a live model or an HTTP server.
+ */
+
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { EvalJudgeUnavailableError } from "../evals/judge.js";
+import { EVAL_ROOT_ENV } from "../evals/loaders.js";
+import type {
+  EvalDefinition,
+  EvalReporter,
+  EvalRunResult,
+  EvalTestContext,
+} from "../evals.js";
+import { defineEval, defineEvalConfig, equals, similarity } from "../evals.js";
+import type { SessionEvent } from "../types.js";
+import * as evalRunner from "./eval-runner.js";
+import * as trajectory from "./trajectory.js";
+
+const runSession = vi.hoisted(() => vi.fn());
+
+vi.mock("./run-client.js", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("./run-client.js")>();
+  return { ...actual, runSession };
+});
+
+// `vi.mock` is hoisted above these imports, so the runner sees the scripted
+// `runSession` even though the imports are static.
+const { discoverEvals, expandEvalModule, runDiscoveredEvals } = evalRunner;
+const { buildTrajectory } = trajectory;
+
+let sequence = 0;
+
+function event(type: string, data: unknown): SessionEvent {
+  sequence++;
+  return {
+    sessionId: "ses_1",
+    seq: sequence,
+    at: new Date(sequence * 1000).toISOString(),
+    turnId: "turn_1",
+    type,
+    data,
+  } as unknown as SessionEvent;
+}
+
+/** A settled turn that called `get_weather` and replied "Sunny, 72F". */
+function weatherEvents(reply = "Sunny, 72F"): SessionEvent[] {
+  return [
+    event("message.received", { text: "weather?" }),
+    event("actions.requested", {
+      calls: [{ callId: "c1", toolName: "get_weather", args: { city: "NYC" } }],
+    }),
+    event("action.result", {
+      callId: "c1",
+      toolName: "get_weather",
+      output: { tempF: 72 },
+      isError: false,
+    }),
+    event("message.completed", { text: reply, finishReason: "stop" }),
+    event("turn.completed", {}),
+    event("session.waiting", {}),
+  ];
+}
+
+function scriptTurn(events: SessionEvent[]): void {
+  runSession.mockImplementation(async () => {
+    const trajectory = buildTrajectory(events);
+    return {
+      sessionId: "ses_1",
+      continuationToken: "tok_1",
+      baseUrl: "http://127.0.0.1:3000",
+      events,
+      trajectory,
+      ok: trajectory.ok,
+    };
+  });
+}
+
+async function runCase(
+  definition: EvalDefinition,
+  options: {
+    reporters?: EvalReporter[];
+    strict?: boolean;
+    judgeCaller?: (args: { prompt: string }) => Promise<string>;
+    judgeModel?: string;
+    skipReport?: boolean;
+    configReporters?: EvalReporter[];
+  } = {}
+): Promise<EvalRunResult[]> {
+  const expanded = expandEvalModule("smoke", definition);
+  return runDiscoveredEvals({
+    projectRoot: "/unused",
+    baseUrl: "http://127.0.0.1:3000",
+    reporters: options.reporters,
+    skipReport: options.skipReport,
+    strict: options.strict,
+    judgeCaller: options.judgeCaller as never,
+    discovered: {
+      evals: expanded.map((e) => ({
+        id: e.id,
+        fileId: "smoke",
+        path: "/repo/evals/smoke.eval.ts",
+        definition: e.definition,
+      })),
+      config: {
+        maxConcurrency: 4,
+        judge:
+          options.judgeModel === undefined
+            ? undefined
+            : { model: options.judgeModel },
+        reporters: options.configReporters,
+      },
+    },
+  });
+}
+
+beforeEach(() => {
+  sequence = 0;
+  runSession.mockReset();
+  scriptTurn(weatherEvents());
+});
+
+describe("verdicts", () => {
+  it("passes a case whose gates all pass", async () => {
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.succeeded();
+          t.calledTool("get_weather", { input: { city: "NYC" }, count: 1 });
+          t.messageIncludes("Sunny");
+        },
+      })
+    );
+    expect(result).toMatchObject({ verdict: "passed", ok: true });
+    expect(result?.assertions.every((a) => a.passed)).toBe(true);
+  });
+
+  it("fails a case with a missed gate and still records every assertion", async () => {
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.calledTool("send_email");
+          t.messageIncludes("Rainy");
+          t.succeeded();
+        },
+      })
+    );
+    expect(result).toMatchObject({ verdict: "failed", ok: false });
+    // Recorded assertions never throw, so one run reports all three outcomes.
+    expect(result?.assertions.map((a) => a.passed)).toEqual([
+      false,
+      false,
+      true,
+    ]);
+  });
+
+  it("scores a case that only misses a soft threshold, leaving ok true", async () => {
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.succeeded();
+          t.check(t.reply, similarity("Rainy and cold")).atLeast(0.9);
+        },
+      })
+    );
+    expect(result).toMatchObject({ verdict: "scored", ok: true });
+    expect(result?.assertions[1]).toMatchObject({
+      severity: "soft",
+      threshold: 0.9,
+      passed: false,
+    });
+  });
+
+  it("keeps the historical fallback when a case records no assertions", async () => {
+    const [passing] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+        },
+      })
+    );
+    expect(passing?.verdict).toBe("passed");
+
+    scriptTurn([
+      event("message.received", { text: "hi" }),
+      event("turn.failed", { message: "model error" }),
+    ]);
+    const [failing] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("hi");
+        },
+      })
+    );
+    expect(failing?.verdict).toBe("failed");
+  });
+
+  it("reports a thrown test body as a failed execution error", async () => {
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.succeeded();
+          throw new Error("helper blew up");
+        },
+      })
+    );
+    expect(result).toMatchObject({ verdict: "failed", ok: false });
+    expect(result?.error).toBe("helper blew up");
+    // Assertions recorded before the throw are still reported.
+    expect(result?.assertions).toHaveLength(1);
+  });
+});
+
+describe("t.skip and t.require", () => {
+  it("reports t.skip separately and keeps ok true", async () => {
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          t.skip("cloud runtime not available here");
+          await t.send("weather?");
+        },
+      })
+    );
+    expect(result).toMatchObject({
+      verdict: "skipped",
+      ok: true,
+      skipReason: "cloud runtime not available here",
+    });
+    expect(runSession).not.toHaveBeenCalled();
+  });
+
+  it("stops the body on a failed require without a duplicate execution error", async () => {
+    const seen: string[] = [];
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          await t.require(t.reply, equals("Rainy"));
+          seen.push("unreachable");
+          t.succeeded();
+        },
+      })
+    );
+    expect(seen).toEqual([]);
+    expect(result).toMatchObject({ verdict: "failed", ok: false });
+    // The recorded gate is the whole story; no duplicate execution error.
+    expect(result?.error).toBeUndefined();
+    expect(result?.assertions).toHaveLength(1);
+    expect(result?.assertions[0]).toMatchObject({
+      name: "require(equals)",
+      passed: false,
+      detail: "got Sunny, 72F",
+    });
+  });
+
+  it("returns the value and continues when require passes", async () => {
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          const reply = await t.require(t.reply, equals("Sunny, 72F"));
+          t.check(reply, equals("Sunny, 72F"));
+        },
+      })
+    );
+    expect(result?.verdict).toBe("passed");
+    expect(result?.assertions).toHaveLength(2);
+  });
+});
+
+describe("turn scoping", () => {
+  it("grades an intermediate turn before the next send overwrites the reply", async () => {
+    const first = weatherEvents("Draft: Best regards");
+    const second = [
+      event("message.received", { text: "send it" }),
+      event("actions.requested", {
+        calls: [{ callId: "c2", toolName: "send_email", args: {} }],
+      }),
+      event("action.result", {
+        callId: "c2",
+        toolName: "send_email",
+        output: "sent",
+        isError: false,
+      }),
+      event("message.completed", { text: "Sent.", finishReason: "stop" }),
+      event("turn.completed", {}),
+    ];
+    const scripted = [first, second];
+    runSession.mockImplementation(async () => {
+      const events = scripted.shift() ?? [];
+      const trajectory = buildTrajectory(events);
+      return {
+        sessionId: "ses_1",
+        continuationToken: "tok",
+        baseUrl: "http://127.0.0.1:3000",
+        events,
+        trajectory,
+        ok: trajectory.ok,
+      };
+    });
+
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          const draft = await t.send("draft it");
+          draft.messageIncludes("Best regards");
+          draft.notCalledTool("send_email");
+
+          await t.send("send it");
+          t.calledTool("send_email");
+          expect(t.turns).toHaveLength(2);
+          expect(t.reply).toBe("Sent.");
+        },
+      })
+    );
+    expect(result?.verdict).toBe("passed");
+    expect(result?.assertions.map((a) => a.name)).toEqual([
+      "turn[1].messageIncludes(Best regards)",
+      "turn[1].notCalledTool(send_email)",
+      "calledTool(send_email)",
+    ]);
+    expect(result?.inputs).toEqual(["draft it", "send it"]);
+  });
+});
+
+describe("judges", () => {
+  const judgeReply = "CHOICE: Y\nREASON: it names a source";
+
+  it("records a judge score as a soft assertion", async () => {
+    const judgeCaller = vi.fn(async () => judgeReply);
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.judge.closedQA("mentions the temperature");
+        },
+      }),
+      { judgeCaller, judgeModel: "gpt-5.4-mini" }
+    );
+    expect(result?.verdict).toBe("passed");
+    expect(result?.assertions[0]).toMatchObject({
+      name: "judge.closedQA(mentions the temperature)",
+      severity: "soft",
+      score: 1,
+      passed: true,
+      detail: "Y (1.00) — it names a source",
+    });
+    expect(judgeCaller).toHaveBeenCalledTimes(1);
+  });
+
+  it("grades t.reply by default and an explicit value with `on`", async () => {
+    const prompts: string[] = [];
+    await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.judge.closedQA("criterion");
+          t.judge.closedQA("criterion", { on: "a different value" });
+        },
+      }),
+      {
+        judgeModel: "gpt-5.4-mini",
+        judgeCaller: async ({ prompt }) => {
+          prompts.push(prompt);
+          return judgeReply;
+        },
+      }
+    );
+    expect(prompts[0]).toContain("Sunny, 72F");
+    expect(prompts[1]).toContain("a different value");
+  });
+
+  it("marks the eval scored when a judge misses its bar", async () => {
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.succeeded();
+          t.judge.factuality("Rainy and 40F").atLeast(0.8);
+        },
+      }),
+      {
+        judgeModel: "gpt-5.4-mini",
+        judgeCaller: async () => "CHOICE: D\nREASON: the values disagree",
+      }
+    );
+    expect(result?.verdict).toBe("scored");
+    expect(result?.assertions[1]).toMatchObject({ score: 0, threshold: 0.8 });
+  });
+
+  it("fails the eval when a judge is promoted to a gate and misses", async () => {
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.judge.factuality("Rainy and 40F").gate(0.8);
+        },
+      }),
+      { judgeModel: "gpt-5.4-mini", judgeCaller: async () => "CHOICE: D" }
+    );
+    expect(result?.verdict).toBe("failed");
+  });
+
+  it("records a failed gate when no judge model is configured anywhere", async () => {
+    const judgeCaller = vi.fn(async () => judgeReply);
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.judge.closedQA("criterion");
+        },
+      }),
+      { judgeCaller }
+    );
+    expect(result?.verdict).toBe("failed");
+    expect(result?.assertions[0]).toMatchObject({
+      passed: false,
+      detail: expect.stringContaining("no judge model configured"),
+    });
+    expect(judgeCaller).not.toHaveBeenCalled();
+  });
+
+  it("prefers the per-eval judge model over the config default", async () => {
+    const models: unknown[] = [];
+    await runCase(
+      defineEval({
+        judge: { model: "claude-opus-4.8" },
+        async test(t) {
+          await t.send("weather?");
+          t.judge.closedQA("criterion");
+          t.judge.closedQA("criterion", { model: "gpt-5.4-mini" });
+        },
+      }),
+      {
+        judgeModel: "config-default",
+        judgeCaller: async (args: { prompt: string; model?: unknown }) => {
+          models.push(args.model);
+          return judgeReply;
+        },
+      }
+    );
+    expect(models).toEqual(["claude-opus-4.8", "gpt-5.4-mini"]);
+  });
+
+  it("skips the eval visibly when the judge has no credentials", async () => {
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.succeeded();
+          t.judge.closedQA("criterion");
+        },
+      }),
+      {
+        judgeModel: "gpt-5.4-mini",
+        judgeCaller: async () => {
+          throw new EvalJudgeUnavailableError("no Cursor credentials");
+        },
+      }
+    );
+    // A missing credential is an environment gap, not a quality regression.
+    expect(result).toMatchObject({ verdict: "skipped", ok: true });
+    expect(result?.skipReason).toContain("no Cursor credentials");
+    expect(result?.logs).toContain("judge skipped: no Cursor credentials");
+  });
+
+  it("still reports a failed gate when the judge is unavailable", async () => {
+    // Otherwise a machine without judge credentials would turn a genuinely red
+    // eval green: the skip would bury the deterministic failure.
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.calledTool("send_email");
+          t.judge.closedQA("criterion");
+        },
+      }),
+      {
+        judgeModel: "gpt-5.4-mini",
+        judgeCaller: async () => {
+          throw new EvalJudgeUnavailableError("no Cursor credentials");
+        },
+      }
+    );
+    expect(result).toMatchObject({ verdict: "failed", ok: false });
+    expect(result?.skipReason).toBeUndefined();
+    expect(result?.logs.join("\n")).toContain("a gate already failed");
+  });
+
+  it("skips rather than scoring when only soft judge signal is missing", async () => {
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.succeeded();
+          t.judge.closedQA("criterion").atLeast(0.6);
+        },
+      }),
+      {
+        judgeModel: "gpt-5.4-mini",
+        judgeCaller: async () => {
+          throw new EvalJudgeUnavailableError("no Cursor credentials");
+        },
+      }
+    );
+    expect(result).toMatchObject({ verdict: "skipped", ok: true });
+  });
+
+  it("fails the assertion when the judge reply is unparseable", async () => {
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.judge.closedQA("criterion");
+        },
+      }),
+      { judgeModel: "gpt-5.4-mini", judgeCaller: async () => "I refuse" }
+    );
+    expect(result?.verdict).toBe("failed");
+    expect(result?.assertions[0]?.detail).toMatch(/had no choice/);
+  });
+
+  it("runs several judges concurrently and waits for all of them", async () => {
+    let inFlight = 0;
+    let maxInFlight = 0;
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.judge.closedQA("first");
+          t.judge.closedQA("second");
+          t.judge.closedQA("third");
+        },
+      }),
+      {
+        judgeModel: "gpt-5.4-mini",
+        judgeCaller: async () => {
+          inFlight++;
+          maxInFlight = Math.max(maxInFlight, inFlight);
+          await new Promise((resolve) => setTimeout(resolve, 5));
+          inFlight--;
+          return judgeReply;
+        },
+      }
+    );
+    expect(maxInFlight).toBeGreaterThan(1);
+    expect(result?.assertions.map((a) => a.score)).toEqual([1, 1, 1]);
+  });
+});
+
+describe("t.judge.model", () => {
+  it("gives a custom judge the configured model and credentials", async () => {
+    const seen: Array<{ prompt: string; model?: unknown }> = [];
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          // The shape a domain judge uses: one raw call, then score the parse.
+          const reply = await t.judge.model("Grade these findings: [...]");
+          t.score("recall", JSON.parse(reply).recall as number).atLeast(0.6);
+        },
+      }),
+      {
+        judgeModel: "gpt-5.4-mini",
+        judgeCaller: async (args: { prompt: string; model?: unknown }) => {
+          seen.push(args);
+          return '{"recall":0.8}';
+        },
+      }
+    );
+    expect(seen).toEqual([
+      { prompt: "Grade these findings: [...]", model: "gpt-5.4-mini" },
+    ]);
+    expect(result?.verdict).toBe("passed");
+    expect(result?.assertions[0]).toMatchObject({
+      name: "score(recall)",
+      score: 0.8,
+      threshold: 0.6,
+      passed: true,
+    });
+  });
+
+  it("throws into the test body when no judge model is configured", async () => {
+    const [result] = await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          await t.judge.model("grade this");
+        },
+      })
+    );
+    expect(result?.verdict).toBe("failed");
+    expect(result?.error).toMatch(/no judge model configured/);
+  });
+});
+
+describe("reporters", () => {
+  function recordingReporter(): {
+    reporter: EvalReporter;
+    calls: string[];
+  } {
+    const calls: string[] = [];
+    return {
+      calls,
+      reporter: {
+        onRunStart: (evals, target) => {
+          calls.push(
+            `start:${evals.map((e) => e.id).join(",")}:${target.mode}`
+          );
+        },
+        onEvalComplete: (result) => {
+          calls.push(`case:${result.id}:${result.verdict}`);
+        },
+        onRunComplete: (summary) => {
+          calls.push(
+            `complete:${summary.passed}p/${summary.failed}f/${summary.scored}s`
+          );
+        },
+      },
+    };
+  }
+
+  const twoCases = defineEval({
+    cases: [
+      {
+        id: "good",
+        async test(t) {
+          await t.send("weather?");
+          t.succeeded();
+        },
+      },
+      {
+        id: "bad",
+        async test(t) {
+          await t.send("weather?");
+          t.calledTool("missing");
+        },
+      },
+    ],
+  });
+
+  it("drives the full lifecycle for a CLI reporter", async () => {
+    const { reporter, calls } = recordingReporter();
+    await runCase(twoCases, { reporters: [reporter] });
+    expect(calls).toEqual([
+      "start:smoke/good,smoke/bad:local",
+      "case:smoke/good:passed",
+      "case:smoke/bad:failed",
+      "complete:1p/1f/0s",
+    ]);
+  });
+
+  it("observes every case from a config reporter", async () => {
+    const { reporter, calls } = recordingReporter();
+    await runCase(twoCases, { configReporters: [reporter] });
+    expect(calls.filter((c) => c.startsWith("case:"))).toEqual([
+      "case:smoke/good:passed",
+      "case:smoke/bad:failed",
+    ]);
+  });
+
+  it("scopes a per-eval reporter to its own case", async () => {
+    const { reporter, calls } = recordingReporter();
+    await runCase(
+      defineEval({
+        cases: [
+          {
+            id: "watched",
+            reporters: [reporter],
+            async test(t) {
+              await t.send("weather?");
+              t.succeeded();
+            },
+          },
+          {
+            id: "unwatched",
+            async test(t) {
+              await t.send("weather?");
+              t.succeeded();
+            },
+          },
+        ],
+      })
+    );
+    // Run hooks still fire, but only the referencing case reports a result.
+    expect(calls).toEqual([
+      "start:smoke/watched,smoke/unwatched:local",
+      "case:smoke/watched:passed",
+      "complete:2p/0f/0s",
+    ]);
+  });
+
+  it("suppresses config and per-eval reporters under skipReport", async () => {
+    const config = recordingReporter();
+    const perEval = recordingReporter();
+    const cli = recordingReporter();
+    await runCase(
+      defineEval({
+        reporters: [perEval.reporter],
+        async test(t) {
+          await t.send("weather?");
+          t.succeeded();
+        },
+      }),
+      {
+        configReporters: [config.reporter],
+        reporters: [cli.reporter],
+        skipReport: true,
+      }
+    );
+    expect(config.calls).toEqual([]);
+    expect(perEval.calls).toEqual([]);
+    // CLI-attached reporters are explicit, so they still run.
+    expect(cli.calls).toContain("case:smoke:passed");
+  });
+
+  it("reports strict mode on the summary so reporters can gate on it", async () => {
+    let sawStrict: boolean | undefined;
+    await runCase(
+      defineEval({
+        async test(t) {
+          await t.send("weather?");
+          t.succeeded();
+        },
+      }),
+      {
+        strict: true,
+        reporters: [
+          {
+            onRunComplete: (summary) => {
+              sawStrict = summary.strict;
+            },
+          },
+        ],
+      }
+    );
+    expect(sawStrict).toBe(true);
+  });
+
+  it("keeps running when a reporter throws", async () => {
+    const results = await runCase(twoCases, {
+      reporters: [
+        {
+          onEvalComplete: () => {
+            throw new Error("upload failed");
+          },
+        },
+      ],
+    });
+    expect(results.map((r) => r.verdict)).toEqual(["passed", "failed"]);
+  });
+});
+
+describe("metadata and metrics", () => {
+  it("carries eval metadata and tags onto the result", async () => {
+    const [result] = await runCase(
+      defineEval({
+        tags: ["smoke"],
+        metadata: { owner: "agents", suite: "weather" },
+        async test(t) {
+          await t.send("weather?");
+          t.succeeded();
+          t.metric("recall", "40.0% (4/10)");
+        },
+      })
+    );
+    expect(result).toMatchObject({
+      tags: ["smoke"],
+      metadata: { owner: "agents", suite: "weather" },
+      metrics: { recall: "40.0% (4/10)" },
+    });
+  });
+
+  it("lets a case-level metadata key override the file-level one", async () => {
+    const [result] = await runCase(
+      defineEval({
+        metadata: { owner: "agents", suite: "weather" },
+        cases: [
+          {
+            id: "override",
+            metadata: { owner: "security" },
+            async test(t) {
+              await t.send("weather?");
+              t.succeeded();
+            },
+          },
+        ],
+      })
+    );
+    expect(result?.metadata).toEqual({ owner: "security", suite: "weather" });
+  });
+});
+
+describe("expandEvalModule", () => {
+  it("expands a single defineEval to the file id", () => {
+    const expanded = expandEvalModule(
+      "weather",
+      defineEval({ test: async () => undefined })
+    );
+    expect(expanded.map((e) => e.id)).toEqual(["weather"]);
+  });
+
+  it("fans a default-exported array out with zero-padded ids", () => {
+    const rows = ["a", "b", "c"];
+    const expanded = expandEvalModule(
+      "sql",
+      rows.map((row) =>
+        defineEval({ description: row, test: async () => undefined })
+      ) as never
+    );
+    expect(expanded.map((e) => e.id)).toEqual([
+      "sql/0000",
+      "sql/0001",
+      "sql/0002",
+    ]);
+    expect(expanded[1]?.definition.description).toBe("b");
+  });
+
+  it("rejects a non-eval default export", () => {
+    expect(() => expandEvalModule("x", { nope: true })).toThrow(
+      /must be defineEval\(\{ \.\.\. \}\) or an array of them/
+    );
+    expect(() => expandEvalModule("x", [])).toThrow(/array is empty/);
+    expect(() => expandEvalModule("x", [{ nope: true }])).toThrow(
+      /array entry 0 must be defineEval/
+    );
+  });
+});
+
+describe("discoverEvals", () => {
+  let projectRoot: string;
+
+  beforeEach(async () => {
+    projectRoot = await mkdtemp(join(tmpdir(), "agentkit-eval-discovery-"));
+    await mkdir(join(projectRoot, "evals", "weather"), { recursive: true });
+  });
+
+  afterEach(async () => {
+    await rm(projectRoot, { recursive: true, force: true });
+  });
+
+  /** Map on-disk eval paths to in-memory definitions. */
+  function importer(
+    modules: Record<string, unknown>
+  ): (path: string) => Promise<Record<string, unknown>> {
+    return async (absolutePath: string) => {
+      for (const [suffix, mod] of Object.entries(modules)) {
+        if (absolutePath.endsWith(suffix)) {
+          return { default: mod };
+        }
+      }
+      throw new Error(`unexpected import: ${absolutePath}`);
+    };
+  }
+
+  it("derives ids from paths and loads the config", async () => {
+    await writeFile(join(projectRoot, "evals/evals.config.ts"), "", "utf8");
+    await writeFile(join(projectRoot, "evals/smoke.eval.ts"), "", "utf8");
+    await writeFile(join(projectRoot, "evals/weather/nyc.eval.ts"), "", "utf8");
+    // A sibling helper that is not an eval file must be ignored.
+    await writeFile(join(projectRoot, "evals/weather/shared.ts"), "", "utf8");
+
+    const { evals, config } = await discoverEvals(
+      projectRoot,
+      importer({
+        "evals.config.ts": defineEvalConfig({ maxConcurrency: 7 }),
+        "smoke.eval.ts": defineEval({ test: async () => undefined }),
+        "nyc.eval.ts": defineEval({ test: async () => undefined }),
+      })
+    );
+    expect(evals.map((e) => e.id)).toEqual(["smoke", "weather/nyc"]);
+    expect(config.maxConcurrency).toBe(7);
+  });
+
+  it("points fixture loaders at the app root", async () => {
+    await writeFile(join(projectRoot, "evals/evals.config.ts"), "", "utf8");
+    await discoverEvals(
+      projectRoot,
+      importer({ "evals.config.ts": defineEvalConfig({ maxConcurrency: 1 }) })
+    );
+    expect(process.env[EVAL_ROOT_ENV]).toBe(projectRoot);
+  });
+
+  it("prefixes the failing file on a bad default export", async () => {
+    await writeFile(join(projectRoot, "evals/broken.eval.ts"), "", "utf8");
+    await expect(
+      discoverEvals(projectRoot, importer({ "broken.eval.ts": { nope: true } }))
+    ).rejects.toThrow(/broken\.eval\.ts: default export must be defineEval/);
+  });
+});
+
+describe("cancellation", () => {
+  it("returns a cancelled result without driving the agent", async () => {
+    const controller = new AbortController();
+    controller.abort();
+    const results = await runDiscoveredEvals({
+      projectRoot: "/unused",
+      baseUrl: "http://127.0.0.1:3000",
+      signal: controller.signal,
+      discovered: {
+        evals: expandEvalModule(
+          "smoke",
+          defineEval({
+            async test(t: EvalTestContext) {
+              await t.send("weather?");
+            },
+          })
+        ).map((e) => ({
+          id: e.id,
+          fileId: "smoke",
+          path: "/repo/evals/smoke.eval.ts",
+          definition: e.definition,
+        })),
+        config: { maxConcurrency: 1 },
+      },
+    });
+    expect(results).toEqual([]);
+    expect(runSession).not.toHaveBeenCalled();
+  });
+});