@cursor/july 0.1.25 → 0.1.27

package/src/internal/eval-runner.run.test.ts

@@ -1,928 +0,0 @@
-/**
- * End-to-end coverage for the eval runner: discovery, the severity model,
- * judges, verdicts, and reporters. `runSession` is mocked so a case can drive
- * a scripted agent without a live model or an HTTP server.
- */
-
-import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { EvalJudgeUnavailableError } from "../evals/judge.js";
-import { EVAL_ROOT_ENV } from "../evals/loaders.js";
-import type {
-  EvalDefinition,
-  EvalReporter,
-  EvalRunResult,
-  EvalTestContext,
-} from "../evals.js";
-import { defineEval, defineEvalConfig, equals, similarity } from "../evals.js";
-import type { SessionEvent } from "../types.js";
-import * as evalRunner from "./eval-runner.js";
-import * as trajectory from "./trajectory.js";
-
-const runSession = vi.hoisted(() => vi.fn());
-
-vi.mock("./run-client.js", async (importOriginal) => {
-  const actual = await importOriginal<typeof import("./run-client.js")>();
-  return { ...actual, runSession };
-});
-
-// `vi.mock` is hoisted above these imports, so the runner sees the scripted
-// `runSession` even though the imports are static.
-const { discoverEvals, expandEvalModule, runDiscoveredEvals } = evalRunner;
-const { buildTrajectory } = trajectory;
-
-let sequence = 0;
-
-function event(type: string, data: unknown): SessionEvent {
-  sequence++;
-  return {
-    sessionId: "ses_1",
-    seq: sequence,
-    at: new Date(sequence * 1000).toISOString(),
-    turnId: "turn_1",
-    type,
-    data,
-  } as unknown as SessionEvent;
-}
-
-/** A settled turn that called `get_weather` and replied "Sunny, 72F". */
-function weatherEvents(reply = "Sunny, 72F"): SessionEvent[] {
-  return [
-    event("message.received", { text: "weather?" }),
-    event("actions.requested", {
-      calls: [{ callId: "c1", toolName: "get_weather", args: { city: "NYC" } }],
-    }),
-    event("action.result", {
-      callId: "c1",
-      toolName: "get_weather",
-      output: { tempF: 72 },
-      isError: false,
-    }),
-    event("message.completed", { text: reply, finishReason: "stop" }),
-    event("turn.completed", {}),
-    event("session.waiting", {}),
-  ];
-}
-
-function scriptTurn(events: SessionEvent[]): void {
-  runSession.mockImplementation(async () => {
-    const trajectory = buildTrajectory(events);
-    return {
-      sessionId: "ses_1",
-      continuationToken: "tok_1",
-      baseUrl: "http://127.0.0.1:3000",
-      events,
-      trajectory,
-      ok: trajectory.ok,
-    };
-  });
-}
-
-async function runCase(
-  definition: EvalDefinition,
-  options: {
-    reporters?: EvalReporter[];
-    strict?: boolean;
-    judgeCaller?: (args: { prompt: string }) => Promise<string>;
-    judgeModel?: string;
-    skipReport?: boolean;
-    configReporters?: EvalReporter[];
-  } = {}
-): Promise<EvalRunResult[]> {
-  const expanded = expandEvalModule("smoke", definition);
-  return runDiscoveredEvals({
-    projectRoot: "/unused",
-    baseUrl: "http://127.0.0.1:3000",
-    reporters: options.reporters,
-    skipReport: options.skipReport,
-    strict: options.strict,
-    judgeCaller: options.judgeCaller as never,
-    discovered: {
-      evals: expanded.map((e) => ({
-        id: e.id,
-        fileId: "smoke",
-        path: "/repo/evals/smoke.eval.ts",
-        definition: e.definition,
-      })),
-      config: {
-        maxConcurrency: 4,
-        judge:
-          options.judgeModel === undefined
-            ? undefined
-            : { model: options.judgeModel },
-        reporters: options.configReporters,
-      },
-    },
-  });
-}
-
-beforeEach(() => {
-  sequence = 0;
-  runSession.mockReset();
-  scriptTurn(weatherEvents());
-});
-
-describe("verdicts", () => {
-  it("passes a case whose gates all pass", async () => {
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.succeeded();
-          t.calledTool("get_weather", { input: { city: "NYC" }, count: 1 });
-          t.messageIncludes("Sunny");
-        },
-      })
-    );
-    expect(result).toMatchObject({ verdict: "passed", ok: true });
-    expect(result?.assertions.every((a) => a.passed)).toBe(true);
-  });
-
-  it("fails a case with a missed gate and still records every assertion", async () => {
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.calledTool("send_email");
-          t.messageIncludes("Rainy");
-          t.succeeded();
-        },
-      })
-    );
-    expect(result).toMatchObject({ verdict: "failed", ok: false });
-    // Recorded assertions never throw, so one run reports all three outcomes.
-    expect(result?.assertions.map((a) => a.passed)).toEqual([
-      false,
-      false,
-      true,
-    ]);
-  });
-
-  it("scores a case that only misses a soft threshold, leaving ok true", async () => {
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.succeeded();
-          t.check(t.reply, similarity("Rainy and cold")).atLeast(0.9);
-        },
-      })
-    );
-    expect(result).toMatchObject({ verdict: "scored", ok: true });
-    expect(result?.assertions[1]).toMatchObject({
-      severity: "soft",
-      threshold: 0.9,
-      passed: false,
-    });
-  });
-
-  it("keeps the historical fallback when a case records no assertions", async () => {
-    const [passing] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-        },
-      })
-    );
-    expect(passing?.verdict).toBe("passed");
-
-    scriptTurn([
-      event("message.received", { text: "hi" }),
-      event("turn.failed", { message: "model error" }),
-    ]);
-    const [failing] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("hi");
-        },
-      })
-    );
-    expect(failing?.verdict).toBe("failed");
-  });
-
-  it("reports a thrown test body as a failed execution error", async () => {
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.succeeded();
-          throw new Error("helper blew up");
-        },
-      })
-    );
-    expect(result).toMatchObject({ verdict: "failed", ok: false });
-    expect(result?.error).toBe("helper blew up");
-    // Assertions recorded before the throw are still reported.
-    expect(result?.assertions).toHaveLength(1);
-  });
-});
-
-describe("t.skip and t.require", () => {
-  it("reports t.skip separately and keeps ok true", async () => {
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          t.skip("cloud runtime not available here");
-          await t.send("weather?");
-        },
-      })
-    );
-    expect(result).toMatchObject({
-      verdict: "skipped",
-      ok: true,
-      skipReason: "cloud runtime not available here",
-    });
-    expect(runSession).not.toHaveBeenCalled();
-  });
-
-  it("stops the body on a failed require without a duplicate execution error", async () => {
-    const seen: string[] = [];
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          await t.require(t.reply, equals("Rainy"));
-          seen.push("unreachable");
-          t.succeeded();
-        },
-      })
-    );
-    expect(seen).toEqual([]);
-    expect(result).toMatchObject({ verdict: "failed", ok: false });
-    // The recorded gate is the whole story; no duplicate execution error.
-    expect(result?.error).toBeUndefined();
-    expect(result?.assertions).toHaveLength(1);
-    expect(result?.assertions[0]).toMatchObject({
-      name: "require(equals)",
-      passed: false,
-      detail: "got Sunny, 72F",
-    });
-  });
-
-  it("returns the value and continues when require passes", async () => {
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          const reply = await t.require(t.reply, equals("Sunny, 72F"));
-          t.check(reply, equals("Sunny, 72F"));
-        },
-      })
-    );
-    expect(result?.verdict).toBe("passed");
-    expect(result?.assertions).toHaveLength(2);
-  });
-});
-
-describe("turn scoping", () => {
-  it("grades an intermediate turn before the next send overwrites the reply", async () => {
-    const first = weatherEvents("Draft: Best regards");
-    const second = [
-      event("message.received", { text: "send it" }),
-      event("actions.requested", {
-        calls: [{ callId: "c2", toolName: "send_email", args: {} }],
-      }),
-      event("action.result", {
-        callId: "c2",
-        toolName: "send_email",
-        output: "sent",
-        isError: false,
-      }),
-      event("message.completed", { text: "Sent.", finishReason: "stop" }),
-      event("turn.completed", {}),
-    ];
-    const scripted = [first, second];
-    runSession.mockImplementation(async () => {
-      const events = scripted.shift() ?? [];
-      const trajectory = buildTrajectory(events);
-      return {
-        sessionId: "ses_1",
-        continuationToken: "tok",
-        baseUrl: "http://127.0.0.1:3000",
-        events,
-        trajectory,
-        ok: trajectory.ok,
-      };
-    });
-
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          const draft = await t.send("draft it");
-          draft.messageIncludes("Best regards");
-          draft.notCalledTool("send_email");
-
-          await t.send("send it");
-          t.calledTool("send_email");
-          expect(t.turns).toHaveLength(2);
-          expect(t.reply).toBe("Sent.");
-        },
-      })
-    );
-    expect(result?.verdict).toBe("passed");
-    expect(result?.assertions.map((a) => a.name)).toEqual([
-      "turn[1].messageIncludes(Best regards)",
-      "turn[1].notCalledTool(send_email)",
-      "calledTool(send_email)",
-    ]);
-    expect(result?.inputs).toEqual(["draft it", "send it"]);
-  });
-});
-
-describe("judges", () => {
-  const judgeReply = "CHOICE: Y\nREASON: it names a source";
-
-  it("records a judge score as a soft assertion", async () => {
-    const judgeCaller = vi.fn(async () => judgeReply);
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.judge.closedQA("mentions the temperature");
-        },
-      }),
-      { judgeCaller, judgeModel: "gpt-5.4-mini" }
-    );
-    expect(result?.verdict).toBe("passed");
-    expect(result?.assertions[0]).toMatchObject({
-      name: "judge.closedQA(mentions the temperature)",
-      severity: "soft",
-      score: 1,
-      passed: true,
-      detail: "Y (1.00) — it names a source",
-    });
-    expect(judgeCaller).toHaveBeenCalledTimes(1);
-  });
-
-  it("grades t.reply by default and an explicit value with `on`", async () => {
-    const prompts: string[] = [];
-    await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.judge.closedQA("criterion");
-          t.judge.closedQA("criterion", { on: "a different value" });
-        },
-      }),
-      {
-        judgeModel: "gpt-5.4-mini",
-        judgeCaller: async ({ prompt }) => {
-          prompts.push(prompt);
-          return judgeReply;
-        },
-      }
-    );
-    expect(prompts[0]).toContain("Sunny, 72F");
-    expect(prompts[1]).toContain("a different value");
-  });
-
-  it("marks the eval scored when a judge misses its bar", async () => {
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.succeeded();
-          t.judge.factuality("Rainy and 40F").atLeast(0.8);
-        },
-      }),
-      {
-        judgeModel: "gpt-5.4-mini",
-        judgeCaller: async () => "CHOICE: D\nREASON: the values disagree",
-      }
-    );
-    expect(result?.verdict).toBe("scored");
-    expect(result?.assertions[1]).toMatchObject({ score: 0, threshold: 0.8 });
-  });
-
-  it("fails the eval when a judge is promoted to a gate and misses", async () => {
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.judge.factuality("Rainy and 40F").gate(0.8);
-        },
-      }),
-      { judgeModel: "gpt-5.4-mini", judgeCaller: async () => "CHOICE: D" }
-    );
-    expect(result?.verdict).toBe("failed");
-  });
-
-  it("records a failed gate when no judge model is configured anywhere", async () => {
-    const judgeCaller = vi.fn(async () => judgeReply);
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.judge.closedQA("criterion");
-        },
-      }),
-      { judgeCaller }
-    );
-    expect(result?.verdict).toBe("failed");
-    expect(result?.assertions[0]).toMatchObject({
-      passed: false,
-      detail: expect.stringContaining("no judge model configured"),
-    });
-    expect(judgeCaller).not.toHaveBeenCalled();
-  });
-
-  it("prefers the per-eval judge model over the config default", async () => {
-    const models: unknown[] = [];
-    await runCase(
-      defineEval({
-        judge: { model: "claude-opus-4.8" },
-        async test(t) {
-          await t.send("weather?");
-          t.judge.closedQA("criterion");
-          t.judge.closedQA("criterion", { model: "gpt-5.4-mini" });
-        },
-      }),
-      {
-        judgeModel: "config-default",
-        judgeCaller: async (args: { prompt: string; model?: unknown }) => {
-          models.push(args.model);
-          return judgeReply;
-        },
-      }
-    );
-    expect(models).toEqual(["claude-opus-4.8", "gpt-5.4-mini"]);
-  });
-
-  it("skips the eval visibly when the judge has no credentials", async () => {
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.succeeded();
-          t.judge.closedQA("criterion");
-        },
-      }),
-      {
-        judgeModel: "gpt-5.4-mini",
-        judgeCaller: async () => {
-          throw new EvalJudgeUnavailableError("no Cursor credentials");
-        },
-      }
-    );
-    // A missing credential is an environment gap, not a quality regression.
-    expect(result).toMatchObject({ verdict: "skipped", ok: true });
-    expect(result?.skipReason).toContain("no Cursor credentials");
-    expect(result?.logs).toContain("judge skipped: no Cursor credentials");
-  });
-
-  it("still reports a failed gate when the judge is unavailable", async () => {
-    // Otherwise a machine without judge credentials would turn a genuinely red
-    // eval green: the skip would bury the deterministic failure.
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.calledTool("send_email");
-          t.judge.closedQA("criterion");
-        },
-      }),
-      {
-        judgeModel: "gpt-5.4-mini",
-        judgeCaller: async () => {
-          throw new EvalJudgeUnavailableError("no Cursor credentials");
-        },
-      }
-    );
-    expect(result).toMatchObject({ verdict: "failed", ok: false });
-    expect(result?.skipReason).toBeUndefined();
-    expect(result?.logs.join("\n")).toContain("a gate already failed");
-  });
-
-  it("skips rather than scoring when only soft judge signal is missing", async () => {
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.succeeded();
-          t.judge.closedQA("criterion").atLeast(0.6);
-        },
-      }),
-      {
-        judgeModel: "gpt-5.4-mini",
-        judgeCaller: async () => {
-          throw new EvalJudgeUnavailableError("no Cursor credentials");
-        },
-      }
-    );
-    expect(result).toMatchObject({ verdict: "skipped", ok: true });
-  });
-
-  it("fails the assertion when the judge reply is unparseable", async () => {
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.judge.closedQA("criterion");
-        },
-      }),
-      { judgeModel: "gpt-5.4-mini", judgeCaller: async () => "I refuse" }
-    );
-    expect(result?.verdict).toBe("failed");
-    expect(result?.assertions[0]?.detail).toMatch(/had no choice/);
-  });
-
-  it("runs several judges concurrently and waits for all of them", async () => {
-    let inFlight = 0;
-    let maxInFlight = 0;
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.judge.closedQA("first");
-          t.judge.closedQA("second");
-          t.judge.closedQA("third");
-        },
-      }),
-      {
-        judgeModel: "gpt-5.4-mini",
-        judgeCaller: async () => {
-          inFlight++;
-          maxInFlight = Math.max(maxInFlight, inFlight);
-          await new Promise((resolve) => setTimeout(resolve, 5));
-          inFlight--;
-          return judgeReply;
-        },
-      }
-    );
-    expect(maxInFlight).toBeGreaterThan(1);
-    expect(result?.assertions.map((a) => a.score)).toEqual([1, 1, 1]);
-  });
-});
-
-describe("t.judge.model", () => {
-  it("gives a custom judge the configured model and credentials", async () => {
-    const seen: Array<{ prompt: string; model?: unknown }> = [];
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          // The shape a domain judge uses: one raw call, then score the parse.
-          const reply = await t.judge.model("Grade these findings: [...]");
-          t.score("recall", JSON.parse(reply).recall as number).atLeast(0.6);
-        },
-      }),
-      {
-        judgeModel: "gpt-5.4-mini",
-        judgeCaller: async (args: { prompt: string; model?: unknown }) => {
-          seen.push(args);
-          return '{"recall":0.8}';
-        },
-      }
-    );
-    expect(seen).toEqual([
-      { prompt: "Grade these findings: [...]", model: "gpt-5.4-mini" },
-    ]);
-    expect(result?.verdict).toBe("passed");
-    expect(result?.assertions[0]).toMatchObject({
-      name: "score(recall)",
-      score: 0.8,
-      threshold: 0.6,
-      passed: true,
-    });
-  });
-
-  it("throws into the test body when no judge model is configured", async () => {
-    const [result] = await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          await t.judge.model("grade this");
-        },
-      })
-    );
-    expect(result?.verdict).toBe("failed");
-    expect(result?.error).toMatch(/no judge model configured/);
-  });
-});
-
-describe("reporters", () => {
-  function recordingReporter(): {
-    reporter: EvalReporter;
-    calls: string[];
-  } {
-    const calls: string[] = [];
-    return {
-      calls,
-      reporter: {
-        onRunStart: (evals, target) => {
-          calls.push(
-            `start:${evals.map((e) => e.id).join(",")}:${target.mode}`
-          );
-        },
-        onEvalComplete: (result) => {
-          calls.push(`case:${result.id}:${result.verdict}`);
-        },
-        onRunComplete: (summary) => {
-          calls.push(
-            `complete:${summary.passed}p/${summary.failed}f/${summary.scored}s`
-          );
-        },
-      },
-    };
-  }
-
-  const twoCases = defineEval({
-    cases: [
-      {
-        id: "good",
-        async test(t) {
-          await t.send("weather?");
-          t.succeeded();
-        },
-      },
-      {
-        id: "bad",
-        async test(t) {
-          await t.send("weather?");
-          t.calledTool("missing");
-        },
-      },
-    ],
-  });
-
-  it("drives the full lifecycle for a CLI reporter", async () => {
-    const { reporter, calls } = recordingReporter();
-    await runCase(twoCases, { reporters: [reporter] });
-    expect(calls).toEqual([
-      "start:smoke/good,smoke/bad:local",
-      "case:smoke/good:passed",
-      "case:smoke/bad:failed",
-      "complete:1p/1f/0s",
-    ]);
-  });
-
-  it("observes every case from a config reporter", async () => {
-    const { reporter, calls } = recordingReporter();
-    await runCase(twoCases, { configReporters: [reporter] });
-    expect(calls.filter((c) => c.startsWith("case:"))).toEqual([
-      "case:smoke/good:passed",
-      "case:smoke/bad:failed",
-    ]);
-  });
-
-  it("scopes a per-eval reporter to its own case", async () => {
-    const { reporter, calls } = recordingReporter();
-    await runCase(
-      defineEval({
-        cases: [
-          {
-            id: "watched",
-            reporters: [reporter],
-            async test(t) {
-              await t.send("weather?");
-              t.succeeded();
-            },
-          },
-          {
-            id: "unwatched",
-            async test(t) {
-              await t.send("weather?");
-              t.succeeded();
-            },
-          },
-        ],
-      })
-    );
-    // Run hooks still fire, but only the referencing case reports a result.
-    expect(calls).toEqual([
-      "start:smoke/watched,smoke/unwatched:local",
-      "case:smoke/watched:passed",
-      "complete:2p/0f/0s",
-    ]);
-  });
-
-  it("suppresses config and per-eval reporters under skipReport", async () => {
-    const config = recordingReporter();
-    const perEval = recordingReporter();
-    const cli = recordingReporter();
-    await runCase(
-      defineEval({
-        reporters: [perEval.reporter],
-        async test(t) {
-          await t.send("weather?");
-          t.succeeded();
-        },
-      }),
-      {
-        configReporters: [config.reporter],
-        reporters: [cli.reporter],
-        skipReport: true,
-      }
-    );
-    expect(config.calls).toEqual([]);
-    expect(perEval.calls).toEqual([]);
-    // CLI-attached reporters are explicit, so they still run.
-    expect(cli.calls).toContain("case:smoke:passed");
-  });
-
-  it("reports strict mode on the summary so reporters can gate on it", async () => {
-    let sawStrict: boolean | undefined;
-    await runCase(
-      defineEval({
-        async test(t) {
-          await t.send("weather?");
-          t.succeeded();
-        },
-      }),
-      {
-        strict: true,
-        reporters: [
-          {
-            onRunComplete: (summary) => {
-              sawStrict = summary.strict;
-            },
-          },
-        ],
-      }
-    );
-    expect(sawStrict).toBe(true);
-  });
-
-  it("keeps running when a reporter throws", async () => {
-    const results = await runCase(twoCases, {
-      reporters: [
-        {
-          onEvalComplete: () => {
-            throw new Error("upload failed");
-          },
-        },
-      ],
-    });
-    expect(results.map((r) => r.verdict)).toEqual(["passed", "failed"]);
-  });
-});
-
-describe("metadata and metrics", () => {
-  it("carries eval metadata and tags onto the result", async () => {
-    const [result] = await runCase(
-      defineEval({
-        tags: ["smoke"],
-        metadata: { owner: "agents", suite: "weather" },
-        async test(t) {
-          await t.send("weather?");
-          t.succeeded();
-          t.metric("recall", "40.0% (4/10)");
-        },
-      })
-    );
-    expect(result).toMatchObject({
-      tags: ["smoke"],
-      metadata: { owner: "agents", suite: "weather" },
-      metrics: { recall: "40.0% (4/10)" },
-    });
-  });
-
-  it("lets a case-level metadata key override the file-level one", async () => {
-    const [result] = await runCase(
-      defineEval({
-        metadata: { owner: "agents", suite: "weather" },
-        cases: [
-          {
-            id: "override",
-            metadata: { owner: "security" },
-            async test(t) {
-              await t.send("weather?");
-              t.succeeded();
-            },
-          },
-        ],
-      })
-    );
-    expect(result?.metadata).toEqual({ owner: "security", suite: "weather" });
-  });
-});
-
-describe("expandEvalModule", () => {
-  it("expands a single defineEval to the file id", () => {
-    const expanded = expandEvalModule(
-      "weather",
-      defineEval({ test: async () => undefined })
-    );
-    expect(expanded.map((e) => e.id)).toEqual(["weather"]);
-  });
-
-  it("fans a default-exported array out with zero-padded ids", () => {
-    const rows = ["a", "b", "c"];
-    const expanded = expandEvalModule(
-      "sql",
-      rows.map((row) =>
-        defineEval({ description: row, test: async () => undefined })
-      ) as never
-    );
-    expect(expanded.map((e) => e.id)).toEqual([
-      "sql/0000",
-      "sql/0001",
-      "sql/0002",
-    ]);
-    expect(expanded[1]?.definition.description).toBe("b");
-  });
-
-  it("rejects a non-eval default export", () => {
-    expect(() => expandEvalModule("x", { nope: true })).toThrow(
-      /must be defineEval\(\{ \.\.\. \}\) or an array of them/
-    );
-    expect(() => expandEvalModule("x", [])).toThrow(/array is empty/);
-    expect(() => expandEvalModule("x", [{ nope: true }])).toThrow(
-      /array entry 0 must be defineEval/
-    );
-  });
-});
-
-describe("discoverEvals", () => {
-  let projectRoot: string;
-
-  beforeEach(async () => {
-    projectRoot = await mkdtemp(join(tmpdir(), "agentkit-eval-discovery-"));
-    await mkdir(join(projectRoot, "evals", "weather"), { recursive: true });
-  });
-
-  afterEach(async () => {
-    await rm(projectRoot, { recursive: true, force: true });
-  });
-
-  /** Map on-disk eval paths to in-memory definitions. */
-  function importer(
-    modules: Record<string, unknown>
-  ): (path: string) => Promise<Record<string, unknown>> {
-    return async (absolutePath: string) => {
-      for (const [suffix, mod] of Object.entries(modules)) {
-        if (absolutePath.endsWith(suffix)) {
-          return { default: mod };
-        }
-      }
-      throw new Error(`unexpected import: ${absolutePath}`);
-    };
-  }
-
-  it("derives ids from paths and loads the config", async () => {
-    await writeFile(join(projectRoot, "evals/evals.config.ts"), "", "utf8");
-    await writeFile(join(projectRoot, "evals/smoke.eval.ts"), "", "utf8");
-    await writeFile(join(projectRoot, "evals/weather/nyc.eval.ts"), "", "utf8");
-    // A sibling helper that is not an eval file must be ignored.
-    await writeFile(join(projectRoot, "evals/weather/shared.ts"), "", "utf8");
-
-    const { evals, config } = await discoverEvals(
-      projectRoot,
-      importer({
-        "evals.config.ts": defineEvalConfig({ maxConcurrency: 7 }),
-        "smoke.eval.ts": defineEval({ test: async () => undefined }),
-        "nyc.eval.ts": defineEval({ test: async () => undefined }),
-      })
-    );
-    expect(evals.map((e) => e.id)).toEqual(["smoke", "weather/nyc"]);
-    expect(config.maxConcurrency).toBe(7);
-  });
-
-  it("points fixture loaders at the app root", async () => {
-    await writeFile(join(projectRoot, "evals/evals.config.ts"), "", "utf8");
-    await discoverEvals(
-      projectRoot,
-      importer({ "evals.config.ts": defineEvalConfig({ maxConcurrency: 1 }) })
-    );
-    expect(process.env[EVAL_ROOT_ENV]).toBe(projectRoot);
-  });
-
-  it("prefixes the failing file on a bad default export", async () => {
-    await writeFile(join(projectRoot, "evals/broken.eval.ts"), "", "utf8");
-    await expect(
-      discoverEvals(projectRoot, importer({ "broken.eval.ts": { nope: true } }))
-    ).rejects.toThrow(/broken\.eval\.ts: default export must be defineEval/);
-  });
-});
-
-describe("cancellation", () => {
-  it("returns a cancelled result without driving the agent", async () => {
-    const controller = new AbortController();
-    controller.abort();
-    const results = await runDiscoveredEvals({
-      projectRoot: "/unused",
-      baseUrl: "http://127.0.0.1:3000",
-      signal: controller.signal,
-      discovered: {
-        evals: expandEvalModule(
-          "smoke",
-          defineEval({
-            async test(t: EvalTestContext) {
-              await t.send("weather?");
-            },
-          })
-        ).map((e) => ({
-          id: e.id,
-          fileId: "smoke",
-          path: "/repo/evals/smoke.eval.ts",
-          definition: e.definition,
-        })),
-        config: { maxConcurrency: 1 },
-      },
-    });
-    expect(results).toEqual([]);
-    expect(runSession).not.toHaveBeenCalled();
-  });
-});