@cursor/july 0.1.20 → 0.1.21

package/src/evals/reporters.test.ts

@@ -0,0 +1,303 @@
+import { mkdtemp, readFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  Artifacts,
+  combineReporters,
+  JUnit,
+  renderJUnitXml,
+  sanitizeCaseId,
+} from "./reporters.js";
+import type { EvalReporter, EvalRunResult, EvalRunSummary } from "./results.js";
+import { summarizeEvalResults } from "./results.js";
+
+const dirs: string[] = [];
+
+async function tempDir(): Promise<string> {
+  const dir = await mkdtemp(join(tmpdir(), "agentkit-eval-reporters-"));
+  dirs.push(dir);
+  return dir;
+}
+
+afterEach(async () => {
+  await Promise.all(
+    dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))
+  );
+});
+
+function result(overrides: Partial<EvalRunResult> = {}): EvalRunResult {
+  return {
+    id: "weather/nyc",
+    path: "/repo/evals/weather.eval.ts",
+    ok: true,
+    verdict: "passed",
+    assertions: [],
+    inputs: ["What is the weather in NYC?"],
+    logs: [],
+    metrics: {},
+    durationMs: 1500,
+    ...overrides,
+  };
+}
+
+function summary(results: EvalRunResult[], strict = false): EvalRunSummary {
+  return summarizeEvalResults(results, {
+    strict,
+    startedAt: "2026-01-01T00:00:00.000Z",
+    finishedAt: "2026-01-01T00:00:05.000Z",
+  });
+}
+
+describe("summarizeEvalResults", () => {
+  it("tallies every verdict and the wall time", () => {
+    const totals = summary([
+      result(),
+      result({ id: "b", verdict: "failed", ok: false }),
+      result({ id: "c", verdict: "scored" }),
+      result({ id: "d", verdict: "skipped", skipReason: "no judge model" }),
+    ]);
+    expect(totals).toMatchObject({
+      total: 4,
+      passed: 1,
+      failed: 1,
+      scored: 1,
+      skipped: 1,
+      durationMs: 5000,
+    });
+  });
+
+  it("infers a verdict from ok for results written before verdicts existed", () => {
+    const totals = summary([
+      { ...result(), verdict: undefined },
+      { ...result({ id: "b", ok: false }), verdict: undefined },
+    ]);
+    expect(totals).toMatchObject({ passed: 1, failed: 1 });
+  });
+});
+
+describe("renderJUnitXml", () => {
+  it("emits one testcase per eval with scores on system-out", () => {
+    const xml = renderJUnitXml(
+      summary([
+        result({
+          assertions: [
+            { name: "succeeded", passed: true },
+            {
+              name: "judge.closedQA(cites a source)",
+              passed: true,
+              severity: "soft",
+              score: 0.75,
+              threshold: 0.6,
+            },
+          ],
+          metrics: { recall: "40.0% (4/10)" },
+          logs: ["matched=[a, b]"],
+        }),
+      ]),
+      "agentkit-evals"
+    );
+    expect(xml).toContain('<testcase name="weather/nyc"');
+    expect(xml).not.toContain("<failure");
+    expect(xml).toContain("judge.closedQA(cites a source)=0.750 (&gt;= 0.6)");
+    expect(xml).toContain("recall=40.0% (4/10)");
+    expect(xml).toContain("matched=[a, b]");
+  });
+
+  it("reports a failed gate as a failure with the assertion detail", () => {
+    const xml = renderJUnitXml(
+      summary([
+        result({
+          verdict: "failed",
+          ok: false,
+          assertions: [
+            {
+              name: "calledTool(get_weather)",
+              passed: false,
+              detail: "called []",
+            },
+          ],
+        }),
+      ]),
+      "suite"
+    );
+    expect(xml).toContain('failures="1"');
+    expect(xml).toContain(
+      '<failure message="calledTool(get_weather) (called [])" type="failed"/>'
+    );
+  });
+
+  it("prefers the execution error message when the test body threw", () => {
+    const xml = renderJUnitXml(
+      summary([
+        result({ verdict: "failed", ok: false, error: "turn timed out" }),
+      ]),
+      "suite"
+    );
+    expect(xml).toContain('<failure message="turn timed out"');
+  });
+
+  it("marks a skipped eval skipped and keeps it out of the failure count", () => {
+    const xml = renderJUnitXml(
+      summary([
+        result({ verdict: "skipped", skipReason: "no judge credentials" }),
+      ]),
+      "suite"
+    );
+    expect(xml).toContain('<skipped message="no judge credentials"/>');
+    expect(xml).toContain('failures="0"');
+    expect(xml).toContain('skipped="1"');
+  });
+
+  it("only fails a scored eval under strict", () => {
+    const scored = [result({ verdict: "scored" })];
+    expect(renderJUnitXml(summary(scored, false), "s")).not.toContain(
+      "<failure"
+    );
+    const strict = renderJUnitXml(summary(scored, true), "s");
+    expect(strict).toContain('type="scored"');
+    expect(strict).toContain('failures="1"');
+  });
+
+  it("escapes XML metacharacters and strips illegal control characters", () => {
+    const xml = renderJUnitXml(
+      summary([
+        result({
+          id: 'weird & "id" <x>',
+          verdict: "failed",
+          ok: false,
+          error: "bad\u0000output",
+        }),
+      ]),
+      "suite"
+    );
+    expect(xml).toContain('name="weird &amp; &quot;id&quot; &lt;x&gt;"');
+    expect(xml).toContain('message="badoutput"');
+    expect(xml).not.toContain("\u0000");
+  });
+});
+
+describe("JUnit reporter", () => {
+  it("writes the XML file, creating parent directories", async () => {
+    const dir = await tempDir();
+    const filePath = join(dir, "nested", "junit.xml");
+    const reporter = JUnit({ filePath });
+    await reporter.onRunComplete?.(summary([result()]));
+    const xml = await readFile(filePath, "utf8");
+    expect(xml).toContain('<testcase name="weather/nyc"');
+  });
+});
+
+describe("Artifacts reporter", () => {
+  it("writes a summary, an index, and per-case detail", async () => {
+    const dir = await tempDir();
+    const reporter = Artifacts({ dir });
+    const passing = result();
+    const failing = result({
+      id: "weather/paris",
+      verdict: "failed",
+      ok: false,
+    });
+
+    await reporter.onRunStart?.(
+      [
+        { id: "weather/nyc", fileId: "weather" },
+        { id: "weather/paris", fileId: "weather" },
+      ],
+      { baseUrl: "http://127.0.0.1:3000", mode: "local" }
+    );
+    await reporter.onEvalComplete?.(passing);
+    await reporter.onEvalComplete?.(failing);
+    await reporter.onRunComplete?.(summary([passing, failing]));
+
+    const runSummary = JSON.parse(
+      await readFile(join(dir, "summary.json"), "utf8")
+    ) as Record<string, unknown>;
+    expect(runSummary).toMatchObject({
+      total: 2,
+      passed: 1,
+      failed: 1,
+      caseCount: 2,
+      target: { baseUrl: "http://127.0.0.1:3000", mode: "local" },
+      discovered: ["weather/nyc", "weather/paris"],
+    });
+    // The full result list stays in the per-case files, not the summary.
+    expect(runSummary.results).toBeUndefined();
+
+    const index = (await readFile(join(dir, "results.jsonl"), "utf8"))
+      .trim()
+      .split("\n")
+      .map((line) => JSON.parse(line) as Record<string, unknown>);
+    expect(index).toEqual([
+      { id: "weather/nyc", verdict: "passed", ok: true, durationMs: 1500 },
+      { id: "weather/paris", verdict: "failed", ok: false, durationMs: 1500 },
+    ]);
+
+    const detail = JSON.parse(
+      await readFile(join(dir, "evals", "weather", "nyc.json"), "utf8")
+    ) as EvalRunResult;
+    expect(detail.inputs).toEqual(["What is the weather in NYC?"]);
+  });
+
+  it("keeps a hand-built case id inside the run directory", () => {
+    expect(sanitizeCaseId("weather/nyc")).toBe("weather/nyc");
+    expect(sanitizeCaseId("../../etc/passwd")).toBe("etc/passwd");
+    expect(sanitizeCaseId("a/../b")).toBe("a/b");
+    expect(sanitizeCaseId("weird name!")).toBe("weird_name_");
+  });
+});
+
+describe("combineReporters", () => {
+  it("fans every hook out to each reporter", async () => {
+    const calls: string[] = [];
+    const make = (name: string): EvalReporter => ({
+      onRunStart: () => {
+        calls.push(`${name}:start`);
+      },
+      onEvalComplete: () => {
+        calls.push(`${name}:case`);
+      },
+      onRunComplete: () => {
+        calls.push(`${name}:complete`);
+      },
+    });
+    const combined = combineReporters([make("a"), make("b")]);
+    await combined.onRunStart?.([], { baseUrl: "x", mode: "local" });
+    await combined.onEvalComplete?.(result());
+    await combined.onRunComplete?.(summary([result()]));
+    expect(calls).toEqual([
+      "a:start",
+      "b:start",
+      "a:case",
+      "b:case",
+      "a:complete",
+      "b:complete",
+    ]);
+  });
+
+  it("isolates a throwing reporter so the run still finishes", async () => {
+    const onError = vi.fn();
+    const healthy = vi.fn();
+    const combined = combineReporters(
+      [
+        {
+          onEvalComplete: () => {
+            throw new Error("upload failed");
+          },
+        },
+        { onEvalComplete: healthy },
+      ],
+      onError
+    );
+    await combined.onEvalComplete?.(result());
+    expect(onError).toHaveBeenCalledTimes(1);
+    expect(healthy).toHaveBeenCalledTimes(1);
+  });
+
+  it("tolerates reporters that implement only some hooks", async () => {
+    const combined = combineReporters([{}]);
+    await expect(
+      combined.onRunComplete?.(summary([result()]))
+    ).resolves.toBeUndefined();
+  });
+});

package/src/evals/run-facts.test.ts

@@ -0,0 +1,259 @@
+import { describe, expect, it } from "vitest";
+import type { SessionEvent } from "../types.js";
+import { deriveRunFacts } from "./run-facts.js";
+
+let sequence = 0;
+
+function event(
+  type: string,
+  data: unknown,
+  extra: { turnId?: string } = {}
+): SessionEvent {
+  sequence++;
+  return {
+    sessionId: "ses_1",
+    seq: sequence,
+    at: new Date(sequence * 1000).toISOString(),
+    turnId: extra.turnId ?? "turn_1",
+    type,
+    data,
+  } as unknown as SessionEvent;
+}
+
+function toolTurn(): SessionEvent[] {
+  return [
+    event("message.received", { text: "weather in NYC?" }),
+    event("actions.requested", {
+      calls: [{ callId: "c1", toolName: "get_weather", args: { city: "NYC" } }],
+    }),
+    event("action.result", {
+      callId: "c1",
+      toolName: "get_weather",
+      output: { tempF: 72 },
+      isError: false,
+    }),
+    event("message.completed", { text: "Sunny, 72F", finishReason: "stop" }),
+    event("turn.completed", {}),
+    event("session.waiting", {}),
+  ];
+}
+
+describe("deriveRunFacts tool lifecycle", () => {
+  it("resolves a requested call with its result instead of duplicating it", () => {
+    const facts = deriveRunFacts(toolTurn());
+    expect(facts.toolCalls).toHaveLength(1);
+    expect(facts.toolCalls[0]).toMatchObject({
+      callId: "c1",
+      toolName: "get_weather",
+      input: { city: "NYC" },
+      output: { tempF: 72 },
+      status: "completed",
+      index: 0,
+    });
+    expect(facts.ok).toBe(true);
+    expect(facts.failedToolCalls).toEqual([]);
+  });
+
+  it("marks a call pending until its result arrives", () => {
+    const events = toolTurn().filter((e) => e.type !== "action.result");
+    const facts = deriveRunFacts(events);
+    expect(facts.toolCalls[0]?.status).toBe("pending");
+  });
+
+  it("marks an errored call failed and lists it under failedToolCalls", () => {
+    const events = [
+      event("actions.requested", {
+        calls: [{ callId: "c1", toolName: "bash", args: { command: "false" } }],
+      }),
+      event("action.result", {
+        callId: "c1",
+        toolName: "bash",
+        output: "exit 1",
+        isError: true,
+      }),
+      event("turn.completed", {}),
+    ];
+    const facts = deriveRunFacts(events);
+    expect(facts.toolCalls[0]?.status).toBe("failed");
+    expect(facts.failedToolCalls.map((c) => c.toolName)).toEqual(["bash"]);
+    // A failed tool does not by itself fail the turn.
+    expect(facts.ok).toBe(true);
+  });
+
+  it("records a result-only call so a dropped request event still counts", () => {
+    const facts = deriveRunFacts([
+      event("action.result", {
+        callId: "c9",
+        toolName: "read_file",
+        output: "hi",
+        isError: false,
+      }),
+      event("turn.completed", {}),
+    ]);
+    expect(facts.toolCalls.map((c) => c.toolName)).toEqual(["read_file"]);
+    expect(facts.toolCalls[0]?.status).toBe("completed");
+  });
+
+  it("keeps calls in request order across turns", () => {
+    const facts = deriveRunFacts([
+      event("actions.requested", {
+        calls: [{ callId: "a", toolName: "first", args: {} }],
+      }),
+      event(
+        "actions.requested",
+        { calls: [{ callId: "b", toolName: "second", args: {} }] },
+        { turnId: "turn_2" }
+      ),
+      event("turn.completed", {}, { turnId: "turn_2" }),
+    ]);
+    expect(facts.toolCalls.map((c) => c.toolName)).toEqual(["first", "second"]);
+    expect(facts.turns).toBe(2);
+  });
+});
+
+describe("deriveRunFacts human-in-the-loop", () => {
+  it("parks on an unanswered approval request", () => {
+    const facts = deriveRunFacts([
+      event("actions.requested", {
+        calls: [{ callId: "c1", toolName: "deploy", args: { env: "prod" } }],
+      }),
+      event("action.approval_requested", {
+        callId: "c1",
+        toolName: "deploy",
+        args: { env: "prod" },
+      }),
+      event("session.waiting", {}),
+    ]);
+    expect(facts.parked).toBe(true);
+    expect(facts.pendingInputRequests).toHaveLength(1);
+    expect(facts.pendingInputRequests[0]).toMatchObject({
+      toolName: "deploy",
+      args: { env: "prod" },
+      resolved: false,
+    });
+    expect(facts.toolCalls[0]?.status).toBe("pending");
+  });
+
+  it("is not parked once the approval resolves", () => {
+    const facts = deriveRunFacts([
+      event("actions.requested", {
+        calls: [{ callId: "c1", toolName: "deploy", args: {} }],
+      }),
+      event("action.approval_requested", { callId: "c1", toolName: "deploy" }),
+      event("action.approval_resolved", {
+        callId: "c1",
+        toolName: "deploy",
+        decision: "approve",
+      }),
+      event("action.result", {
+        callId: "c1",
+        toolName: "deploy",
+        output: "ok",
+        isError: false,
+      }),
+      event("turn.completed", {}),
+    ]);
+    expect(facts.parked).toBe(false);
+    expect(facts.toolCalls[0]).toMatchObject({
+      status: "completed",
+      approvalDecision: "approve",
+    });
+  });
+
+  it("keeps a denied call rejected even after its synthetic error result", () => {
+    const facts = deriveRunFacts([
+      event("actions.requested", {
+        calls: [{ callId: "c1", toolName: "deploy", args: {} }],
+      }),
+      event("action.approval_requested", { callId: "c1", toolName: "deploy" }),
+      event("action.approval_resolved", {
+        callId: "c1",
+        toolName: "deploy",
+        decision: "deny",
+      }),
+      event("action.result", {
+        callId: "c1",
+        toolName: "deploy",
+        output: "denied by user",
+        isError: true,
+      }),
+      event("turn.completed", {}),
+    ]);
+    expect(facts.toolCalls[0]?.status).toBe("rejected");
+    // A rejection is a policy outcome, not a broken tool.
+    expect(facts.failedToolCalls).toEqual([]);
+  });
+});
+
+describe("deriveRunFacts subagents and failures", () => {
+  it("tracks a subagent delegation and its output", () => {
+    const facts = deriveRunFacts([
+      event("actions.requested", {
+        calls: [{ callId: "t1", toolName: "task", args: { name: "research" } }],
+      }),
+      event("subagent.called", {
+        callId: "t1",
+        name: "research",
+        description: "look it up",
+      }),
+      event("message.completed", {
+        text: "nested chatter",
+        finishReason: "stop",
+        parentCallId: "t1",
+      }),
+      event("subagent.completed", { callId: "t1", name: "research" }),
+      event("action.result", {
+        callId: "t1",
+        toolName: "task",
+        output: "found it",
+        isError: false,
+      }),
+      event("message.completed", { text: "Done.", finishReason: "stop" }),
+      event("turn.completed", {}),
+    ]);
+    expect(facts.subagents).toHaveLength(1);
+    expect(facts.subagents[0]).toMatchObject({
+      name: "research",
+      status: "completed",
+      output: "found it",
+    });
+    // Nested subagent text is not part of the agent's own reply.
+    expect(facts.assistantMessages).toEqual(["Done."]);
+  });
+
+  it("fails the run on a failed turn", () => {
+    const facts = deriveRunFacts([
+      event("message.received", { text: "hi" }),
+      event("turn.failed", { message: "model error" }),
+    ]);
+    expect(facts.ok).toBe(false);
+    expect(facts.turnsFailed).toBe(1);
+    expect(facts.failureMessage).toBe("model error");
+  });
+
+  it("fails the run on a session failure with no turn id", () => {
+    const events = [
+      event("message.received", { text: "hi" }),
+      {
+        ...event("session.failed", { message: "transport died" }),
+        turnId: undefined,
+      },
+    ] as SessionEvent[];
+    const facts = deriveRunFacts(events);
+    expect(facts.ok).toBe(false);
+    expect(facts.failureMessage).toBe("transport died");
+  });
+
+  it("is not ok with no turns at all", () => {
+    expect(deriveRunFacts([]).ok).toBe(false);
+  });
+
+  it("joins assistant messages in order", () => {
+    const facts = deriveRunFacts([
+      event("message.completed", { text: "first", finishReason: "tool_call" }),
+      event("message.completed", { text: "second", finishReason: "stop" }),
+      event("turn.completed", {}),
+    ]);
+    expect(facts.assistantText).toBe("first\nsecond");
+  });
+});