@cursor/july 0.1.26 → 0.1.28

package/src/evals/reporters.test.ts

@@ -1,303 +0,0 @@
-import { mkdtemp, readFile, rm } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { afterEach, describe, expect, it, vi } from "vitest";
-import {
-  Artifacts,
-  combineReporters,
-  JUnit,
-  renderJUnitXml,
-  sanitizeCaseId,
-} from "./reporters.js";
-import type { EvalReporter, EvalRunResult, EvalRunSummary } from "./results.js";
-import { summarizeEvalResults } from "./results.js";
-
-const dirs: string[] = [];
-
-async function tempDir(): Promise<string> {
-  const dir = await mkdtemp(join(tmpdir(), "agentkit-eval-reporters-"));
-  dirs.push(dir);
-  return dir;
-}
-
-afterEach(async () => {
-  await Promise.all(
-    dirs.splice(0).map((dir) => rm(dir, { recursive: true, force: true }))
-  );
-});
-
-function result(overrides: Partial<EvalRunResult> = {}): EvalRunResult {
-  return {
-    id: "weather/nyc",
-    path: "/repo/evals/weather.eval.ts",
-    ok: true,
-    verdict: "passed",
-    assertions: [],
-    inputs: ["What is the weather in NYC?"],
-    logs: [],
-    metrics: {},
-    durationMs: 1500,
-    ...overrides,
-  };
-}
-
-function summary(results: EvalRunResult[], strict = false): EvalRunSummary {
-  return summarizeEvalResults(results, {
-    strict,
-    startedAt: "2026-01-01T00:00:00.000Z",
-    finishedAt: "2026-01-01T00:00:05.000Z",
-  });
-}
-
-describe("summarizeEvalResults", () => {
-  it("tallies every verdict and the wall time", () => {
-    const totals = summary([
-      result(),
-      result({ id: "b", verdict: "failed", ok: false }),
-      result({ id: "c", verdict: "scored" }),
-      result({ id: "d", verdict: "skipped", skipReason: "no judge model" }),
-    ]);
-    expect(totals).toMatchObject({
-      total: 4,
-      passed: 1,
-      failed: 1,
-      scored: 1,
-      skipped: 1,
-      durationMs: 5000,
-    });
-  });
-
-  it("infers a verdict from ok for results written before verdicts existed", () => {
-    const totals = summary([
-      { ...result(), verdict: undefined },
-      { ...result({ id: "b", ok: false }), verdict: undefined },
-    ]);
-    expect(totals).toMatchObject({ passed: 1, failed: 1 });
-  });
-});
-
-describe("renderJUnitXml", () => {
-  it("emits one testcase per eval with scores on system-out", () => {
-    const xml = renderJUnitXml(
-      summary([
-        result({
-          assertions: [
-            { name: "succeeded", passed: true },
-            {
-              name: "judge.closedQA(cites a source)",
-              passed: true,
-              severity: "soft",
-              score: 0.75,
-              threshold: 0.6,
-            },
-          ],
-          metrics: { recall: "40.0% (4/10)" },
-          logs: ["matched=[a, b]"],
-        }),
-      ]),
-      "agentkit-evals"
-    );
-    expect(xml).toContain('<testcase name="weather/nyc"');
-    expect(xml).not.toContain("<failure");
-    expect(xml).toContain("judge.closedQA(cites a source)=0.750 (&gt;= 0.6)");
-    expect(xml).toContain("recall=40.0% (4/10)");
-    expect(xml).toContain("matched=[a, b]");
-  });
-
-  it("reports a failed gate as a failure with the assertion detail", () => {
-    const xml = renderJUnitXml(
-      summary([
-        result({
-          verdict: "failed",
-          ok: false,
-          assertions: [
-            {
-              name: "calledTool(get_weather)",
-              passed: false,
-              detail: "called []",
-            },
-          ],
-        }),
-      ]),
-      "suite"
-    );
-    expect(xml).toContain('failures="1"');
-    expect(xml).toContain(
-      '<failure message="calledTool(get_weather) (called [])" type="failed"/>'
-    );
-  });
-
-  it("prefers the execution error message when the test body threw", () => {
-    const xml = renderJUnitXml(
-      summary([
-        result({ verdict: "failed", ok: false, error: "turn timed out" }),
-      ]),
-      "suite"
-    );
-    expect(xml).toContain('<failure message="turn timed out"');
-  });
-
-  it("marks a skipped eval skipped and keeps it out of the failure count", () => {
-    const xml = renderJUnitXml(
-      summary([
-        result({ verdict: "skipped", skipReason: "no judge credentials" }),
-      ]),
-      "suite"
-    );
-    expect(xml).toContain('<skipped message="no judge credentials"/>');
-    expect(xml).toContain('failures="0"');
-    expect(xml).toContain('skipped="1"');
-  });
-
-  it("only fails a scored eval under strict", () => {
-    const scored = [result({ verdict: "scored" })];
-    expect(renderJUnitXml(summary(scored, false), "s")).not.toContain(
-      "<failure"
-    );
-    const strict = renderJUnitXml(summary(scored, true), "s");
-    expect(strict).toContain('type="scored"');
-    expect(strict).toContain('failures="1"');
-  });
-
-  it("escapes XML metacharacters and strips illegal control characters", () => {
-    const xml = renderJUnitXml(
-      summary([
-        result({
-          id: 'weird & "id" <x>',
-          verdict: "failed",
-          ok: false,
-          error: "bad\u0000output",
-        }),
-      ]),
-      "suite"
-    );
-    expect(xml).toContain('name="weird &amp; &quot;id&quot; &lt;x&gt;"');
-    expect(xml).toContain('message="badoutput"');
-    expect(xml).not.toContain("\u0000");
-  });
-});
-
-describe("JUnit reporter", () => {
-  it("writes the XML file, creating parent directories", async () => {
-    const dir = await tempDir();
-    const filePath = join(dir, "nested", "junit.xml");
-    const reporter = JUnit({ filePath });
-    await reporter.onRunComplete?.(summary([result()]));
-    const xml = await readFile(filePath, "utf8");
-    expect(xml).toContain('<testcase name="weather/nyc"');
-  });
-});
-
-describe("Artifacts reporter", () => {
-  it("writes a summary, an index, and per-case detail", async () => {
-    const dir = await tempDir();
-    const reporter = Artifacts({ dir });
-    const passing = result();
-    const failing = result({
-      id: "weather/paris",
-      verdict: "failed",
-      ok: false,
-    });
-
-    await reporter.onRunStart?.(
-      [
-        { id: "weather/nyc", fileId: "weather" },
-        { id: "weather/paris", fileId: "weather" },
-      ],
-      { baseUrl: "http://127.0.0.1:3000", mode: "local" }
-    );
-    await reporter.onEvalComplete?.(passing);
-    await reporter.onEvalComplete?.(failing);
-    await reporter.onRunComplete?.(summary([passing, failing]));
-
-    const runSummary = JSON.parse(
-      await readFile(join(dir, "summary.json"), "utf8")
-    ) as Record<string, unknown>;
-    expect(runSummary).toMatchObject({
-      total: 2,
-      passed: 1,
-      failed: 1,
-      caseCount: 2,
-      target: { baseUrl: "http://127.0.0.1:3000", mode: "local" },
-      discovered: ["weather/nyc", "weather/paris"],
-    });
-    // The full result list stays in the per-case files, not the summary.
-    expect(runSummary.results).toBeUndefined();
-
-    const index = (await readFile(join(dir, "results.jsonl"), "utf8"))
-      .trim()
-      .split("\n")
-      .map((line) => JSON.parse(line) as Record<string, unknown>);
-    expect(index).toEqual([
-      { id: "weather/nyc", verdict: "passed", ok: true, durationMs: 1500 },
-      { id: "weather/paris", verdict: "failed", ok: false, durationMs: 1500 },
-    ]);
-
-    const detail = JSON.parse(
-      await readFile(join(dir, "evals", "weather", "nyc.json"), "utf8")
-    ) as EvalRunResult;
-    expect(detail.inputs).toEqual(["What is the weather in NYC?"]);
-  });
-
-  it("keeps a hand-built case id inside the run directory", () => {
-    expect(sanitizeCaseId("weather/nyc")).toBe("weather/nyc");
-    expect(sanitizeCaseId("../../etc/passwd")).toBe("etc/passwd");
-    expect(sanitizeCaseId("a/../b")).toBe("a/b");
-    expect(sanitizeCaseId("weird name!")).toBe("weird_name_");
-  });
-});
-
-describe("combineReporters", () => {
-  it("fans every hook out to each reporter", async () => {
-    const calls: string[] = [];
-    const make = (name: string): EvalReporter => ({
-      onRunStart: () => {
-        calls.push(`${name}:start`);
-      },
-      onEvalComplete: () => {
-        calls.push(`${name}:case`);
-      },
-      onRunComplete: () => {
-        calls.push(`${name}:complete`);
-      },
-    });
-    const combined = combineReporters([make("a"), make("b")]);
-    await combined.onRunStart?.([], { baseUrl: "x", mode: "local" });
-    await combined.onEvalComplete?.(result());
-    await combined.onRunComplete?.(summary([result()]));
-    expect(calls).toEqual([
-      "a:start",
-      "b:start",
-      "a:case",
-      "b:case",
-      "a:complete",
-      "b:complete",
-    ]);
-  });
-
-  it("isolates a throwing reporter so the run still finishes", async () => {
-    const onError = vi.fn();
-    const healthy = vi.fn();
-    const combined = combineReporters(
-      [
-        {
-          onEvalComplete: () => {
-            throw new Error("upload failed");
-          },
-        },
-        { onEvalComplete: healthy },
-      ],
-      onError
-    );
-    await combined.onEvalComplete?.(result());
-    expect(onError).toHaveBeenCalledTimes(1);
-    expect(healthy).toHaveBeenCalledTimes(1);
-  });
-
-  it("tolerates reporters that implement only some hooks", async () => {
-    const combined = combineReporters([{}]);
-    await expect(
-      combined.onRunComplete?.(summary([result()]))
-    ).resolves.toBeUndefined();
-  });
-});

package/src/evals/run-facts.test.ts

@@ -1,259 +0,0 @@
-import { describe, expect, it } from "vitest";
-import type { SessionEvent } from "../types.js";
-import { deriveRunFacts } from "./run-facts.js";
-
-let sequence = 0;
-
-function event(
-  type: string,
-  data: unknown,
-  extra: { turnId?: string } = {}
-): SessionEvent {
-  sequence++;
-  return {
-    sessionId: "ses_1",
-    seq: sequence,
-    at: new Date(sequence * 1000).toISOString(),
-    turnId: extra.turnId ?? "turn_1",
-    type,
-    data,
-  } as unknown as SessionEvent;
-}
-
-function toolTurn(): SessionEvent[] {
-  return [
-    event("message.received", { text: "weather in NYC?" }),
-    event("actions.requested", {
-      calls: [{ callId: "c1", toolName: "get_weather", args: { city: "NYC" } }],
-    }),
-    event("action.result", {
-      callId: "c1",
-      toolName: "get_weather",
-      output: { tempF: 72 },
-      isError: false,
-    }),
-    event("message.completed", { text: "Sunny, 72F", finishReason: "stop" }),
-    event("turn.completed", {}),
-    event("session.waiting", {}),
-  ];
-}
-
-describe("deriveRunFacts tool lifecycle", () => {
-  it("resolves a requested call with its result instead of duplicating it", () => {
-    const facts = deriveRunFacts(toolTurn());
-    expect(facts.toolCalls).toHaveLength(1);
-    expect(facts.toolCalls[0]).toMatchObject({
-      callId: "c1",
-      toolName: "get_weather",
-      input: { city: "NYC" },
-      output: { tempF: 72 },
-      status: "completed",
-      index: 0,
-    });
-    expect(facts.ok).toBe(true);
-    expect(facts.failedToolCalls).toEqual([]);
-  });
-
-  it("marks a call pending until its result arrives", () => {
-    const events = toolTurn().filter((e) => e.type !== "action.result");
-    const facts = deriveRunFacts(events);
-    expect(facts.toolCalls[0]?.status).toBe("pending");
-  });
-
-  it("marks an errored call failed and lists it under failedToolCalls", () => {
-    const events = [
-      event("actions.requested", {
-        calls: [{ callId: "c1", toolName: "bash", args: { command: "false" } }],
-      }),
-      event("action.result", {
-        callId: "c1",
-        toolName: "bash",
-        output: "exit 1",
-        isError: true,
-      }),
-      event("turn.completed", {}),
-    ];
-    const facts = deriveRunFacts(events);
-    expect(facts.toolCalls[0]?.status).toBe("failed");
-    expect(facts.failedToolCalls.map((c) => c.toolName)).toEqual(["bash"]);
-    // A failed tool does not by itself fail the turn.
-    expect(facts.ok).toBe(true);
-  });
-
-  it("records a result-only call so a dropped request event still counts", () => {
-    const facts = deriveRunFacts([
-      event("action.result", {
-        callId: "c9",
-        toolName: "read_file",
-        output: "hi",
-        isError: false,
-      }),
-      event("turn.completed", {}),
-    ]);
-    expect(facts.toolCalls.map((c) => c.toolName)).toEqual(["read_file"]);
-    expect(facts.toolCalls[0]?.status).toBe("completed");
-  });
-
-  it("keeps calls in request order across turns", () => {
-    const facts = deriveRunFacts([
-      event("actions.requested", {
-        calls: [{ callId: "a", toolName: "first", args: {} }],
-      }),
-      event(
-        "actions.requested",
-        { calls: [{ callId: "b", toolName: "second", args: {} }] },
-        { turnId: "turn_2" }
-      ),
-      event("turn.completed", {}, { turnId: "turn_2" }),
-    ]);
-    expect(facts.toolCalls.map((c) => c.toolName)).toEqual(["first", "second"]);
-    expect(facts.turns).toBe(2);
-  });
-});
-
-describe("deriveRunFacts human-in-the-loop", () => {
-  it("parks on an unanswered approval request", () => {
-    const facts = deriveRunFacts([
-      event("actions.requested", {
-        calls: [{ callId: "c1", toolName: "deploy", args: { env: "prod" } }],
-      }),
-      event("action.approval_requested", {
-        callId: "c1",
-        toolName: "deploy",
-        args: { env: "prod" },
-      }),
-      event("session.waiting", {}),
-    ]);
-    expect(facts.parked).toBe(true);
-    expect(facts.pendingInputRequests).toHaveLength(1);
-    expect(facts.pendingInputRequests[0]).toMatchObject({
-      toolName: "deploy",
-      args: { env: "prod" },
-      resolved: false,
-    });
-    expect(facts.toolCalls[0]?.status).toBe("pending");
-  });
-
-  it("is not parked once the approval resolves", () => {
-    const facts = deriveRunFacts([
-      event("actions.requested", {
-        calls: [{ callId: "c1", toolName: "deploy", args: {} }],
-      }),
-      event("action.approval_requested", { callId: "c1", toolName: "deploy" }),
-      event("action.approval_resolved", {
-        callId: "c1",
-        toolName: "deploy",
-        decision: "approve",
-      }),
-      event("action.result", {
-        callId: "c1",
-        toolName: "deploy",
-        output: "ok",
-        isError: false,
-      }),
-      event("turn.completed", {}),
-    ]);
-    expect(facts.parked).toBe(false);
-    expect(facts.toolCalls[0]).toMatchObject({
-      status: "completed",
-      approvalDecision: "approve",
-    });
-  });
-
-  it("keeps a denied call rejected even after its synthetic error result", () => {
-    const facts = deriveRunFacts([
-      event("actions.requested", {
-        calls: [{ callId: "c1", toolName: "deploy", args: {} }],
-      }),
-      event("action.approval_requested", { callId: "c1", toolName: "deploy" }),
-      event("action.approval_resolved", {
-        callId: "c1",
-        toolName: "deploy",
-        decision: "deny",
-      }),
-      event("action.result", {
-        callId: "c1",
-        toolName: "deploy",
-        output: "denied by user",
-        isError: true,
-      }),
-      event("turn.completed", {}),
-    ]);
-    expect(facts.toolCalls[0]?.status).toBe("rejected");
-    // A rejection is a policy outcome, not a broken tool.
-    expect(facts.failedToolCalls).toEqual([]);
-  });
-});
-
-describe("deriveRunFacts subagents and failures", () => {
-  it("tracks a subagent delegation and its output", () => {
-    const facts = deriveRunFacts([
-      event("actions.requested", {
-        calls: [{ callId: "t1", toolName: "task", args: { name: "research" } }],
-      }),
-      event("subagent.called", {
-        callId: "t1",
-        name: "research",
-        description: "look it up",
-      }),
-      event("message.completed", {
-        text: "nested chatter",
-        finishReason: "stop",
-        parentCallId: "t1",
-      }),
-      event("subagent.completed", { callId: "t1", name: "research" }),
-      event("action.result", {
-        callId: "t1",
-        toolName: "task",
-        output: "found it",
-        isError: false,
-      }),
-      event("message.completed", { text: "Done.", finishReason: "stop" }),
-      event("turn.completed", {}),
-    ]);
-    expect(facts.subagents).toHaveLength(1);
-    expect(facts.subagents[0]).toMatchObject({
-      name: "research",
-      status: "completed",
-      output: "found it",
-    });
-    // Nested subagent text is not part of the agent's own reply.
-    expect(facts.assistantMessages).toEqual(["Done."]);
-  });
-
-  it("fails the run on a failed turn", () => {
-    const facts = deriveRunFacts([
-      event("message.received", { text: "hi" }),
-      event("turn.failed", { message: "model error" }),
-    ]);
-    expect(facts.ok).toBe(false);
-    expect(facts.turnsFailed).toBe(1);
-    expect(facts.failureMessage).toBe("model error");
-  });
-
-  it("fails the run on a session failure with no turn id", () => {
-    const events = [
-      event("message.received", { text: "hi" }),
-      {
-        ...event("session.failed", { message: "transport died" }),
-        turnId: undefined,
-      },
-    ] as SessionEvent[];
-    const facts = deriveRunFacts(events);
-    expect(facts.ok).toBe(false);
-    expect(facts.failureMessage).toBe("transport died");
-  });
-
-  it("is not ok with no turns at all", () => {
-    expect(deriveRunFacts([]).ok).toBe(false);
-  });
-
-  it("joins assistant messages in order", () => {
-    const facts = deriveRunFacts([
-      event("message.completed", { text: "first", finishReason: "tool_call" }),
-      event("message.completed", { text: "second", finishReason: "stop" }),
-      event("turn.completed", {}),
-    ]);
-    expect(facts.assistantText).toBe("first\nsecond");
-  });
-});