@cursor/july 0.1.26 → 0.1.28

package/src/internal/eval-run-store.storage.test.ts

@@ -1,211 +0,0 @@
-import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { afterEach, describe, expect, it } from "vitest";
-import type { EvalRunSnapshot } from "../evals.js";
-import { defineStorage } from "../storage.js";
-import { EvalRunStore } from "./eval-run-store.js";
-import { StorageCoordinator } from "./storage-coordinator.js";
-
-function completedRun(runId: string): EvalRunSnapshot {
-  return {
-    runId,
-    status: "completed",
-    startedAt: "2026-01-01T00:00:00.000Z",
-    finishedAt: "2026-01-01T00:00:01.000Z",
-    summary: { passed: 1, failed: 0, total: 1, done: 1 },
-    cases: [],
-    config: { maxPlaygroundRuns: 20, durableRuns: true },
-  };
-}
-
-describe("EvalRunStore defineStorage hydrate", () => {
-  let dir: string | undefined;
-
-  afterEach(async () => {
-    if (dir !== undefined) {
-      await rm(dir, { recursive: true, force: true });
-      dir = undefined;
-    }
-  });
-
-  it("hydrates playground history from the defineStorage evals table", async () => {
-    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
-    await mkdir(join(dir, "evals"), { recursive: true });
-    await writeFile(
-      join(dir, "evals", "evals.config.js"),
-      "export default { maxConcurrency: 1 };\n",
-      "utf8"
-    );
-
-    const table = new Map<string, EvalRunSnapshot>();
-    const coordinator = new StorageCoordinator({
-      definition: defineStorage({
-        put: () => {},
-        evals: {
-          put: (run) => {
-            table.set(run.runId, run);
-          },
-          delete: (runId) => {
-            table.delete(runId);
-          },
-          list: () => [...table.values()],
-        },
-      }),
-      agentName: "test-agent",
-      projectRoot: dir,
-      logger: () => {},
-    });
-
-    const storage = coordinator.evalRuns();
-    expect(storage).toBeDefined();
-    storage!.save(completedRun("evalrun_from_sink"));
-    await coordinator.whenIdle();
-
-    const store = new EvalRunStore(dir, () => {}, coordinator.evalRuns());
-    await store.hydrate();
-    const listed = await store.listRuns();
-    expect(listed).toHaveLength(1);
-    expect(listed[0]?.runId).toBe("evalrun_from_sink");
-    expect(listed[0]?.status).toBe("completed");
-  });
-
-  it("marks interrupted running batches as failed on hydrate", async () => {
-    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
-    await mkdir(join(dir, "evals"), { recursive: true });
-    await writeFile(
-      join(dir, "evals", "evals.config.js"),
-      "export default { maxConcurrency: 1 };\n",
-      "utf8"
-    );
-
-    const interrupted: EvalRunSnapshot = {
-      ...completedRun("evalrun_interrupted"),
-      status: "running",
-      finishedAt: undefined,
-      cases: [
-        { id: "smoke", fileId: "smoke", status: "running" },
-        { id: "other", fileId: "other", status: "pending" },
-      ],
-    };
-    const saved: EvalRunSnapshot[] = [];
-    const store = new EvalRunStore(dir, () => {}, {
-      save: (run) => {
-        saved.push(structuredClone(run));
-      },
-      delete: () => {},
-      list: async () => [interrupted],
-    });
-    await store.hydrate();
-
-    const listed = await store.listRuns();
-    expect(listed[0]?.status).toBe("failed");
-    expect(listed[0]?.cases.every((c) => c.status === "done")).toBe(true);
-    // The summary is reconciled with the repaired case rows (both cases
-    // were interrupted, so nothing passed) instead of keeping stale counts.
-    expect(listed[0]?.summary).toEqual({
-      passed: 0,
-      failed: 2,
-      scored: 0,
-      skipped: 0,
-      total: 2,
-      done: 2,
-    });
-    // The repaired snapshot was written back to storage.
-    expect(saved.map((run) => run.runId)).toContain("evalrun_interrupted");
-    expect(saved.at(-1)?.status).toBe("failed");
-  });
-
-  it("prunes hydrated history even when eval discovery fails", async () => {
-    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
-    await mkdir(join(dir, "evals"), { recursive: true });
-    // Discovery throws at hydrate time; pruning must still run with the
-    // default playground-history window (20).
-    await writeFile(
-      join(dir, "evals", "broken.eval.js"),
-      "throw new Error('boom');\n",
-      "utf8"
-    );
-
-    const stored = Array.from({ length: 25 }, (_, i) => ({
-      ...completedRun(`evalrun_${String(i).padStart(2, "0")}`),
-      startedAt: `2026-01-01T00:${String(i).padStart(2, "0")}:00.000Z`,
-    }));
-    const deleted: string[] = [];
-    const store = new EvalRunStore(dir, () => {}, {
-      save: () => {},
-      delete: (runId) => {
-        deleted.push(runId);
-      },
-      list: async () => stored,
-    });
-    await store.hydrate();
-
-    const listed = await store.listRuns();
-    expect(listed).toHaveLength(20);
-    expect(deleted).toHaveLength(5);
-    // Oldest five (by startedAt) were pruned.
-    expect(deleted.sort()).toEqual([
-      "evalrun_00",
-      "evalrun_01",
-      "evalrun_02",
-      "evalrun_03",
-      "evalrun_04",
-    ]);
-  });
-
-  it("close() durably marks an in-flight batch as interrupted", async () => {
-    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
-    await mkdir(join(dir, "evals"), { recursive: true });
-    await writeFile(
-      join(dir, "evals", "evals.config.js"),
-      "export default { maxConcurrency: 1 };\n",
-      "utf8"
-    );
-
-    const running: EvalRunSnapshot = {
-      ...completedRun("evalrun_active"),
-      status: "running",
-      finishedAt: undefined,
-      summary: { passed: 1, failed: 0, total: 2, done: 1 },
-      cases: [
-        { id: "done", fileId: "done", status: "done", ok: true },
-        { id: "midway", fileId: "midway", status: "running" },
-      ],
-    };
-    const saved: EvalRunSnapshot[] = [];
-    const store = new EvalRunStore(dir, () => {}, {
-      save: (run) => {
-        saved.push(structuredClone(run));
-      },
-      delete: () => {},
-      list: async () => [running],
-    });
-    await store.hydrate();
-    // hydrate already repaired it; reset to a live in-flight shape.
-    const run = (await store.listRuns())[0]!;
-    run.status = "running";
-    run.error = undefined;
-    run.cases[1]!.status = "running";
-    run.cases[1]!.ok = undefined;
-    run.cases[1]!.error = undefined;
-    (store as unknown as { activeRunId?: string }).activeRunId = run.runId;
-    saved.length = 0;
-
-    store.close();
-
-    expect(saved).toHaveLength(1);
-    expect(saved[0]?.status).toBe("failed");
-    expect(saved[0]?.summary).toEqual({
-      passed: 1,
-      failed: 1,
-      scored: 0,
-      skipped: 0,
-      total: 2,
-      done: 2,
-    });
-    // Idempotent: a second close has nothing left to write.
-    store.close();
-    expect(saved).toHaveLength(1);
-  });
-});

package/src/internal/eval-runner.http.test.ts

@@ -1,403 +0,0 @@
-/**
- * Runs the eval runner against a real HTTP agent stub.
- *
- * `eval-runner.run.test.ts` mocks the session transport to cover grading logic;
- * this file keeps the transport real, so the assertions are graded from events
- * that actually arrived over the NDJSON session stream. That is the part a mock
- * cannot prove.
- */
-
-import { createServer, type Server } from "node:http";
-import type { AddressInfo } from "node:net";
-import { afterEach, describe, expect, it } from "vitest";
-import type { EvalDefinition, EvalRunResult } from "../evals.js";
-import { defineEval, includes } from "../evals.js";
-import type { SessionEvent } from "../types.js";
-import { expandEvalModule, runDiscoveredEvals } from "./eval-runner.js";
-
-interface StubAgent {
-  url: string;
-  /** Messages the runner sent, in order. */
-  received: string[];
-  close(): Promise<void>;
-}
-
-/** One scripted turn: the events the stub streams back for that message. */
-type TurnScript = (message: string, turnId: string) => ScriptedEvent[];
-
-/**
- * A scripted event without its envelope cursor; `appendTurn` fills in
- * `sessionId` and `index`. Loosely typed on purpose: the stub composes events
- * generically, which the `type`/`data` discriminated union cannot express.
- */
-interface ScriptedEvent {
-  at: string;
-  turnId?: string;
-  type: string;
-  data: unknown;
-}
-
-async function startStubAgent(script: TurnScript): Promise<StubAgent> {
-  const received: string[] = [];
-  const sessions = new Map<string, SessionEvent[]>();
-  let nextSession = 0;
-  let nextTurn = 0;
-
-  const appendTurn = (sessionId: string, message: string): void => {
-    const events = sessions.get(sessionId)!;
-    nextTurn++;
-    for (const event of script(message, `turn_${nextTurn}`)) {
-      // `index` is the monotonic per-session cursor a client reconnects with
-      // via `?startIndex=`, so it must be the event's position in this session.
-      events.push({
-        ...event,
-        sessionId,
-        index: events.length,
-      } as SessionEvent);
-    }
-  };
-
-  const server: Server = createServer((req, res) => {
-    const url = new URL(req.url ?? "/", "http://127.0.0.1");
-    const chunks: Buffer[] = [];
-    req.on("data", (chunk: Buffer) => chunks.push(chunk));
-    req.on("end", () => {
-      const body =
-        chunks.length === 0
-          ? {}
-          : (JSON.parse(Buffer.concat(chunks).toString("utf8")) as {
-              message?: string;
-            });
-      const streamMatch = /^\/v1\/session\/([^/]+)\/stream$/.exec(url.pathname);
-      if (streamMatch !== null) {
-        const events = sessions.get(streamMatch[1]!) ?? [];
-        const startIndex = Number(url.searchParams.get("startIndex") ?? "0");
-        res.writeHead(200, { "content-type": "application/x-ndjson" });
-        for (const event of events.slice(startIndex)) {
-          res.write(`${JSON.stringify(event)}\n`);
-        }
-        res.end();
-        return;
-      }
-      if (url.pathname === "/v1/session") {
-        nextSession++;
-        const sessionId = `ses_${nextSession}`;
-        sessions.set(sessionId, []);
-        received.push(body.message ?? "");
-        appendTurn(sessionId, body.message ?? "");
-        res.writeHead(200, { "content-type": "application/json" });
-        res.end(
-          JSON.stringify({ ok: true, sessionId, continuationToken: "tok_1" })
-        );
-        return;
-      }
-      const followUp = /^\/v1\/session\/([^/]+)$/.exec(url.pathname);
-      if (followUp !== null) {
-        received.push(body.message ?? "");
-        appendTurn(followUp[1]!, body.message ?? "");
-        res.writeHead(200, { "content-type": "application/json" });
-        res.end(JSON.stringify({ ok: true, continuationToken: "tok_2" }));
-        return;
-      }
-      res.writeHead(404).end();
-    });
-  });
-
-  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
-  const { port } = server.address() as AddressInfo;
-  return {
-    url: `http://127.0.0.1:${port}`,
-    received,
-    close: () =>
-      new Promise<void>((resolve) => {
-        server.close(() => resolve());
-      }),
-  };
-}
-
-function event(
-  type: string,
-  data: unknown,
-  turnId: string | undefined
-): ScriptedEvent {
-  return { at: new Date().toISOString(), turnId, type, data };
-}
-
-/** A weather turn: one completed tool call, one reply, then park. */
-const weatherScript: TurnScript = (message, turnId) => [
-  event("message.received", { text: message }, turnId),
-  event(
-    "actions.requested",
-    {
-      calls: [
-        {
-          callId: `${turnId}_c1`,
-          toolName: "get_weather",
-          args: { city: "NYC" },
-        },
-      ],
-    },
-    turnId
-  ),
-  event(
-    "action.result",
-    {
-      callId: `${turnId}_c1`,
-      toolName: "get_weather",
-      output: { tempF: 72 },
-      isError: false,
-    },
-    turnId
-  ),
-  event(
-    "message.completed",
-    { text: "Sunny, 72F in NYC.", finishReason: "stop" },
-    turnId
-  ),
-  event("turn.completed", {}, turnId),
-  event("session.waiting", {}, undefined),
-];
-
-let stub: StubAgent | undefined;
-
-afterEach(async () => {
-  await stub?.close();
-  stub = undefined;
-});
-
-async function run(
-  definition: EvalDefinition,
-  baseUrl: string
-): Promise<EvalRunResult[]> {
-  return runDiscoveredEvals({
-    projectRoot: "/unused",
-    baseUrl,
-    discovered: {
-      evals: expandEvalModule("http", definition).map((e) => ({
-        id: e.id,
-        fileId: "http",
-        path: "/repo/evals/http.eval.ts",
-        definition: e.definition,
-      })),
-      config: { maxConcurrency: 2 },
-    },
-  });
-}
-
-describe("runner over a real session stream", () => {
-  it("grades assertions from events that arrived over HTTP", async () => {
-    stub = await startStubAgent(weatherScript);
-    const [result] = await run(
-      defineEval({
-        async test(t) {
-          await t.send("What is the weather in NYC?");
-          t.succeeded();
-          t.calledTool("get_weather", {
-            input: { city: "NYC" },
-            output: { tempF: 72 },
-            status: "completed",
-            count: 1,
-          });
-          t.noFailedActions();
-          t.messageIncludes("Sunny");
-          t.check(t.reply, includes("72F"));
-          t.maxToolCalls(1);
-        },
-      }),
-      stub.url
-    );
-    expect(result?.verdict).toBe("passed");
-    expect(result?.assertions.every((a) => a.passed)).toBe(true);
-    expect(stub.received).toEqual(["What is the weather in NYC?"]);
-    // The transport-derived views the CLI and playground print.
-    expect(result?.tools).toEqual(["get_weather"]);
-    expect(result?.toolCalls).toEqual([
-      {
-        toolName: "get_weather",
-        args: { city: "NYC" },
-        output: { tempF: 72 },
-        isError: false,
-      },
-    ]);
-    expect(result?.finalText).toBe("Sunny, 72F in NYC.");
-    expect(result?.sessionId).toBe("ses_1");
-  });
-
-  it("resumes the stream so a second send reads the second turn", async () => {
-    // Each turn replies with the message it received and calls a distinct tool,
-    // so a send that replayed the earlier turn would be unmistakable. The
-    // stream replays from an index and stops at the first terminal event, so
-    // this is the regression test for resuming at the right cursor.
-    stub = await startStubAgent((message, turnId) => [
-      event("message.received", { text: message }, turnId),
-      event(
-        "actions.requested",
-        {
-          calls: [
-            { callId: `${turnId}_c1`, toolName: `tool_${message}`, args: {} },
-          ],
-        },
-        turnId
-      ),
-      event(
-        "action.result",
-        {
-          callId: `${turnId}_c1`,
-          toolName: `tool_${message}`,
-          output: "ok",
-          isError: false,
-        },
-        turnId
-      ),
-      event(
-        "message.completed",
-        { text: `reply to ${message}`, finishReason: "stop" },
-        turnId
-      ),
-      event("turn.completed", {}, turnId),
-      event("session.waiting", {}, undefined),
-    ]);
-
-    const [result] = await run(
-      defineEval({
-        async test(t) {
-          const first = await t.send("first");
-          t.check(first.message, includes("reply to first"));
-
-          const second = await t.send("second");
-          t.check(second.message, includes("reply to second"));
-          t.check(t.reply, includes("reply to second"));
-
-          // Each turn sees only its own tool call.
-          first.calledTool("tool_first", { count: 1 });
-          first.notCalledTool("tool_second");
-          second.calledTool("tool_second", { count: 1 });
-          second.notCalledTool("tool_first");
-
-          // The run sees both, exactly once each.
-          t.calledTool("tool_first", { count: 1 });
-          t.calledTool("tool_second", { count: 1 });
-          t.maxToolCalls(2);
-          t.toolOrder(["tool_first", "tool_second"]);
-          t.check(second.sessionId, includes(first.sessionId));
-        },
-      }),
-      stub.url
-    );
-    expect(result?.verdict).toBe("passed");
-    const failed = result!.assertions.filter((a) => !a.passed);
-    expect(failed).toEqual([]);
-    expect(stub.received).toEqual(["first", "second"]);
-    expect(result?.inputs).toEqual(["first", "second"]);
-    expect(result?.finalText).toBe("reply to second");
-  });
-
-  it("fails a run whose turn failed on the wire", async () => {
-    stub = await startStubAgent((message, turnId) => [
-      event("message.received", { text: message }, turnId),
-      event("turn.failed", { message: "provider unavailable" }, turnId),
-      event("session.waiting", {}, undefined),
-    ]);
-    const [result] = await run(
-      defineEval({
-        async test(t) {
-          await t.send("hello");
-          t.succeeded();
-        },
-      }),
-      stub.url
-    );
-    expect(result?.verdict).toBe("failed");
-    expect(result?.assertions[0]).toMatchObject({
-      name: "succeeded",
-      passed: false,
-      detail: "provider unavailable",
-    });
-  });
-
-  it("detects a run parked on an unanswered approval request", async () => {
-    stub = await startStubAgent((message, turnId) => [
-      event("message.received", { text: message }, turnId),
-      event(
-        "actions.requested",
-        {
-          calls: [{ callId: "c1", toolName: "deploy", args: { env: "prod" } }],
-        },
-        turnId
-      ),
-      event(
-        "action.approval_requested",
-        { callId: "c1", toolName: "deploy", args: { env: "prod" } },
-        turnId
-      ),
-      event("session.waiting", {}, undefined),
-    ]);
-    const [result] = await run(
-      defineEval({
-        async test(t) {
-          await t.send("deploy to prod");
-          t.parked();
-          const request = t.requireInputRequest({ toolName: "deploy" });
-          t.check(request.args, includes("prod"));
-          t.calledTool("deploy", { status: "pending", count: 1 });
-        },
-      }),
-      stub.url
-    );
-    expect(result?.verdict).toBe("passed");
-    expect(result?.assertions.map((a) => a.passed)).toEqual([
-      true,
-      true,
-      true,
-      true,
-    ]);
-  });
-
-  it("records a failed tool call without failing the turn", async () => {
-    stub = await startStubAgent((message, turnId) => [
-      event("message.received", { text: message }, turnId),
-      event(
-        "actions.requested",
-        {
-          calls: [
-            { callId: "c1", toolName: "bash", args: { command: "false" } },
-          ],
-        },
-        turnId
-      ),
-      event(
-        "action.result",
-        { callId: "c1", toolName: "bash", output: "exit 1", isError: true },
-        turnId
-      ),
-      event(
-        "message.completed",
-        { text: "That command failed.", finishReason: "stop" },
-        turnId
-      ),
-      event("turn.completed", {}, turnId),
-      event("session.waiting", {}, undefined),
-    ]);
-    const [result] = await run(
-      defineEval({
-        async test(t) {
-          await t.send("run false");
-          t.succeeded();
-          t.calledTool("bash", { status: "failed", count: 1 });
-          t.noFailedActions();
-        },
-      }),
-      stub.url
-    );
-    expect(result?.verdict).toBe("failed");
-    const byName = new Map(result!.assertions.map((a) => [a.name, a]));
-    expect(byName.get("succeeded")?.passed).toBe(true);
-    expect(
-      byName.get("calledTool(bash, {status=failed, count=1})")?.passed
-    ).toBe(true);
-    expect(byName.get("noFailedActions")).toMatchObject({
-      passed: false,
-      detail: "bash",
-    });
-  });
-});