@cursor/july 0.1.19 → 0.1.21

package/src/internal/eval-judge-model.test.ts

@@ -0,0 +1,187 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { EvalJudgeUnavailableError } from "../evals/judge.js";
+import {
+  assertJudgeBackendMatch,
+  callJudgeModel,
+  resolveJudgeModel,
+} from "./eval-judge-model.js";
+
+const create = vi.hoisted(() => vi.fn());
+const resolveApiKey = vi.hoisted(() => vi.fn());
+
+vi.mock("@cursor/sdk", () => ({ Agent: { create } }));
+
+vi.mock("./cursor/credentials.js", async (importOriginal) => {
+  const actual =
+    await importOriginal<typeof import("./cursor/credentials.js")>();
+  return { ...actual, resolveApiKey };
+});
+
+beforeEach(() => {
+  create.mockReset();
+  resolveApiKey.mockReset();
+  resolveApiKey.mockResolvedValue({
+    apiKey: "key_1",
+    source: "env",
+    backendUrl: "https://api2.cursor.sh",
+  });
+  create.mockResolvedValue({
+    send: async () => ({
+      wait: async () => ({ status: "completed", result: "CHOICE: Y" }),
+    }),
+    close: () => undefined,
+  });
+});
+
+describe("resolveJudgeModel", () => {
+  it("resolves innermost-first and ignores a blank env value", () => {
+    expect(
+      resolveJudgeModel({
+        call: "call-model",
+        evalLevel: "eval-model",
+        config: "config-model",
+        env: "env-model",
+      })
+    ).toBe("call-model");
+    expect(
+      resolveJudgeModel({ evalLevel: "eval-model", config: "config-model" })
+    ).toBe("eval-model");
+    expect(resolveJudgeModel({ config: "config-model" })).toBe("config-model");
+    expect(resolveJudgeModel({ env: "  env-model  " })).toBe("env-model");
+    expect(resolveJudgeModel({ env: "   " })).toBeUndefined();
+    expect(resolveJudgeModel({})).toBeUndefined();
+  });
+});
+
+describe("assertJudgeBackendMatch", () => {
+  it("allows a credential scoped to the backend the SDK will use", () => {
+    expect(() =>
+      assertJudgeBackendMatch(
+        "https://api2.cursor.sh",
+        "https://api2.cursor.sh"
+      )
+    ).not.toThrow();
+  });
+
+  it("ignores a trailing-slash difference", () => {
+    expect(() =>
+      assertJudgeBackendMatch(
+        "https://api2.cursor.sh/",
+        "https://api2.cursor.sh"
+      )
+    ).not.toThrow();
+  });
+
+  it("declines when the credential belongs to a different backend", () => {
+    // The SDK takes no backend option, so sending the key anyway would ship it
+    // (and the judge prompt) to a host it was not minted for.
+    expect(() =>
+      assertJudgeBackendMatch(
+        "https://staging.example.com",
+        "https://api2.cursor.sh"
+      )
+    ).toThrow(EvalJudgeUnavailableError);
+    expect(() =>
+      assertJudgeBackendMatch(
+        "https://staging.example.com",
+        "https://api2.cursor.sh"
+      )
+    ).toThrow(/does not read AGENT_SERVE_CURSOR_BACKEND_URL/);
+  });
+});
+
+describe("callJudgeModel", () => {
+  it("skips visibly when there are no credentials", async () => {
+    resolveApiKey.mockResolvedValue(undefined);
+    await expect(
+      callJudgeModel({ prompt: "grade this", model: "gpt-5.4-mini" })
+    ).rejects.toThrow(EvalJudgeUnavailableError);
+    expect(create).not.toHaveBeenCalled();
+  });
+
+  it("confines the grading session so an injected reply cannot reach the host", async () => {
+    await callJudgeModel({ prompt: "grade this", model: "gpt-5.4-mini" });
+
+    expect(create).toHaveBeenCalledTimes(1);
+    const options = create.mock.calls[0]![0] as {
+      mcpServers: Record<string, unknown>;
+      local: {
+        cwd: string;
+        settingSources: string[];
+        sandboxOptions: { enabled: boolean };
+      };
+    };
+    // Grading embeds untrusted agent output, so the judge gets no MCP surface,
+    // no ambient settings, an empty scratch cwd, and a sandbox.
+    expect(options.mcpServers).toEqual({});
+    expect(options.local.settingSources).toEqual([]);
+    expect(options.local.sandboxOptions).toEqual({ enabled: true });
+    expect(options.local.cwd).toMatch(/agentkit-eval-judge-/);
+  });
+
+  it("never opens a session when the credential's backend disagrees", async () => {
+    resolveApiKey.mockResolvedValue({
+      apiKey: "key_1",
+      source: "login",
+      backendUrl: "https://staging.example.com",
+    });
+    await expect(
+      callJudgeModel({ prompt: "p", model: "gpt-5.4-mini" })
+    ).rejects.toThrow(EvalJudgeUnavailableError);
+    // The point of failing closed: the key never reaches the SDK.
+    expect(create).not.toHaveBeenCalled();
+  });
+
+  it("passes a bare model id through as a selection object", async () => {
+    await callJudgeModel({ prompt: "p", model: "gpt-5.4-mini" });
+    expect(create.mock.calls[0]![0]).toMatchObject({
+      apiKey: "key_1",
+      model: { id: "gpt-5.4-mini" },
+    });
+  });
+
+  it("preserves model params when given a selection object", async () => {
+    await callJudgeModel({
+      prompt: "p",
+      model: { id: "gpt-5.4-mini", params: [{ id: "effort", value: "low" }] },
+    });
+    expect(create.mock.calls[0]![0]).toMatchObject({
+      model: { id: "gpt-5.4-mini", params: [{ id: "effort", value: "low" }] },
+    });
+  });
+
+  it("returns the judge reply", async () => {
+    await expect(
+      callJudgeModel({ prompt: "p", model: "gpt-5.4-mini" })
+    ).resolves.toBe("CHOICE: Y");
+  });
+
+  it("surfaces a failed grading run as an error", async () => {
+    create.mockResolvedValue({
+      send: async () => ({
+        wait: async () => ({
+          status: "error",
+          error: { message: "provider unavailable" },
+        }),
+      }),
+      close: () => undefined,
+    });
+    await expect(
+      callJudgeModel({ prompt: "p", model: "gpt-5.4-mini" })
+    ).rejects.toThrow(/eval judge run failed: provider unavailable/);
+  });
+
+  it("closes the session even when grading throws", async () => {
+    const close = vi.fn();
+    create.mockResolvedValue({
+      send: async () => {
+        throw new Error("boom");
+      },
+      close,
+    });
+    await expect(
+      callJudgeModel({ prompt: "p", model: "gpt-5.4-mini" })
+    ).rejects.toThrow("boom");
+    expect(close).toHaveBeenCalledTimes(1);
+  });
+});

package/src/internal/eval-run-store.cancel.test.ts

@@ -0,0 +1,142 @@
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  EvalRunNotRunningError,
+  EvalRunStore,
+  EvalRunUnknownError,
+} from "./eval-run-store.js";
+
+vi.mock("./eval-runner.js", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("./eval-runner.js")>();
+  return {
+    ...actual,
+    runDiscoveredEvals: vi.fn(
+      async (options: {
+        signal?: AbortSignal;
+        onCaseStart?: (id: string) => void;
+        onCaseDone?: (result: {
+          id: string;
+          path: string;
+          ok: boolean;
+          assertions: [];
+          logs: string[];
+          metrics: Record<string, string | number>;
+          inputs: string[];
+          toolCalls: [];
+          tools: string[];
+          durationMs: number;
+        }) => void;
+        discovered?: {
+          evals: Array<{ id: string; path: string }>;
+        };
+      }) => {
+        const first = options.discovered?.evals[0];
+        if (first !== undefined) {
+          options.onCaseStart?.(first.id);
+        }
+        await new Promise<void>((resolve, reject) => {
+          const onAbort = (): void => {
+            options.signal?.removeEventListener("abort", onAbort);
+            reject(new Error("cancelled"));
+          };
+          if (options.signal?.aborted) {
+            onAbort();
+            return;
+          }
+          options.signal?.addEventListener("abort", onAbort, { once: true });
+          // Keep the batch "running" until cancel aborts.
+          setTimeout(() => {
+            options.signal?.removeEventListener("abort", onAbort);
+            if (first !== undefined) {
+              options.onCaseDone?.({
+                id: first.id,
+                path: first.path,
+                ok: true,
+                assertions: [],
+                logs: [],
+                metrics: {},
+                inputs: [],
+                toolCalls: [],
+                tools: [],
+                durationMs: 1,
+              });
+            }
+            resolve();
+          }, 5_000);
+        });
+        return [];
+      }
+    ),
+  };
+});
+
+describe("EvalRunStore.cancel", () => {
+  let dir: string | undefined;
+
+  afterEach(async () => {
+    if (dir !== undefined) {
+      await rm(dir, { recursive: true, force: true });
+      dir = undefined;
+    }
+    vi.clearAllMocks();
+  });
+
+  it("cancels a running batch by Eval ID and clears the active run", async () => {
+    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-cancel-"));
+    await mkdir(join(dir, "evals"), { recursive: true });
+    await writeFile(
+      join(dir, "evals", "evals.config.js"),
+      "export default { maxConcurrency: 1 };\n",
+      "utf8"
+    );
+    await writeFile(
+      join(dir, "evals", "smoke.eval.js"),
+      `export default {
+  __agentServe: "eval",
+  async test() {}
+};
+`,
+      "utf8"
+    );
+
+    const store = new EvalRunStore(dir, () => {});
+    store.loopbackUrl = "http://127.0.0.1:9/smoke";
+    const started = await store.start();
+    expect(started.status).toBe("running");
+    expect(store.getActiveRunId()).toBe(started.runId);
+
+    const cancelled = await store.cancel(started.runId);
+    expect(cancelled.status).toBe("cancelled");
+    expect(cancelled.error).toContain("cancelled");
+    expect(store.getActiveRunId()).toBeUndefined();
+    expect(await store.cancel(started.runId)).toMatchObject({
+      status: "cancelled",
+    });
+    await expect(store.cancel("evalrun_missing")).rejects.toBeInstanceOf(
+      EvalRunUnknownError
+    );
+    await expect(store.cancel(started.runId)).resolves.toMatchObject({
+      status: "cancelled",
+    });
+
+    // After cancel, a completed-looking batch cannot be cancelled again as running.
+    const completedId = started.runId;
+    const snap = await store.get(completedId);
+    expect(snap?.status).toBe("cancelled");
+    // Force a non-running status to exercise NotRunningError via a fresh id.
+    const fakeId = "evalrun_done_only";
+    (
+      store as unknown as {
+        runs: Map<string, { runId: string; status: string }>;
+      }
+    ).runs.set(fakeId, {
+      runId: fakeId,
+      status: "completed",
+    });
+    await expect(store.cancel(fakeId)).rejects.toBeInstanceOf(
+      EvalRunNotRunningError
+    );
+  });
+});

package/src/internal/eval-run-store.storage.test.ts

@@ -0,0 +1,211 @@
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import type { EvalRunSnapshot } from "../evals.js";
+import { defineStorage } from "../storage.js";
+import { EvalRunStore } from "./eval-run-store.js";
+import { StorageCoordinator } from "./storage-coordinator.js";
+
+function completedRun(runId: string): EvalRunSnapshot {
+  return {
+    runId,
+    status: "completed",
+    startedAt: "2026-01-01T00:00:00.000Z",
+    finishedAt: "2026-01-01T00:00:01.000Z",
+    summary: { passed: 1, failed: 0, total: 1, done: 1 },
+    cases: [],
+    config: { maxPlaygroundRuns: 20, durableRuns: true },
+  };
+}
+
+describe("EvalRunStore defineStorage hydrate", () => {
+  let dir: string | undefined;
+
+  afterEach(async () => {
+    if (dir !== undefined) {
+      await rm(dir, { recursive: true, force: true });
+      dir = undefined;
+    }
+  });
+
+  it("hydrates playground history from the defineStorage evals table", async () => {
+    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
+    await mkdir(join(dir, "evals"), { recursive: true });
+    await writeFile(
+      join(dir, "evals", "evals.config.js"),
+      "export default { maxConcurrency: 1 };\n",
+      "utf8"
+    );
+
+    const table = new Map<string, EvalRunSnapshot>();
+    const coordinator = new StorageCoordinator({
+      definition: defineStorage({
+        put: () => {},
+        evals: {
+          put: (run) => {
+            table.set(run.runId, run);
+          },
+          delete: (runId) => {
+            table.delete(runId);
+          },
+          list: () => [...table.values()],
+        },
+      }),
+      agentName: "test-agent",
+      projectRoot: dir,
+      logger: () => {},
+    });
+
+    const storage = coordinator.evalRuns();
+    expect(storage).toBeDefined();
+    storage!.save(completedRun("evalrun_from_sink"));
+    await coordinator.whenIdle();
+
+    const store = new EvalRunStore(dir, () => {}, coordinator.evalRuns());
+    await store.hydrate();
+    const listed = await store.listRuns();
+    expect(listed).toHaveLength(1);
+    expect(listed[0]?.runId).toBe("evalrun_from_sink");
+    expect(listed[0]?.status).toBe("completed");
+  });
+
+  it("marks interrupted running batches as failed on hydrate", async () => {
+    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
+    await mkdir(join(dir, "evals"), { recursive: true });
+    await writeFile(
+      join(dir, "evals", "evals.config.js"),
+      "export default { maxConcurrency: 1 };\n",
+      "utf8"
+    );
+
+    const interrupted: EvalRunSnapshot = {
+      ...completedRun("evalrun_interrupted"),
+      status: "running",
+      finishedAt: undefined,
+      cases: [
+        { id: "smoke", fileId: "smoke", status: "running" },
+        { id: "other", fileId: "other", status: "pending" },
+      ],
+    };
+    const saved: EvalRunSnapshot[] = [];
+    const store = new EvalRunStore(dir, () => {}, {
+      save: (run) => {
+        saved.push(structuredClone(run));
+      },
+      delete: () => {},
+      list: async () => [interrupted],
+    });
+    await store.hydrate();
+
+    const listed = await store.listRuns();
+    expect(listed[0]?.status).toBe("failed");
+    expect(listed[0]?.cases.every((c) => c.status === "done")).toBe(true);
+    // The summary is reconciled with the repaired case rows (both cases
+    // were interrupted, so nothing passed) instead of keeping stale counts.
+    expect(listed[0]?.summary).toEqual({
+      passed: 0,
+      failed: 2,
+      scored: 0,
+      skipped: 0,
+      total: 2,
+      done: 2,
+    });
+    // The repaired snapshot was written back to storage.
+    expect(saved.map((run) => run.runId)).toContain("evalrun_interrupted");
+    expect(saved.at(-1)?.status).toBe("failed");
+  });
+
+  it("prunes hydrated history even when eval discovery fails", async () => {
+    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
+    await mkdir(join(dir, "evals"), { recursive: true });
+    // Discovery throws at hydrate time; pruning must still run with the
+    // default playground-history window (20).
+    await writeFile(
+      join(dir, "evals", "broken.eval.js"),
+      "throw new Error('boom');\n",
+      "utf8"
+    );
+
+    const stored = Array.from({ length: 25 }, (_, i) => ({
+      ...completedRun(`evalrun_${String(i).padStart(2, "0")}`),
+      startedAt: `2026-01-01T00:${String(i).padStart(2, "0")}:00.000Z`,
+    }));
+    const deleted: string[] = [];
+    const store = new EvalRunStore(dir, () => {}, {
+      save: () => {},
+      delete: (runId) => {
+        deleted.push(runId);
+      },
+      list: async () => stored,
+    });
+    await store.hydrate();
+
+    const listed = await store.listRuns();
+    expect(listed).toHaveLength(20);
+    expect(deleted).toHaveLength(5);
+    // Oldest five (by startedAt) were pruned.
+    expect(deleted.sort()).toEqual([
+      "evalrun_00",
+      "evalrun_01",
+      "evalrun_02",
+      "evalrun_03",
+      "evalrun_04",
+    ]);
+  });
+
+  it("close() durably marks an in-flight batch as interrupted", async () => {
+    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
+    await mkdir(join(dir, "evals"), { recursive: true });
+    await writeFile(
+      join(dir, "evals", "evals.config.js"),
+      "export default { maxConcurrency: 1 };\n",
+      "utf8"
+    );
+
+    const running: EvalRunSnapshot = {
+      ...completedRun("evalrun_active"),
+      status: "running",
+      finishedAt: undefined,
+      summary: { passed: 1, failed: 0, total: 2, done: 1 },
+      cases: [
+        { id: "done", fileId: "done", status: "done", ok: true },
+        { id: "midway", fileId: "midway", status: "running" },
+      ],
+    };
+    const saved: EvalRunSnapshot[] = [];
+    const store = new EvalRunStore(dir, () => {}, {
+      save: (run) => {
+        saved.push(structuredClone(run));
+      },
+      delete: () => {},
+      list: async () => [running],
+    });
+    await store.hydrate();
+    // hydrate already repaired it; reset to a live in-flight shape.
+    const run = (await store.listRuns())[0]!;
+    run.status = "running";
+    run.error = undefined;
+    run.cases[1]!.status = "running";
+    run.cases[1]!.ok = undefined;
+    run.cases[1]!.error = undefined;
+    (store as unknown as { activeRunId?: string }).activeRunId = run.runId;
+    saved.length = 0;
+
+    store.close();
+
+    expect(saved).toHaveLength(1);
+    expect(saved[0]?.status).toBe("failed");
+    expect(saved[0]?.summary).toEqual({
+      passed: 1,
+      failed: 1,
+      scored: 0,
+      skipped: 0,
+      total: 2,
+      done: 2,
+    });
+    // Idempotent: a second close has nothing left to write.
+    store.close();
+    expect(saved).toHaveLength(1);
+  });
+});