@cursor/july 0.1.23 → 0.1.24

package/src/internal/eval-judge-model.test.ts

@@ -1,187 +0,0 @@
-import { beforeEach, describe, expect, it, vi } from "vitest";
-import { EvalJudgeUnavailableError } from "../evals/judge.js";
-import {
-  assertJudgeBackendMatch,
-  callJudgeModel,
-  resolveJudgeModel,
-} from "./eval-judge-model.js";
-
-const create = vi.hoisted(() => vi.fn());
-const resolveApiKey = vi.hoisted(() => vi.fn());
-
-vi.mock("@cursor/sdk", () => ({ Agent: { create } }));
-
-vi.mock("./cursor/credentials.js", async (importOriginal) => {
-  const actual =
-    await importOriginal<typeof import("./cursor/credentials.js")>();
-  return { ...actual, resolveApiKey };
-});
-
-beforeEach(() => {
-  create.mockReset();
-  resolveApiKey.mockReset();
-  resolveApiKey.mockResolvedValue({
-    apiKey: "key_1",
-    source: "env",
-    backendUrl: "https://api2.cursor.sh",
-  });
-  create.mockResolvedValue({
-    send: async () => ({
-      wait: async () => ({ status: "completed", result: "CHOICE: Y" }),
-    }),
-    close: () => undefined,
-  });
-});
-
-describe("resolveJudgeModel", () => {
-  it("resolves innermost-first and ignores a blank env value", () => {
-    expect(
-      resolveJudgeModel({
-        call: "call-model",
-        evalLevel: "eval-model",
-        config: "config-model",
-        env: "env-model",
-      })
-    ).toBe("call-model");
-    expect(
-      resolveJudgeModel({ evalLevel: "eval-model", config: "config-model" })
-    ).toBe("eval-model");
-    expect(resolveJudgeModel({ config: "config-model" })).toBe("config-model");
-    expect(resolveJudgeModel({ env: "  env-model  " })).toBe("env-model");
-    expect(resolveJudgeModel({ env: "   " })).toBeUndefined();
-    expect(resolveJudgeModel({})).toBeUndefined();
-  });
-});
-
-describe("assertJudgeBackendMatch", () => {
-  it("allows a credential scoped to the backend the SDK will use", () => {
-    expect(() =>
-      assertJudgeBackendMatch(
-        "https://api2.cursor.sh",
-        "https://api2.cursor.sh"
-      )
-    ).not.toThrow();
-  });
-
-  it("ignores a trailing-slash difference", () => {
-    expect(() =>
-      assertJudgeBackendMatch(
-        "https://api2.cursor.sh/",
-        "https://api2.cursor.sh"
-      )
-    ).not.toThrow();
-  });
-
-  it("declines when the credential belongs to a different backend", () => {
-    // The SDK takes no backend option, so sending the key anyway would ship it
-    // (and the judge prompt) to a host it was not minted for.
-    expect(() =>
-      assertJudgeBackendMatch(
-        "https://staging.example.com",
-        "https://api2.cursor.sh"
-      )
-    ).toThrow(EvalJudgeUnavailableError);
-    expect(() =>
-      assertJudgeBackendMatch(
-        "https://staging.example.com",
-        "https://api2.cursor.sh"
-      )
-    ).toThrow(/does not read AGENT_SERVE_CURSOR_BACKEND_URL/);
-  });
-});
-
-describe("callJudgeModel", () => {
-  it("skips visibly when there are no credentials", async () => {
-    resolveApiKey.mockResolvedValue(undefined);
-    await expect(
-      callJudgeModel({ prompt: "grade this", model: "gpt-5.4-mini" })
-    ).rejects.toThrow(EvalJudgeUnavailableError);
-    expect(create).not.toHaveBeenCalled();
-  });
-
-  it("confines the grading session so an injected reply cannot reach the host", async () => {
-    await callJudgeModel({ prompt: "grade this", model: "gpt-5.4-mini" });
-
-    expect(create).toHaveBeenCalledTimes(1);
-    const options = create.mock.calls[0]![0] as {
-      mcpServers: Record<string, unknown>;
-      local: {
-        cwd: string;
-        settingSources: string[];
-        sandboxOptions: { enabled: boolean };
-      };
-    };
-    // Grading embeds untrusted agent output, so the judge gets no MCP surface,
-    // no ambient settings, an empty scratch cwd, and a sandbox.
-    expect(options.mcpServers).toEqual({});
-    expect(options.local.settingSources).toEqual([]);
-    expect(options.local.sandboxOptions).toEqual({ enabled: true });
-    expect(options.local.cwd).toMatch(/agentkit-eval-judge-/);
-  });
-
-  it("never opens a session when the credential's backend disagrees", async () => {
-    resolveApiKey.mockResolvedValue({
-      apiKey: "key_1",
-      source: "login",
-      backendUrl: "https://staging.example.com",
-    });
-    await expect(
-      callJudgeModel({ prompt: "p", model: "gpt-5.4-mini" })
-    ).rejects.toThrow(EvalJudgeUnavailableError);
-    // The point of failing closed: the key never reaches the SDK.
-    expect(create).not.toHaveBeenCalled();
-  });
-
-  it("passes a bare model id through as a selection object", async () => {
-    await callJudgeModel({ prompt: "p", model: "gpt-5.4-mini" });
-    expect(create.mock.calls[0]![0]).toMatchObject({
-      apiKey: "key_1",
-      model: { id: "gpt-5.4-mini" },
-    });
-  });
-
-  it("preserves model params when given a selection object", async () => {
-    await callJudgeModel({
-      prompt: "p",
-      model: { id: "gpt-5.4-mini", params: [{ id: "effort", value: "low" }] },
-    });
-    expect(create.mock.calls[0]![0]).toMatchObject({
-      model: { id: "gpt-5.4-mini", params: [{ id: "effort", value: "low" }] },
-    });
-  });
-
-  it("returns the judge reply", async () => {
-    await expect(
-      callJudgeModel({ prompt: "p", model: "gpt-5.4-mini" })
-    ).resolves.toBe("CHOICE: Y");
-  });
-
-  it("surfaces a failed grading run as an error", async () => {
-    create.mockResolvedValue({
-      send: async () => ({
-        wait: async () => ({
-          status: "error",
-          error: { message: "provider unavailable" },
-        }),
-      }),
-      close: () => undefined,
-    });
-    await expect(
-      callJudgeModel({ prompt: "p", model: "gpt-5.4-mini" })
-    ).rejects.toThrow(/eval judge run failed: provider unavailable/);
-  });
-
-  it("closes the session even when grading throws", async () => {
-    const close = vi.fn();
-    create.mockResolvedValue({
-      send: async () => {
-        throw new Error("boom");
-      },
-      close,
-    });
-    await expect(
-      callJudgeModel({ prompt: "p", model: "gpt-5.4-mini" })
-    ).rejects.toThrow("boom");
-    expect(close).toHaveBeenCalledTimes(1);
-  });
-});

package/src/internal/eval-run-store.cancel.test.ts

@@ -1,142 +0,0 @@
-import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { afterEach, describe, expect, it, vi } from "vitest";
-import {
-  EvalRunNotRunningError,
-  EvalRunStore,
-  EvalRunUnknownError,
-} from "./eval-run-store.js";
-
-vi.mock("./eval-runner.js", async (importOriginal) => {
-  const actual = await importOriginal<typeof import("./eval-runner.js")>();
-  return {
-    ...actual,
-    runDiscoveredEvals: vi.fn(
-      async (options: {
-        signal?: AbortSignal;
-        onCaseStart?: (id: string) => void;
-        onCaseDone?: (result: {
-          id: string;
-          path: string;
-          ok: boolean;
-          assertions: [];
-          logs: string[];
-          metrics: Record<string, string | number>;
-          inputs: string[];
-          toolCalls: [];
-          tools: string[];
-          durationMs: number;
-        }) => void;
-        discovered?: {
-          evals: Array<{ id: string; path: string }>;
-        };
-      }) => {
-        const first = options.discovered?.evals[0];
-        if (first !== undefined) {
-          options.onCaseStart?.(first.id);
-        }
-        await new Promise<void>((resolve, reject) => {
-          const onAbort = (): void => {
-            options.signal?.removeEventListener("abort", onAbort);
-            reject(new Error("cancelled"));
-          };
-          if (options.signal?.aborted) {
-            onAbort();
-            return;
-          }
-          options.signal?.addEventListener("abort", onAbort, { once: true });
-          // Keep the batch "running" until cancel aborts.
-          setTimeout(() => {
-            options.signal?.removeEventListener("abort", onAbort);
-            if (first !== undefined) {
-              options.onCaseDone?.({
-                id: first.id,
-                path: first.path,
-                ok: true,
-                assertions: [],
-                logs: [],
-                metrics: {},
-                inputs: [],
-                toolCalls: [],
-                tools: [],
-                durationMs: 1,
-              });
-            }
-            resolve();
-          }, 5_000);
-        });
-        return [];
-      }
-    ),
-  };
-});
-
-describe("EvalRunStore.cancel", () => {
-  let dir: string | undefined;
-
-  afterEach(async () => {
-    if (dir !== undefined) {
-      await rm(dir, { recursive: true, force: true });
-      dir = undefined;
-    }
-    vi.clearAllMocks();
-  });
-
-  it("cancels a running batch by Eval ID and clears the active run", async () => {
-    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-cancel-"));
-    await mkdir(join(dir, "evals"), { recursive: true });
-    await writeFile(
-      join(dir, "evals", "evals.config.js"),
-      "export default { maxConcurrency: 1 };\n",
-      "utf8"
-    );
-    await writeFile(
-      join(dir, "evals", "smoke.eval.js"),
-      `export default {
-  __agentServe: "eval",
-  async test() {}
-};
-`,
-      "utf8"
-    );
-
-    const store = new EvalRunStore(dir, () => {});
-    store.loopbackUrl = "http://127.0.0.1:9/smoke";
-    const started = await store.start();
-    expect(started.status).toBe("running");
-    expect(store.getActiveRunId()).toBe(started.runId);
-
-    const cancelled = await store.cancel(started.runId);
-    expect(cancelled.status).toBe("cancelled");
-    expect(cancelled.error).toContain("cancelled");
-    expect(store.getActiveRunId()).toBeUndefined();
-    expect(await store.cancel(started.runId)).toMatchObject({
-      status: "cancelled",
-    });
-    await expect(store.cancel("evalrun_missing")).rejects.toBeInstanceOf(
-      EvalRunUnknownError
-    );
-    await expect(store.cancel(started.runId)).resolves.toMatchObject({
-      status: "cancelled",
-    });
-
-    // After cancel, a completed-looking batch cannot be cancelled again as running.
-    const completedId = started.runId;
-    const snap = await store.get(completedId);
-    expect(snap?.status).toBe("cancelled");
-    // Force a non-running status to exercise NotRunningError via a fresh id.
-    const fakeId = "evalrun_done_only";
-    (
-      store as unknown as {
-        runs: Map<string, { runId: string; status: string }>;
-      }
-    ).runs.set(fakeId, {
-      runId: fakeId,
-      status: "completed",
-    });
-    await expect(store.cancel(fakeId)).rejects.toBeInstanceOf(
-      EvalRunNotRunningError
-    );
-  });
-});

package/src/internal/eval-run-store.storage.test.ts

@@ -1,211 +0,0 @@
-import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { afterEach, describe, expect, it } from "vitest";
-import type { EvalRunSnapshot } from "../evals.js";
-import { defineStorage } from "../storage.js";
-import { EvalRunStore } from "./eval-run-store.js";
-import { StorageCoordinator } from "./storage-coordinator.js";
-
-function completedRun(runId: string): EvalRunSnapshot {
-  return {
-    runId,
-    status: "completed",
-    startedAt: "2026-01-01T00:00:00.000Z",
-    finishedAt: "2026-01-01T00:00:01.000Z",
-    summary: { passed: 1, failed: 0, total: 1, done: 1 },
-    cases: [],
-    config: { maxPlaygroundRuns: 20, durableRuns: true },
-  };
-}
-
-describe("EvalRunStore defineStorage hydrate", () => {
-  let dir: string | undefined;
-
-  afterEach(async () => {
-    if (dir !== undefined) {
-      await rm(dir, { recursive: true, force: true });
-      dir = undefined;
-    }
-  });
-
-  it("hydrates playground history from the defineStorage evals table", async () => {
-    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
-    await mkdir(join(dir, "evals"), { recursive: true });
-    await writeFile(
-      join(dir, "evals", "evals.config.js"),
-      "export default { maxConcurrency: 1 };\n",
-      "utf8"
-    );
-
-    const table = new Map<string, EvalRunSnapshot>();
-    const coordinator = new StorageCoordinator({
-      definition: defineStorage({
-        put: () => {},
-        evals: {
-          put: (run) => {
-            table.set(run.runId, run);
-          },
-          delete: (runId) => {
-            table.delete(runId);
-          },
-          list: () => [...table.values()],
-        },
-      }),
-      agentName: "test-agent",
-      projectRoot: dir,
-      logger: () => {},
-    });
-
-    const storage = coordinator.evalRuns();
-    expect(storage).toBeDefined();
-    storage!.save(completedRun("evalrun_from_sink"));
-    await coordinator.whenIdle();
-
-    const store = new EvalRunStore(dir, () => {}, coordinator.evalRuns());
-    await store.hydrate();
-    const listed = await store.listRuns();
-    expect(listed).toHaveLength(1);
-    expect(listed[0]?.runId).toBe("evalrun_from_sink");
-    expect(listed[0]?.status).toBe("completed");
-  });
-
-  it("marks interrupted running batches as failed on hydrate", async () => {
-    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
-    await mkdir(join(dir, "evals"), { recursive: true });
-    await writeFile(
-      join(dir, "evals", "evals.config.js"),
-      "export default { maxConcurrency: 1 };\n",
-      "utf8"
-    );
-
-    const interrupted: EvalRunSnapshot = {
-      ...completedRun("evalrun_interrupted"),
-      status: "running",
-      finishedAt: undefined,
-      cases: [
-        { id: "smoke", fileId: "smoke", status: "running" },
-        { id: "other", fileId: "other", status: "pending" },
-      ],
-    };
-    const saved: EvalRunSnapshot[] = [];
-    const store = new EvalRunStore(dir, () => {}, {
-      save: (run) => {
-        saved.push(structuredClone(run));
-      },
-      delete: () => {},
-      list: async () => [interrupted],
-    });
-    await store.hydrate();
-
-    const listed = await store.listRuns();
-    expect(listed[0]?.status).toBe("failed");
-    expect(listed[0]?.cases.every((c) => c.status === "done")).toBe(true);
-    // The summary is reconciled with the repaired case rows (both cases
-    // were interrupted, so nothing passed) instead of keeping stale counts.
-    expect(listed[0]?.summary).toEqual({
-      passed: 0,
-      failed: 2,
-      scored: 0,
-      skipped: 0,
-      total: 2,
-      done: 2,
-    });
-    // The repaired snapshot was written back to storage.
-    expect(saved.map((run) => run.runId)).toContain("evalrun_interrupted");
-    expect(saved.at(-1)?.status).toBe("failed");
-  });
-
-  it("prunes hydrated history even when eval discovery fails", async () => {
-    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
-    await mkdir(join(dir, "evals"), { recursive: true });
-    // Discovery throws at hydrate time; pruning must still run with the
-    // default playground-history window (20).
-    await writeFile(
-      join(dir, "evals", "broken.eval.js"),
-      "throw new Error('boom');\n",
-      "utf8"
-    );
-
-    const stored = Array.from({ length: 25 }, (_, i) => ({
-      ...completedRun(`evalrun_${String(i).padStart(2, "0")}`),
-      startedAt: `2026-01-01T00:${String(i).padStart(2, "0")}:00.000Z`,
-    }));
-    const deleted: string[] = [];
-    const store = new EvalRunStore(dir, () => {}, {
-      save: () => {},
-      delete: (runId) => {
-        deleted.push(runId);
-      },
-      list: async () => stored,
-    });
-    await store.hydrate();
-
-    const listed = await store.listRuns();
-    expect(listed).toHaveLength(20);
-    expect(deleted).toHaveLength(5);
-    // Oldest five (by startedAt) were pruned.
-    expect(deleted.sort()).toEqual([
-      "evalrun_00",
-      "evalrun_01",
-      "evalrun_02",
-      "evalrun_03",
-      "evalrun_04",
-    ]);
-  });
-
-  it("close() durably marks an in-flight batch as interrupted", async () => {
-    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
-    await mkdir(join(dir, "evals"), { recursive: true });
-    await writeFile(
-      join(dir, "evals", "evals.config.js"),
-      "export default { maxConcurrency: 1 };\n",
-      "utf8"
-    );
-
-    const running: EvalRunSnapshot = {
-      ...completedRun("evalrun_active"),
-      status: "running",
-      finishedAt: undefined,
-      summary: { passed: 1, failed: 0, total: 2, done: 1 },
-      cases: [
-        { id: "done", fileId: "done", status: "done", ok: true },
-        { id: "midway", fileId: "midway", status: "running" },
-      ],
-    };
-    const saved: EvalRunSnapshot[] = [];
-    const store = new EvalRunStore(dir, () => {}, {
-      save: (run) => {
-        saved.push(structuredClone(run));
-      },
-      delete: () => {},
-      list: async () => [running],
-    });
-    await store.hydrate();
-    // hydrate already repaired it; reset to a live in-flight shape.
-    const run = (await store.listRuns())[0]!;
-    run.status = "running";
-    run.error = undefined;
-    run.cases[1]!.status = "running";
-    run.cases[1]!.ok = undefined;
-    run.cases[1]!.error = undefined;
-    (store as unknown as { activeRunId?: string }).activeRunId = run.runId;
-    saved.length = 0;
-
-    store.close();
-
-    expect(saved).toHaveLength(1);
-    expect(saved[0]?.status).toBe("failed");
-    expect(saved[0]?.summary).toEqual({
-      passed: 1,
-      failed: 1,
-      scored: 0,
-      skipped: 0,
-      total: 2,
-      done: 2,
-    });
-    // Idempotent: a second close has nothing left to write.
-    store.close();
-    expect(saved).toHaveLength(1);
-  });
-});