@cursor/july 0.1.24 → 0.1.26

package/src/internal/eval-run-store.cancel.test.ts

@@ -0,0 +1,142 @@
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+  EvalRunNotRunningError,
+  EvalRunStore,
+  EvalRunUnknownError,
+} from "./eval-run-store.js";
+
+vi.mock("./eval-runner.js", async (importOriginal) => {
+  const actual = await importOriginal<typeof import("./eval-runner.js")>();
+  return {
+    ...actual,
+    runDiscoveredEvals: vi.fn(
+      async (options: {
+        signal?: AbortSignal;
+        onCaseStart?: (id: string) => void;
+        onCaseDone?: (result: {
+          id: string;
+          path: string;
+          ok: boolean;
+          assertions: [];
+          logs: string[];
+          metrics: Record<string, string | number>;
+          inputs: string[];
+          toolCalls: [];
+          tools: string[];
+          durationMs: number;
+        }) => void;
+        discovered?: {
+          evals: Array<{ id: string; path: string }>;
+        };
+      }) => {
+        const first = options.discovered?.evals[0];
+        if (first !== undefined) {
+          options.onCaseStart?.(first.id);
+        }
+        await new Promise<void>((resolve, reject) => {
+          const onAbort = (): void => {
+            options.signal?.removeEventListener("abort", onAbort);
+            reject(new Error("cancelled"));
+          };
+          if (options.signal?.aborted) {
+            onAbort();
+            return;
+          }
+          options.signal?.addEventListener("abort", onAbort, { once: true });
+          // Keep the batch "running" until cancel aborts.
+          setTimeout(() => {
+            options.signal?.removeEventListener("abort", onAbort);
+            if (first !== undefined) {
+              options.onCaseDone?.({
+                id: first.id,
+                path: first.path,
+                ok: true,
+                assertions: [],
+                logs: [],
+                metrics: {},
+                inputs: [],
+                toolCalls: [],
+                tools: [],
+                durationMs: 1,
+              });
+            }
+            resolve();
+          }, 5_000);
+        });
+        return [];
+      }
+    ),
+  };
+});
+
+describe("EvalRunStore.cancel", () => {
+  let dir: string | undefined;
+
+  afterEach(async () => {
+    if (dir !== undefined) {
+      await rm(dir, { recursive: true, force: true });
+      dir = undefined;
+    }
+    vi.clearAllMocks();
+  });
+
+  it("cancels a running batch by Eval ID and clears the active run", async () => {
+    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-cancel-"));
+    await mkdir(join(dir, "evals"), { recursive: true });
+    await writeFile(
+      join(dir, "evals", "evals.config.js"),
+      "export default { maxConcurrency: 1 };\n",
+      "utf8"
+    );
+    await writeFile(
+      join(dir, "evals", "smoke.eval.js"),
+      `export default {
+  __agentServe: "eval",
+  async test() {}
+};
+`,
+      "utf8"
+    );
+
+    const store = new EvalRunStore(dir, () => {});
+    store.loopbackUrl = "http://127.0.0.1:9/smoke";
+    const started = await store.start();
+    expect(started.status).toBe("running");
+    expect(store.getActiveRunId()).toBe(started.runId);
+
+    const cancelled = await store.cancel(started.runId);
+    expect(cancelled.status).toBe("cancelled");
+    expect(cancelled.error).toContain("cancelled");
+    expect(store.getActiveRunId()).toBeUndefined();
+    expect(await store.cancel(started.runId)).toMatchObject({
+      status: "cancelled",
+    });
+    await expect(store.cancel("evalrun_missing")).rejects.toBeInstanceOf(
+      EvalRunUnknownError
+    );
+    await expect(store.cancel(started.runId)).resolves.toMatchObject({
+      status: "cancelled",
+    });
+
+    // After cancel, a completed-looking batch cannot be cancelled again as running.
+    const completedId = started.runId;
+    const snap = await store.get(completedId);
+    expect(snap?.status).toBe("cancelled");
+    // Force a non-running status to exercise NotRunningError via a fresh id.
+    const fakeId = "evalrun_done_only";
+    (
+      store as unknown as {
+        runs: Map<string, { runId: string; status: string }>;
+      }
+    ).runs.set(fakeId, {
+      runId: fakeId,
+      status: "completed",
+    });
+    await expect(store.cancel(fakeId)).rejects.toBeInstanceOf(
+      EvalRunNotRunningError
+    );
+  });
+});

package/src/internal/eval-run-store.storage.test.ts

@@ -0,0 +1,211 @@
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import type { EvalRunSnapshot } from "../evals.js";
+import { defineStorage } from "../storage.js";
+import { EvalRunStore } from "./eval-run-store.js";
+import { StorageCoordinator } from "./storage-coordinator.js";
+
+function completedRun(runId: string): EvalRunSnapshot {
+  return {
+    runId,
+    status: "completed",
+    startedAt: "2026-01-01T00:00:00.000Z",
+    finishedAt: "2026-01-01T00:00:01.000Z",
+    summary: { passed: 1, failed: 0, total: 1, done: 1 },
+    cases: [],
+    config: { maxPlaygroundRuns: 20, durableRuns: true },
+  };
+}
+
+describe("EvalRunStore defineStorage hydrate", () => {
+  let dir: string | undefined;
+
+  afterEach(async () => {
+    if (dir !== undefined) {
+      await rm(dir, { recursive: true, force: true });
+      dir = undefined;
+    }
+  });
+
+  it("hydrates playground history from the defineStorage evals table", async () => {
+    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
+    await mkdir(join(dir, "evals"), { recursive: true });
+    await writeFile(
+      join(dir, "evals", "evals.config.js"),
+      "export default { maxConcurrency: 1 };\n",
+      "utf8"
+    );
+
+    const table = new Map<string, EvalRunSnapshot>();
+    const coordinator = new StorageCoordinator({
+      definition: defineStorage({
+        put: () => {},
+        evals: {
+          put: (run) => {
+            table.set(run.runId, run);
+          },
+          delete: (runId) => {
+            table.delete(runId);
+          },
+          list: () => [...table.values()],
+        },
+      }),
+      agentName: "test-agent",
+      projectRoot: dir,
+      logger: () => {},
+    });
+
+    const storage = coordinator.evalRuns();
+    expect(storage).toBeDefined();
+    storage!.save(completedRun("evalrun_from_sink"));
+    await coordinator.whenIdle();
+
+    const store = new EvalRunStore(dir, () => {}, coordinator.evalRuns());
+    await store.hydrate();
+    const listed = await store.listRuns();
+    expect(listed).toHaveLength(1);
+    expect(listed[0]?.runId).toBe("evalrun_from_sink");
+    expect(listed[0]?.status).toBe("completed");
+  });
+
+  it("marks interrupted running batches as failed on hydrate", async () => {
+    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
+    await mkdir(join(dir, "evals"), { recursive: true });
+    await writeFile(
+      join(dir, "evals", "evals.config.js"),
+      "export default { maxConcurrency: 1 };\n",
+      "utf8"
+    );
+
+    const interrupted: EvalRunSnapshot = {
+      ...completedRun("evalrun_interrupted"),
+      status: "running",
+      finishedAt: undefined,
+      cases: [
+        { id: "smoke", fileId: "smoke", status: "running" },
+        { id: "other", fileId: "other", status: "pending" },
+      ],
+    };
+    const saved: EvalRunSnapshot[] = [];
+    const store = new EvalRunStore(dir, () => {}, {
+      save: (run) => {
+        saved.push(structuredClone(run));
+      },
+      delete: () => {},
+      list: async () => [interrupted],
+    });
+    await store.hydrate();
+
+    const listed = await store.listRuns();
+    expect(listed[0]?.status).toBe("failed");
+    expect(listed[0]?.cases.every((c) => c.status === "done")).toBe(true);
+    // The summary is reconciled with the repaired case rows (both cases
+    // were interrupted, so nothing passed) instead of keeping stale counts.
+    expect(listed[0]?.summary).toEqual({
+      passed: 0,
+      failed: 2,
+      scored: 0,
+      skipped: 0,
+      total: 2,
+      done: 2,
+    });
+    // The repaired snapshot was written back to storage.
+    expect(saved.map((run) => run.runId)).toContain("evalrun_interrupted");
+    expect(saved.at(-1)?.status).toBe("failed");
+  });
+
+  it("prunes hydrated history even when eval discovery fails", async () => {
+    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
+    await mkdir(join(dir, "evals"), { recursive: true });
+    // Discovery throws at hydrate time; pruning must still run with the
+    // default playground-history window (20).
+    await writeFile(
+      join(dir, "evals", "broken.eval.js"),
+      "throw new Error('boom');\n",
+      "utf8"
+    );
+
+    const stored = Array.from({ length: 25 }, (_, i) => ({
+      ...completedRun(`evalrun_${String(i).padStart(2, "0")}`),
+      startedAt: `2026-01-01T00:${String(i).padStart(2, "0")}:00.000Z`,
+    }));
+    const deleted: string[] = [];
+    const store = new EvalRunStore(dir, () => {}, {
+      save: () => {},
+      delete: (runId) => {
+        deleted.push(runId);
+      },
+      list: async () => stored,
+    });
+    await store.hydrate();
+
+    const listed = await store.listRuns();
+    expect(listed).toHaveLength(20);
+    expect(deleted).toHaveLength(5);
+    // Oldest five (by startedAt) were pruned.
+    expect(deleted.sort()).toEqual([
+      "evalrun_00",
+      "evalrun_01",
+      "evalrun_02",
+      "evalrun_03",
+      "evalrun_04",
+    ]);
+  });
+
+  it("close() durably marks an in-flight batch as interrupted", async () => {
+    dir = await mkdtemp(join(tmpdir(), "agent-serve-eval-storage-"));
+    await mkdir(join(dir, "evals"), { recursive: true });
+    await writeFile(
+      join(dir, "evals", "evals.config.js"),
+      "export default { maxConcurrency: 1 };\n",
+      "utf8"
+    );
+
+    const running: EvalRunSnapshot = {
+      ...completedRun("evalrun_active"),
+      status: "running",
+      finishedAt: undefined,
+      summary: { passed: 1, failed: 0, total: 2, done: 1 },
+      cases: [
+        { id: "done", fileId: "done", status: "done", ok: true },
+        { id: "midway", fileId: "midway", status: "running" },
+      ],
+    };
+    const saved: EvalRunSnapshot[] = [];
+    const store = new EvalRunStore(dir, () => {}, {
+      save: (run) => {
+        saved.push(structuredClone(run));
+      },
+      delete: () => {},
+      list: async () => [running],
+    });
+    await store.hydrate();
+    // hydrate already repaired it; reset to a live in-flight shape.
+    const run = (await store.listRuns())[0]!;
+    run.status = "running";
+    run.error = undefined;
+    run.cases[1]!.status = "running";
+    run.cases[1]!.ok = undefined;
+    run.cases[1]!.error = undefined;
+    (store as unknown as { activeRunId?: string }).activeRunId = run.runId;
+    saved.length = 0;
+
+    store.close();
+
+    expect(saved).toHaveLength(1);
+    expect(saved[0]?.status).toBe("failed");
+    expect(saved[0]?.summary).toEqual({
+      passed: 1,
+      failed: 1,
+      scored: 0,
+      skipped: 0,
+      total: 2,
+      done: 2,
+    });
+    // Idempotent: a second close has nothing left to write.
+    store.close();
+    expect(saved).toHaveLength(1);
+  });
+});