@cursor/july 0.1.19 → 0.1.21

package/src/internal/evals-client.test.ts

@@ -0,0 +1,307 @@
+import { describe, expect, it, vi } from "vitest";
+import type { EvalRunSnapshot } from "../evals.js";
+import {
+  getRemoteEvalRun,
+  listRemoteEvalRuns,
+  listRemoteEvals,
+  RemoteEvalError,
+  runRemoteEvals,
+  snapshotToEvalRunResults,
+  startRemoteEvalRun,
+} from "./evals-client.js";
+
+function completedSnapshot(
+  overrides: Partial<EvalRunSnapshot> = {}
+): EvalRunSnapshot {
+  return {
+    runId: "evalrun_abc",
+    status: "completed",
+    startedAt: "2026-07-20T00:00:00.000Z",
+    finishedAt: "2026-07-20T00:00:05.000Z",
+    summary: { passed: 1, failed: 0, total: 1, done: 1 },
+    cases: [
+      {
+        id: "smoke",
+        fileId: "smoke",
+        status: "done",
+        ok: true,
+        assertions: [{ name: "succeeded", passed: true }],
+        durationMs: 1200,
+        inputs: ["hi"],
+        logs: [],
+      },
+    ],
+    config: { maxConcurrency: 2, durableRuns: true },
+    ...overrides,
+  };
+}
+
+describe("startRemoteEvalRun", () => {
+  it("POSTs filterIds/tags/timeoutMs/verbose and returns the snapshot", async () => {
+    const fetchImpl = vi.fn(async () =>
+      Response.json(
+        completedSnapshot({ status: "running", finishedAt: undefined }),
+        {
+          status: 202,
+        }
+      )
+    );
+
+    const snap = await startRemoteEvalRun({
+      baseUrl: "http://127.0.0.1:3000/weather-agent",
+      filterIds: ["smoke"],
+      tags: ["ci"],
+      timeoutMs: 45_000,
+      verbose: true,
+      fetchImpl: fetchImpl as unknown as typeof fetch,
+      auth: { headers: { "x-anyrun-port-token": "tok" } },
+    });
+
+    expect(snap.runId).toBe("evalrun_abc");
+    expect(fetchImpl).toHaveBeenCalledOnce();
+    expect(fetchImpl).toHaveBeenCalledWith(
+      "http://127.0.0.1:3000/weather-agent/v1/dev/evals/runs",
+      expect.objectContaining({
+        method: "POST",
+        headers: expect.objectContaining({
+          "content-type": "application/json",
+          "x-anyrun-port-token": "tok",
+        }),
+        body: JSON.stringify({
+          filterIds: ["smoke"],
+          tags: ["ci"],
+          timeoutMs: 45_000,
+          verbose: true,
+        }),
+      })
+    );
+  });
+
+  it("maps no_matching_evals distinctly from other 404s", async () => {
+    await expect(
+      startRemoteEvalRun({
+        baseUrl: "http://x",
+        fetchImpl: (async () =>
+          Response.json(
+            { ok: false, error: "no_matching_evals" },
+            { status: 404 }
+          )) as typeof fetch,
+      })
+    ).rejects.toMatchObject({
+      name: "RemoteEvalError",
+      status: 404,
+      code: "no_matching_evals",
+      message: "No matching evals found.",
+    });
+
+    await expect(
+      startRemoteEvalRun({
+        baseUrl: "http://x",
+        fetchImpl: (async () =>
+          Response.json(
+            { ok: false, error: "not_found" },
+            { status: 404 }
+          )) as typeof fetch,
+      })
+    ).rejects.toMatchObject({
+      name: "RemoteEvalError",
+      status: 404,
+      code: "start_failed",
+      message: "Could not start eval run: not_found",
+    });
+
+    await expect(
+      startRemoteEvalRun({
+        baseUrl: "http://x",
+        fetchImpl: (async () =>
+          Response.json(
+            {
+              ok: false,
+              error: "eval_run_in_progress",
+              activeRunId: "evalrun_busy",
+            },
+            { status: 409 }
+          )) as typeof fetch,
+      })
+    ).rejects.toBeInstanceOf(RemoteEvalError);
+  });
+});
+
+describe("getRemoteEvalRun / listRemoteEvalRuns", () => {
+  it("fetches one run and lists recent runs", async () => {
+    const done = completedSnapshot();
+    const fetchImpl = vi.fn(async (input: RequestInfo | URL) => {
+      const url = String(input);
+      if (url.endsWith("/v1/dev/evals/runs")) {
+        return Response.json({ runs: [done], activeRunId: "evalrun_abc" });
+      }
+      if (url.endsWith("/v1/dev/evals/runs/evalrun_abc")) {
+        return Response.json(done);
+      }
+      return Response.json({ ok: false }, { status: 404 });
+    });
+
+    const snap = await getRemoteEvalRun({
+      baseUrl: "http://agent",
+      runId: "evalrun_abc",
+      fetchImpl: fetchImpl as unknown as typeof fetch,
+    });
+    expect(snap.runId).toBe("evalrun_abc");
+
+    const listed = await listRemoteEvalRuns({
+      baseUrl: "http://agent",
+      fetchImpl: fetchImpl as unknown as typeof fetch,
+    });
+    expect(listed.activeRunId).toBe("evalrun_abc");
+    expect(listed.runs).toHaveLength(1);
+  });
+});
+
+describe("listRemoteEvals", () => {
+  it("filters by id prefix and tags", async () => {
+    const fetchImpl = vi.fn(async () =>
+      Response.json({
+        evals: [
+          { id: "smoke", fileId: "smoke", tags: ["ci"] },
+          { id: "weather/nyc", fileId: "weather", tags: ["weather"] },
+          { id: "weather/paris", fileId: "weather", tags: ["weather", "ci"] },
+        ],
+        config: { maxConcurrency: 2, maxPlaygroundRuns: 20, durableRuns: true },
+      })
+    );
+
+    const listed = await listRemoteEvals({
+      baseUrl: "http://agent",
+      filterIds: ["weather"],
+      tags: ["ci"],
+      fetchImpl: fetchImpl as unknown as typeof fetch,
+    });
+    expect(listed.map((e) => e.id)).toEqual(["weather/paris"]);
+  });
+});
+
+describe("runRemoteEvals", () => {
+  it("starts, polls until completed, and maps case results", async () => {
+    const running = completedSnapshot({
+      status: "running",
+      finishedAt: undefined,
+      summary: { passed: 0, failed: 0, total: 1, done: 0 },
+      cases: [
+        {
+          id: "smoke",
+          fileId: "smoke",
+          status: "running",
+          inputs: [],
+          logs: [],
+        },
+      ],
+    });
+    const done = completedSnapshot();
+    let polls = 0;
+    const fetchImpl = vi.fn(async (input: RequestInfo | URL) => {
+      const url = String(input);
+      if (url.endsWith("/v1/dev/evals/runs") && !url.includes("evalrun_")) {
+        return Response.json(running, { status: 202 });
+      }
+      polls += 1;
+      return Response.json(polls === 1 ? running : done);
+    });
+
+    const snapshots: EvalRunSnapshot[] = [];
+    const { snapshot, results } = await runRemoteEvals({
+      baseUrl: "http://agent",
+      pollMs: 1,
+      fetchImpl: fetchImpl as unknown as typeof fetch,
+      onSnapshot: (s) => snapshots.push(s),
+    });
+
+    expect(snapshot.status).toBe("completed");
+    expect(snapshot.config.durableRuns).toBe(true);
+    expect(results).toHaveLength(1);
+    expect(results[0]).toMatchObject({
+      id: "smoke",
+      ok: true,
+      durationMs: 1200,
+    });
+    expect(snapshots.length).toBeGreaterThanOrEqual(2);
+  });
+});
+
+describe("snapshotToEvalRunResults", () => {
+  it("defaults missing case fields for CLI printing", () => {
+    const results = snapshotToEvalRunResults(
+      completedSnapshot({
+        cases: [
+          {
+            id: "pending-ish",
+            fileId: "pending-ish",
+            status: "done",
+          },
+        ],
+      })
+    );
+    expect(results[0]).toEqual({
+      id: "pending-ish",
+      path: "pending-ish",
+      ok: false,
+      // A case with no verdict on the wire is graded from `ok`.
+      verdict: "failed",
+      assertions: [],
+      inputs: [],
+      logs: [],
+      metrics: {},
+      durationMs: 0,
+    });
+  });
+
+  it("carries verdict, skip reason, and metadata back from a remote batch", () => {
+    const results = snapshotToEvalRunResults(
+      completedSnapshot({
+        cases: [
+          {
+            id: "scored",
+            fileId: "scored",
+            status: "done",
+            ok: true,
+            verdict: "scored",
+            metadata: { owner: "agents" },
+            assertions: [
+              {
+                name: "score(recall)",
+                passed: false,
+                severity: "soft",
+                score: 0.4,
+                threshold: 0.6,
+              },
+            ],
+          },
+          {
+            id: "skipped",
+            fileId: "skipped",
+            status: "done",
+            ok: true,
+            verdict: "skipped",
+            skipReason: "no judge credentials",
+          },
+        ],
+      })
+    );
+    expect(results[0]).toMatchObject({
+      verdict: "scored",
+      ok: true,
+      metadata: { owner: "agents" },
+      assertions: [
+        {
+          name: "score(recall)",
+          severity: "soft",
+          score: 0.4,
+          threshold: 0.6,
+        },
+      ],
+    });
+    expect(results[1]).toMatchObject({
+      verdict: "skipped",
+      skipReason: "no judge credentials",
+    });
+  });
+});

package/src/internal/event-mapper.test.ts

@@ -0,0 +1,243 @@
+import type { ToolCall } from "@cursor/sdk";
+import { describe, expect, it } from "vitest";
+import type { SessionEventPayload } from "../types.js";
+import { TurnEventMapper } from "./event-mapper.js";
+
+/** Test harness updates — includes tool-call-delta ahead of the published SDK. */
+type TestUpdate =
+  | { type: "text-delta"; text: string }
+  | {
+      type: "tool-call-started";
+      callId: string;
+      modelCallId: string;
+      toolCall: ToolCall;
+    }
+  | {
+      type: "tool-call-completed";
+      callId: string;
+      modelCallId: string;
+      toolCall: ToolCall;
+    }
+  | {
+      type: "tool-call-delta";
+      callId: string;
+      modelCallId: string;
+      taskUpdate: Record<string, unknown> & { type: string };
+    };
+
+function collect(): {
+  events: SessionEventPayload[];
+  mapper: TurnEventMapper;
+} {
+  const events: SessionEventPayload[] = [];
+  const mapper = new TurnEventMapper((payload) => {
+    events.push(payload);
+  });
+  return { events, mapper };
+}
+
+function taskTool(name: string, description: string): ToolCall {
+  return {
+    type: "task",
+    args: {
+      description,
+      prompt: "do the work",
+      subagentType: { kind: "custom", name },
+    },
+  };
+}
+
+function shellTool(command: string): ToolCall {
+  return {
+    type: "shell",
+    args: { command },
+  };
+}
+
+function apply(mapper: TurnEventMapper, update: TestUpdate): void {
+  mapper.handleUpdate(update as never);
+}
+
+describe("TurnEventMapper subagent nesting", () => {
+  it("emits bookends and nested progress with parentCallId", () => {
+    const { events, mapper } = collect();
+
+    apply(mapper, {
+      type: "tool-call-started",
+      callId: "task-1",
+      modelCallId: "m1",
+      toolCall: taskTool("researcher", "Look up the weather"),
+    });
+    apply(mapper, {
+      type: "tool-call-delta",
+      callId: "task-1",
+      modelCallId: "m1",
+      taskUpdate: { type: "thinking-delta", text: "planning…" },
+    });
+    apply(mapper, {
+      type: "tool-call-delta",
+      callId: "task-1",
+      modelCallId: "m1",
+      taskUpdate: {
+        type: "thinking-completed",
+        thinkingDurationMs: 12,
+      },
+    });
+    apply(mapper, {
+      type: "tool-call-delta",
+      callId: "task-1",
+      modelCallId: "m1",
+      taskUpdate: {
+        type: "tool-call-started",
+        callId: "shell-1",
+        modelCallId: "m2",
+        toolCall: shellTool("curl weather"),
+      },
+    });
+    apply(mapper, {
+      type: "tool-call-delta",
+      callId: "task-1",
+      modelCallId: "m1",
+      taskUpdate: {
+        type: "tool-call-completed",
+        callId: "shell-1",
+        modelCallId: "m2",
+        toolCall: {
+          ...shellTool("curl weather"),
+          result: {
+            status: "success",
+            value: { exitCode: 0, stdout: "72F", stderr: "" },
+          },
+        },
+      },
+    });
+    apply(mapper, {
+      type: "tool-call-delta",
+      callId: "task-1",
+      modelCallId: "m1",
+      taskUpdate: { type: "text-delta", text: "It is 72F." },
+    });
+    apply(mapper, {
+      type: "tool-call-completed",
+      callId: "task-1",
+      modelCallId: "m1",
+      toolCall: {
+        type: "task",
+        args: taskTool("researcher", "Look up the weather").args,
+        result: {
+          status: "success",
+          value: {
+            isBackground: false,
+            backgroundReason: "unspecified",
+            agentId: "sub-1",
+          },
+        },
+      } as ToolCall,
+    });
+
+    expect(events.map((e) => e.type)).toEqual([
+      "actions.requested",
+      "subagent.called",
+      "reasoning.appended",
+      "reasoning.completed",
+      "actions.requested",
+      "action.result",
+      "message.appended",
+      "action.result",
+      "message.completed",
+      "subagent.completed",
+    ]);
+
+    expect(events[0]).toMatchObject({
+      type: "actions.requested",
+      data: { calls: [{ callId: "task-1", toolName: "task" }] },
+    });
+    expect(events[1]).toMatchObject({
+      type: "subagent.called",
+      data: { callId: "task-1", name: "researcher" },
+    });
+    expect(events[2]).toMatchObject({
+      type: "reasoning.appended",
+      data: { text: "planning…", parentCallId: "task-1" },
+    });
+    expect(events[4]).toMatchObject({
+      type: "actions.requested",
+      data: {
+        parentCallId: "task-1",
+        calls: [
+          { callId: "shell-1", toolName: "shell", parentCallId: "task-1" },
+        ],
+      },
+    });
+    expect(events[5]).toMatchObject({
+      type: "action.result",
+      data: { callId: "shell-1", parentCallId: "task-1", isError: false },
+    });
+    expect(events[6]).toMatchObject({
+      type: "message.appended",
+      data: { text: "It is 72F.", parentCallId: "task-1" },
+    });
+    expect(events[8]).toMatchObject({
+      type: "message.completed",
+      data: { text: "It is 72F.", parentCallId: "task-1" },
+    });
+    expect(events[9]).toMatchObject({
+      type: "subagent.completed",
+      data: { callId: "task-1", name: "researcher" },
+    });
+  });
+
+  it("keeps parent-level text separate from nested subagent text", () => {
+    const { events, mapper } = collect();
+    const updates: TestUpdate[] = [
+      { type: "text-delta", text: "Parent says hi. " },
+      {
+        type: "tool-call-started",
+        callId: "task-1",
+        modelCallId: "m1",
+        toolCall: taskTool("researcher", "research"),
+      },
+      {
+        type: "tool-call-delta",
+        callId: "task-1",
+        modelCallId: "m1",
+        taskUpdate: { type: "text-delta", text: "Nested says hi." },
+      },
+      {
+        type: "tool-call-completed",
+        callId: "task-1",
+        modelCallId: "m1",
+        toolCall: {
+          type: "task",
+          args: taskTool("researcher", "research").args,
+          result: {
+            status: "success",
+            value: {
+              isBackground: false,
+              backgroundReason: "unspecified",
+            },
+          },
+        } as ToolCall,
+      },
+    ];
+    for (const update of updates) {
+      apply(mapper, update);
+    }
+    mapper.finish();
+
+    const parentText = events.filter(
+      (e) => e.type === "message.completed" && e.data.parentCallId === undefined
+    );
+    const nestedText = events.filter(
+      (e) => e.type === "message.completed" && e.data.parentCallId === "task-1"
+    );
+    expect(parentText).toHaveLength(1);
+    expect(parentText[0]).toMatchObject({
+      data: { text: "Parent says hi. " },
+    });
+    expect(nestedText).toHaveLength(1);
+    expect(nestedText[0]).toMatchObject({
+      data: { text: "Nested says hi.", parentCallId: "task-1" },
+    });
+  });
+});