@cursor/july 0.1.20 → 0.1.21

package/src/internal/eval-runner.http.test.ts

@@ -0,0 +1,403 @@
+/**
+ * Runs the eval runner against a real HTTP agent stub.
+ *
+ * `eval-runner.run.test.ts` mocks the session transport to cover grading logic;
+ * this file keeps the transport real, so the assertions are graded from events
+ * that actually arrived over the NDJSON session stream. That is the part a mock
+ * cannot prove.
+ */
+
+import { createServer, type Server } from "node:http";
+import type { AddressInfo } from "node:net";
+import { afterEach, describe, expect, it } from "vitest";
+import type { EvalDefinition, EvalRunResult } from "../evals.js";
+import { defineEval, includes } from "../evals.js";
+import type { SessionEvent } from "../types.js";
+import { expandEvalModule, runDiscoveredEvals } from "./eval-runner.js";
+
+interface StubAgent {
+  url: string;
+  /** Messages the runner sent, in order. */
+  received: string[];
+  close(): Promise<void>;
+}
+
+/** One scripted turn: the events the stub streams back for that message. */
+type TurnScript = (message: string, turnId: string) => ScriptedEvent[];
+
+/**
+ * A scripted event without its envelope cursor; `appendTurn` fills in
+ * `sessionId` and `index`. Loosely typed on purpose: the stub composes events
+ * generically, which the `type`/`data` discriminated union cannot express.
+ */
+interface ScriptedEvent {
+  at: string;
+  turnId?: string;
+  type: string;
+  data: unknown;
+}
+
+async function startStubAgent(script: TurnScript): Promise<StubAgent> {
+  const received: string[] = [];
+  const sessions = new Map<string, SessionEvent[]>();
+  let nextSession = 0;
+  let nextTurn = 0;
+
+  const appendTurn = (sessionId: string, message: string): void => {
+    const events = sessions.get(sessionId)!;
+    nextTurn++;
+    for (const event of script(message, `turn_${nextTurn}`)) {
+      // `index` is the monotonic per-session cursor a client reconnects with
+      // via `?startIndex=`, so it must be the event's position in this session.
+      events.push({
+        ...event,
+        sessionId,
+        index: events.length,
+      } as SessionEvent);
+    }
+  };
+
+  const server: Server = createServer((req, res) => {
+    const url = new URL(req.url ?? "/", "http://127.0.0.1");
+    const chunks: Buffer[] = [];
+    req.on("data", (chunk: Buffer) => chunks.push(chunk));
+    req.on("end", () => {
+      const body =
+        chunks.length === 0
+          ? {}
+          : (JSON.parse(Buffer.concat(chunks).toString("utf8")) as {
+              message?: string;
+            });
+      const streamMatch = /^\/v1\/session\/([^/]+)\/stream$/.exec(url.pathname);
+      if (streamMatch !== null) {
+        const events = sessions.get(streamMatch[1]!) ?? [];
+        const startIndex = Number(url.searchParams.get("startIndex") ?? "0");
+        res.writeHead(200, { "content-type": "application/x-ndjson" });
+        for (const event of events.slice(startIndex)) {
+          res.write(`${JSON.stringify(event)}\n`);
+        }
+        res.end();
+        return;
+      }
+      if (url.pathname === "/v1/session") {
+        nextSession++;
+        const sessionId = `ses_${nextSession}`;
+        sessions.set(sessionId, []);
+        received.push(body.message ?? "");
+        appendTurn(sessionId, body.message ?? "");
+        res.writeHead(200, { "content-type": "application/json" });
+        res.end(
+          JSON.stringify({ ok: true, sessionId, continuationToken: "tok_1" })
+        );
+        return;
+      }
+      const followUp = /^\/v1\/session\/([^/]+)$/.exec(url.pathname);
+      if (followUp !== null) {
+        received.push(body.message ?? "");
+        appendTurn(followUp[1]!, body.message ?? "");
+        res.writeHead(200, { "content-type": "application/json" });
+        res.end(JSON.stringify({ ok: true, continuationToken: "tok_2" }));
+        return;
+      }
+      res.writeHead(404).end();
+    });
+  });
+
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const { port } = server.address() as AddressInfo;
+  return {
+    url: `http://127.0.0.1:${port}`,
+    received,
+    close: () =>
+      new Promise<void>((resolve) => {
+        server.close(() => resolve());
+      }),
+  };
+}
+
+function event(
+  type: string,
+  data: unknown,
+  turnId: string | undefined
+): ScriptedEvent {
+  return { at: new Date().toISOString(), turnId, type, data };
+}
+
+/** A weather turn: one completed tool call, one reply, then park. */
+const weatherScript: TurnScript = (message, turnId) => [
+  event("message.received", { text: message }, turnId),
+  event(
+    "actions.requested",
+    {
+      calls: [
+        {
+          callId: `${turnId}_c1`,
+          toolName: "get_weather",
+          args: { city: "NYC" },
+        },
+      ],
+    },
+    turnId
+  ),
+  event(
+    "action.result",
+    {
+      callId: `${turnId}_c1`,
+      toolName: "get_weather",
+      output: { tempF: 72 },
+      isError: false,
+    },
+    turnId
+  ),
+  event(
+    "message.completed",
+    { text: "Sunny, 72F in NYC.", finishReason: "stop" },
+    turnId
+  ),
+  event("turn.completed", {}, turnId),
+  event("session.waiting", {}, undefined),
+];
+
+let stub: StubAgent | undefined;
+
+afterEach(async () => {
+  await stub?.close();
+  stub = undefined;
+});
+
+async function run(
+  definition: EvalDefinition,
+  baseUrl: string
+): Promise<EvalRunResult[]> {
+  return runDiscoveredEvals({
+    projectRoot: "/unused",
+    baseUrl,
+    discovered: {
+      evals: expandEvalModule("http", definition).map((e) => ({
+        id: e.id,
+        fileId: "http",
+        path: "/repo/evals/http.eval.ts",
+        definition: e.definition,
+      })),
+      config: { maxConcurrency: 2 },
+    },
+  });
+}
+
+describe("runner over a real session stream", () => {
+  it("grades assertions from events that arrived over HTTP", async () => {
+    stub = await startStubAgent(weatherScript);
+    const [result] = await run(
+      defineEval({
+        async test(t) {
+          await t.send("What is the weather in NYC?");
+          t.succeeded();
+          t.calledTool("get_weather", {
+            input: { city: "NYC" },
+            output: { tempF: 72 },
+            status: "completed",
+            count: 1,
+          });
+          t.noFailedActions();
+          t.messageIncludes("Sunny");
+          t.check(t.reply, includes("72F"));
+          t.maxToolCalls(1);
+        },
+      }),
+      stub.url
+    );
+    expect(result?.verdict).toBe("passed");
+    expect(result?.assertions.every((a) => a.passed)).toBe(true);
+    expect(stub.received).toEqual(["What is the weather in NYC?"]);
+    // The transport-derived views the CLI and playground print.
+    expect(result?.tools).toEqual(["get_weather"]);
+    expect(result?.toolCalls).toEqual([
+      {
+        toolName: "get_weather",
+        args: { city: "NYC" },
+        output: { tempF: 72 },
+        isError: false,
+      },
+    ]);
+    expect(result?.finalText).toBe("Sunny, 72F in NYC.");
+    expect(result?.sessionId).toBe("ses_1");
+  });
+
+  it("resumes the stream so a second send reads the second turn", async () => {
+    // Each turn replies with the message it received and calls a distinct tool,
+    // so a send that replayed the earlier turn would be unmistakable. The
+    // stream replays from an index and stops at the first terminal event, so
+    // this is the regression test for resuming at the right cursor.
+    stub = await startStubAgent((message, turnId) => [
+      event("message.received", { text: message }, turnId),
+      event(
+        "actions.requested",
+        {
+          calls: [
+            { callId: `${turnId}_c1`, toolName: `tool_${message}`, args: {} },
+          ],
+        },
+        turnId
+      ),
+      event(
+        "action.result",
+        {
+          callId: `${turnId}_c1`,
+          toolName: `tool_${message}`,
+          output: "ok",
+          isError: false,
+        },
+        turnId
+      ),
+      event(
+        "message.completed",
+        { text: `reply to ${message}`, finishReason: "stop" },
+        turnId
+      ),
+      event("turn.completed", {}, turnId),
+      event("session.waiting", {}, undefined),
+    ]);
+
+    const [result] = await run(
+      defineEval({
+        async test(t) {
+          const first = await t.send("first");
+          t.check(first.message, includes("reply to first"));
+
+          const second = await t.send("second");
+          t.check(second.message, includes("reply to second"));
+          t.check(t.reply, includes("reply to second"));
+
+          // Each turn sees only its own tool call.
+          first.calledTool("tool_first", { count: 1 });
+          first.notCalledTool("tool_second");
+          second.calledTool("tool_second", { count: 1 });
+          second.notCalledTool("tool_first");
+
+          // The run sees both, exactly once each.
+          t.calledTool("tool_first", { count: 1 });
+          t.calledTool("tool_second", { count: 1 });
+          t.maxToolCalls(2);
+          t.toolOrder(["tool_first", "tool_second"]);
+          t.check(second.sessionId, includes(first.sessionId));
+        },
+      }),
+      stub.url
+    );
+    expect(result?.verdict).toBe("passed");
+    const failed = result!.assertions.filter((a) => !a.passed);
+    expect(failed).toEqual([]);
+    expect(stub.received).toEqual(["first", "second"]);
+    expect(result?.inputs).toEqual(["first", "second"]);
+    expect(result?.finalText).toBe("reply to second");
+  });
+
+  it("fails a run whose turn failed on the wire", async () => {
+    stub = await startStubAgent((message, turnId) => [
+      event("message.received", { text: message }, turnId),
+      event("turn.failed", { message: "provider unavailable" }, turnId),
+      event("session.waiting", {}, undefined),
+    ]);
+    const [result] = await run(
+      defineEval({
+        async test(t) {
+          await t.send("hello");
+          t.succeeded();
+        },
+      }),
+      stub.url
+    );
+    expect(result?.verdict).toBe("failed");
+    expect(result?.assertions[0]).toMatchObject({
+      name: "succeeded",
+      passed: false,
+      detail: "provider unavailable",
+    });
+  });
+
+  it("detects a run parked on an unanswered approval request", async () => {
+    stub = await startStubAgent((message, turnId) => [
+      event("message.received", { text: message }, turnId),
+      event(
+        "actions.requested",
+        {
+          calls: [{ callId: "c1", toolName: "deploy", args: { env: "prod" } }],
+        },
+        turnId
+      ),
+      event(
+        "action.approval_requested",
+        { callId: "c1", toolName: "deploy", args: { env: "prod" } },
+        turnId
+      ),
+      event("session.waiting", {}, undefined),
+    ]);
+    const [result] = await run(
+      defineEval({
+        async test(t) {
+          await t.send("deploy to prod");
+          t.parked();
+          const request = t.requireInputRequest({ toolName: "deploy" });
+          t.check(request.args, includes("prod"));
+          t.calledTool("deploy", { status: "pending", count: 1 });
+        },
+      }),
+      stub.url
+    );
+    expect(result?.verdict).toBe("passed");
+    expect(result?.assertions.map((a) => a.passed)).toEqual([
+      true,
+      true,
+      true,
+      true,
+    ]);
+  });
+
+  it("records a failed tool call without failing the turn", async () => {
+    stub = await startStubAgent((message, turnId) => [
+      event("message.received", { text: message }, turnId),
+      event(
+        "actions.requested",
+        {
+          calls: [
+            { callId: "c1", toolName: "bash", args: { command: "false" } },
+          ],
+        },
+        turnId
+      ),
+      event(
+        "action.result",
+        { callId: "c1", toolName: "bash", output: "exit 1", isError: true },
+        turnId
+      ),
+      event(
+        "message.completed",
+        { text: "That command failed.", finishReason: "stop" },
+        turnId
+      ),
+      event("turn.completed", {}, turnId),
+      event("session.waiting", {}, undefined),
+    ]);
+    const [result] = await run(
+      defineEval({
+        async test(t) {
+          await t.send("run false");
+          t.succeeded();
+          t.calledTool("bash", { status: "failed", count: 1 });
+          t.noFailedActions();
+        },
+      }),
+      stub.url
+    );
+    expect(result?.verdict).toBe("failed");
+    const byName = new Map(result!.assertions.map((a) => [a.name, a]));
+    expect(byName.get("succeeded")?.passed).toBe(true);
+    expect(
+      byName.get("calledTool(bash, {status=failed, count=1})")?.passed
+    ).toBe(true);
+    expect(byName.get("noFailedActions")).toMatchObject({
+      passed: false,
+      detail: "bash",
+    });
+  });
+});