@cursor/july 0.1.22 → 0.1.24

package/src/internal/eval-runner.http.test.ts

@@ -1,403 +0,0 @@
-/**
- * Runs the eval runner against a real HTTP agent stub.
- *
- * `eval-runner.run.test.ts` mocks the session transport to cover grading logic;
- * this file keeps the transport real, so the assertions are graded from events
- * that actually arrived over the NDJSON session stream. That is the part a mock
- * cannot prove.
- */
-
-import { createServer, type Server } from "node:http";
-import type { AddressInfo } from "node:net";
-import { afterEach, describe, expect, it } from "vitest";
-import type { EvalDefinition, EvalRunResult } from "../evals.js";
-import { defineEval, includes } from "../evals.js";
-import type { SessionEvent } from "../types.js";
-import { expandEvalModule, runDiscoveredEvals } from "./eval-runner.js";
-
-interface StubAgent {
-  url: string;
-  /** Messages the runner sent, in order. */
-  received: string[];
-  close(): Promise<void>;
-}
-
-/** One scripted turn: the events the stub streams back for that message. */
-type TurnScript = (message: string, turnId: string) => ScriptedEvent[];
-
-/**
- * A scripted event without its envelope cursor; `appendTurn` fills in
- * `sessionId` and `index`. Loosely typed on purpose: the stub composes events
- * generically, which the `type`/`data` discriminated union cannot express.
- */
-interface ScriptedEvent {
-  at: string;
-  turnId?: string;
-  type: string;
-  data: unknown;
-}
-
-async function startStubAgent(script: TurnScript): Promise<StubAgent> {
-  const received: string[] = [];
-  const sessions = new Map<string, SessionEvent[]>();
-  let nextSession = 0;
-  let nextTurn = 0;
-
-  const appendTurn = (sessionId: string, message: string): void => {
-    const events = sessions.get(sessionId)!;
-    nextTurn++;
-    for (const event of script(message, `turn_${nextTurn}`)) {
-      // `index` is the monotonic per-session cursor a client reconnects with
-      // via `?startIndex=`, so it must be the event's position in this session.
-      events.push({
-        ...event,
-        sessionId,
-        index: events.length,
-      } as SessionEvent);
-    }
-  };
-
-  const server: Server = createServer((req, res) => {
-    const url = new URL(req.url ?? "/", "http://127.0.0.1");
-    const chunks: Buffer[] = [];
-    req.on("data", (chunk: Buffer) => chunks.push(chunk));
-    req.on("end", () => {
-      const body =
-        chunks.length === 0
-          ? {}
-          : (JSON.parse(Buffer.concat(chunks).toString("utf8")) as {
-              message?: string;
-            });
-      const streamMatch = /^\/v1\/session\/([^/]+)\/stream$/.exec(url.pathname);
-      if (streamMatch !== null) {
-        const events = sessions.get(streamMatch[1]!) ?? [];
-        const startIndex = Number(url.searchParams.get("startIndex") ?? "0");
-        res.writeHead(200, { "content-type": "application/x-ndjson" });
-        for (const event of events.slice(startIndex)) {
-          res.write(`${JSON.stringify(event)}\n`);
-        }
-        res.end();
-        return;
-      }
-      if (url.pathname === "/v1/session") {
-        nextSession++;
-        const sessionId = `ses_${nextSession}`;
-        sessions.set(sessionId, []);
-        received.push(body.message ?? "");
-        appendTurn(sessionId, body.message ?? "");
-        res.writeHead(200, { "content-type": "application/json" });
-        res.end(
-          JSON.stringify({ ok: true, sessionId, continuationToken: "tok_1" })
-        );
-        return;
-      }
-      const followUp = /^\/v1\/session\/([^/]+)$/.exec(url.pathname);
-      if (followUp !== null) {
-        received.push(body.message ?? "");
-        appendTurn(followUp[1]!, body.message ?? "");
-        res.writeHead(200, { "content-type": "application/json" });
-        res.end(JSON.stringify({ ok: true, continuationToken: "tok_2" }));
-        return;
-      }
-      res.writeHead(404).end();
-    });
-  });
-
-  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
-  const { port } = server.address() as AddressInfo;
-  return {
-    url: `http://127.0.0.1:${port}`,
-    received,
-    close: () =>
-      new Promise<void>((resolve) => {
-        server.close(() => resolve());
-      }),
-  };
-}
-
-function event(
-  type: string,
-  data: unknown,
-  turnId: string | undefined
-): ScriptedEvent {
-  return { at: new Date().toISOString(), turnId, type, data };
-}
-
-/** A weather turn: one completed tool call, one reply, then park. */
-const weatherScript: TurnScript = (message, turnId) => [
-  event("message.received", { text: message }, turnId),
-  event(
-    "actions.requested",
-    {
-      calls: [
-        {
-          callId: `${turnId}_c1`,
-          toolName: "get_weather",
-          args: { city: "NYC" },
-        },
-      ],
-    },
-    turnId
-  ),
-  event(
-    "action.result",
-    {
-      callId: `${turnId}_c1`,
-      toolName: "get_weather",
-      output: { tempF: 72 },
-      isError: false,
-    },
-    turnId
-  ),
-  event(
-    "message.completed",
-    { text: "Sunny, 72F in NYC.", finishReason: "stop" },
-    turnId
-  ),
-  event("turn.completed", {}, turnId),
-  event("session.waiting", {}, undefined),
-];
-
-let stub: StubAgent | undefined;
-
-afterEach(async () => {
-  await stub?.close();
-  stub = undefined;
-});
-
-async function run(
-  definition: EvalDefinition,
-  baseUrl: string
-): Promise<EvalRunResult[]> {
-  return runDiscoveredEvals({
-    projectRoot: "/unused",
-    baseUrl,
-    discovered: {
-      evals: expandEvalModule("http", definition).map((e) => ({
-        id: e.id,
-        fileId: "http",
-        path: "/repo/evals/http.eval.ts",
-        definition: e.definition,
-      })),
-      config: { maxConcurrency: 2 },
-    },
-  });
-}
-
-describe("runner over a real session stream", () => {
-  it("grades assertions from events that arrived over HTTP", async () => {
-    stub = await startStubAgent(weatherScript);
-    const [result] = await run(
-      defineEval({
-        async test(t) {
-          await t.send("What is the weather in NYC?");
-          t.succeeded();
-          t.calledTool("get_weather", {
-            input: { city: "NYC" },
-            output: { tempF: 72 },
-            status: "completed",
-            count: 1,
-          });
-          t.noFailedActions();
-          t.messageIncludes("Sunny");
-          t.check(t.reply, includes("72F"));
-          t.maxToolCalls(1);
-        },
-      }),
-      stub.url
-    );
-    expect(result?.verdict).toBe("passed");
-    expect(result?.assertions.every((a) => a.passed)).toBe(true);
-    expect(stub.received).toEqual(["What is the weather in NYC?"]);
-    // The transport-derived views the CLI and playground print.
-    expect(result?.tools).toEqual(["get_weather"]);
-    expect(result?.toolCalls).toEqual([
-      {
-        toolName: "get_weather",
-        args: { city: "NYC" },
-        output: { tempF: 72 },
-        isError: false,
-      },
-    ]);
-    expect(result?.finalText).toBe("Sunny, 72F in NYC.");
-    expect(result?.sessionId).toBe("ses_1");
-  });
-
-  it("resumes the stream so a second send reads the second turn", async () => {
-    // Each turn replies with the message it received and calls a distinct tool,
-    // so a send that replayed the earlier turn would be unmistakable. The
-    // stream replays from an index and stops at the first terminal event, so
-    // this is the regression test for resuming at the right cursor.
-    stub = await startStubAgent((message, turnId) => [
-      event("message.received", { text: message }, turnId),
-      event(
-        "actions.requested",
-        {
-          calls: [
-            { callId: `${turnId}_c1`, toolName: `tool_${message}`, args: {} },
-          ],
-        },
-        turnId
-      ),
-      event(
-        "action.result",
-        {
-          callId: `${turnId}_c1`,
-          toolName: `tool_${message}`,
-          output: "ok",
-          isError: false,
-        },
-        turnId
-      ),
-      event(
-        "message.completed",
-        { text: `reply to ${message}`, finishReason: "stop" },
-        turnId
-      ),
-      event("turn.completed", {}, turnId),
-      event("session.waiting", {}, undefined),
-    ]);
-
-    const [result] = await run(
-      defineEval({
-        async test(t) {
-          const first = await t.send("first");
-          t.check(first.message, includes("reply to first"));
-
-          const second = await t.send("second");
-          t.check(second.message, includes("reply to second"));
-          t.check(t.reply, includes("reply to second"));
-
-          // Each turn sees only its own tool call.
-          first.calledTool("tool_first", { count: 1 });
-          first.notCalledTool("tool_second");
-          second.calledTool("tool_second", { count: 1 });
-          second.notCalledTool("tool_first");
-
-          // The run sees both, exactly once each.
-          t.calledTool("tool_first", { count: 1 });
-          t.calledTool("tool_second", { count: 1 });
-          t.maxToolCalls(2);
-          t.toolOrder(["tool_first", "tool_second"]);
-          t.check(second.sessionId, includes(first.sessionId));
-        },
-      }),
-      stub.url
-    );
-    expect(result?.verdict).toBe("passed");
-    const failed = result!.assertions.filter((a) => !a.passed);
-    expect(failed).toEqual([]);
-    expect(stub.received).toEqual(["first", "second"]);
-    expect(result?.inputs).toEqual(["first", "second"]);
-    expect(result?.finalText).toBe("reply to second");
-  });
-
-  it("fails a run whose turn failed on the wire", async () => {
-    stub = await startStubAgent((message, turnId) => [
-      event("message.received", { text: message }, turnId),
-      event("turn.failed", { message: "provider unavailable" }, turnId),
-      event("session.waiting", {}, undefined),
-    ]);
-    const [result] = await run(
-      defineEval({
-        async test(t) {
-          await t.send("hello");
-          t.succeeded();
-        },
-      }),
-      stub.url
-    );
-    expect(result?.verdict).toBe("failed");
-    expect(result?.assertions[0]).toMatchObject({
-      name: "succeeded",
-      passed: false,
-      detail: "provider unavailable",
-    });
-  });
-
-  it("detects a run parked on an unanswered approval request", async () => {
-    stub = await startStubAgent((message, turnId) => [
-      event("message.received", { text: message }, turnId),
-      event(
-        "actions.requested",
-        {
-          calls: [{ callId: "c1", toolName: "deploy", args: { env: "prod" } }],
-        },
-        turnId
-      ),
-      event(
-        "action.approval_requested",
-        { callId: "c1", toolName: "deploy", args: { env: "prod" } },
-        turnId
-      ),
-      event("session.waiting", {}, undefined),
-    ]);
-    const [result] = await run(
-      defineEval({
-        async test(t) {
-          await t.send("deploy to prod");
-          t.parked();
-          const request = t.requireInputRequest({ toolName: "deploy" });
-          t.check(request.args, includes("prod"));
-          t.calledTool("deploy", { status: "pending", count: 1 });
-        },
-      }),
-      stub.url
-    );
-    expect(result?.verdict).toBe("passed");
-    expect(result?.assertions.map((a) => a.passed)).toEqual([
-      true,
-      true,
-      true,
-      true,
-    ]);
-  });
-
-  it("records a failed tool call without failing the turn", async () => {
-    stub = await startStubAgent((message, turnId) => [
-      event("message.received", { text: message }, turnId),
-      event(
-        "actions.requested",
-        {
-          calls: [
-            { callId: "c1", toolName: "bash", args: { command: "false" } },
-          ],
-        },
-        turnId
-      ),
-      event(
-        "action.result",
-        { callId: "c1", toolName: "bash", output: "exit 1", isError: true },
-        turnId
-      ),
-      event(
-        "message.completed",
-        { text: "That command failed.", finishReason: "stop" },
-        turnId
-      ),
-      event("turn.completed", {}, turnId),
-      event("session.waiting", {}, undefined),
-    ]);
-    const [result] = await run(
-      defineEval({
-        async test(t) {
-          await t.send("run false");
-          t.succeeded();
-          t.calledTool("bash", { status: "failed", count: 1 });
-          t.noFailedActions();
-        },
-      }),
-      stub.url
-    );
-    expect(result?.verdict).toBe("failed");
-    const byName = new Map(result!.assertions.map((a) => [a.name, a]));
-    expect(byName.get("succeeded")?.passed).toBe(true);
-    expect(
-      byName.get("calledTool(bash, {status=failed, count=1})")?.passed
-    ).toBe(true);
-    expect(byName.get("noFailedActions")).toMatchObject({
-      passed: false,
-      detail: "bash",
-    });
-  });
-});