@cursor/july 0.1.26 → 0.1.27

package/src/evals/assertions.test.ts

@@ -1,617 +0,0 @@
-import { describe, expect, it } from "vitest";
-import type { SessionEvent } from "../types.js";
-import {
-  computeVerdict,
-  createAssertions,
-  EvalAssertionRecorder,
-  EvalRequireFailure,
-  verdictFailsRun,
-} from "./assertions.js";
-import { equals, includes, similarity } from "./expect.js";
-
-let sequence = 0;
-
-function event(type: string, data: unknown, turnId = "turn_1"): SessionEvent {
-  sequence++;
-  return {
-    sessionId: "ses_1",
-    seq: sequence,
-    at: new Date(sequence * 1000).toISOString(),
-    turnId,
-    type,
-    data,
-  } as unknown as SessionEvent;
-}
-
-function weatherRun(): SessionEvent[] {
-  return [
-    event("message.received", { text: "weather in NYC?" }),
-    event("actions.requested", {
-      calls: [{ callId: "c1", toolName: "get_weather", args: { city: "NYC" } }],
-    }),
-    event("action.result", {
-      callId: "c1",
-      toolName: "get_weather",
-      output: { tempF: 72 },
-      isError: false,
-    }),
-    event("message.completed", { text: "Sunny, 72F", finishReason: "stop" }),
-    event("turn.completed", {}),
-    event("session.waiting", {}),
-  ];
-}
-
-function setup(events: SessionEvent[] = weatherRun()) {
-  const recorder = new EvalAssertionRecorder();
-  const t = createAssertions({ recorder, events: () => events });
-  return { recorder, t };
-}
-
-describe("severity handles", () => {
-  it("records gates by default and keeps the wire shape compact", () => {
-    const { recorder, t } = setup();
-    t.succeeded();
-    const [result] = recorder.results();
-    expect(result).toMatchObject({ name: "succeeded", passed: true });
-    // `gate` is the default everywhere, so it is omitted rather than repeated.
-    expect(result?.severity).toBeUndefined();
-  });
-
-  it("demotes a gate to a tracked soft assertion", () => {
-    const { recorder, t } = setup();
-    t.calledTool("nope").soft();
-    const [result] = recorder.results();
-    expect(result?.severity).toBe("soft");
-    // The miss is still recorded honestly; being soft only means it does not
-    // decide the verdict, so the CLI shows `~` and the eval still passes.
-    expect(result?.passed).toBe(false);
-    expect(computeVerdict(recorder.results())).toBe("passed");
-  });
-
-  it("keeps a tracked score from failing the verdict when it has no bar", () => {
-    const { recorder, t } = setup();
-    t.check("Rainy", similarity("Sunny, 72F"));
-    expect(recorder.results()[0]?.passed).toBe(true);
-    expect(recorder.results()[0]?.score).toBeLessThan(0.5);
-    expect(computeVerdict(recorder.results())).toBe("passed");
-  });
-
-  it("scores an eval when a soft threshold is missed", () => {
-    const { recorder, t } = setup();
-    t.check("Rainy", similarity("Sunny, 72F")).atLeast(0.9);
-    const [result] = recorder.results();
-    expect(result?.severity).toBe("soft");
-    expect(result?.threshold).toBe(0.9);
-    expect(result?.passed).toBe(false);
-    expect(computeVerdict(recorder.results())).toBe("scored");
-  });
-
-  it("passes a soft assertion that clears its bar", () => {
-    const { recorder, t } = setup();
-    t.check("Sunny, 72F", similarity("Sunny, 72F")).atLeast(0.9);
-    expect(recorder.results()[0]?.passed).toBe(true);
-    expect(computeVerdict(recorder.results())).toBe("passed");
-  });
-
-  it("promotes a score to a hard gate with .gate(threshold)", () => {
-    const { recorder, t } = setup();
-    t.check("Rainy", similarity("Sunny, 72F")).gate(0.9);
-    expect(recorder.results()[0]?.severity).toBeUndefined();
-    expect(computeVerdict(recorder.results())).toBe("failed");
-  });
-
-  it("converts a boolean assertion to a score when given a threshold", () => {
-    const { recorder, t } = setup();
-    t.calledTool("get_weather").gate(1);
-    expect(recorder.results()[0]?.passed).toBe(true);
-    const second = setup();
-    second.t.calledTool("missing").gate(1);
-    expect(second.recorder.results()[0]?.passed).toBe(false);
-  });
-
-  it("re-reads severity on the last call, so chains are order-independent", () => {
-    const { recorder, t } = setup();
-    t.check("Rainy", similarity("Sunny")).atLeast(0.9).soft();
-    // `.soft()` with no threshold keeps the 0.9 bar set by `.atLeast`.
-    expect(recorder.results()[0]?.threshold).toBe(0.9);
-    expect(computeVerdict(recorder.results())).toBe("scored");
-  });
-});
-
-describe("computeVerdict / verdictFailsRun", () => {
-  it("prefers a failed gate over a missed soft bar", () => {
-    expect(
-      computeVerdict([
-        { name: "a", passed: false },
-        { name: "b", passed: false, severity: "soft", threshold: 0.5 },
-      ])
-    ).toBe("failed");
-  });
-
-  it("treats an assertion with no severity as a gate", () => {
-    expect(computeVerdict([{ name: "legacy", passed: false }])).toBe("failed");
-    expect(computeVerdict([{ name: "legacy", passed: true }])).toBe("passed");
-  });
-
-  it("only fails the run on scored under --strict", () => {
-    expect(verdictFailsRun("scored", false)).toBe(false);
-    expect(verdictFailsRun("scored", true)).toBe(true);
-    expect(verdictFailsRun("failed", false)).toBe(true);
-    expect(verdictFailsRun("skipped", true)).toBe(false);
-    expect(verdictFailsRun("passed", true)).toBe(false);
-  });
-});
-
-describe("run-level assertions", () => {
-  it("succeeded passes a healthy run left open for the next message", () => {
-    const { recorder, t } = setup();
-    t.succeeded();
-    expect(recorder.results()[0]?.passed).toBe(true);
-  });
-
-  it("succeeded fails a run parked on an unanswered approval", () => {
-    const { recorder, t } = setup([
-      event("message.received", { text: "deploy" }),
-      event("actions.requested", {
-        calls: [{ callId: "c1", toolName: "deploy", args: {} }],
-      }),
-      event("action.approval_requested", { callId: "c1", toolName: "deploy" }),
-      event("session.waiting", {}),
-    ]);
-    t.succeeded();
-    t.parked();
-    const [succeeded, parked] = recorder.results();
-    expect(succeeded?.passed).toBe(false);
-    expect(succeeded?.detail).toMatch(/unanswered approval/);
-    expect(parked?.passed).toBe(true);
-  });
-
-  it("calledTool defaults to request-based matching for compatibility", () => {
-    // A pending call still counts as "the agent used this tool".
-    const { recorder, t } = setup([
-      event("actions.requested", {
-        calls: [{ callId: "c1", toolName: "slow", args: {} }],
-      }),
-      event("turn.completed", {}),
-    ]);
-    t.calledTool("slow");
-    t.calledTool("slow", { status: "completed" });
-    const [anyState, completed] = recorder.results();
-    expect(anyState?.passed).toBe(true);
-    expect(completed?.passed).toBe(false);
-  });
-
-  it("infers the parameter type of an inline matcher predicate", () => {
-    const { recorder, t } = setup();
-    // No annotation on `value`: the matcher union deliberately keeps its
-    // function member so this compiles under noImplicitAny.
-    t.calledTool("get_weather", {
-      input: (value) => (value as { city: string }).city === "NYC",
-    });
-    t.calledTool("get_weather", { count: (count) => count >= 1 });
-    // Nested inside an object literal, which is where a matcher predicate most
-    // often lands (`{ input: { field: predicate } }`).
-    t.calledTool("get_weather", {
-      input: { city: (value) => String(value).startsWith("NY") },
-    });
-    expect(recorder.results().map((r) => r.passed)).toEqual([true, true, true]);
-  });
-
-  it("narrows calledTool with input, output, and count matchers", () => {
-    const { recorder, t } = setup();
-    t.calledTool("get_weather", { input: { city: "NYC" }, count: 1 });
-    t.calledTool("get_weather", { output: { tempF: 72 } });
-    t.calledTool("get_weather", { input: { city: /^NY/ } });
-    t.calledTool("get_weather", { count: 2 });
-    const results = recorder.results();
-    expect(results.map((r) => r.passed)).toEqual([true, true, true, false]);
-    expect(results[0]?.name).toBe(
-      'calledTool(get_weather, {input={"city":"NYC"}, count=1})'
-    );
-  });
-
-  it("taggedArtifact matches artifact.tagged events by kind and predicate", () => {
-    const { recorder, t } = setup([
-      ...weatherRun(),
-      event("artifact.tagged", {
-        id: "art_1",
-        kind: "reviewed-pr",
-        key: "pr-1",
-        title: "PR #1",
-        data: { url: "https://github.com/org/repo/pull/1" },
-        source: "model",
-      }),
-    ]);
-    t.taggedArtifact();
-    t.taggedArtifact("reviewed-pr");
-    t.taggedArtifact("report");
-    t.taggedArtifact("reviewed-pr", (record) => record.source === "model");
-    t.taggedArtifact("reviewed-pr", (record) => record.key === "other");
-    const results = recorder.results();
-    expect(results.map((r) => r.passed)).toEqual([
-      true,
-      true,
-      false,
-      true,
-      false,
-    ]);
-    expect(results[1]?.name).toBe("taggedArtifact(reviewed-pr)");
-    expect(results[2]?.detail).toBe("tagged kinds: [reviewed-pr]");
-  });
-
-  it("taggedArtifact fails when nothing was tagged", () => {
-    const { recorder, t } = setup();
-    t.taggedArtifact();
-    const [result] = recorder.results();
-    expect(result?.passed).toBe(false);
-    expect(result?.detail).toBe("tagged kinds: []");
-  });
-
-  it("notCalledTool matches any lifecycle state", () => {
-    const { recorder, t } = setup();
-    t.notCalledTool("echo");
-    t.notCalledTool("get_weather");
-    expect(recorder.results().map((r) => r.passed)).toEqual([true, false]);
-  });
-
-  it("messageIncludes reads the joined assistant text", () => {
-    const { recorder, t } = setup([
-      event("message.completed", {
-        text: "part one",
-        finishReason: "tool_call",
-      }),
-      event("message.completed", { text: "part two", finishReason: "stop" }),
-      event("turn.completed", {}),
-    ]);
-    t.messageIncludes("part one");
-    t.messageIncludes(/part two/);
-    t.messageIncludes("part three");
-    expect(recorder.results().map((r) => r.passed)).toEqual([
-      true,
-      true,
-      false,
-    ]);
-  });
-
-  it("toolOrder allows unrelated calls between the required ones", () => {
-    const { recorder, t } = setup([
-      event("actions.requested", {
-        calls: [
-          { callId: "a", toolName: "prepare", args: {} },
-          { callId: "b", toolName: "unrelated", args: {} },
-          { callId: "c", toolName: "finalize", args: {} },
-        ],
-      }),
-      event("turn.completed", {}),
-    ]);
-    t.toolOrder(["prepare", "finalize"]);
-    t.toolOrder(["finalize", "prepare"]);
-    const [forward, backward] = recorder.results();
-    expect(forward?.passed).toBe(true);
-    expect(backward?.passed).toBe(false);
-    expect(backward?.detail).toMatch(/missing prepare after position/);
-  });
-
-  it("usedNoTools, maxToolCalls, and noFailedActions read the whole run", () => {
-    const { recorder, t } = setup([
-      event("actions.requested", {
-        calls: [
-          { callId: "a", toolName: "one", args: {} },
-          { callId: "b", toolName: "two", args: {} },
-        ],
-      }),
-      event("action.result", {
-        callId: "b",
-        toolName: "two",
-        output: "boom",
-        isError: true,
-      }),
-      event("turn.completed", {}),
-    ]);
-    t.usedNoTools();
-    t.maxToolCalls(2);
-    t.maxToolCalls(1);
-    t.noFailedActions();
-    expect(recorder.results().map((r) => r.passed)).toEqual([
-      false,
-      true,
-      false,
-      false,
-    ]);
-    expect(recorder.results()[3]?.detail).toBe("two");
-  });
-
-  it("calledSubagent matches identity, output, and status", () => {
-    const { recorder, t } = setup([
-      event("actions.requested", {
-        calls: [{ callId: "t1", toolName: "task", args: {} }],
-      }),
-      event("subagent.called", { callId: "t1", name: "research" }),
-      event("subagent.completed", { callId: "t1", name: "research" }),
-      event("action.result", {
-        callId: "t1",
-        toolName: "task",
-        output: "72F in NYC",
-        isError: false,
-      }),
-      event("turn.completed", {}),
-    ]);
-    t.calledSubagent("research", { output: /72F/, status: "completed" });
-    t.calledSubagent("research", { count: 2 });
-    t.calledSubagent("other");
-    expect(recorder.results().map((r) => r.passed)).toEqual([
-      true,
-      false,
-      false,
-    ]);
-  });
-
-  it("loadedSkill matches any tool call that opened the skill file", () => {
-    const { recorder, t } = setup([
-      event("actions.requested", {
-        calls: [
-          {
-            callId: "c1",
-            toolName: "read_file",
-            args: { path: ".cursor/skills/alert-investigation/SKILL.md" },
-          },
-        ],
-      }),
-      event("turn.completed", {}),
-    ]);
-    t.loadedSkill("alert-investigation");
-    t.loadedSkill("other-skill");
-    expect(recorder.results().map((r) => r.passed)).toEqual([true, false]);
-  });
-});
-
-describe("event assertions", () => {
-  it("matches presence, absence, data, and count", () => {
-    const { recorder, t } = setup();
-    t.event("turn.completed");
-    t.event("message.completed", { data: { text: /Sunny/ } });
-    t.event("message.completed", { count: 2 });
-    t.notEvent("turn.failed");
-    t.notEvent("turn.completed");
-    expect(recorder.results().map((r) => r.passed)).toEqual([
-      true,
-      true,
-      false,
-      true,
-      false,
-    ]);
-  });
-
-  it("eventOrder requires matching groups in stream order", () => {
-    const { recorder, t } = setup();
-    t.eventOrder([
-      { type: "actions.requested" },
-      { type: "action.result" },
-      { type: "turn.completed" },
-    ]);
-    t.eventOrder([{ type: "turn.completed" }, { type: "actions.requested" }]);
-    expect(recorder.results().map((r) => r.passed)).toEqual([true, false]);
-  });
-
-  it("eventOrder honors a numeric count per group", () => {
-    const { recorder, t } = setup([
-      event("subagent.called", { callId: "a", name: "r" }),
-      event("subagent.called", { callId: "b", name: "r" }),
-      event("turn.completed", {}),
-    ]);
-    t.eventOrder([
-      { type: "subagent.called", count: 2 },
-      { type: "turn.completed" },
-    ]);
-    t.eventOrder([
-      { type: "subagent.called", count: 3 },
-      { type: "turn.completed" },
-    ]);
-    expect(recorder.results().map((r) => r.passed)).toEqual([true, false]);
-  });
-
-  it("eventsSatisfy records a predicate throw as a failure detail", () => {
-    const { recorder, t } = setup();
-    t.eventsSatisfy("has a reply", (events) =>
-      events.some((e) => e.type === "message.completed")
-    );
-    t.eventsSatisfy("throws", () => {
-      throw new Error("bad predicate");
-    });
-    const results = recorder.results();
-    expect(results[0]?.passed).toBe(true);
-    expect(results[1]).toMatchObject({
-      passed: false,
-      detail: "bad predicate",
-    });
-  });
-});
-
-describe("require* lookups", () => {
-  it("requireToolCall returns the matching call", () => {
-    const { recorder, t } = setup();
-    const call = t.requireToolCall("get_weather");
-    expect(call.output).toEqual({ tempF: 72 });
-    expect(recorder.results()[0]?.passed).toBe(true);
-  });
-
-  it("requireToolCall records a gate and stops the body when nothing matches", () => {
-    const { recorder, t } = setup();
-    expect(() => t.requireToolCall("missing")).toThrow(EvalRequireFailure);
-    expect(recorder.results()[0]).toMatchObject({
-      name: "requireToolCall(missing)",
-      passed: false,
-    });
-  });
-
-  it("requireInputRequest demands exactly one pending approval", () => {
-    const { recorder, t } = setup([
-      event("actions.requested", {
-        calls: [{ callId: "c1", toolName: "deploy", args: { env: "prod" } }],
-      }),
-      event("action.approval_requested", {
-        callId: "c1",
-        toolName: "deploy",
-        args: { env: "prod" },
-      }),
-      event("session.waiting", {}),
-    ]);
-    const request = t.requireInputRequest({ toolName: "deploy" });
-    expect(request.args).toEqual({ env: "prod" });
-    expect(() => t.requireInputRequest({ toolName: "other" })).toThrow(
-      EvalRequireFailure
-    );
-    expect(recorder.results()[1]?.detail).toMatch(/found 0/);
-  });
-});
-
-describe("turn scoping", () => {
-  it("prefixes turn assertions and narrows them to that turn's events", () => {
-    const recorder = new EvalAssertionRecorder();
-    const firstTurn = [
-      event("actions.requested", {
-        calls: [{ callId: "c1", toolName: "search", args: {} }],
-      }),
-      event("message.completed", { text: "draft", finishReason: "stop" }),
-      event("turn.completed", {}),
-    ];
-    const secondTurn = [
-      event("actions.requested", {
-        calls: [{ callId: "c2", toolName: "send_email", args: {} }],
-      }),
-      event(
-        "message.completed",
-        { text: "sent", finishReason: "stop" },
-        "turn_2"
-      ),
-      event("turn.completed", {}, "turn_2"),
-    ];
-    const turnOne = createAssertions({
-      recorder,
-      events: () => firstTurn,
-      scope: "turn[1]",
-    });
-    const run = createAssertions({
-      recorder,
-      events: () => [...firstTurn, ...secondTurn],
-    });
-
-    turnOne.calledTool("search");
-    turnOne.notCalledTool("send_email");
-    run.calledTool("send_email");
-
-    const results = recorder.results();
-    expect(results.map((r) => r.name)).toEqual([
-      "turn[1].calledTool(search)",
-      "turn[1].notCalledTool(send_email)",
-      "calledTool(send_email)",
-    ]);
-    expect(results.map((r) => r.passed)).toEqual([true, true, true]);
-    expect(results[0]?.scope).toBe("turn[1]");
-    expect(results[2]?.scope).toBeUndefined();
-  });
-});
-
-describe("score", () => {
-  it("records a tracked score that does not gate until given a bar", () => {
-    const { recorder, t } = setup();
-    t.score("recall", 0.4);
-    expect(recorder.results()[0]).toMatchObject({
-      name: "score(recall)",
-      severity: "soft",
-      score: 0.4,
-      passed: true,
-    });
-    expect(computeVerdict(recorder.results())).toBe("passed");
-  });
-
-  it("marks the eval scored once the score misses its bar", () => {
-    const { recorder, t } = setup();
-    t.score("recall", 0.4).atLeast(0.6);
-    expect(recorder.results()[0]?.passed).toBe(false);
-    expect(computeVerdict(recorder.results())).toBe("scored");
-  });
-
-  it("can be promoted to a hard gate", () => {
-    const { recorder, t } = setup();
-    t.score("recall", 0.4).gate(0.6);
-    expect(computeVerdict(recorder.results())).toBe("failed");
-  });
-
-  it("fails loudly on a non-finite score rather than reporting NaN", () => {
-    const { recorder, t } = setup();
-    t.score("recall", Number.NaN);
-    expect(recorder.results()[0]).toMatchObject({
-      passed: false,
-      score: undefined,
-      detail: "score is not a finite number: NaN",
-    });
-  });
-});
-
-describe("check", () => {
-  it("keeps the historical assertion name so output stays stable", () => {
-    const { recorder, t } = setup();
-    t.check("Sunny, 72F", includes("Sunny"));
-    t.check(72, equals(72));
-    expect(recorder.results().map((r) => r.name)).toEqual([
-      "check(includes)",
-      "check(equals)",
-    ]);
-  });
-});
-
-describe("recordAsync", () => {
-  it("fills the slot in when the async score settles", async () => {
-    const recorder = new EvalAssertionRecorder();
-    recorder
-      .recordAsync(
-        { name: "judge.closedQA(x)", passed: true, severity: "soft" },
-        Promise.resolve({ passed: true, score: 0.4, detail: "N (0.40)" })
-      )
-      .atLeast(0.6);
-    await recorder.waitForPending();
-    expect(recorder.results()[0]).toMatchObject({
-      severity: "soft",
-      score: 0.4,
-      threshold: 0.6,
-      passed: false,
-      detail: "N (0.40)",
-    });
-    expect(computeVerdict(recorder.results())).toBe("scored");
-  });
-
-  it("turns a rejected score into a failed gate with the error message", async () => {
-    const recorder = new EvalAssertionRecorder();
-    recorder
-      .recordAsync(
-        { name: "judge.factuality(x)", passed: true, severity: "soft" },
-        Promise.reject(new Error("judge reply had no choice"))
-      )
-      .atLeast(0.7);
-    await recorder.waitForPending();
-    expect(recorder.results()[0]).toMatchObject({
-      passed: false,
-      detail: "judge reply had no choice",
-    });
-    // Grading errors must not hide behind soft severity.
-    expect(recorder.results()[0]?.severity).toBeUndefined();
-    expect(computeVerdict(recorder.results())).toBe("failed");
-  });
-
-  it("waits for judges started while an earlier judge was settling", async () => {
-    const recorder = new EvalAssertionRecorder();
-    recorder.recordAsync(
-      { name: "first", passed: true, severity: "soft" },
-      Promise.resolve({}).then(() => {
-        recorder.recordAsync(
-          { name: "second", passed: true, severity: "soft" },
-          Promise.resolve({ score: 1 })
-        );
-        return { score: 1 };
-      })
-    );
-    await recorder.waitForPending();
-    expect(recorder.results().map((r) => r.score)).toEqual([1, 1]);
-  });
-});