@cursor/july 0.1.24 → 0.1.25

package/src/evals/assertions.test.ts

@@ -0,0 +1,617 @@
+import { describe, expect, it } from "vitest";
+import type { SessionEvent } from "../types.js";
+import {
+  computeVerdict,
+  createAssertions,
+  EvalAssertionRecorder,
+  EvalRequireFailure,
+  verdictFailsRun,
+} from "./assertions.js";
+import { equals, includes, similarity } from "./expect.js";
+
+let sequence = 0;
+
+function event(type: string, data: unknown, turnId = "turn_1"): SessionEvent {
+  sequence++;
+  return {
+    sessionId: "ses_1",
+    seq: sequence,
+    at: new Date(sequence * 1000).toISOString(),
+    turnId,
+    type,
+    data,
+  } as unknown as SessionEvent;
+}
+
+function weatherRun(): SessionEvent[] {
+  return [
+    event("message.received", { text: "weather in NYC?" }),
+    event("actions.requested", {
+      calls: [{ callId: "c1", toolName: "get_weather", args: { city: "NYC" } }],
+    }),
+    event("action.result", {
+      callId: "c1",
+      toolName: "get_weather",
+      output: { tempF: 72 },
+      isError: false,
+    }),
+    event("message.completed", { text: "Sunny, 72F", finishReason: "stop" }),
+    event("turn.completed", {}),
+    event("session.waiting", {}),
+  ];
+}
+
+function setup(events: SessionEvent[] = weatherRun()) {
+  const recorder = new EvalAssertionRecorder();
+  const t = createAssertions({ recorder, events: () => events });
+  return { recorder, t };
+}
+
+describe("severity handles", () => {
+  it("records gates by default and keeps the wire shape compact", () => {
+    const { recorder, t } = setup();
+    t.succeeded();
+    const [result] = recorder.results();
+    expect(result).toMatchObject({ name: "succeeded", passed: true });
+    // `gate` is the default everywhere, so it is omitted rather than repeated.
+    expect(result?.severity).toBeUndefined();
+  });
+
+  it("demotes a gate to a tracked soft assertion", () => {
+    const { recorder, t } = setup();
+    t.calledTool("nope").soft();
+    const [result] = recorder.results();
+    expect(result?.severity).toBe("soft");
+    // The miss is still recorded honestly; being soft only means it does not
+    // decide the verdict, so the CLI shows `~` and the eval still passes.
+    expect(result?.passed).toBe(false);
+    expect(computeVerdict(recorder.results())).toBe("passed");
+  });
+
+  it("keeps a tracked score from failing the verdict when it has no bar", () => {
+    const { recorder, t } = setup();
+    t.check("Rainy", similarity("Sunny, 72F"));
+    expect(recorder.results()[0]?.passed).toBe(true);
+    expect(recorder.results()[0]?.score).toBeLessThan(0.5);
+    expect(computeVerdict(recorder.results())).toBe("passed");
+  });
+
+  it("scores an eval when a soft threshold is missed", () => {
+    const { recorder, t } = setup();
+    t.check("Rainy", similarity("Sunny, 72F")).atLeast(0.9);
+    const [result] = recorder.results();
+    expect(result?.severity).toBe("soft");
+    expect(result?.threshold).toBe(0.9);
+    expect(result?.passed).toBe(false);
+    expect(computeVerdict(recorder.results())).toBe("scored");
+  });
+
+  it("passes a soft assertion that clears its bar", () => {
+    const { recorder, t } = setup();
+    t.check("Sunny, 72F", similarity("Sunny, 72F")).atLeast(0.9);
+    expect(recorder.results()[0]?.passed).toBe(true);
+    expect(computeVerdict(recorder.results())).toBe("passed");
+  });
+
+  it("promotes a score to a hard gate with .gate(threshold)", () => {
+    const { recorder, t } = setup();
+    t.check("Rainy", similarity("Sunny, 72F")).gate(0.9);
+    expect(recorder.results()[0]?.severity).toBeUndefined();
+    expect(computeVerdict(recorder.results())).toBe("failed");
+  });
+
+  it("converts a boolean assertion to a score when given a threshold", () => {
+    const { recorder, t } = setup();
+    t.calledTool("get_weather").gate(1);
+    expect(recorder.results()[0]?.passed).toBe(true);
+    const second = setup();
+    second.t.calledTool("missing").gate(1);
+    expect(second.recorder.results()[0]?.passed).toBe(false);
+  });
+
+  it("re-reads severity on the last call, so chains are order-independent", () => {
+    const { recorder, t } = setup();
+    t.check("Rainy", similarity("Sunny")).atLeast(0.9).soft();
+    // `.soft()` with no threshold keeps the 0.9 bar set by `.atLeast`.
+    expect(recorder.results()[0]?.threshold).toBe(0.9);
+    expect(computeVerdict(recorder.results())).toBe("scored");
+  });
+});
+
+describe("computeVerdict / verdictFailsRun", () => {
+  it("prefers a failed gate over a missed soft bar", () => {
+    expect(
+      computeVerdict([
+        { name: "a", passed: false },
+        { name: "b", passed: false, severity: "soft", threshold: 0.5 },
+      ])
+    ).toBe("failed");
+  });
+
+  it("treats an assertion with no severity as a gate", () => {
+    expect(computeVerdict([{ name: "legacy", passed: false }])).toBe("failed");
+    expect(computeVerdict([{ name: "legacy", passed: true }])).toBe("passed");
+  });
+
+  it("only fails the run on scored under --strict", () => {
+    expect(verdictFailsRun("scored", false)).toBe(false);
+    expect(verdictFailsRun("scored", true)).toBe(true);
+    expect(verdictFailsRun("failed", false)).toBe(true);
+    expect(verdictFailsRun("skipped", true)).toBe(false);
+    expect(verdictFailsRun("passed", true)).toBe(false);
+  });
+});
+
+describe("run-level assertions", () => {
+  it("succeeded passes a healthy run left open for the next message", () => {
+    const { recorder, t } = setup();
+    t.succeeded();
+    expect(recorder.results()[0]?.passed).toBe(true);
+  });
+
+  it("succeeded fails a run parked on an unanswered approval", () => {
+    const { recorder, t } = setup([
+      event("message.received", { text: "deploy" }),
+      event("actions.requested", {
+        calls: [{ callId: "c1", toolName: "deploy", args: {} }],
+      }),
+      event("action.approval_requested", { callId: "c1", toolName: "deploy" }),
+      event("session.waiting", {}),
+    ]);
+    t.succeeded();
+    t.parked();
+    const [succeeded, parked] = recorder.results();
+    expect(succeeded?.passed).toBe(false);
+    expect(succeeded?.detail).toMatch(/unanswered approval/);
+    expect(parked?.passed).toBe(true);
+  });
+
+  it("calledTool defaults to request-based matching for compatibility", () => {
+    // A pending call still counts as "the agent used this tool".
+    const { recorder, t } = setup([
+      event("actions.requested", {
+        calls: [{ callId: "c1", toolName: "slow", args: {} }],
+      }),
+      event("turn.completed", {}),
+    ]);
+    t.calledTool("slow");
+    t.calledTool("slow", { status: "completed" });
+    const [anyState, completed] = recorder.results();
+    expect(anyState?.passed).toBe(true);
+    expect(completed?.passed).toBe(false);
+  });
+
+  it("infers the parameter type of an inline matcher predicate", () => {
+    const { recorder, t } = setup();
+    // No annotation on `value`: the matcher union deliberately keeps its
+    // function member so this compiles under noImplicitAny.
+    t.calledTool("get_weather", {
+      input: (value) => (value as { city: string }).city === "NYC",
+    });
+    t.calledTool("get_weather", { count: (count) => count >= 1 });
+    // Nested inside an object literal, which is where a matcher predicate most
+    // often lands (`{ input: { field: predicate } }`).
+    t.calledTool("get_weather", {
+      input: { city: (value) => String(value).startsWith("NY") },
+    });
+    expect(recorder.results().map((r) => r.passed)).toEqual([true, true, true]);
+  });
+
+  it("narrows calledTool with input, output, and count matchers", () => {
+    const { recorder, t } = setup();
+    t.calledTool("get_weather", { input: { city: "NYC" }, count: 1 });
+    t.calledTool("get_weather", { output: { tempF: 72 } });
+    t.calledTool("get_weather", { input: { city: /^NY/ } });
+    t.calledTool("get_weather", { count: 2 });
+    const results = recorder.results();
+    expect(results.map((r) => r.passed)).toEqual([true, true, true, false]);
+    expect(results[0]?.name).toBe(
+      'calledTool(get_weather, {input={"city":"NYC"}, count=1})'
+    );
+  });
+
+  it("taggedArtifact matches artifact.tagged events by kind and predicate", () => {
+    const { recorder, t } = setup([
+      ...weatherRun(),
+      event("artifact.tagged", {
+        id: "art_1",
+        kind: "reviewed-pr",
+        key: "pr-1",
+        title: "PR #1",
+        data: { url: "https://github.com/org/repo/pull/1" },
+        source: "model",
+      }),
+    ]);
+    t.taggedArtifact();
+    t.taggedArtifact("reviewed-pr");
+    t.taggedArtifact("report");
+    t.taggedArtifact("reviewed-pr", (record) => record.source === "model");
+    t.taggedArtifact("reviewed-pr", (record) => record.key === "other");
+    const results = recorder.results();
+    expect(results.map((r) => r.passed)).toEqual([
+      true,
+      true,
+      false,
+      true,
+      false,
+    ]);
+    expect(results[1]?.name).toBe("taggedArtifact(reviewed-pr)");
+    expect(results[2]?.detail).toBe("tagged kinds: [reviewed-pr]");
+  });
+
+  it("taggedArtifact fails when nothing was tagged", () => {
+    const { recorder, t } = setup();
+    t.taggedArtifact();
+    const [result] = recorder.results();
+    expect(result?.passed).toBe(false);
+    expect(result?.detail).toBe("tagged kinds: []");
+  });
+
+  it("notCalledTool matches any lifecycle state", () => {
+    const { recorder, t } = setup();
+    t.notCalledTool("echo");
+    t.notCalledTool("get_weather");
+    expect(recorder.results().map((r) => r.passed)).toEqual([true, false]);
+  });
+
+  it("messageIncludes reads the joined assistant text", () => {
+    const { recorder, t } = setup([
+      event("message.completed", {
+        text: "part one",
+        finishReason: "tool_call",
+      }),
+      event("message.completed", { text: "part two", finishReason: "stop" }),
+      event("turn.completed", {}),
+    ]);
+    t.messageIncludes("part one");
+    t.messageIncludes(/part two/);
+    t.messageIncludes("part three");
+    expect(recorder.results().map((r) => r.passed)).toEqual([
+      true,
+      true,
+      false,
+    ]);
+  });
+
+  it("toolOrder allows unrelated calls between the required ones", () => {
+    const { recorder, t } = setup([
+      event("actions.requested", {
+        calls: [
+          { callId: "a", toolName: "prepare", args: {} },
+          { callId: "b", toolName: "unrelated", args: {} },
+          { callId: "c", toolName: "finalize", args: {} },
+        ],
+      }),
+      event("turn.completed", {}),
+    ]);
+    t.toolOrder(["prepare", "finalize"]);
+    t.toolOrder(["finalize", "prepare"]);
+    const [forward, backward] = recorder.results();
+    expect(forward?.passed).toBe(true);
+    expect(backward?.passed).toBe(false);
+    expect(backward?.detail).toMatch(/missing prepare after position/);
+  });
+
+  it("usedNoTools, maxToolCalls, and noFailedActions read the whole run", () => {
+    const { recorder, t } = setup([
+      event("actions.requested", {
+        calls: [
+          { callId: "a", toolName: "one", args: {} },
+          { callId: "b", toolName: "two", args: {} },
+        ],
+      }),
+      event("action.result", {
+        callId: "b",
+        toolName: "two",
+        output: "boom",
+        isError: true,
+      }),
+      event("turn.completed", {}),
+    ]);
+    t.usedNoTools();
+    t.maxToolCalls(2);
+    t.maxToolCalls(1);
+    t.noFailedActions();
+    expect(recorder.results().map((r) => r.passed)).toEqual([
+      false,
+      true,
+      false,
+      false,
+    ]);
+    expect(recorder.results()[3]?.detail).toBe("two");
+  });
+
+  it("calledSubagent matches identity, output, and status", () => {
+    const { recorder, t } = setup([
+      event("actions.requested", {
+        calls: [{ callId: "t1", toolName: "task", args: {} }],
+      }),
+      event("subagent.called", { callId: "t1", name: "research" }),
+      event("subagent.completed", { callId: "t1", name: "research" }),
+      event("action.result", {
+        callId: "t1",
+        toolName: "task",
+        output: "72F in NYC",
+        isError: false,
+      }),
+      event("turn.completed", {}),
+    ]);
+    t.calledSubagent("research", { output: /72F/, status: "completed" });
+    t.calledSubagent("research", { count: 2 });
+    t.calledSubagent("other");
+    expect(recorder.results().map((r) => r.passed)).toEqual([
+      true,
+      false,
+      false,
+    ]);
+  });
+
+  it("loadedSkill matches any tool call that opened the skill file", () => {
+    const { recorder, t } = setup([
+      event("actions.requested", {
+        calls: [
+          {
+            callId: "c1",
+            toolName: "read_file",
+            args: { path: ".cursor/skills/alert-investigation/SKILL.md" },
+          },
+        ],
+      }),
+      event("turn.completed", {}),
+    ]);
+    t.loadedSkill("alert-investigation");
+    t.loadedSkill("other-skill");
+    expect(recorder.results().map((r) => r.passed)).toEqual([true, false]);
+  });
+});
+
+describe("event assertions", () => {
+  it("matches presence, absence, data, and count", () => {
+    const { recorder, t } = setup();
+    t.event("turn.completed");
+    t.event("message.completed", { data: { text: /Sunny/ } });
+    t.event("message.completed", { count: 2 });
+    t.notEvent("turn.failed");
+    t.notEvent("turn.completed");
+    expect(recorder.results().map((r) => r.passed)).toEqual([
+      true,
+      true,
+      false,
+      true,
+      false,
+    ]);
+  });
+
+  it("eventOrder requires matching groups in stream order", () => {
+    const { recorder, t } = setup();
+    t.eventOrder([
+      { type: "actions.requested" },
+      { type: "action.result" },
+      { type: "turn.completed" },
+    ]);
+    t.eventOrder([{ type: "turn.completed" }, { type: "actions.requested" }]);
+    expect(recorder.results().map((r) => r.passed)).toEqual([true, false]);
+  });
+
+  it("eventOrder honors a numeric count per group", () => {
+    const { recorder, t } = setup([
+      event("subagent.called", { callId: "a", name: "r" }),
+      event("subagent.called", { callId: "b", name: "r" }),
+      event("turn.completed", {}),
+    ]);
+    t.eventOrder([
+      { type: "subagent.called", count: 2 },
+      { type: "turn.completed" },
+    ]);
+    t.eventOrder([
+      { type: "subagent.called", count: 3 },
+      { type: "turn.completed" },
+    ]);
+    expect(recorder.results().map((r) => r.passed)).toEqual([true, false]);
+  });
+
+  it("eventsSatisfy records a predicate throw as a failure detail", () => {
+    const { recorder, t } = setup();
+    t.eventsSatisfy("has a reply", (events) =>
+      events.some((e) => e.type === "message.completed")
+    );
+    t.eventsSatisfy("throws", () => {
+      throw new Error("bad predicate");
+    });
+    const results = recorder.results();
+    expect(results[0]?.passed).toBe(true);
+    expect(results[1]).toMatchObject({
+      passed: false,
+      detail: "bad predicate",
+    });
+  });
+});
+
+describe("require* lookups", () => {
+  it("requireToolCall returns the matching call", () => {
+    const { recorder, t } = setup();
+    const call = t.requireToolCall("get_weather");
+    expect(call.output).toEqual({ tempF: 72 });
+    expect(recorder.results()[0]?.passed).toBe(true);
+  });
+
+  it("requireToolCall records a gate and stops the body when nothing matches", () => {
+    const { recorder, t } = setup();
+    expect(() => t.requireToolCall("missing")).toThrow(EvalRequireFailure);
+    expect(recorder.results()[0]).toMatchObject({
+      name: "requireToolCall(missing)",
+      passed: false,
+    });
+  });
+
+  it("requireInputRequest demands exactly one pending approval", () => {
+    const { recorder, t } = setup([
+      event("actions.requested", {
+        calls: [{ callId: "c1", toolName: "deploy", args: { env: "prod" } }],
+      }),
+      event("action.approval_requested", {
+        callId: "c1",
+        toolName: "deploy",
+        args: { env: "prod" },
+      }),
+      event("session.waiting", {}),
+    ]);
+    const request = t.requireInputRequest({ toolName: "deploy" });
+    expect(request.args).toEqual({ env: "prod" });
+    expect(() => t.requireInputRequest({ toolName: "other" })).toThrow(
+      EvalRequireFailure
+    );
+    expect(recorder.results()[1]?.detail).toMatch(/found 0/);
+  });
+});
+
+describe("turn scoping", () => {
+  it("prefixes turn assertions and narrows them to that turn's events", () => {
+    const recorder = new EvalAssertionRecorder();
+    const firstTurn = [
+      event("actions.requested", {
+        calls: [{ callId: "c1", toolName: "search", args: {} }],
+      }),
+      event("message.completed", { text: "draft", finishReason: "stop" }),
+      event("turn.completed", {}),
+    ];
+    const secondTurn = [
+      event("actions.requested", {
+        calls: [{ callId: "c2", toolName: "send_email", args: {} }],
+      }),
+      event(
+        "message.completed",
+        { text: "sent", finishReason: "stop" },
+        "turn_2"
+      ),
+      event("turn.completed", {}, "turn_2"),
+    ];
+    const turnOne = createAssertions({
+      recorder,
+      events: () => firstTurn,
+      scope: "turn[1]",
+    });
+    const run = createAssertions({
+      recorder,
+      events: () => [...firstTurn, ...secondTurn],
+    });
+
+    turnOne.calledTool("search");
+    turnOne.notCalledTool("send_email");
+    run.calledTool("send_email");
+
+    const results = recorder.results();
+    expect(results.map((r) => r.name)).toEqual([
+      "turn[1].calledTool(search)",
+      "turn[1].notCalledTool(send_email)",
+      "calledTool(send_email)",
+    ]);
+    expect(results.map((r) => r.passed)).toEqual([true, true, true]);
+    expect(results[0]?.scope).toBe("turn[1]");
+    expect(results[2]?.scope).toBeUndefined();
+  });
+});
+
+describe("score", () => {
+  it("records a tracked score that does not gate until given a bar", () => {
+    const { recorder, t } = setup();
+    t.score("recall", 0.4);
+    expect(recorder.results()[0]).toMatchObject({
+      name: "score(recall)",
+      severity: "soft",
+      score: 0.4,
+      passed: true,
+    });
+    expect(computeVerdict(recorder.results())).toBe("passed");
+  });
+
+  it("marks the eval scored once the score misses its bar", () => {
+    const { recorder, t } = setup();
+    t.score("recall", 0.4).atLeast(0.6);
+    expect(recorder.results()[0]?.passed).toBe(false);
+    expect(computeVerdict(recorder.results())).toBe("scored");
+  });
+
+  it("can be promoted to a hard gate", () => {
+    const { recorder, t } = setup();
+    t.score("recall", 0.4).gate(0.6);
+    expect(computeVerdict(recorder.results())).toBe("failed");
+  });
+
+  it("fails loudly on a non-finite score rather than reporting NaN", () => {
+    const { recorder, t } = setup();
+    t.score("recall", Number.NaN);
+    expect(recorder.results()[0]).toMatchObject({
+      passed: false,
+      score: undefined,
+      detail: "score is not a finite number: NaN",
+    });
+  });
+});
+
+describe("check", () => {
+  it("keeps the historical assertion name so output stays stable", () => {
+    const { recorder, t } = setup();
+    t.check("Sunny, 72F", includes("Sunny"));
+    t.check(72, equals(72));
+    expect(recorder.results().map((r) => r.name)).toEqual([
+      "check(includes)",
+      "check(equals)",
+    ]);
+  });
+});
+
+describe("recordAsync", () => {
+  it("fills the slot in when the async score settles", async () => {
+    const recorder = new EvalAssertionRecorder();
+    recorder
+      .recordAsync(
+        { name: "judge.closedQA(x)", passed: true, severity: "soft" },
+        Promise.resolve({ passed: true, score: 0.4, detail: "N (0.40)" })
+      )
+      .atLeast(0.6);
+    await recorder.waitForPending();
+    expect(recorder.results()[0]).toMatchObject({
+      severity: "soft",
+      score: 0.4,
+      threshold: 0.6,
+      passed: false,
+      detail: "N (0.40)",
+    });
+    expect(computeVerdict(recorder.results())).toBe("scored");
+  });
+
+  it("turns a rejected score into a failed gate with the error message", async () => {
+    const recorder = new EvalAssertionRecorder();
+    recorder
+      .recordAsync(
+        { name: "judge.factuality(x)", passed: true, severity: "soft" },
+        Promise.reject(new Error("judge reply had no choice"))
+      )
+      .atLeast(0.7);
+    await recorder.waitForPending();
+    expect(recorder.results()[0]).toMatchObject({
+      passed: false,
+      detail: "judge reply had no choice",
+    });
+    // Grading errors must not hide behind soft severity.
+    expect(recorder.results()[0]?.severity).toBeUndefined();
+    expect(computeVerdict(recorder.results())).toBe("failed");
+  });
+
+  it("waits for judges started while an earlier judge was settling", async () => {
+    const recorder = new EvalAssertionRecorder();
+    recorder.recordAsync(
+      { name: "first", passed: true, severity: "soft" },
+      Promise.resolve({}).then(() => {
+        recorder.recordAsync(
+          { name: "second", passed: true, severity: "soft" },
+          Promise.resolve({ score: 1 })
+        );
+        return { score: 1 };
+      })
+    );
+    await recorder.waitForPending();
+    expect(recorder.results().map((r) => r.score)).toEqual([1, 1]);
+  });
+});

package/src/evals/assertions.ts

@@ -15,7 +15,7 @@
  * by `t.send(...)`, which narrows the events under inspection to that turn.
  */
 
-import type { SessionEvent } from "../types.js";
+import type { SessionEvent, SessionEventPayload } from "../types.js";
 import type { Expectation } from "./expect.js";
 import { evaluateExpectation, expectationSeverity } from "./expect.js";
 import type {
@@ -37,6 +37,12 @@ import { deriveRunFacts } from "./run-facts.js";
 /** Hard gates fail the eval; soft assertions are tracked scores. */
 export type EvalAssertionSeverity = "gate" | "soft";
 
+/** One `artifact.tagged` payload, as seen by `t.taggedArtifact(...)`. */
+export type EvalTaggedArtifactFact = Extract<
+  SessionEventPayload,
+  { type: "artifact.tagged" }
+>["data"];
+
 /**
  * Overall grade for one eval case.
  *
@@ -314,6 +320,14 @@ export interface EvalAssertions {
     name: string,
     matcher?: EvalSubagentMatcher
   ): EvalAssertionHandle;
+  /**
+   * Gate: at least one artifact was tagged (`ctx.artifacts.tag` or the
+   * `tag_artifact` built-in), optionally of `kind` and matching `predicate`.
+   */
+  taggedArtifact(
+    kind?: string,
+    predicate?: (record: EvalTaggedArtifactFact) => boolean
+  ): EvalAssertionHandle;
   /** Gate: at least one matching event of `type` occurred. */
   event(type: string, matcher?: EvalEventMatcher): EvalAssertionHandle;
   /** Gate: no matching event of `type` occurred. */
@@ -567,6 +581,28 @@ export function createAssertions(
       });
     },
 
+    taggedArtifact(
+      kind?: string,
+      predicate?: (record: EvalTaggedArtifactFact) => boolean
+    ) {
+      const tagged = events().flatMap((event) =>
+        event.type === "artifact.tagged" ? [event.data] : []
+      );
+      const matched = tagged.filter(
+        (record) =>
+          (kind === undefined || record.kind === kind) &&
+          (predicate === undefined || predicate(record) === true)
+      );
+      return record({
+        name: `taggedArtifact(${kind ?? ""})`,
+        passed: matched.length > 0,
+        detail:
+          matched.length > 0
+            ? undefined
+            : `tagged kinds: [${tagged.map((r) => r.kind).join(", ")}]`,
+      });
+    },
+
     event(type: string, matcher?: EvalEventMatcher) {
       const matched = matchingEvents(type, matcher);
       const passed = matchCount(matched.length, matcher?.count);