@cursor/july 0.1.16 → 0.1.18

package/src/evals/assertions.ts

@@ -0,0 +1,704 @@
+/**
+ * Assertion recording and the scoped assertion vocabulary.
+ *
+ * Recorded assertions never throw: the recorder collects every result so one
+ * run reports all failures instead of dying on the first. Severity rides on the
+ * returned handle, so there is no separate thresholds map:
+ *
+ * ```ts
+ * t.succeeded();                                  // gate (hard)
+ * t.calledTool("get_weather").soft();             // tracked, does not gate
+ * t.check(t.reply, similarity("Sunny")).atLeast(0.8);
+ * ```
+ *
+ * The same vocabulary is available on the run (`t`) and on each turn returned
+ * by `t.send(...)`, which narrows the events under inspection to that turn.
+ */
+
+import type { SessionEvent } from "../types.js";
+import type { Expectation } from "./expect.js";
+import { evaluateExpectation, expectationSeverity } from "./expect.js";
+import type {
+  EvalCountMatcher,
+  EvalEventMatcher,
+  EvalEventOrderMatcher,
+  EvalMatcher,
+  EvalSubagentMatcher,
+  EvalToolCallMatcher,
+} from "./matchers.js";
+import { describeMatcherObject, matchCount, matchValue } from "./matchers.js";
+import type {
+  EvalInputRequestFact,
+  EvalRunFacts,
+  EvalToolCallFact,
+} from "./run-facts.js";
+import { deriveRunFacts } from "./run-facts.js";
+
+/** Hard gates fail the eval; soft assertions are tracked scores. */
+export type EvalAssertionSeverity = "gate" | "soft";
+
+/**
+ * Overall grade for one eval case.
+ *
+ * - `passed` — every gate passed and no soft threshold was missed.
+ * - `failed` — a gate failed, or the test body threw.
+ * - `scored` — only soft thresholds were missed: visible, and fatal only under
+ *   `--strict`.
+ * - `skipped` — `t.skip(reason)`, or a judge with no credentials available.
+ */
+export type EvalVerdict = "passed" | "failed" | "scored" | "skipped";
+
+export interface EvalAssertionResult {
+  name: string;
+  passed: boolean;
+  detail?: string;
+  /** Defaults to `gate`; omitted on the wire for gates to stay compact. */
+  severity?: EvalAssertionSeverity;
+  /** 0–1 score for scored assertions (`similarity`, judges). */
+  score?: number;
+  /** Bar a scored assertion must clear. */
+  threshold?: number;
+  /** Set when the assertion inspects one turn rather than the whole run. */
+  scope?: string;
+}
+
+/**
+ * Chainable severity control returned by every assertion.
+ *
+ * - `.gate(threshold?)` is hard: a miss fails the eval and `eval` exits 1.
+ * - `.soft(threshold?)` is tracked: a below-threshold miss marks the eval
+ *   `scored`, fatal only under `--strict`. With no threshold it never fails.
+ * - `.atLeast(threshold)` is soft with a bar.
+ */
+export interface EvalAssertionHandle {
+  gate(threshold?: number): EvalAssertionHandle;
+  soft(threshold?: number): EvalAssertionHandle;
+  atLeast(threshold: number): EvalAssertionHandle;
+}
+
+interface AssertionSlot extends EvalAssertionResult {
+  severity: EvalAssertionSeverity;
+  /** Recorded boolean outcome, before any threshold is applied. */
+  rawPassed: boolean;
+}
+
+/** Thrown by `t.require(...)` to stop the test body without an error result. */
+export class EvalRequireFailure extends Error {
+  constructor(readonly assertionName: string) {
+    super(`required assertion failed: ${assertionName}`);
+    this.name = "EvalRequireFailure";
+  }
+}
+
+/** Thrown by `t.skip(reason)` to end the case as skipped. */
+export class EvalSkipSignal extends Error {
+  constructor(readonly reason: string) {
+    super(reason);
+    this.name = "EvalSkipSignal";
+  }
+}
+
+export interface RecordAssertionInput {
+  name: string;
+  passed: boolean;
+  detail?: string;
+  severity?: EvalAssertionSeverity;
+  score?: number;
+  threshold?: number;
+  scope?: string;
+}
+
+/**
+ * Collects assertion results and any in-flight judge scoring for one case.
+ */
+export class EvalAssertionRecorder {
+  private readonly slots: AssertionSlot[] = [];
+  private readonly pending: Array<Promise<void>> = [];
+
+  record(input: RecordAssertionInput): EvalAssertionHandle {
+    const slot: AssertionSlot = {
+      name: input.name,
+      passed: input.passed,
+      rawPassed: input.passed,
+      detail: input.detail,
+      severity: input.severity ?? "gate",
+      score: input.score,
+      threshold: input.threshold,
+      scope: input.scope,
+    };
+    applyThreshold(slot);
+    this.slots.push(slot);
+    return handleFor(slot);
+  }
+
+  /**
+   * Reserve a slot now and fill it in when `settle` resolves. Judges start when
+   * recorded so several graders overlap; {@link waitForPending} joins them.
+   */
+  recordAsync(
+    input: RecordAssertionInput,
+    settle: Promise<Partial<RecordAssertionInput>>
+  ): EvalAssertionHandle {
+    const handle = this.record(input);
+    const slot = this.slots[this.slots.length - 1]!;
+    this.pending.push(
+      settle.then(
+        (patch) => {
+          Object.assign(slot, patch);
+          if (patch.passed !== undefined) {
+            slot.rawPassed = patch.passed;
+          }
+          applyThreshold(slot);
+        },
+        (error: unknown) => {
+          // Grading blew up, which is a broken assertion rather than a low
+          // score. Soft severity covers "the model scored badly", so promote to
+          // a gate here or the failure would be invisible in the verdict.
+          slot.rawPassed = false;
+          slot.passed = false;
+          slot.score = undefined;
+          slot.threshold = undefined;
+          slot.severity = "gate";
+          slot.detail = error instanceof Error ? error.message : String(error);
+        }
+      )
+    );
+    return handle;
+  }
+
+  /** Await every async assertion started during the test body. */
+  async waitForPending(): Promise<void> {
+    while (this.pending.length > 0) {
+      const inFlight = this.pending.splice(0, this.pending.length);
+      await Promise.all(inFlight);
+    }
+  }
+
+  results(): EvalAssertionResult[] {
+    return this.slots.map((slot) => ({
+      name: slot.name,
+      passed: slot.passed,
+      detail: slot.detail,
+      // Keep the wire shape compact: `gate` is the default everywhere.
+      severity: slot.severity === "gate" ? undefined : slot.severity,
+      score: slot.score,
+      threshold: slot.threshold,
+      scope: slot.scope,
+    }));
+  }
+
+  get count(): number {
+    return this.slots.length;
+  }
+}
+
+/**
+ * Re-derive `passed` from the score whenever a threshold is in play.
+ *
+ * A tracked score with no bar always passes — there is nothing to miss. A
+ * boolean assertion keeps its recorded outcome even when demoted to soft, so
+ * the report stays honest; severity decides the verdict, not the `passed` flag.
+ */
+function applyThreshold(slot: AssertionSlot): void {
+  if (slot.threshold === undefined) {
+    slot.passed = slot.score !== undefined ? true : slot.rawPassed;
+    return;
+  }
+  const score = slot.score ?? (slot.rawPassed ? 1 : 0);
+  slot.passed = score >= slot.threshold;
+}
+
+function handleFor(slot: AssertionSlot): EvalAssertionHandle {
+  const handle: EvalAssertionHandle = {
+    gate(threshold?: number) {
+      slot.severity = "gate";
+      if (threshold !== undefined) {
+        slot.threshold = threshold;
+      }
+      applyThreshold(slot);
+      return handle;
+    },
+    soft(threshold?: number) {
+      slot.severity = "soft";
+      if (threshold !== undefined) {
+        slot.threshold = threshold;
+      }
+      applyThreshold(slot);
+      return handle;
+    },
+    atLeast(threshold: number) {
+      return handle.soft(threshold);
+    },
+  };
+  return handle;
+}
+
+/**
+ * Grade a case from its recorded assertions.
+ *
+ * With no assertions at all, the caller's fallback applies (at least one
+ * successful turn), which keeps pre-severity eval files working unchanged.
+ */
+export function computeVerdict(
+  results: readonly EvalAssertionResult[]
+): Exclude<EvalVerdict, "skipped"> {
+  const failedGate = results.some(
+    (r) => (r.severity ?? "gate") === "gate" && !r.passed
+  );
+  if (failedGate) {
+    return "failed";
+  }
+  const missedSoftBar = results.some(
+    (r) =>
+      (r.severity ?? "gate") === "soft" &&
+      r.threshold !== undefined &&
+      !r.passed
+  );
+  return missedSoftBar ? "scored" : "passed";
+}
+
+/** Whether a verdict should fail the process exit code. */
+export function verdictFailsRun(
+  verdict: EvalVerdict,
+  strict: boolean
+): boolean {
+  if (verdict === "failed") {
+    return true;
+  }
+  return strict && verdict === "scored";
+}
+
+/**
+ * The deterministic assertion vocabulary, available on the run (`t`) and on
+ * each turn returned by `t.send(...)`.
+ */
+export interface EvalAssertions {
+  /**
+   * Gate: the run did not fail and is not parked on an unanswered approval.
+   * A session left open for the next message is a healthy end state.
+   */
+  succeeded(): EvalAssertionHandle;
+  /** Gate: the run cleanly parked on an unanswered human approval request. */
+  parked(): EvalAssertionHandle;
+  /** Gate: joined assistant text contains `token`. */
+  messageIncludes(token: string | RegExp): EvalAssertionHandle;
+  /**
+   * Gate: a matching call to `name` happened.
+   *
+   * With no matcher this is request-based (any lifecycle state), so it keeps
+   * meaning "the agent used this tool". Pass `{ status: "completed" }` to
+   * require the call actually returned, or `{ input, output, count }` to
+   * constrain it further.
+   */
+  calledTool(name: string, matcher?: EvalToolCallMatcher): EvalAssertionHandle;
+  /** Gate: no request for `name` in any lifecycle state. */
+  notCalledTool(name: string): EvalAssertionHandle;
+  /**
+   * Gate: the agent opened the skill's `SKILL.md`.
+   *
+   * Skills are materialized as files rather than exposed as a tool, so this
+   * matches any tool call whose input references `skills/<name>/SKILL.md`
+   * (a file read, a grep, or a shell `cat` all count).
+   */
+  loadedSkill(name: string): EvalAssertionHandle;
+  /** Gate: tool requests appear in this relative order (extra calls allowed). */
+  toolOrder(names: readonly string[]): EvalAssertionHandle;
+  /** Gate: no tool calls at all. */
+  usedNoTools(): EvalAssertionHandle;
+  /** Gate: at most `max` tool calls. */
+  maxToolCalls(max: number): EvalAssertionHandle;
+  /** Gate: no tool call reported an error. */
+  noFailedActions(): EvalAssertionHandle;
+  /** Gate: a matching subagent delegation happened. */
+  calledSubagent(
+    name: string,
+    matcher?: EvalSubagentMatcher
+  ): EvalAssertionHandle;
+  /** Gate: at least one matching event of `type` occurred. */
+  event(type: string, matcher?: EvalEventMatcher): EvalAssertionHandle;
+  /** Gate: no matching event of `type` occurred. */
+  notEvent(type: string, matcher?: EvalEventMatcher): EvalAssertionHandle;
+  /** Gate: matching event groups occur in this relative order. */
+  eventOrder(matchers: readonly EvalEventOrderMatcher[]): EvalAssertionHandle;
+  /** Gate: escape hatch for any predicate over the typed event stream. */
+  eventsSatisfy(
+    label: string,
+    predicate: (events: readonly SessionEvent[]) => boolean
+  ): EvalAssertionHandle;
+  /** Grade an explicit value with an `Expectation` builder. */
+  check(value: unknown, assertion: Expectation): EvalAssertionHandle;
+  /**
+   * Record a 0–1 score the eval computed itself, for a benchmark the built-in
+   * builders cannot express (recall against a gold set, a custom judge).
+   *
+   * Soft and tracked-only by default, so a new benchmark can report its number
+   * for a while before anyone gates on it. Add the bar when the number is
+   * trustworthy: `t.score("recall", recall).atLeast(0.6)`.
+   */
+  score(name: string, value: number): EvalAssertionHandle;
+  /**
+   * Record a gate and return the matching tool call so dependent code can read
+   * its input/output. Stops the test body when no call matches.
+   */
+  requireToolCall(
+    name: string,
+    matcher?: EvalToolCallMatcher
+  ): EvalToolCallFact;
+  /**
+   * Record a gate that exactly one approval request is pending, and return it.
+   * Stops the test body otherwise.
+   */
+  requireInputRequest(filter?: {
+    toolName?: EvalMatcher<string>;
+    args?: EvalMatcher;
+  }): EvalInputRequestFact;
+}
+
+export interface CreateAssertionsOptions {
+  recorder: EvalAssertionRecorder;
+  /** Events under inspection. Read lazily so run scope sees later turns. */
+  events: () => readonly SessionEvent[];
+  /** Label prefix for turn-scoped assertions, e.g. `turn[2]`. */
+  scope?: string;
+}
+
+/** Build the assertion vocabulary bound to one scope. */
+export function createAssertions(
+  options: CreateAssertionsOptions
+): EvalAssertions {
+  const { recorder, events, scope } = options;
+  const facts = (): EvalRunFacts => deriveRunFacts(events());
+  const name = (text: string): string =>
+    scope === undefined ? text : `${scope}.${text}`;
+
+  const record = (input: RecordAssertionInput): EvalAssertionHandle =>
+    recorder.record({ ...input, name: name(input.name), scope });
+
+  const matchingToolCalls = (
+    toolName: string,
+    matcher: EvalToolCallMatcher | undefined
+  ): EvalToolCallFact[] =>
+    facts().toolCalls.filter((call) => {
+      if (call.toolName !== toolName) {
+        return false;
+      }
+      if (matcher === undefined) {
+        return true;
+      }
+      if (matcher.status !== undefined && call.status !== matcher.status) {
+        return false;
+      }
+      if (!matchValue(call.input, matcher.input)) {
+        return false;
+      }
+      if (!matchValue(call.output, matcher.output)) {
+        return false;
+      }
+      return true;
+    });
+
+  const matchingEvents = (
+    type: string,
+    matcher: EvalEventMatcher | undefined
+  ): SessionEvent[] =>
+    events().filter(
+      (event) => event.type === type && matchValue(event.data, matcher?.data)
+    );
+
+  return {
+    succeeded() {
+      const run = facts();
+      const detail = !run.ok
+        ? (run.failureMessage ?? "turn failed")
+        : run.parked
+          ? `parked on ${run.pendingInputRequests.length} unanswered approval request(s)`
+          : undefined;
+      return record({
+        name: "succeeded",
+        passed: run.ok && !run.parked,
+        detail,
+      });
+    },
+
+    parked() {
+      const run = facts();
+      const pending = run.pendingInputRequests;
+      return record({
+        name: "parked",
+        passed: run.parked && run.turnsFailed === 0,
+        detail: run.parked
+          ? pending.map((r) => r.toolName).join(", ")
+          : "no unanswered approval request",
+      });
+    },
+
+    messageIncludes(token: string | RegExp) {
+      const text = facts().assistantText;
+      const passed =
+        typeof token === "string" ? text.includes(token) : token.test(text);
+      return record({
+        name: `messageIncludes(${String(token)})`,
+        passed,
+        detail: passed
+          ? undefined
+          : `text=${JSON.stringify(text.slice(0, 120))}`,
+      });
+    },
+
+    calledTool(toolName: string, matcher?: EvalToolCallMatcher) {
+      const matched = matchingToolCalls(toolName, matcher);
+      const passed = matchCount(matched.length, matcher?.count);
+      const observed = facts()
+        .toolCalls.map((c) => c.toolName)
+        .join(", ");
+      return record({
+        name: `calledTool(${toolName}${describeMatcherObject(
+          matcher as Record<string, unknown> | undefined
+        )})`,
+        passed,
+        detail: passed
+          ? undefined
+          : `matched ${matched.length}; called [${observed}]`,
+      });
+    },
+
+    notCalledTool(toolName: string) {
+      const matched = matchingToolCalls(toolName, undefined);
+      return record({
+        name: `notCalledTool(${toolName})`,
+        passed: matched.length === 0,
+        detail: matched.length === 0 ? undefined : "tool was called",
+      });
+    },
+
+    loadedSkill(skill: string) {
+      const needle = `skills/${skill}/SKILL.md`;
+      const matched = facts().toolCalls.filter((call) =>
+        JSON.stringify(call.input ?? "").includes(needle)
+      );
+      return record({
+        name: `loadedSkill(${skill})`,
+        passed: matched.length > 0,
+        detail:
+          matched.length > 0 ? undefined : `no tool call referenced ${needle}`,
+      });
+    },
+
+    toolOrder(names: readonly string[]) {
+      const called = facts().toolCalls.map((c) => c.toolName);
+      let cursor = 0;
+      for (const expected of names) {
+        const found = called.indexOf(expected, cursor);
+        if (found === -1) {
+          return record({
+            name: `toolOrder([${names.join(", ")}])`,
+            passed: false,
+            detail: `missing ${expected} after position ${cursor}; called [${called.join(", ")}]`,
+          });
+        }
+        cursor = found + 1;
+      }
+      return record({
+        name: `toolOrder([${names.join(", ")}])`,
+        passed: true,
+      });
+    },
+
+    usedNoTools() {
+      const called = facts().toolCalls;
+      return record({
+        name: "usedNoTools",
+        passed: called.length === 0,
+        detail:
+          called.length === 0
+            ? undefined
+            : `called [${called.map((c) => c.toolName).join(", ")}]`,
+      });
+    },
+
+    maxToolCalls(max: number) {
+      const total = facts().toolCalls.length;
+      return record({
+        name: `maxToolCalls(${max})`,
+        passed: total <= max,
+        detail: total <= max ? undefined : `made ${total} tool calls`,
+      });
+    },
+
+    noFailedActions() {
+      const failed = facts().failedToolCalls;
+      return record({
+        name: "noFailedActions",
+        passed: failed.length === 0,
+        detail:
+          failed.length === 0
+            ? undefined
+            : failed.map((c) => c.toolName).join(", "),
+      });
+    },
+
+    calledSubagent(subagentName: string, matcher?: EvalSubagentMatcher) {
+      const matched = facts().subagents.filter((sub) => {
+        if (sub.name !== subagentName) {
+          return false;
+        }
+        if (matcher === undefined) {
+          return true;
+        }
+        if (matcher.status !== undefined && sub.status !== matcher.status) {
+          return false;
+        }
+        if (!matchValue(sub.callId, matcher.callId)) {
+          return false;
+        }
+        return matchValue(sub.output, matcher.output);
+      });
+      const passed = matchCount(matched.length, matcher?.count);
+      return record({
+        name: `calledSubagent(${subagentName}${describeMatcherObject(
+          matcher as Record<string, unknown> | undefined
+        )})`,
+        passed,
+        detail: passed
+          ? undefined
+          : `matched ${matched.length}; delegated [${facts()
+              .subagents.map((s) => s.name ?? s.callId)
+              .join(", ")}]`,
+      });
+    },
+
+    event(type: string, matcher?: EvalEventMatcher) {
+      const matched = matchingEvents(type, matcher);
+      const passed = matchCount(matched.length, matcher?.count);
+      return record({
+        name: `event(${type}${describeMatcherObject(
+          matcher as Record<string, unknown> | undefined
+        )})`,
+        passed,
+        detail: passed ? undefined : `matched ${matched.length}`,
+      });
+    },
+
+    notEvent(type: string, matcher?: EvalEventMatcher) {
+      const matched = matchingEvents(type, matcher);
+      return record({
+        name: `notEvent(${type}${describeMatcherObject(
+          matcher as Record<string, unknown> | undefined
+        )})`,
+        passed: matched.length === 0,
+        detail: matched.length === 0 ? undefined : `matched ${matched.length}`,
+      });
+    },
+
+    eventOrder(matchers: readonly EvalEventOrderMatcher[]) {
+      const stream = events();
+      const label = `eventOrder([${matchers.map((m) => m.type).join(", ")}])`;
+      let cursor = 0;
+      for (const matcher of matchers) {
+        const required = typeof matcher.count === "number" ? matcher.count : 1;
+        let seen = 0;
+        while (seen < required && cursor < stream.length) {
+          const event = stream[cursor]!;
+          cursor++;
+          if (
+            event.type === matcher.type &&
+            matchValue(event.data, matcher.data)
+          ) {
+            seen++;
+          }
+        }
+        if (seen < required) {
+          return record({
+            name: label,
+            passed: false,
+            detail: `only ${seen}/${required} ${matcher.type} event(s) in order`,
+          });
+        }
+      }
+      return record({ name: label, passed: true });
+    },
+
+    eventsSatisfy(
+      label: string,
+      predicate: (events: readonly SessionEvent[]) => boolean
+    ) {
+      let passed = false;
+      let detail: string | undefined;
+      try {
+        passed = predicate(events()) === true;
+      } catch (error) {
+        detail = error instanceof Error ? error.message : String(error);
+      }
+      return record({
+        name: `eventsSatisfy(${label})`,
+        passed,
+        detail: passed ? undefined : (detail ?? label),
+      });
+    },
+
+    check(value: unknown, assertion: Expectation) {
+      const outcome = evaluateExpectation(value, assertion);
+      return record({
+        name: `check(${assertion.kind})`,
+        passed: outcome.passed,
+        detail: outcome.detail,
+        score: outcome.score,
+        severity: expectationSeverity(assertion),
+      });
+    },
+
+    score(scoreName: string, value: number) {
+      const finite = Number.isFinite(value);
+      return record({
+        name: `score(${scoreName})`,
+        // A NaN score is a bug in the eval's own math, so fail loudly rather
+        // than reporting a number nobody can act on.
+        passed: finite,
+        severity: "soft",
+        score: finite ? value : undefined,
+        detail: finite ? undefined : `score is not a finite number: ${value}`,
+      });
+    },
+
+    requireToolCall(toolName: string, matcher?: EvalToolCallMatcher) {
+      const matched = matchingToolCalls(toolName, matcher);
+      const assertionName = `requireToolCall(${toolName})`;
+      const first = matched[0];
+      record({
+        name: assertionName,
+        passed: first !== undefined,
+        detail: first === undefined ? "no matching tool call" : undefined,
+      });
+      if (first === undefined) {
+        throw new EvalRequireFailure(name(assertionName));
+      }
+      return first;
+    },
+
+    requireInputRequest(filter?: {
+      toolName?: EvalMatcher<string>;
+      args?: EvalMatcher;
+    }) {
+      const pending = facts().pendingInputRequests.filter(
+        (request) =>
+          matchValue(request.toolName, filter?.toolName) &&
+          matchValue(request.args, filter?.args)
+      );
+      const assertionName = "requireInputRequest";
+      const passed = pending.length === 1;
+      record({
+        name: assertionName,
+        passed,
+        detail: passed
+          ? undefined
+          : `expected exactly 1 pending approval request, found ${pending.length}`,
+      });
+      if (!passed) {
+        throw new EvalRequireFailure(name(assertionName));
+      }
+      return pending[0]!;
+    },
+  };
+}
+
+export type { EvalCountMatcher, EvalInputRequestFact, EvalToolCallFact };