@cursor/bdk 0.2.2 → 0.2.4

package/src/evals/assertions.ts

@@ -1,688 +0,0 @@
-/**
- * Assertion recording and the scoped assertion vocabulary.
- *
- * Recorded assertions never throw: the recorder collects every result so one
- * run reports all failures instead of dying on the first. Severity rides on the
- * returned handle, so there is no separate thresholds map:
- *
- * ```ts
- * t.succeeded();                                  // gate (hard)
- * t.calledTool("get_weather").soft();             // tracked, does not gate
- * t.check(t.reply, similarity("Sunny")).atLeast(0.8);
- * ```
- *
- * The same vocabulary is available on the run (`t`) and on each turn returned
- * by `t.send(...)`, which narrows the events under inspection to that turn.
- */
-
-import type { SessionEvent, SessionEventPayload } from "../types.js";
-import type { Expectation } from "./expect.js";
-import { evaluateExpectation, expectationSeverity } from "./expect.js";
-import type {
-  EvalCountMatcher,
-  EvalEventMatcher,
-  EvalEventOrderMatcher,
-  EvalMatcher,
-  EvalSubagentMatcher,
-  EvalToolCallMatcher,
-} from "./matchers.js";
-import { describeMatcherObject, matchCount, matchValue } from "./matchers.js";
-import type { EvalInputRequestFact, EvalRunFacts, EvalToolCallFact } from "./run-facts.js";
-import { deriveRunFacts } from "./run-facts.js";
-
-/** Hard gates fail the eval; soft assertions are tracked scores. */
-export type EvalAssertionSeverity = "gate" | "soft";
-
-/** One `artifact.tagged` payload, as seen by `t.taggedArtifact(...)`. */
-export type EvalTaggedArtifactFact = Extract<
-  SessionEventPayload,
-  { type: "artifact.tagged" }
->["data"];
-
-/**
- * Overall grade for one eval case.
- *
- * - `passed` — every gate passed and no soft threshold was missed.
- * - `failed` — a gate failed, or the test body threw.
- * - `scored` — only soft thresholds were missed: visible, and fatal only under
- *   `--strict`.
- * - `skipped` — `t.skip(reason)`, or a judge with no credentials available.
- */
-export type EvalVerdict = "passed" | "failed" | "scored" | "skipped";
-
-export interface EvalAssertionResult {
-  name: string;
-  passed: boolean;
-  detail?: string;
-  /** Defaults to `gate`; omitted on the wire for gates to stay compact. */
-  severity?: EvalAssertionSeverity;
-  /** 0–1 score for scored assertions (`similarity`, judges). */
-  score?: number;
-  /** Bar a scored assertion must clear. */
-  threshold?: number;
-  /** Set when the assertion inspects one turn rather than the whole run. */
-  scope?: string;
-}
-
-/**
- * Chainable severity control returned by every assertion.
- *
- * - `.gate(threshold?)` is hard: a miss fails the eval and `eval` exits 1.
- * - `.soft(threshold?)` is tracked: a below-threshold miss marks the eval
- *   `scored`, fatal only under `--strict`. With no threshold it never fails.
- * - `.atLeast(threshold)` is soft with a bar.
- */
-export interface EvalAssertionHandle {
-  gate(threshold?: number): EvalAssertionHandle;
-  soft(threshold?: number): EvalAssertionHandle;
-  atLeast(threshold: number): EvalAssertionHandle;
-}
-
-interface AssertionSlot extends EvalAssertionResult {
-  severity: EvalAssertionSeverity;
-  /** Recorded boolean outcome, before any threshold is applied. */
-  rawPassed: boolean;
-}
-
-/** Thrown by `t.require(...)` to stop the test body without an error result. */
-export class EvalRequireFailure extends Error {
-  constructor(readonly assertionName: string) {
-    super(`required assertion failed: ${assertionName}`);
-    this.name = "EvalRequireFailure";
-  }
-}
-
-/** Thrown by `t.skip(reason)` to end the case as skipped. */
-export class EvalSkipSignal extends Error {
-  constructor(readonly reason: string) {
-    super(reason);
-    this.name = "EvalSkipSignal";
-  }
-}
-
-export interface RecordAssertionInput {
-  name: string;
-  passed: boolean;
-  detail?: string;
-  severity?: EvalAssertionSeverity;
-  score?: number;
-  threshold?: number;
-  scope?: string;
-}
-
-/**
- * Collects assertion results and any in-flight judge scoring for one case.
- */
-export class EvalAssertionRecorder {
-  private readonly slots: AssertionSlot[] = [];
-  private readonly pending: Array<Promise<void>> = [];
-
-  record(input: RecordAssertionInput): EvalAssertionHandle {
-    const slot: AssertionSlot = {
-      name: input.name,
-      passed: input.passed,
-      rawPassed: input.passed,
-      detail: input.detail,
-      severity: input.severity ?? "gate",
-      score: input.score,
-      threshold: input.threshold,
-      scope: input.scope,
-    };
-    applyThreshold(slot);
-    this.slots.push(slot);
-    return handleFor(slot);
-  }
-
-  /**
-   * Reserve a slot now and fill it in when `settle` resolves. Judges start when
-   * recorded so several graders overlap; {@link waitForPending} joins them.
-   */
-  recordAsync(
-    input: RecordAssertionInput,
-    settle: Promise<Partial<RecordAssertionInput>>,
-  ): EvalAssertionHandle {
-    const handle = this.record(input);
-    const slot = this.slots[this.slots.length - 1]!;
-    this.pending.push(
-      settle.then(
-        patch => {
-          Object.assign(slot, patch);
-          if (patch.passed !== undefined) {
-            slot.rawPassed = patch.passed;
-          }
-          applyThreshold(slot);
-        },
-        (error: unknown) => {
-          // Grading blew up, which is a broken assertion rather than a low
-          // score. Soft severity covers "the model scored badly", so promote to
-          // a gate here or the failure would be invisible in the verdict.
-          slot.rawPassed = false;
-          slot.passed = false;
-          slot.score = undefined;
-          slot.threshold = undefined;
-          slot.severity = "gate";
-          slot.detail = error instanceof Error ? error.message : String(error);
-        },
-      ),
-    );
-    return handle;
-  }
-
-  /** Await every async assertion started during the test body. */
-  async waitForPending(): Promise<void> {
-    while (this.pending.length > 0) {
-      const inFlight = this.pending.splice(0, this.pending.length);
-      await Promise.all(inFlight);
-    }
-  }
-
-  results(): EvalAssertionResult[] {
-    return this.slots.map(slot => ({
-      name: slot.name,
-      passed: slot.passed,
-      detail: slot.detail,
-      // Keep the wire shape compact: `gate` is the default everywhere.
-      severity: slot.severity === "gate" ? undefined : slot.severity,
-      score: slot.score,
-      threshold: slot.threshold,
-      scope: slot.scope,
-    }));
-  }
-
-  get count(): number {
-    return this.slots.length;
-  }
-}
-
-/**
- * Re-derive `passed` from the score whenever a threshold is in play.
- *
- * A tracked score with no bar always passes — there is nothing to miss. A
- * boolean assertion keeps its recorded outcome even when demoted to soft, so
- * the report stays honest; severity decides the verdict, not the `passed` flag.
- */
-function applyThreshold(slot: AssertionSlot): void {
-  if (slot.threshold === undefined) {
-    slot.passed = slot.score !== undefined ? true : slot.rawPassed;
-    return;
-  }
-  const score = slot.score ?? (slot.rawPassed ? 1 : 0);
-  slot.passed = score >= slot.threshold;
-}
-
-function handleFor(slot: AssertionSlot): EvalAssertionHandle {
-  const handle: EvalAssertionHandle = {
-    gate(threshold?: number) {
-      slot.severity = "gate";
-      if (threshold !== undefined) {
-        slot.threshold = threshold;
-      }
-      applyThreshold(slot);
-      return handle;
-    },
-    soft(threshold?: number) {
-      slot.severity = "soft";
-      if (threshold !== undefined) {
-        slot.threshold = threshold;
-      }
-      applyThreshold(slot);
-      return handle;
-    },
-    atLeast(threshold: number) {
-      return handle.soft(threshold);
-    },
-  };
-  return handle;
-}
-
-/**
- * Grade a case from its recorded assertions.
- *
- * With no assertions at all, the caller's fallback applies (at least one
- * successful turn), which keeps pre-severity eval files working unchanged.
- */
-export function computeVerdict(
-  results: readonly EvalAssertionResult[],
-): Exclude<EvalVerdict, "skipped"> {
-  const failedGate = results.some(r => (r.severity ?? "gate") === "gate" && !r.passed);
-  if (failedGate) {
-    return "failed";
-  }
-  const missedSoftBar = results.some(
-    r => (r.severity ?? "gate") === "soft" && r.threshold !== undefined && !r.passed,
-  );
-  return missedSoftBar ? "scored" : "passed";
-}
-
-/** Whether a verdict should fail the process exit code. */
-export function verdictFailsRun(verdict: EvalVerdict, strict: boolean): boolean {
-  if (verdict === "failed") {
-    return true;
-  }
-  return strict && verdict === "scored";
-}
-
-/**
- * The deterministic assertion vocabulary, available on the run (`t`) and on
- * each turn returned by `t.send(...)`.
- */
-export interface EvalAssertions {
-  /**
-   * Gate: the run did not fail and is not parked on an unanswered approval.
-   * A session left open for the next message is a healthy end state.
-   */
-  succeeded(): EvalAssertionHandle;
-  /** Gate: the run cleanly parked on an unanswered human approval request. */
-  parked(): EvalAssertionHandle;
-  /** Gate: joined assistant text contains `token`. */
-  messageIncludes(token: string | RegExp): EvalAssertionHandle;
-  /**
-   * Gate: a matching call to `name` happened.
-   *
-   * With no matcher this is request-based (any lifecycle state), so it keeps
-   * meaning "the agent used this tool". Pass `{ status: "completed" }` to
-   * require the call actually returned, or `{ input, output, count }` to
-   * constrain it further.
-   */
-  calledTool(name: string, matcher?: EvalToolCallMatcher): EvalAssertionHandle;
-  /** Gate: no request for `name` in any lifecycle state. */
-  notCalledTool(name: string): EvalAssertionHandle;
-  /**
-   * Gate: the agent opened the skill's `SKILL.md`.
-   *
-   * Skills are materialized as files rather than exposed as a tool, so this
-   * matches any tool call whose input references `skills/<name>/SKILL.md`
-   * (a file read, a grep, or a shell `cat` all count).
-   */
-  loadedSkill(name: string): EvalAssertionHandle;
-  /** Gate: tool requests appear in this relative order (extra calls allowed). */
-  toolOrder(names: readonly string[]): EvalAssertionHandle;
-  /** Gate: no tool calls at all. */
-  usedNoTools(): EvalAssertionHandle;
-  /** Gate: at most `max` tool calls. */
-  maxToolCalls(max: number): EvalAssertionHandle;
-  /** Gate: no tool call reported an error. */
-  noFailedActions(): EvalAssertionHandle;
-  /** Gate: a matching subagent delegation happened. */
-  calledSubagent(name: string, matcher?: EvalSubagentMatcher): EvalAssertionHandle;
-  /**
-   * Gate: at least one artifact was tagged (`ctx.artifacts.tag` or the
-   * `tag_artifact` built-in), optionally of `kind` and matching `predicate`.
-   */
-  taggedArtifact(
-    kind?: string,
-    predicate?: (record: EvalTaggedArtifactFact) => boolean,
-  ): EvalAssertionHandle;
-  /** Gate: at least one matching event of `type` occurred. */
-  event(type: string, matcher?: EvalEventMatcher): EvalAssertionHandle;
-  /** Gate: no matching event of `type` occurred. */
-  notEvent(type: string, matcher?: EvalEventMatcher): EvalAssertionHandle;
-  /** Gate: matching event groups occur in this relative order. */
-  eventOrder(matchers: readonly EvalEventOrderMatcher[]): EvalAssertionHandle;
-  /** Gate: escape hatch for any predicate over the typed event stream. */
-  eventsSatisfy(
-    label: string,
-    predicate: (events: readonly SessionEvent[]) => boolean,
-  ): EvalAssertionHandle;
-  /** Grade an explicit value with an `Expectation` builder. */
-  check(value: unknown, assertion: Expectation): EvalAssertionHandle;
-  /**
-   * Record a 0–1 score the eval computed itself, for a benchmark the built-in
-   * builders cannot express (recall against a gold set, a custom judge).
-   *
-   * Soft and tracked-only by default, so a new benchmark can report its number
-   * for a while before anyone gates on it. Add the bar when the number is
-   * trustworthy: `t.score("recall", recall).atLeast(0.6)`.
-   */
-  score(name: string, value: number): EvalAssertionHandle;
-  /**
-   * Record a gate and return the matching tool call so dependent code can read
-   * its input/output. Stops the test body when no call matches.
-   */
-  requireToolCall(name: string, matcher?: EvalToolCallMatcher): EvalToolCallFact;
-  /**
-   * Record a gate that exactly one approval request is pending, and return it.
-   * Stops the test body otherwise.
-   */
-  requireInputRequest(filter?: {
-    toolName?: EvalMatcher<string>;
-    args?: EvalMatcher;
-  }): EvalInputRequestFact;
-}
-
-export interface CreateAssertionsOptions {
-  recorder: EvalAssertionRecorder;
-  /** Events under inspection. Read lazily so run scope sees later turns. */
-  events: () => readonly SessionEvent[];
-  /** Label prefix for turn-scoped assertions, e.g. `turn[2]`. */
-  scope?: string;
-}
-
-/** Build the assertion vocabulary bound to one scope. */
-export function createAssertions(options: CreateAssertionsOptions): EvalAssertions {
-  const { recorder, events, scope } = options;
-  const facts = (): EvalRunFacts => deriveRunFacts(events());
-  const name = (text: string): string => (scope === undefined ? text : `${scope}.${text}`);
-
-  const record = (input: RecordAssertionInput): EvalAssertionHandle =>
-    recorder.record({ ...input, name: name(input.name), scope });
-
-  const matchingToolCalls = (
-    toolName: string,
-    matcher: EvalToolCallMatcher | undefined,
-  ): EvalToolCallFact[] =>
-    facts().toolCalls.filter(call => {
-      if (call.toolName !== toolName) {
-        return false;
-      }
-      if (matcher === undefined) {
-        return true;
-      }
-      if (matcher.status !== undefined && call.status !== matcher.status) {
-        return false;
-      }
-      if (!matchValue(call.input, matcher.input)) {
-        return false;
-      }
-      if (!matchValue(call.output, matcher.output)) {
-        return false;
-      }
-      return true;
-    });
-
-  const matchingEvents = (type: string, matcher: EvalEventMatcher | undefined): SessionEvent[] =>
-    events().filter(event => event.type === type && matchValue(event.data, matcher?.data));
-
-  return {
-    succeeded() {
-      const run = facts();
-      const detail = !run.ok
-        ? (run.failureMessage ?? "turn failed")
-        : run.parked
-          ? `parked on ${run.pendingInputRequests.length} unanswered approval request(s)`
-          : undefined;
-      return record({
-        name: "succeeded",
-        passed: run.ok && !run.parked,
-        detail,
-      });
-    },
-
-    parked() {
-      const run = facts();
-      const pending = run.pendingInputRequests;
-      return record({
-        name: "parked",
-        passed: run.parked && run.turnsFailed === 0,
-        detail: run.parked
-          ? pending.map(r => r.toolName).join(", ")
-          : "no unanswered approval request",
-      });
-    },
-
-    messageIncludes(token: string | RegExp) {
-      const text = facts().assistantText;
-      const passed = typeof token === "string" ? text.includes(token) : token.test(text);
-      return record({
-        name: `messageIncludes(${String(token)})`,
-        passed,
-        detail: passed ? undefined : `text=${JSON.stringify(text.slice(0, 120))}`,
-      });
-    },
-
-    calledTool(toolName: string, matcher?: EvalToolCallMatcher) {
-      const matched = matchingToolCalls(toolName, matcher);
-      const passed = matchCount(matched.length, matcher?.count);
-      const observed = facts()
-        .toolCalls.map(c => c.toolName)
-        .join(", ");
-      return record({
-        name: `calledTool(${toolName}${describeMatcherObject(
-          matcher as Record<string, unknown> | undefined,
-        )})`,
-        passed,
-        detail: passed ? undefined : `matched ${matched.length}; called [${observed}]`,
-      });
-    },
-
-    notCalledTool(toolName: string) {
-      const matched = matchingToolCalls(toolName, undefined);
-      return record({
-        name: `notCalledTool(${toolName})`,
-        passed: matched.length === 0,
-        detail: matched.length === 0 ? undefined : "tool was called",
-      });
-    },
-
-    loadedSkill(skill: string) {
-      const needle = `skills/${skill}/SKILL.md`;
-      const matched = facts().toolCalls.filter(call =>
-        JSON.stringify(call.input ?? "").includes(needle),
-      );
-      return record({
-        name: `loadedSkill(${skill})`,
-        passed: matched.length > 0,
-        detail: matched.length > 0 ? undefined : `no tool call referenced ${needle}`,
-      });
-    },
-
-    toolOrder(names: readonly string[]) {
-      const called = facts().toolCalls.map(c => c.toolName);
-      let cursor = 0;
-      for (const expected of names) {
-        const found = called.indexOf(expected, cursor);
-        if (found === -1) {
-          return record({
-            name: `toolOrder([${names.join(", ")}])`,
-            passed: false,
-            detail: `missing ${expected} after position ${cursor}; called [${called.join(", ")}]`,
-          });
-        }
-        cursor = found + 1;
-      }
-      return record({
-        name: `toolOrder([${names.join(", ")}])`,
-        passed: true,
-      });
-    },
-
-    usedNoTools() {
-      const called = facts().toolCalls;
-      return record({
-        name: "usedNoTools",
-        passed: called.length === 0,
-        detail:
-          called.length === 0 ? undefined : `called [${called.map(c => c.toolName).join(", ")}]`,
-      });
-    },
-
-    maxToolCalls(max: number) {
-      const total = facts().toolCalls.length;
-      return record({
-        name: `maxToolCalls(${max})`,
-        passed: total <= max,
-        detail: total <= max ? undefined : `made ${total} tool calls`,
-      });
-    },
-
-    noFailedActions() {
-      const failed = facts().failedToolCalls;
-      return record({
-        name: "noFailedActions",
-        passed: failed.length === 0,
-        detail: failed.length === 0 ? undefined : failed.map(c => c.toolName).join(", "),
-      });
-    },
-
-    calledSubagent(subagentName: string, matcher?: EvalSubagentMatcher) {
-      const matched = facts().subagents.filter(sub => {
-        if (sub.name !== subagentName) {
-          return false;
-        }
-        if (matcher === undefined) {
-          return true;
-        }
-        if (matcher.status !== undefined && sub.status !== matcher.status) {
-          return false;
-        }
-        if (!matchValue(sub.callId, matcher.callId)) {
-          return false;
-        }
-        return matchValue(sub.output, matcher.output);
-      });
-      const passed = matchCount(matched.length, matcher?.count);
-      return record({
-        name: `calledSubagent(${subagentName}${describeMatcherObject(
-          matcher as Record<string, unknown> | undefined,
-        )})`,
-        passed,
-        detail: passed
-          ? undefined
-          : `matched ${matched.length}; delegated [${facts()
-              .subagents.map(s => s.name ?? s.callId)
-              .join(", ")}]`,
-      });
-    },
-
-    taggedArtifact(kind?: string, predicate?: (record: EvalTaggedArtifactFact) => boolean) {
-      const tagged = events().flatMap(event =>
-        event.type === "artifact.tagged" ? [event.data] : [],
-      );
-      const matched = tagged.filter(
-        record =>
-          (kind === undefined || record.kind === kind) &&
-          (predicate === undefined || predicate(record) === true),
-      );
-      return record({
-        name: `taggedArtifact(${kind ?? ""})`,
-        passed: matched.length > 0,
-        detail:
-          matched.length > 0 ? undefined : `tagged kinds: [${tagged.map(r => r.kind).join(", ")}]`,
-      });
-    },
-
-    event(type: string, matcher?: EvalEventMatcher) {
-      const matched = matchingEvents(type, matcher);
-      const passed = matchCount(matched.length, matcher?.count);
-      return record({
-        name: `event(${type}${describeMatcherObject(
-          matcher as Record<string, unknown> | undefined,
-        )})`,
-        passed,
-        detail: passed ? undefined : `matched ${matched.length}`,
-      });
-    },
-
-    notEvent(type: string, matcher?: EvalEventMatcher) {
-      const matched = matchingEvents(type, matcher);
-      return record({
-        name: `notEvent(${type}${describeMatcherObject(
-          matcher as Record<string, unknown> | undefined,
-        )})`,
-        passed: matched.length === 0,
-        detail: matched.length === 0 ? undefined : `matched ${matched.length}`,
-      });
-    },
-
-    eventOrder(matchers: readonly EvalEventOrderMatcher[]) {
-      const stream = events();
-      const label = `eventOrder([${matchers.map(m => m.type).join(", ")}])`;
-      let cursor = 0;
-      for (const matcher of matchers) {
-        const required = typeof matcher.count === "number" ? matcher.count : 1;
-        let seen = 0;
-        while (seen < required && cursor < stream.length) {
-          const event = stream[cursor]!;
-          cursor++;
-          if (event.type === matcher.type && matchValue(event.data, matcher.data)) {
-            seen++;
-          }
-        }
-        if (seen < required) {
-          return record({
-            name: label,
-            passed: false,
-            detail: `only ${seen}/${required} ${matcher.type} event(s) in order`,
-          });
-        }
-      }
-      return record({ name: label, passed: true });
-    },
-
-    eventsSatisfy(label: string, predicate: (events: readonly SessionEvent[]) => boolean) {
-      let passed = false;
-      let detail: string | undefined;
-      try {
-        passed = predicate(events()) === true;
-      } catch (error) {
-        detail = error instanceof Error ? error.message : String(error);
-      }
-      return record({
-        name: `eventsSatisfy(${label})`,
-        passed,
-        detail: passed ? undefined : (detail ?? label),
-      });
-    },
-
-    check(value: unknown, assertion: Expectation) {
-      const outcome = evaluateExpectation(value, assertion);
-      return record({
-        name: `check(${assertion.kind})`,
-        passed: outcome.passed,
-        detail: outcome.detail,
-        score: outcome.score,
-        severity: expectationSeverity(assertion),
-      });
-    },
-
-    score(scoreName: string, value: number) {
-      const finite = Number.isFinite(value);
-      return record({
-        name: `score(${scoreName})`,
-        // A NaN score is a bug in the eval's own math, so fail loudly rather
-        // than reporting a number nobody can act on.
-        passed: finite,
-        severity: "soft",
-        score: finite ? value : undefined,
-        detail: finite ? undefined : `score is not a finite number: ${value}`,
-      });
-    },
-
-    requireToolCall(toolName: string, matcher?: EvalToolCallMatcher) {
-      const matched = matchingToolCalls(toolName, matcher);
-      const assertionName = `requireToolCall(${toolName})`;
-      const first = matched[0];
-      record({
-        name: assertionName,
-        passed: first !== undefined,
-        detail: first === undefined ? "no matching tool call" : undefined,
-      });
-      if (first === undefined) {
-        throw new EvalRequireFailure(name(assertionName));
-      }
-      return first;
-    },
-
-    requireInputRequest(filter?: { toolName?: EvalMatcher<string>; args?: EvalMatcher }) {
-      const pending = facts().pendingInputRequests.filter(
-        request =>
-          matchValue(request.toolName, filter?.toolName) && matchValue(request.args, filter?.args),
-      );
-      const assertionName = "requireInputRequest";
-      const passed = pending.length === 1;
-      record({
-        name: assertionName,
-        passed,
-        detail: passed
-          ? undefined
-          : `expected exactly 1 pending approval request, found ${pending.length}`,
-      });
-      if (!passed) {
-        throw new EvalRequireFailure(name(assertionName));
-      }
-      return pending[0]!;
-    },
-  };
-}
-
-export type { EvalCountMatcher, EvalInputRequestFact, EvalToolCallFact };