@cursor/bdk 0.2.3 → 0.2.5

package/src/evals/matchers.ts

@@ -1,223 +0,0 @@
-/**
- * Matcher mini-language shared by tool-call, subagent, and event assertions.
- *
- * A matcher is a literal, a RegExp, or a predicate:
- *
- * ```ts
- * t.calledTool("bash", { input: { command: /^pwd/ }, count: 1 });
- * t.calledTool("echo", { count: (n) => n >= 2 });
- * t.calledTool("read", { output: (value) => String(value).includes("hello") });
- * ```
- *
- * Object literals partial-deep-match, so `{ input: { city: "NYC" } }` matches a
- * call whose args also carry unrelated keys. Nested values may themselves be
- * matchers.
- */
-
-/**
- * Literal forms a matcher can take.
- *
- * Spelled out rather than written as `unknown` so the union keeps its function
- * member: `unknown` would absorb it, and TypeScript could no longer infer the
- * parameter type of an inline predicate. Nested values are matchers in turn, so
- * a predicate buried in an object literal gets its parameter type too.
- */
-export type EvalMatcherLiteral =
-  | string
-  | number
-  | boolean
-  | null
-  | Date
-  | readonly EvalMatcher[]
-  | { readonly [key: string]: EvalMatcher | undefined };
-
-/** A literal to partial-match, a RegExp to test, or a predicate. */
-export type EvalMatcher<T = unknown> = EvalMatcherLiteral | RegExp | ((value: T) => boolean);
-
-/** An exact count, or a predicate over the observed count. */
-export type EvalCountMatcher = number | ((count: number) => boolean);
-
-/** Lifecycle state of one tool call, derived from the session event stream. */
-export type EvalToolCallStatus =
-  /** `action.result` arrived without `isError`. */
-  | "completed"
-  /** `action.result` arrived with `isError: true`. */
-  | "failed"
-  /** Requested, no result yet (includes calls parked on human approval). */
-  | "pending"
-  /** A human denied the approval request for this call. */
-  | "rejected";
-
-/** Constraints for {@link import("../evals.js").EvalAssertions.calledTool}. */
-export interface EvalToolCallMatcher {
-  /** Match the tool's arguments (partial-deep for object literals). */
-  input?: EvalMatcher;
-  /** Match the tool's result payload. */
-  output?: EvalMatcher;
-  /**
-   * Restrict to one lifecycle state. Omitted matches **any** state, which is
-   * request-based and therefore backwards compatible with `calledTool(name)`.
-   */
-  status?: EvalToolCallStatus;
-  /** Require an exact number of matching calls, or satisfy a predicate. */
-  count?: EvalCountMatcher;
-}
-
-/** Constraints for {@link import("../evals.js").EvalAssertions.calledSubagent}. */
-export interface EvalSubagentMatcher {
-  /** Match the delegating `task` call id. */
-  callId?: EvalMatcher<string>;
-  /** Match the subagent's final output text. */
-  output?: EvalMatcher;
-  /** `completed` once `subagent.completed` arrived, else `pending`. */
-  status?: "completed" | "pending";
-  count?: EvalCountMatcher;
-}
-
-/** Constraints for {@link import("../evals.js").EvalAssertions.event}. */
-export interface EvalEventMatcher {
-  /** Partial-deep match against the event's `data` payload. */
-  data?: EvalMatcher;
-  count?: EvalCountMatcher;
-}
-
-/** One entry of {@link import("../evals.js").EvalAssertions.eventOrder}. */
-export interface EvalEventOrderMatcher extends EvalEventMatcher {
-  type: string;
-}
-
-function isPlainObject(value: unknown): value is Record<string, unknown> {
-  return (
-    typeof value === "object" &&
-    value !== null &&
-    !Array.isArray(value) &&
-    !(value instanceof RegExp) &&
-    !(value instanceof Date)
-  );
-}
-
-/** Stable string form used when a RegExp is tested against a non-string. */
-export function stringifyForMatch(value: unknown): string {
-  if (typeof value === "string") {
-    return value;
-  }
-  if (value === undefined) {
-    return "";
-  }
-  if (value === null) {
-    return "null";
-  }
-  if (typeof value === "object") {
-    try {
-      return JSON.stringify(value) ?? "";
-    } catch {
-      return String(value);
-    }
-  }
-  return String(value);
-}
-
-/**
- * Whether `actual` satisfies `matcher`.
- *
- * - `undefined` matcher matches anything (the constraint was not supplied).
- * - A function is called with `actual`.
- * - A RegExp is tested against {@link stringifyForMatch}.
- * - An object or array literal partial-deep-matches, and its own values may be
- *   matchers.
- * - Anything else compares by deep equality.
- */
-export function matchValue<T = unknown>(actual: T, matcher: EvalMatcher<T> | undefined): boolean {
-  if (matcher === undefined) {
-    return true;
-  }
-  if (typeof matcher === "function") {
-    return matcher(actual) === true;
-  }
-  if (matcher instanceof RegExp) {
-    return matcher.test(stringifyForMatch(actual));
-  }
-  if (Array.isArray(matcher)) {
-    if (!Array.isArray(actual) || actual.length !== matcher.length) {
-      return false;
-    }
-    return matcher.every((entry, index) => matchValue(actual[index], entry as EvalMatcher));
-  }
-  if (isPlainObject(matcher)) {
-    if (!isPlainObject(actual)) {
-      return false;
-    }
-    return Object.entries(matcher).every(([key, entry]) =>
-      matchValue(actual[key], entry as EvalMatcher),
-    );
-  }
-  return deepEqual(actual, matcher);
-}
-
-/** Whether an observed count satisfies a {@link EvalCountMatcher}. */
-export function matchCount(observed: number, matcher: EvalCountMatcher | undefined): boolean {
-  if (matcher === undefined) {
-    return observed > 0;
-  }
-  if (typeof matcher === "function") {
-    return matcher(observed) === true;
-  }
-  return observed === matcher;
-}
-
-/** Human-readable matcher summary for assertion names and failure details. */
-export function describeMatcher(matcher: EvalMatcher | undefined): string {
-  if (matcher === undefined) {
-    return "any";
-  }
-  if (typeof matcher === "function") {
-    return "predicate";
-  }
-  if (matcher instanceof RegExp) {
-    return String(matcher);
-  }
-  return stringifyForMatch(matcher);
-}
-
-/** Compact `{ input: …, count: 2 }` summary for assertion names. */
-export function describeMatcherObject(matcher: Record<string, unknown> | undefined): string {
-  if (matcher === undefined) {
-    return "";
-  }
-  const parts = Object.entries(matcher)
-    .filter(([, value]) => value !== undefined)
-    .map(([key, value]) => `${key}=${describeMatcher(value as EvalMatcher)}`);
-  return parts.length === 0 ? "" : `, {${parts.join(", ")}}`;
-}
-
-export function deepEqual(a: unknown, b: unknown): boolean {
-  if (Object.is(a, b)) {
-    return true;
-  }
-  if (a instanceof RegExp && b instanceof RegExp) {
-    return String(a) === String(b);
-  }
-  if (typeof a !== "object" || typeof b !== "object" || a === null || b === null) {
-    return false;
-  }
-  if (Array.isArray(a) !== Array.isArray(b)) {
-    return false;
-  }
-  if (Array.isArray(a) && Array.isArray(b)) {
-    if (a.length !== b.length) {
-      return false;
-    }
-    return a.every((item, i) => deepEqual(item, b[i]));
-  }
-  const aKeys = Object.keys(a as object);
-  const bKeys = Object.keys(b as object);
-  if (aKeys.length !== bKeys.length) {
-    return false;
-  }
-  for (const key of aKeys) {
-    if (!deepEqual((a as Record<string, unknown>)[key], (b as Record<string, unknown>)[key])) {
-      return false;
-    }
-  }
-  return true;
-}

package/src/evals/reporters.ts

@@ -1,257 +0,0 @@
-/**
- * Built-in eval reporters.
- *
- * The CLI already prints a console summary; reporters add destinations on top.
- * Attach them in `evals/evals.config.ts` to observe every case in the run, or
- * on a single `defineEval({ reporters })` to scope a destination to that case.
- *
- * ```ts
- * // evals/evals.config.ts
- * import { defineEvalConfig } from "@cursor/bdk/evals";
- * import { JUnit } from "@cursor/bdk/evals/reporters";
- *
- * export default defineEvalConfig({
- *   maxConcurrency: 20,
- *   reporters: [JUnit({ filePath: ".agent-serve/junit.xml" })],
- * });
- * ```
- *
- * `agent-sdk eval --junit <path>` and `--artifacts <dir>` attach these same two
- * reporters without touching the eval files, which is usually the better fit:
- * CI owns the output paths, not the eval author.
- */
-
-import { mkdir, writeFile } from "node:fs/promises";
-import { dirname, join, resolve } from "node:path";
-import type {
-  EvalReporter,
-  EvalReporterEvalInfo,
-  EvalReporterTarget,
-  EvalRunResult,
-  EvalRunSummary,
-} from "./results.js";
-
-export type {
-  EvalReporter,
-  EvalReporterEvalInfo,
-  EvalReporterTarget,
-  EvalRunResult,
-  EvalRunSummary,
-} from "./results.js";
-
-export interface JUnitReporterOptions {
-  /** Where to write the XML. Relative paths resolve from the process cwd. */
-  filePath: string;
-  /** `<testsuite name>`; defaults to `agentkit-evals`. */
-  suiteName?: string;
-}
-
-/**
- * Write JUnit XML so CI shows per-case annotations.
- *
- * Each case becomes one `<testcase>` named by its path-derived id. Failed gates
- * and execution errors become `<failure>`, `t.skip(reason)` becomes `<skipped>`,
- * and a soft threshold miss becomes a `<failure>` only under `--strict` (it is
- * reported as a `<system-out>` score line otherwise).
- */
-export function JUnit(options: JUnitReporterOptions): EvalReporter {
-  return {
-    async onRunComplete(summary) {
-      const xml = renderJUnitXml(summary, options.suiteName ?? "agentkit-evals");
-      const target = resolve(options.filePath);
-      await mkdir(dirname(target), { recursive: true });
-      await writeFile(target, xml, "utf8");
-    },
-  };
-}
-
-/** Render a {@link EvalRunSummary} as JUnit XML. Exported for tests. */
-export function renderJUnitXml(summary: EvalRunSummary, suiteName: string): string {
-  const lines: string[] = [
-    '<?xml version="1.0" encoding="UTF-8"?>',
-    `<testsuites name="${escapeXml(suiteName)}" tests="${summary.total}" failures="${
-      summary.failed + (summary.strict ? summary.scored : 0)
-    }" skipped="${summary.skipped}" time="${(summary.durationMs / 1000).toFixed(3)}">`,
-    `  <testsuite name="${escapeXml(suiteName)}" tests="${summary.total}" failures="${
-      summary.failed + (summary.strict ? summary.scored : 0)
-    }" skipped="${summary.skipped}" time="${(summary.durationMs / 1000).toFixed(3)}">`,
-  ];
-  for (const result of summary.results) {
-    const verdict = result.verdict ?? (result.ok ? "passed" : "failed");
-    const time = (result.durationMs / 1000).toFixed(3);
-    lines.push(
-      `    <testcase name="${escapeXml(result.id)}" classname="${escapeXml(
-        suiteName,
-      )}" time="${time}">`,
-    );
-    if (verdict === "skipped") {
-      lines.push(`      <skipped message="${escapeXml(result.skipReason ?? "skipped")}"/>`);
-    } else if (verdict === "failed" || (verdict === "scored" && summary.strict)) {
-      const failures = result.assertions.filter(a => !a.passed);
-      const message =
-        result.error ??
-        failures
-          .map(a => `${a.name}${a.detail === undefined ? "" : ` (${a.detail})`}`)
-          .join("; ") ??
-        verdict;
-      lines.push(
-        `      <failure message="${escapeXml(
-          message === "" ? verdict : message,
-        )}" type="${escapeXml(verdict)}"/>`,
-      );
-    }
-    const systemOut = renderSystemOut(result);
-    if (systemOut !== "") {
-      lines.push(`      <system-out>${escapeXml(systemOut)}</system-out>`);
-    }
-    lines.push("    </testcase>");
-  }
-  lines.push("  </testsuite>", "</testsuites>", "");
-  return lines.join("\n");
-}
-
-function renderSystemOut(result: EvalRunResult): string {
-  const parts: string[] = [];
-  for (const assertion of result.assertions) {
-    if (assertion.score === undefined) {
-      continue;
-    }
-    const bar = assertion.threshold === undefined ? "" : ` (>= ${assertion.threshold})`;
-    parts.push(`${assertion.name}=${assertion.score.toFixed(3)}${bar}`);
-  }
-  for (const [name, value] of Object.entries(result.metrics)) {
-    parts.push(`${name}=${String(value)}`);
-  }
-  parts.push(...result.logs);
-  return parts.join("\n");
-}
-
-function escapeXml(text: string): string {
-  return stripIllegalXmlChars(text)
-    .replace(/&/g, "&amp;")
-    .replace(/</g, "&lt;")
-    .replace(/>/g, "&gt;")
-    .replace(/"/g, "&quot;")
-    .replace(/'/g, "&apos;");
-}
-
-/**
- * Drop characters XML 1.0 forbids outright.
- *
- * Tool output routinely carries stray control bytes, and they are illegal in
- * XML even when escaped — a CI provider would reject the whole report. Tab,
- * newline, and carriage return are the only legal ones below 0x20.
- */
-function stripIllegalXmlChars(text: string): string {
-  let out = "";
-  for (const char of text) {
-    const code = char.codePointAt(0)!;
-    if (code >= 0x20 || code === 0x09 || code === 0x0a || code === 0x0d) {
-      out += char;
-    }
-  }
-  return out;
-}
-
-export interface ArtifactsReporterOptions {
-  /** Run directory. Relative paths resolve from the process cwd. */
-  dir: string;
-}
-
-/**
- * Persist the full story of a run to disk.
- *
- * Console output stays tight on purpose; when a case fails, the artifact has
- * everything — every assertion with its score, the captured event stream, the
- * tool calls, and `t.log` lines.
- *
- * ```text
- * <dir>/summary.json                 run totals + config
- * <dir>/results.jsonl                one line per case, in completion order
- * <dir>/evals/<case-id>.json         per-case detail
- * ```
- */
-export function Artifacts(options: ArtifactsReporterOptions): EvalReporter {
-  const root = resolve(options.dir);
-  const jsonlPath = join(root, "results.jsonl");
-  const lines: string[] = [];
-  let target: EvalReporterTarget | undefined;
-  let discovered: readonly EvalReporterEvalInfo[] = [];
-
-  return {
-    async onRunStart(evals, runTarget) {
-      discovered = evals;
-      target = runTarget;
-      await mkdir(join(root, "evals"), { recursive: true });
-    },
-    async onEvalComplete(result) {
-      lines.push(
-        JSON.stringify({
-          id: result.id,
-          verdict: result.verdict ?? (result.ok ? "passed" : "failed"),
-          ok: result.ok,
-          durationMs: result.durationMs,
-        }),
-      );
-      const file = join(root, "evals", `${sanitizeCaseId(result.id)}.json`);
-      await mkdir(dirname(file), { recursive: true });
-      await writeFile(file, `${JSON.stringify(result, null, 2)}\n`, "utf8");
-      await writeFile(jsonlPath, `${lines.join("\n")}\n`, "utf8");
-    },
-    async onRunComplete(summary) {
-      await mkdir(root, { recursive: true });
-      const { results, ...totals } = summary;
-      await writeFile(
-        join(root, "summary.json"),
-        `${JSON.stringify(
-          {
-            ...totals,
-            caseCount: results.length,
-            target,
-            discovered: discovered.map(e => e.id),
-          },
-          null,
-          2,
-        )}\n`,
-        "utf8",
-      );
-    },
-  };
-}
-
-/**
- * Turn a case id into a safe relative path.
- *
- * Ids are already path-shaped (`weather/nyc`), and `defineEval` rejects `..` in
- * case ids, but the artifact writer sanitizes anyway so a hand-built result can
- * never escape the run directory.
- */
-export function sanitizeCaseId(id: string): string {
-  return id
-    .split("/")
-    .map(segment => segment.replace(/[^A-Za-z0-9._-]/g, "_"))
-    .filter(segment => segment !== "" && segment !== "." && segment !== "..")
-    .join("/");
-}
-
-/** Fan one lifecycle call out to several reporters, isolating failures. */
-export function combineReporters(
-  reporters: readonly EvalReporter[],
-  onError?: (error: unknown) => void,
-): EvalReporter {
-  const forEach = async (run: (reporter: EvalReporter) => void | Promise<void>): Promise<void> => {
-    for (const reporter of reporters) {
-      try {
-        await run(reporter);
-      } catch (error) {
-        // A broken destination must not fail the eval run itself.
-        onError?.(error);
-      }
-    }
-  };
-  return {
-    onRunStart: (evals, target) => forEach(r => r.onRunStart?.(evals, target)),
-    onEvalComplete: result => forEach(r => r.onEvalComplete?.(result)),
-    onRunComplete: summary => forEach(r => r.onRunComplete?.(summary)),
-  };
-}

package/src/evals/results.ts

@@ -1,130 +0,0 @@
-/**
- * Result and reporter types shared by the runner, the CLI, and reporters.
- *
- * Pure types only: reporters that touch the filesystem live in
- * `@cursor/bdk/evals/reporters`, so the authoring surface stays importable
- * from anywhere.
- */
-
-import type { EvalAssertionResult, EvalVerdict } from "./assertions.js";
-
-/** One finished eval case. */
-export interface EvalRunResult {
-  id: string;
-  path: string;
-  description?: string;
-  /**
-   * No hard gate failed. Unchanged from before severity existed: a soft
-   * threshold miss (`verdict: "scored"`) leaves this `true` and is fatal only
-   * under `--strict`.
-   */
-  ok: boolean;
-  /** Full grade, including `scored` and `skipped`. */
-  verdict?: EvalVerdict;
-  /** Reason passed to `t.skip(...)`. */
-  skipReason?: string;
-  assertions: EvalAssertionResult[];
-  sessionId?: string;
-  /** Messages passed to `t.send` (eval inputs). */
-  inputs: string[];
-  finalText?: string;
-  tools?: string[];
-  /** Tool calls with args/results from the trajectory. */
-  toolCalls?: Array<{
-    toolName: string;
-    args?: unknown;
-    output?: unknown;
-    isError: boolean;
-  }>;
-  error?: string;
-  logs: string[];
-  /** Structured scores from `t.metric(...)`. */
-  metrics: Record<string, string | number>;
-  /** Author-supplied metadata from `defineEval({ metadata })`. */
-  metadata?: Record<string, unknown>;
-  tags?: string[];
-  durationMs: number;
-}
-
-/** Aggregate outcome of one `eval` invocation. */
-export interface EvalRunSummary {
-  total: number;
-  passed: number;
-  failed: number;
-  /** Cases that only missed a soft threshold. */
-  scored: number;
-  skipped: number;
-  /** Whether soft threshold misses were treated as failures. */
-  strict: boolean;
-  startedAt: string;
-  finishedAt: string;
-  durationMs: number;
-  results: readonly EvalRunResult[];
-}
-
-/** One discovered case, as handed to {@link EvalReporter.onRunStart}. */
-export interface EvalReporterEvalInfo {
-  id: string;
-  fileId: string;
-  description?: string;
-  tags?: string[];
-}
-
-/** Where the run executed. */
-export interface EvalReporterTarget {
-  baseUrl: string;
-  /** `local` boots an ephemeral server; the others target a running one. */
-  mode: "local" | "url" | "prod";
-}
-
-/**
- * A destination for eval results.
- *
- * The runner grades everything itself; a reporter only ships results out. All
- * three hooks are optional and may return a promise for async work such as an
- * upload or a file write.
- *
- * ```ts
- * // evals/evals.config.ts
- * export default defineEvalConfig({
- *   maxConcurrency: 20,
- *   reporters: [JUnit({ filePath: ".agent-serve/junit.xml" })],
- * });
- * ```
- */
-export interface EvalReporter {
-  /** Fires once before any case runs. */
-  onRunStart?(
-    evals: readonly EvalReporterEvalInfo[],
-    target: EvalReporterTarget,
-  ): void | Promise<void>;
-  /** Fires after each observed case, with its assertions and verdict. */
-  onEvalComplete?(result: EvalRunResult): void | Promise<void>;
-  /** Fires once with the aggregated summary. */
-  onRunComplete?(summary: EvalRunSummary): void | Promise<void>;
-}
-
-/** Tally verdicts for {@link EvalRunSummary}. */
-export function summarizeEvalResults(
-  results: readonly EvalRunResult[],
-  options: { strict: boolean; startedAt: string; finishedAt: string },
-): EvalRunSummary {
-  const verdictOf = (result: EvalRunResult): EvalVerdict =>
-    result.verdict ?? (result.ok ? "passed" : "failed");
-  const counts = { passed: 0, failed: 0, scored: 0, skipped: 0 };
-  for (const result of results) {
-    counts[verdictOf(result)]++;
-  }
-  return {
-    total: results.length,
-    passed: counts.passed,
-    failed: counts.failed,
-    scored: counts.scored,
-    skipped: counts.skipped,
-    strict: options.strict,
-    startedAt: options.startedAt,
-    finishedAt: options.finishedAt,
-    durationMs: new Date(options.finishedAt).getTime() - new Date(options.startedAt).getTime(),
-    results,
-  };
-}