@cursor/july 0.1.16 → 0.1.17

package/src/evals/loaders.ts

@@ -0,0 +1,113 @@
+/**
+ * Fixture loaders for dataset-driven evals.
+ *
+ * Eval modules are ESM, so top-level `await` can load a dataset and fan one
+ * file out over it:
+ *
+ * ```ts
+ * // evals/sql.eval.ts
+ * import { defineEval } from "@cursor/july/evals";
+ * import { loadYaml } from "@cursor/july/evals/loaders";
+ * import { equals } from "@cursor/july/evals";
+ *
+ * const rows = await loadYaml<{ task: string; prompt: string; sql: string }[]>(
+ *   "evals/data/cases.yaml"
+ * );
+ *
+ * export default rows.map((row) =>
+ *   defineEval({
+ *     description: row.task,
+ *     async test(t) {
+ *       await t.send(row.prompt);
+ *       t.succeeded();
+ *       t.check(t.reply, equals(row.sql));
+ *     },
+ *   })
+ * );
+ * ```
+ *
+ * Paths resolve from the app root (the directory holding `agent/` and
+ * `evals/`), not from the eval file, so a fixture path reads the same however
+ * deep the eval sits. These loaders are for fixtures — do not use them for
+ * runtime agent code.
+ */
+
+import { readFile } from "node:fs/promises";
+import { isAbsolute, resolve } from "node:path";
+
+/**
+ * App root used to resolve relative fixture paths.
+ *
+ * `eval` sets `AGENT_SERVE_EVAL_ROOT` to the project it discovered, which keeps
+ * fixture paths stable no matter what cwd the CLI was invoked from. Falls back
+ * to the process cwd.
+ */
+export const EVAL_ROOT_ENV = "AGENT_SERVE_EVAL_ROOT";
+
+export function evalFixtureRoot(): string {
+  const root = process.env[EVAL_ROOT_ENV];
+  return root === undefined || root.trim() === "" ? process.cwd() : root;
+}
+
+/** Resolve a fixture path against the app root. */
+export function resolveFixturePath(path: string): string {
+  return isAbsolute(path) ? path : resolve(evalFixtureRoot(), path);
+}
+
+async function readFixture(path: string): Promise<string> {
+  const absolute = resolveFixturePath(path);
+  try {
+    return await readFile(absolute, "utf8");
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new Error(`failed to load eval fixture ${absolute}: ${detail}`);
+  }
+}
+
+/** Load a JSON fixture. */
+export async function loadJson<T = unknown>(path: string): Promise<T> {
+  const text = await readFixture(path);
+  try {
+    return JSON.parse(text) as T;
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new Error(`${resolveFixturePath(path)} is not valid JSON: ${detail}`);
+  }
+}
+
+/** Load a newline-delimited JSON fixture, skipping blank lines. */
+export async function loadJsonl<T = unknown>(path: string): Promise<T[]> {
+  const text = await readFixture(path);
+  const rows: T[] = [];
+  let lineNumber = 0;
+  for (const line of text.split("\n")) {
+    lineNumber++;
+    const trimmed = line.trim();
+    if (trimmed === "") {
+      continue;
+    }
+    try {
+      rows.push(JSON.parse(trimmed) as T);
+    } catch (error) {
+      const detail = error instanceof Error ? error.message : String(error);
+      throw new Error(
+        `${resolveFixturePath(path)}:${lineNumber} is not valid JSON: ${detail}`
+      );
+    }
+  }
+  return rows;
+}
+
+/** Load a YAML fixture. */
+export async function loadYaml<T = unknown>(path: string): Promise<T> {
+  const text = await readFixture(path);
+  // Imported lazily so a project that only uses JSON fixtures never pays for
+  // the YAML parser at eval-discovery time.
+  const { load } = await import("js-yaml");
+  try {
+    return load(text) as T;
+  } catch (error) {
+    const detail = error instanceof Error ? error.message : String(error);
+    throw new Error(`${resolveFixturePath(path)} is not valid YAML: ${detail}`);
+  }
+}

package/src/evals/matchers.ts

@@ -0,0 +1,246 @@
+/**
+ * Matcher mini-language shared by tool-call, subagent, and event assertions.
+ *
+ * A matcher is a literal, a RegExp, or a predicate:
+ *
+ * ```ts
+ * t.calledTool("bash", { input: { command: /^pwd/ }, count: 1 });
+ * t.calledTool("echo", { count: (n) => n >= 2 });
+ * t.calledTool("read", { output: (value) => String(value).includes("hello") });
+ * ```
+ *
+ * Object literals partial-deep-match, so `{ input: { city: "NYC" } }` matches a
+ * call whose args also carry unrelated keys. Nested values may themselves be
+ * matchers.
+ */
+
+/**
+ * Literal forms a matcher can take.
+ *
+ * Spelled out rather than written as `unknown` so the union keeps its function
+ * member: `unknown` would absorb it, and TypeScript could no longer infer the
+ * parameter type of an inline predicate. Nested values are matchers in turn, so
+ * a predicate buried in an object literal gets its parameter type too.
+ */
+export type EvalMatcherLiteral =
+  | string
+  | number
+  | boolean
+  | null
+  | Date
+  | readonly EvalMatcher[]
+  | { readonly [key: string]: EvalMatcher | undefined };
+
+/** A literal to partial-match, a RegExp to test, or a predicate. */
+export type EvalMatcher<T = unknown> =
+  | EvalMatcherLiteral
+  | RegExp
+  | ((value: T) => boolean);
+
+/** An exact count, or a predicate over the observed count. */
+export type EvalCountMatcher = number | ((count: number) => boolean);
+
+/** Lifecycle state of one tool call, derived from the session event stream. */
+export type EvalToolCallStatus =
+  /** `action.result` arrived without `isError`. */
+  | "completed"
+  /** `action.result` arrived with `isError: true`. */
+  | "failed"
+  /** Requested, no result yet (includes calls parked on human approval). */
+  | "pending"
+  /** A human denied the approval request for this call. */
+  | "rejected";
+
+/** Constraints for {@link import("../evals.js").EvalAssertions.calledTool}. */
+export interface EvalToolCallMatcher {
+  /** Match the tool's arguments (partial-deep for object literals). */
+  input?: EvalMatcher;
+  /** Match the tool's result payload. */
+  output?: EvalMatcher;
+  /**
+   * Restrict to one lifecycle state. Omitted matches **any** state, which is
+   * request-based and therefore backwards compatible with `calledTool(name)`.
+   */
+  status?: EvalToolCallStatus;
+  /** Require an exact number of matching calls, or satisfy a predicate. */
+  count?: EvalCountMatcher;
+}
+
+/** Constraints for {@link import("../evals.js").EvalAssertions.calledSubagent}. */
+export interface EvalSubagentMatcher {
+  /** Match the delegating `task` call id. */
+  callId?: EvalMatcher<string>;
+  /** Match the subagent's final output text. */
+  output?: EvalMatcher;
+  /** `completed` once `subagent.completed` arrived, else `pending`. */
+  status?: "completed" | "pending";
+  count?: EvalCountMatcher;
+}
+
+/** Constraints for {@link import("../evals.js").EvalAssertions.event}. */
+export interface EvalEventMatcher {
+  /** Partial-deep match against the event's `data` payload. */
+  data?: EvalMatcher;
+  count?: EvalCountMatcher;
+}
+
+/** One entry of {@link import("../evals.js").EvalAssertions.eventOrder}. */
+export interface EvalEventOrderMatcher extends EvalEventMatcher {
+  type: string;
+}
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  return (
+    typeof value === "object" &&
+    value !== null &&
+    !Array.isArray(value) &&
+    !(value instanceof RegExp) &&
+    !(value instanceof Date)
+  );
+}
+
+/** Stable string form used when a RegExp is tested against a non-string. */
+export function stringifyForMatch(value: unknown): string {
+  if (typeof value === "string") {
+    return value;
+  }
+  if (value === undefined) {
+    return "";
+  }
+  if (value === null) {
+    return "null";
+  }
+  if (typeof value === "object") {
+    try {
+      return JSON.stringify(value) ?? "";
+    } catch {
+      return String(value);
+    }
+  }
+  return String(value);
+}
+
+/**
+ * Whether `actual` satisfies `matcher`.
+ *
+ * - `undefined` matcher matches anything (the constraint was not supplied).
+ * - A function is called with `actual`.
+ * - A RegExp is tested against {@link stringifyForMatch}.
+ * - An object or array literal partial-deep-matches, and its own values may be
+ *   matchers.
+ * - Anything else compares by deep equality.
+ */
+export function matchValue<T = unknown>(
+  actual: T,
+  matcher: EvalMatcher<T> | undefined
+): boolean {
+  if (matcher === undefined) {
+    return true;
+  }
+  if (typeof matcher === "function") {
+    return matcher(actual) === true;
+  }
+  if (matcher instanceof RegExp) {
+    return matcher.test(stringifyForMatch(actual));
+  }
+  if (Array.isArray(matcher)) {
+    if (!Array.isArray(actual) || actual.length !== matcher.length) {
+      return false;
+    }
+    return matcher.every((entry, index) =>
+      matchValue(actual[index], entry as EvalMatcher)
+    );
+  }
+  if (isPlainObject(matcher)) {
+    if (!isPlainObject(actual)) {
+      return false;
+    }
+    return Object.entries(matcher).every(([key, entry]) =>
+      matchValue(actual[key], entry as EvalMatcher)
+    );
+  }
+  return deepEqual(actual, matcher);
+}
+
+/** Whether an observed count satisfies a {@link EvalCountMatcher}. */
+export function matchCount(
+  observed: number,
+  matcher: EvalCountMatcher | undefined
+): boolean {
+  if (matcher === undefined) {
+    return observed > 0;
+  }
+  if (typeof matcher === "function") {
+    return matcher(observed) === true;
+  }
+  return observed === matcher;
+}
+
+/** Human-readable matcher summary for assertion names and failure details. */
+export function describeMatcher(matcher: EvalMatcher | undefined): string {
+  if (matcher === undefined) {
+    return "any";
+  }
+  if (typeof matcher === "function") {
+    return "predicate";
+  }
+  if (matcher instanceof RegExp) {
+    return String(matcher);
+  }
+  return stringifyForMatch(matcher);
+}
+
+/** Compact `{ input: …, count: 2 }` summary for assertion names. */
+export function describeMatcherObject(
+  matcher: Record<string, unknown> | undefined
+): string {
+  if (matcher === undefined) {
+    return "";
+  }
+  const parts = Object.entries(matcher)
+    .filter(([, value]) => value !== undefined)
+    .map(([key, value]) => `${key}=${describeMatcher(value as EvalMatcher)}`);
+  return parts.length === 0 ? "" : `, {${parts.join(", ")}}`;
+}
+
+export function deepEqual(a: unknown, b: unknown): boolean {
+  if (Object.is(a, b)) {
+    return true;
+  }
+  if (a instanceof RegExp && b instanceof RegExp) {
+    return String(a) === String(b);
+  }
+  if (
+    typeof a !== "object" ||
+    typeof b !== "object" ||
+    a === null ||
+    b === null
+  ) {
+    return false;
+  }
+  if (Array.isArray(a) !== Array.isArray(b)) {
+    return false;
+  }
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) {
+      return false;
+    }
+    return a.every((item, i) => deepEqual(item, b[i]));
+  }
+  const aKeys = Object.keys(a as object);
+  const bKeys = Object.keys(b as object);
+  if (aKeys.length !== bKeys.length) {
+    return false;
+  }
+  for (const key of aKeys) {
+    if (
+      !deepEqual(
+        (a as Record<string, unknown>)[key],
+        (b as Record<string, unknown>)[key]
+      )
+    ) {
+      return false;
+    }
+  }
+  return true;
+}

package/src/evals/reporters.ts

@@ -0,0 +1,274 @@
+/**
+ * Built-in eval reporters.
+ *
+ * The CLI already prints a console summary; reporters add destinations on top.
+ * Attach them in `evals/evals.config.ts` to observe every case in the run, or
+ * on a single `defineEval({ reporters })` to scope a destination to that case.
+ *
+ * ```ts
+ * // evals/evals.config.ts
+ * import { defineEvalConfig } from "@cursor/july/evals";
+ * import { JUnit } from "@cursor/july/evals/reporters";
+ *
+ * export default defineEvalConfig({
+ *   maxConcurrency: 20,
+ *   reporters: [JUnit({ filePath: ".agent-serve/junit.xml" })],
+ * });
+ * ```
+ *
+ * `agentkit eval --junit <path>` and `--artifacts <dir>` attach these same two
+ * reporters without touching the eval files, which is usually the better fit:
+ * CI owns the output paths, not the eval author.
+ */
+
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname, join, resolve } from "node:path";
+import type {
+  EvalReporter,
+  EvalReporterEvalInfo,
+  EvalReporterTarget,
+  EvalRunResult,
+  EvalRunSummary,
+} from "./results.js";
+
+export type {
+  EvalReporter,
+  EvalReporterEvalInfo,
+  EvalReporterTarget,
+  EvalRunResult,
+  EvalRunSummary,
+} from "./results.js";
+
+export interface JUnitReporterOptions {
+  /** Where to write the XML. Relative paths resolve from the process cwd. */
+  filePath: string;
+  /** `<testsuite name>`; defaults to `agentkit-evals`. */
+  suiteName?: string;
+}
+
+/**
+ * Write JUnit XML so CI shows per-case annotations.
+ *
+ * Each case becomes one `<testcase>` named by its path-derived id. Failed gates
+ * and execution errors become `<failure>`, `t.skip(reason)` becomes `<skipped>`,
+ * and a soft threshold miss becomes a `<failure>` only under `--strict` (it is
+ * reported as a `<system-out>` score line otherwise).
+ */
+export function JUnit(options: JUnitReporterOptions): EvalReporter {
+  return {
+    async onRunComplete(summary) {
+      const xml = renderJUnitXml(
+        summary,
+        options.suiteName ?? "agentkit-evals"
+      );
+      const target = resolve(options.filePath);
+      await mkdir(dirname(target), { recursive: true });
+      await writeFile(target, xml, "utf8");
+    },
+  };
+}
+
+/** Render a {@link EvalRunSummary} as JUnit XML. Exported for tests. */
+export function renderJUnitXml(
+  summary: EvalRunSummary,
+  suiteName: string
+): string {
+  const lines: string[] = [
+    '<?xml version="1.0" encoding="UTF-8"?>',
+    `<testsuites name="${escapeXml(suiteName)}" tests="${summary.total}" failures="${
+      summary.failed + (summary.strict ? summary.scored : 0)
+    }" skipped="${summary.skipped}" time="${(summary.durationMs / 1000).toFixed(3)}">`,
+    `  <testsuite name="${escapeXml(suiteName)}" tests="${summary.total}" failures="${
+      summary.failed + (summary.strict ? summary.scored : 0)
+    }" skipped="${summary.skipped}" time="${(summary.durationMs / 1000).toFixed(3)}">`,
+  ];
+  for (const result of summary.results) {
+    const verdict = result.verdict ?? (result.ok ? "passed" : "failed");
+    const time = (result.durationMs / 1000).toFixed(3);
+    lines.push(
+      `    <testcase name="${escapeXml(result.id)}" classname="${escapeXml(
+        suiteName
+      )}" time="${time}">`
+    );
+    if (verdict === "skipped") {
+      lines.push(
+        `      <skipped message="${escapeXml(result.skipReason ?? "skipped")}"/>`
+      );
+    } else if (
+      verdict === "failed" ||
+      (verdict === "scored" && summary.strict)
+    ) {
+      const failures = result.assertions.filter((a) => !a.passed);
+      const message =
+        result.error ??
+        failures
+          .map(
+            (a) => `${a.name}${a.detail === undefined ? "" : ` (${a.detail})`}`
+          )
+          .join("; ") ??
+        verdict;
+      lines.push(
+        `      <failure message="${escapeXml(
+          message === "" ? verdict : message
+        )}" type="${escapeXml(verdict)}"/>`
+      );
+    }
+    const systemOut = renderSystemOut(result);
+    if (systemOut !== "") {
+      lines.push(`      <system-out>${escapeXml(systemOut)}</system-out>`);
+    }
+    lines.push("    </testcase>");
+  }
+  lines.push("  </testsuite>", "</testsuites>", "");
+  return lines.join("\n");
+}
+
+function renderSystemOut(result: EvalRunResult): string {
+  const parts: string[] = [];
+  for (const assertion of result.assertions) {
+    if (assertion.score === undefined) {
+      continue;
+    }
+    const bar =
+      assertion.threshold === undefined ? "" : ` (>= ${assertion.threshold})`;
+    parts.push(`${assertion.name}=${assertion.score.toFixed(3)}${bar}`);
+  }
+  for (const [name, value] of Object.entries(result.metrics)) {
+    parts.push(`${name}=${String(value)}`);
+  }
+  parts.push(...result.logs);
+  return parts.join("\n");
+}
+
+function escapeXml(text: string): string {
+  return stripIllegalXmlChars(text)
+    .replace(/&/g, "&amp;")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;")
+    .replace(/'/g, "&apos;");
+}
+
+/**
+ * Drop characters XML 1.0 forbids outright.
+ *
+ * Tool output routinely carries stray control bytes, and they are illegal in
+ * XML even when escaped — a CI provider would reject the whole report. Tab,
+ * newline, and carriage return are the only legal ones below 0x20.
+ */
+function stripIllegalXmlChars(text: string): string {
+  let out = "";
+  for (const char of text) {
+    const code = char.codePointAt(0)!;
+    if (code >= 0x20 || code === 0x09 || code === 0x0a || code === 0x0d) {
+      out += char;
+    }
+  }
+  return out;
+}
+
+export interface ArtifactsReporterOptions {
+  /** Run directory. Relative paths resolve from the process cwd. */
+  dir: string;
+}
+
+/**
+ * Persist the full story of a run to disk.
+ *
+ * Console output stays tight on purpose; when a case fails, the artifact has
+ * everything — every assertion with its score, the captured event stream, the
+ * tool calls, and `t.log` lines.
+ *
+ * ```text
+ * <dir>/summary.json                 run totals + config
+ * <dir>/results.jsonl                one line per case, in completion order
+ * <dir>/evals/<case-id>.json         per-case detail
+ * ```
+ */
+export function Artifacts(options: ArtifactsReporterOptions): EvalReporter {
+  const root = resolve(options.dir);
+  const jsonlPath = join(root, "results.jsonl");
+  const lines: string[] = [];
+  let target: EvalReporterTarget | undefined;
+  let discovered: readonly EvalReporterEvalInfo[] = [];
+
+  return {
+    async onRunStart(evals, runTarget) {
+      discovered = evals;
+      target = runTarget;
+      await mkdir(join(root, "evals"), { recursive: true });
+    },
+    async onEvalComplete(result) {
+      lines.push(
+        JSON.stringify({
+          id: result.id,
+          verdict: result.verdict ?? (result.ok ? "passed" : "failed"),
+          ok: result.ok,
+          durationMs: result.durationMs,
+        })
+      );
+      const file = join(root, "evals", `${sanitizeCaseId(result.id)}.json`);
+      await mkdir(dirname(file), { recursive: true });
+      await writeFile(file, `${JSON.stringify(result, null, 2)}\n`, "utf8");
+      await writeFile(jsonlPath, `${lines.join("\n")}\n`, "utf8");
+    },
+    async onRunComplete(summary) {
+      await mkdir(root, { recursive: true });
+      const { results, ...totals } = summary;
+      await writeFile(
+        join(root, "summary.json"),
+        `${JSON.stringify(
+          {
+            ...totals,
+            caseCount: results.length,
+            target,
+            discovered: discovered.map((e) => e.id),
+          },
+          null,
+          2
+        )}\n`,
+        "utf8"
+      );
+    },
+  };
+}
+
+/**
+ * Turn a case id into a safe relative path.
+ *
+ * Ids are already path-shaped (`weather/nyc`), and `defineEval` rejects `..` in
+ * case ids, but the artifact writer sanitizes anyway so a hand-built result can
+ * never escape the run directory.
+ */
+export function sanitizeCaseId(id: string): string {
+  return id
+    .split("/")
+    .map((segment) => segment.replace(/[^A-Za-z0-9._-]/g, "_"))
+    .filter((segment) => segment !== "" && segment !== "." && segment !== "..")
+    .join("/");
+}
+
+/** Fan one lifecycle call out to several reporters, isolating failures. */
+export function combineReporters(
+  reporters: readonly EvalReporter[],
+  onError?: (error: unknown) => void
+): EvalReporter {
+  const forEach = async (
+    run: (reporter: EvalReporter) => void | Promise<void>
+  ): Promise<void> => {
+    for (const reporter of reporters) {
+      try {
+        await run(reporter);
+      } catch (error) {
+        // A broken destination must not fail the eval run itself.
+        onError?.(error);
+      }
+    }
+  };
+  return {
+    onRunStart: (evals, target) =>
+      forEach((r) => r.onRunStart?.(evals, target)),
+    onEvalComplete: (result) => forEach((r) => r.onEvalComplete?.(result)),
+    onRunComplete: (summary) => forEach((r) => r.onRunComplete?.(summary)),
+  };
+}