@graphorin/evals 0.6.0 → 0.7.0

package/src/reporters/json.ts

@@ -0,0 +1,18 @@
+/**
+ * JSON reporter. Returns the canonical JSON-serialised representation
+ * of an {@link EvalReport} so downstream tooling (dashboards,
+ * spreadsheets, regression checkers) can consume it deterministically.
+ *
+ * @packageDocumentation
+ */
+
+import type { EvalReport } from '@graphorin/observability/eval';
+
+/** @stable */
+export function renderJsonReport<I, O>(
+  report: EvalReport<I, O>,
+  options: { readonly pretty?: boolean } = {},
+): string {
+  const indent = options.pretty === true ? 2 : 0;
+  return JSON.stringify(report, null, indent);
+}

package/src/reporters/junit.ts

@@ -0,0 +1,53 @@
+/**
+ * JUnit XML reporter. Renders an {@link EvalReport} as a JUnit
+ * `<testsuite>` document so CI systems (GitHub Actions, GitLab,
+ * CircleCI, Jenkins) can render pass / fail counts in the standard
+ * test-report widget.
+ *
+ * @packageDocumentation
+ */
+
+import type { EvalReport } from '@graphorin/observability/eval';
+
+/** @stable */
+export function renderJunitReport<I, O>(
+  report: EvalReport<I, O>,
+  options: { readonly suiteName?: string } = {},
+): string {
+  const suiteName = options.suiteName ?? 'graphorin-evals';
+  const totalTests = report.results.reduce((acc, r) => acc + r.scores.length, 0);
+  const failures = report.results.reduce(
+    (acc, r) => acc + r.scores.filter((s) => !s.result.pass).length,
+    0,
+  );
+  const totalTime = (report.summary.avgDurationMs * report.summary.total) / 1000;
+  const lines: string[] = [];
+  lines.push('<?xml version="1.0" encoding="UTF-8"?>');
+  lines.push(
+    `<testsuite name="${escapeXml(suiteName)}" tests="${totalTests}" failures="${failures}" time="${totalTime.toFixed(3)}">`,
+  );
+  for (const r of report.results) {
+    for (const { scorer, result } of r.scores) {
+      const time = (r.durationMs / 1000).toFixed(3);
+      lines.push(
+        `  <testcase classname="${escapeXml(scorer)}" name="${escapeXml(r.caseId)}" time="${time}">`,
+      );
+      if (!result.pass) {
+        const message = escapeXml(result.reason ?? 'failed');
+        lines.push(`    <failure message="${message}">${message}</failure>`);
+      }
+      lines.push('  </testcase>');
+    }
+  }
+  lines.push('</testsuite>');
+  return lines.join('\n');
+}
+
+function escapeXml(value: string): string {
+  return value
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;');
+}

package/src/reporters/markdown.ts

@@ -0,0 +1,65 @@
+/**
+ * Markdown reporter. Renders an {@link EvalReport} as a markdown
+ * document suitable for embedding in pull-request descriptions or
+ * documentation sites.
+ *
+ * @packageDocumentation
+ */
+
+import type { EvalReport } from '@graphorin/observability/eval';
+
+/** @stable */
+export function renderMarkdownReport<I, O>(report: EvalReport<I, O>): string {
+  const lines: string[] = [];
+  lines.push('# graphorin/evals report');
+  lines.push('');
+  lines.push('## Summary');
+  lines.push('');
+  lines.push('| Metric | Value |');
+  lines.push('| --- | --- |');
+  lines.push(`| Total cases | ${report.summary.total} |`);
+  lines.push(`| Passed | ${report.summary.passed} |`);
+  lines.push(`| Failed | ${report.summary.failed} |`);
+  lines.push(`| Avg duration (ms) | ${report.summary.avgDurationMs.toFixed(2)} |`);
+  lines.push('');
+  lines.push('## Per-scorer');
+  lines.push('');
+  lines.push('| Scorer | Pass | Fail | Avg score |');
+  lines.push('| --- | ---: | ---: | ---: |');
+  for (const [scorer, row] of Object.entries(report.summary.byScorer)) {
+    const avg = row.avgScore === null ? 'n/a' : row.avgScore.toFixed(4);
+    lines.push(`| \`${scorer}\` | ${row.passed} | ${row.failed} | ${avg} |`);
+  }
+  lines.push('');
+  lines.push('## Failures');
+  lines.push('');
+  let any = false;
+  for (const r of report.results) {
+    const failures = r.scores.filter((s) => !s.result.pass);
+    if (failures.length === 0) continue;
+    any = true;
+    lines.push(`### \`${r.caseId}\` _(\`${r.durationMs}\` ms)_`);
+    lines.push('');
+    for (const f of failures) {
+      const reason = escapeMarkdownInline(f.result.reason ?? '_no reason_');
+      lines.push(`- **${f.scorer}** - ${reason}`);
+    }
+    lines.push('');
+  }
+  if (!any) {
+    lines.push('_No failures._');
+    lines.push('');
+  }
+  return lines.join('\n');
+}
+
+/**
+ * Escape characters that would break a Markdown table row when rendered
+ * inline. Backslashes must be escaped first so that an existing trailing
+ * `\` in the input cannot combine with our inserted `\|` to form a
+ * literal pipe. Newlines are flattened to spaces because table cells
+ * cannot contain hard line breaks.
+ */
+function escapeMarkdownInline(value: string): string {
+  return value.replace(/\\/g, '\\\\').replace(/\|/g, '\\|').replace(/\r?\n/g, ' ');
+}

package/src/reporters/terminal.ts

@@ -0,0 +1,46 @@
+/**
+ * Terminal reporter. Renders an {@link EvalReport} as plain ANSI-free
+ * text suitable for stdout / CI logs.
+ *
+ * @packageDocumentation
+ */
+
+import type { EvalReport } from '@graphorin/observability/eval';
+
+/** @stable */
+export function renderTerminalReport<I, O>(report: EvalReport<I, O>): string {
+  const lines: string[] = [];
+  const { summary } = report;
+  lines.push('graphorin/evals - terminal report');
+  lines.push('=================================');
+  lines.push(`total:   ${summary.total}`);
+  lines.push(`passed:  ${summary.passed}`);
+  lines.push(`failed:  ${summary.failed}`);
+  lines.push(`avg ms:  ${summary.avgDurationMs.toFixed(2)}`);
+  lines.push('');
+  lines.push('per-scorer:');
+  for (const [scorer, row] of Object.entries(summary.byScorer)) {
+    const avg = row.avgScore === null ? 'n/a' : row.avgScore.toFixed(4);
+    lines.push(
+      `  ${scorer.padEnd(24)}  pass=${row.passed.toString().padStart(4)}  fail=${row.failed.toString().padStart(4)}  avg=${avg}`,
+    );
+  }
+  lines.push('');
+  lines.push('failures (first 10):');
+  let failuresShown = 0;
+  for (const r of report.results) {
+    if (failuresShown >= 10) break;
+    const failures = r.scores.filter((s) => !s.result.pass);
+    if (failures.length === 0) continue;
+    failuresShown += 1;
+    lines.push(`  • ${r.caseId} (${r.durationMs} ms)`);
+    for (const f of failures) {
+      const reason = f.result.reason ?? '<no reason>';
+      lines.push(`      ${f.scorer}: ${reason}`);
+    }
+  }
+  if (failuresShown === 0) {
+    lines.push('  (none)');
+  }
+  return lines.join('\n');
+}

package/src/runner.ts

@@ -0,0 +1,233 @@
+/**
+ * Parallel eval runner. Walks every case in the dataset (across N
+ * iterations), executes them with bounded concurrency, applies every
+ * scorer to each output, and returns an aggregated {@link EvalReport}.
+ *
+ * The runner respects:
+ *
+ *  - `concurrency` - bounded worker pool. Default `1` (sequential).
+ *  - `signal` - propagated to `agent.run(...)` if the agent accepts a
+ *    second `{ signal }` argument; the runner stops issuing new work
+ *    on the next loop iteration.
+ *  - `onProgress` - invoked after every case with a heartbeat so
+ *    long-running terminals can paint a progress bar.
+ *
+ * @packageDocumentation
+ */
+
+import { passHatK, wilsonInterval } from './stats.js';
+import type { Case, EvalCaseResult, EvalReport, RunOptions, ScoreResult, Scorer } from './types.js';
+
+/**
+ * @stable
+ */
+export async function runEvals<I, O>(opts: RunOptions<I, O>): Promise<EvalReport<I, O>> {
+  const iterations = Math.max(1, opts.iterations ?? 1);
+  const concurrency = Math.max(1, opts.concurrency ?? 1);
+  const signal = opts.signal;
+  const cases = opts.dataset.cases;
+  const total = cases.length * iterations;
+
+  type WorkItem = { idx: number; iter: number; sample: Case<I, O> };
+  const queue: WorkItem[] = [];
+  for (let iter = 0; iter < iterations; iter++) {
+    for (let idx = 0; idx < cases.length; idx++) {
+      const sample = cases[idx];
+      if (sample === undefined) continue;
+      queue.push({ idx, iter, sample });
+    }
+  }
+
+  const results: EvalCaseResult<I, O>[] = new Array(total);
+  let nextWorkIndex = 0;
+  let completed = 0;
+
+  async function worker(): Promise<void> {
+    while (true) {
+      // EB-14: stop dispatching new work on abort, but don't throw - whatever
+      // already completed must survive into a partial report.
+      if (isAborted(signal)) return;
+      const myIndex = nextWorkIndex++;
+      if (myIndex >= queue.length) return;
+      const item = queue[myIndex];
+      if (item === undefined) return;
+      // EB-6: a caller-provided id must still be disambiguated per iteration -
+      // otherwise iterations>1 emits multiple results under one caseId and
+      // JUnit/HTML reporters render indistinguishable testcases.
+      const baseId = item.sample.id ?? `case-${item.idx}`;
+      const caseId = iterations === 1 ? baseId : `${baseId}-iter-${item.iter}`;
+      const startedAt = Date.now();
+      let output: O;
+      try {
+        output = await opts.agent.run(
+          item.sample.input,
+          signal !== undefined ? { signal } : undefined,
+        );
+      } catch (err) {
+        // An abort-induced agent failure is not a real scorer failure - drop it
+        // so it doesn't pollute the partial report with spurious fails (EB-14).
+        if (isAborted(signal)) return;
+        const durationMs = Date.now() - startedAt;
+        const failResult: EvalCaseResult<I, O> = {
+          caseId,
+          input: item.sample.input,
+          output: undefined as unknown as O,
+          durationMs,
+          scores: opts.scorers.map((s) => ({
+            scorer: s.name,
+            result: {
+              pass: false,
+              reason: `agent.run threw: ${err instanceof Error ? err.message : String(err)}`,
+            },
+          })),
+        };
+        results[myIndex] = failResult;
+        completed += 1;
+        opts.onProgress?.({
+          index: completed,
+          total,
+          caseId,
+          durationMs,
+          passed: false,
+        });
+        continue;
+      }
+      const durationMs = Date.now() - startedAt;
+      const scores: EvalCaseResult<I, O>['scores'][number][] = [];
+      let allPassed = true;
+      for (const scorer of opts.scorers) {
+        // No mid-case abort check: once agent.run completes, finish scoring so
+        // the case lands intact in the partial report rather than half-done.
+        const result = await safeScore(scorer, item.sample, output, durationMs);
+        scores.push({ scorer: scorer.name, result });
+        if (!result.pass) allPassed = false;
+      }
+      results[myIndex] = {
+        caseId,
+        input: item.sample.input,
+        output,
+        durationMs,
+        scores,
+      };
+      completed += 1;
+      opts.onProgress?.({
+        index: completed,
+        total,
+        caseId,
+        durationMs,
+        passed: allPassed,
+      });
+    }
+  }
+
+  const workerCount = Math.min(concurrency, queue.length);
+  await Promise.all(Array.from({ length: workerCount }, () => worker()));
+
+  // EB-14: an aborted run resolves with a PARTIAL report (only the cases that
+  // finished) marked `aborted: true`, instead of discarding everything - a
+  // long judged run shouldn't lose all completed work to a Ctrl+C. Callers who
+  // prefer the old throw-on-abort opt in with `throwOnAbort`.
+  if (isAborted(signal)) {
+    if (opts.throwOnAbort === true) throwIfAborted(signal);
+    const done = results.filter((r): r is EvalCaseResult<I, O> => r !== undefined);
+    return { ...summarize(done, opts.scorers), aborted: true };
+  }
+  return summarize(results, opts.scorers);
+}
+
+async function safeScore<I, O>(
+  scorer: Scorer<I, O>,
+  sampleCase: Case<I, O>,
+  output: O,
+  durationMs: number,
+): Promise<ScoreResult> {
+  try {
+    return await scorer.score({ case: sampleCase, output, durationMs });
+  } catch (err) {
+    return {
+      pass: false,
+      reason: `Scorer "${scorer.name}" threw: ${err instanceof Error ? err.message : String(err)}`,
+    };
+  }
+}
+
+function summarize<I, O>(
+  results: ReadonlyArray<EvalCaseResult<I, O>>,
+  scorers: ReadonlyArray<Scorer<I, O>>,
+): EvalReport<I, O> {
+  const total = results.length;
+  let passed = 0;
+  let failed = 0;
+  let durationSum = 0;
+  const byScorer: Record<
+    string,
+    { passed: number; failed: number; scoreSum: number; scoreCount: number }
+  > = {};
+  for (const scorer of scorers) {
+    byScorer[scorer.name] = { passed: 0, failed: 0, scoreSum: 0, scoreCount: 0 };
+  }
+  for (const r of results) {
+    durationSum += r.durationMs;
+    let passEntire = true;
+    for (const { scorer, result } of r.scores) {
+      const bucket = byScorer[scorer] ?? { passed: 0, failed: 0, scoreSum: 0, scoreCount: 0 };
+      if (result.pass) bucket.passed += 1;
+      else {
+        bucket.failed += 1;
+        passEntire = false;
+      }
+      if (typeof result.score === 'number' && Number.isFinite(result.score)) {
+        bucket.scoreSum += result.score;
+        bucket.scoreCount += 1;
+      }
+      byScorer[scorer] = bucket;
+    }
+    if (passEntire) passed += 1;
+    else failed += 1;
+  }
+  const summaryByScorer: Record<
+    string,
+    { passed: number; failed: number; avgScore: number | null }
+  > = {};
+  for (const [scorer, b] of Object.entries(byScorer)) {
+    summaryByScorer[scorer] = {
+      passed: b.passed,
+      failed: b.failed,
+      avgScore: b.scoreCount === 0 ? null : b.scoreSum / b.scoreCount,
+    };
+  }
+  // E8 (evals-05): a bare pass count over a small suite communicates false
+  // precision - attach the 95% Wilson interval, and under `iterations > 1`
+  // the pass^k stability metric (mean pass rate hides a flaky case).
+  const outcomes = results.map((r) => ({
+    caseId: r.caseId,
+    pass: r.scores.every((s) => s.result.pass),
+  }));
+  const stability = passHatK(outcomes);
+  return {
+    results,
+    summary: {
+      total,
+      passed,
+      failed,
+      avgDurationMs: total === 0 ? 0 : durationSum / total,
+      byScorer: Object.freeze(summaryByScorer),
+      passRateCi: wilsonInterval(passed, total),
+      ...(stability.k > 1 ? { passHatK: stability } : {}),
+    },
+  };
+}
+
+// A plain function call so TypeScript re-reads `signal.aborted` each time -
+// the flag flips during an `await` (external mutation) and inline
+// `signal?.aborted === true` checks would otherwise be narrowed away.
+function isAborted(signal: AbortSignal | undefined): boolean {
+  return signal?.aborted === true;
+}
+
+function throwIfAborted(signal: AbortSignal | undefined): void {
+  if (signal?.aborted === true) {
+    const reason = signal.reason;
+    throw reason instanceof Error ? reason : new Error('Eval run aborted');
+  }
+}

package/src/scorers/code/exact-match.ts

@@ -0,0 +1,95 @@
+/**
+ * `exactMatch` - passes when `output` deeply equals `case.expected`.
+ * Useful for deterministic outputs (numeric / parsed JSON / classifier
+ * label).
+ *
+ * @packageDocumentation
+ */
+
+import type { Scorer } from '@graphorin/observability/eval';
+
+/** @stable */
+export interface ExactMatchOptions {
+  /** Optional name override. Default `'exact-match'`. */
+  readonly name?: string;
+  /** When `true`, treat strings case-insensitively. Default `false`. */
+  readonly caseInsensitive?: boolean;
+  /** When `true`, trim whitespace before comparing strings. Default `false`. */
+  readonly trim?: boolean;
+}
+
+/**
+ * Build an exact-match scorer.
+ *
+ * @stable
+ */
+export function exactMatch<I = unknown, O = unknown>(
+  options: ExactMatchOptions = {},
+): Scorer<I, O> {
+  const name = options.name ?? 'exact-match';
+  return {
+    name,
+    async score({ case: c, output }) {
+      const expected = c.expected;
+      if (expected === undefined) {
+        return {
+          pass: false,
+          reason: 'exactMatch requires a `case.expected` value.',
+        };
+      }
+      const a = normalize(output, options);
+      const b = normalize(expected, options);
+      const pass = deepEqual(a, b);
+      if (pass) return { pass, score: 1 };
+      return {
+        pass,
+        score: 0,
+        reason: `expected ${truncate(JSON.stringify(b))}, received ${truncate(JSON.stringify(a))}`,
+      };
+    },
+  };
+}
+
+function normalize(value: unknown, opts: ExactMatchOptions): unknown {
+  if (typeof value === 'string') {
+    let v = value;
+    if (opts.trim === true) v = v.trim();
+    if (opts.caseInsensitive === true) v = v.toLowerCase();
+    return v;
+  }
+  return value;
+}
+
+function deepEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  if (typeof a !== typeof b) return false;
+  if (a === null || b === null) return a === b;
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) {
+      if (!deepEqual(a[i], b[i])) return false;
+    }
+    return true;
+  }
+  if (typeof a === 'object' && typeof b === 'object') {
+    const aKeys = Object.keys(a as Record<string, unknown>).sort();
+    const bKeys = Object.keys(b as Record<string, unknown>).sort();
+    if (aKeys.length !== bKeys.length) return false;
+    for (let i = 0; i < aKeys.length; i++) {
+      if (aKeys[i] !== bKeys[i]) return false;
+      if (
+        !deepEqual(
+          (a as Record<string, unknown>)[aKeys[i] as string],
+          (b as Record<string, unknown>)[bKeys[i] as string],
+        )
+      )
+        return false;
+    }
+    return true;
+  }
+  return false;
+}
+
+function truncate(s: string, max: number = 80): string {
+  return s.length > max ? `${s.slice(0, max)}…` : s;
+}

package/src/scorers/code/index.ts

@@ -0,0 +1,10 @@
+/**
+ * Pure-code scorers: exact-match, regex, json-path, predicate.
+ *
+ * @packageDocumentation
+ */
+
+export { type ExactMatchOptions, exactMatch } from './exact-match.js';
+export { type JsonPathOptions, jsonPath } from './json-path.js';
+export { type PredicateOptions, predicate } from './predicate.js';
+export { type RegexMatchOptions, regexMatch } from './regex.js';

package/src/scorers/code/json-path.ts

@@ -0,0 +1,101 @@
+/**
+ * `jsonPath` - passes when the value at a JSON-pointer-shaped path
+ * deep-equals the caller-supplied target. Supports the dot-notation
+ * subset (`a.b.c`) and array indices (`a.0.name`).
+ *
+ * @packageDocumentation
+ */
+
+import type { Scorer } from '@graphorin/observability/eval';
+
+/** @stable */
+export interface JsonPathOptions {
+  /** Dot-separated path (`'data.user.id'`). */
+  readonly path: string;
+  /** Expected value. Compared by deep-equality. */
+  readonly equals: unknown;
+  /** Optional name override. */
+  readonly name?: string;
+}
+
+/** @stable */
+export function jsonPath<I = unknown>(options: JsonPathOptions): Scorer<I, unknown> {
+  const name = options.name ?? `json-path:${options.path}`;
+  return {
+    name,
+    async score({ output }) {
+      const actual = readPath(output, options.path);
+      const pass = deepEqual(actual, options.equals);
+      if (pass) return { pass, score: 1 };
+      return {
+        pass,
+        score: 0,
+        reason:
+          `at '${options.path}' expected ${truncate(stringifySafe(options.equals))}, ` +
+          `received ${truncate(stringifySafe(actual))}.`,
+      };
+    },
+  };
+}
+
+function readPath(root: unknown, path: string): unknown {
+  if (path.length === 0) return root;
+  const segments = path.split('.');
+  let cursor: unknown = root;
+  for (const segment of segments) {
+    if (cursor === null || cursor === undefined) return undefined;
+    if (Array.isArray(cursor)) {
+      const idx = Number.parseInt(segment, 10);
+      if (!Number.isFinite(idx)) return undefined;
+      cursor = cursor[idx];
+    } else if (typeof cursor === 'object') {
+      cursor = (cursor as Record<string, unknown>)[segment];
+    } else {
+      return undefined;
+    }
+  }
+  return cursor;
+}
+
+function deepEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  if (typeof a !== typeof b) return false;
+  if (a === null || b === null) return a === b;
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) {
+      if (!deepEqual(a[i], b[i])) return false;
+    }
+    return true;
+  }
+  if (typeof a === 'object' && typeof b === 'object') {
+    const aKeys = Object.keys(a as Record<string, unknown>).sort();
+    const bKeys = Object.keys(b as Record<string, unknown>).sort();
+    if (aKeys.length !== bKeys.length) return false;
+    for (let i = 0; i < aKeys.length; i++) {
+      if (aKeys[i] !== bKeys[i]) return false;
+      if (
+        !deepEqual(
+          (a as Record<string, unknown>)[aKeys[i] as string],
+          (b as Record<string, unknown>)[bKeys[i] as string],
+        )
+      )
+        return false;
+    }
+    return true;
+  }
+  return false;
+}
+
+function truncate(s: string, max: number = 80): string {
+  return s.length > max ? `${s.slice(0, max)}…` : s;
+}
+
+function stringifySafe(value: unknown): string {
+  if (value === undefined) return 'undefined';
+  try {
+    return JSON.stringify(value) ?? String(value);
+  } catch {
+    return String(value);
+  }
+}

package/src/scorers/code/predicate.ts

@@ -0,0 +1,32 @@
+/**
+ * `predicate` - passes when the caller-supplied predicate returns
+ * truthy. The escape hatch for ad-hoc, project-specific scoring rules.
+ *
+ * @packageDocumentation
+ */
+
+import type { Case, ScoreResult, Scorer } from '@graphorin/observability/eval';
+
+/** @stable */
+export interface PredicateOptions<I, O> {
+  readonly name: string;
+  readonly check: (args: {
+    readonly case: Case<I, O>;
+    readonly output: O;
+    readonly durationMs: number;
+  }) => Promise<boolean | ScoreResult> | boolean | ScoreResult;
+}
+
+/** @stable */
+export function predicate<I = unknown, O = unknown>(options: PredicateOptions<I, O>): Scorer<I, O> {
+  return {
+    name: options.name,
+    async score(args) {
+      const result = await options.check(args);
+      if (typeof result === 'boolean') {
+        return { pass: result, score: result ? 1 : 0 };
+      }
+      return result;
+    },
+  };
+}