@graphorin/evals 0.6.0 → 0.7.0

package/src/scorers/trajectory/types.ts

@@ -0,0 +1,44 @@
+/**
+ * Shared types for the trajectory scorer family. A {@link Trajectory} is
+ * the recorded sequence of tool calls a harness made while attempting a
+ * task, plus an optional goal-state snapshot and the final text output.
+ *
+ * The scorers in this folder are pure functions over a `Trajectory` - no
+ * network, no model - so they can gate harness reliability in CI. Build a
+ * `Trajectory` by folding an agent's `AgentEvent` stream (correlate
+ * `tool.call.start` / `tool.call.end` with `tool.execute.end` /
+ * `tool.execute.error` by `toolCallId`).
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * One executed tool call as observed on the `AgentEvent` stream.
+ *
+ * @stable
+ */
+export interface TrajectoryToolCall {
+  readonly toolCallId: string;
+  readonly toolName: string;
+  /** The arguments the model emitted (the resolved `tool.call.end.finalArgs`). */
+  readonly args: unknown;
+  /** `'ok'` when the call returned; `'error'` when the executor surfaced a `ToolError`. */
+  readonly status: 'ok' | 'error';
+  /** The tool output, present when `status === 'ok'`. */
+  readonly result?: unknown;
+  /** The surfaced error, present when `status === 'error'`. */
+  readonly error?: { readonly kind?: string; readonly message?: string };
+}
+
+/**
+ * The full record of a single harness attempt at a task.
+ *
+ * @stable
+ */
+export interface Trajectory {
+  readonly calls: ReadonlyArray<TrajectoryToolCall>;
+  /** Goal-state snapshot compared by {@link finalStateCorrect}. */
+  readonly finalState?: unknown;
+  /** The assistant's final text, when the run completed. */
+  readonly finalOutput?: string;
+}

package/src/scorers/trajectory/util.ts

@@ -0,0 +1,81 @@
+/**
+ * Internal helpers shared by the trajectory scorers: structural deep
+ * equality, dot-path reads, and an order-insensitive canonical key used to
+ * de-duplicate tool calls by `(name, args)`. Private to the family - not
+ * exported from the package surface.
+ *
+ * @packageDocumentation
+ */
+
+/** Structural deep-equality with order-insensitive object key comparison. */
+export function deepEqual(a: unknown, b: unknown): boolean {
+  if (a === b) return true;
+  if (typeof a !== typeof b) return false;
+  if (a === null || b === null) return a === b;
+  if (Array.isArray(a) && Array.isArray(b)) {
+    if (a.length !== b.length) return false;
+    for (let i = 0; i < a.length; i++) {
+      if (!deepEqual(a[i], b[i])) return false;
+    }
+    return true;
+  }
+  if (typeof a === 'object' && typeof b === 'object') {
+    const aObj = a as Record<string, unknown>;
+    const bObj = b as Record<string, unknown>;
+    const aKeys = Object.keys(aObj).sort();
+    const bKeys = Object.keys(bObj).sort();
+    if (aKeys.length !== bKeys.length) return false;
+    for (let i = 0; i < aKeys.length; i++) {
+      const key = aKeys[i];
+      if (key === undefined || key !== bKeys[i]) return false;
+      if (!deepEqual(aObj[key], bObj[key])) return false;
+    }
+    return true;
+  }
+  return false;
+}
+
+/** Read a dot-separated path (`'orders.0.status'`) out of a value. */
+export function readPath(root: unknown, path: string): unknown {
+  if (path.length === 0) return root;
+  let cursor: unknown = root;
+  for (const segment of path.split('.')) {
+    if (cursor === null || cursor === undefined) return undefined;
+    if (Array.isArray(cursor)) {
+      const idx = Number.parseInt(segment, 10);
+      if (!Number.isFinite(idx)) return undefined;
+      cursor = cursor[idx];
+    } else if (typeof cursor === 'object') {
+      cursor = (cursor as Record<string, unknown>)[segment];
+    } else {
+      return undefined;
+    }
+  }
+  return cursor;
+}
+
+/** Order-insensitive canonical JSON, so `{a,b}` and `{b,a}` key alike. */
+export function canonicalize(value: unknown): string {
+  if (value === null || typeof value !== 'object') {
+    return JSON.stringify(value) ?? 'null';
+  }
+  if (Array.isArray(value)) {
+    return `[${value.map(canonicalize).join(',')}]`;
+  }
+  const obj = value as Record<string, unknown>;
+  const keys = Object.keys(obj).sort();
+  return `{${keys.map((k) => `${JSON.stringify(k)}:${canonicalize(obj[k])}`).join(',')}}`;
+}
+
+export function truncate(s: string, max = 80): string {
+  return s.length > max ? `${s.slice(0, max)}…` : s;
+}
+
+export function stringifySafe(value: unknown): string {
+  if (value === undefined) return 'undefined';
+  try {
+    return JSON.stringify(value) ?? String(value);
+  } catch {
+    return String(value);
+  }
+}

package/src/stats.ts

@@ -0,0 +1,196 @@
+/**
+ * Shared eval statistics (audit 2026-07-04, E8; closes the evals-05 /
+ * evals-08 gap): confidence intervals, pass^k, and paired significance
+ * for benchmark deltas. Everything here is pure and dependency-free so
+ * the runner, the regression gate, and the benchmarks can share one
+ * implementation instead of each inlining its own mean/stddev.
+ *
+ * Statistical choices, briefly:
+ * - **Wilson score interval** for pass-rate CIs. Eval suites are small
+ *   (n in the tens) and pass rates hug 0/1, exactly where the normal
+ *   approximation interval collapses; Wilson stays sane there.
+ * - **pass^k** ("pass-hat-k", the fraction of base cases that pass in
+ *   ALL k repeat iterations) as the stability metric for `iterations>1`
+ *   runs - a mean pass rate hides a case that flips every other run.
+ * - **McNemar's test** (continuity-corrected normal approximation) for
+ *   "did the pass rate really change vs the baseline": the same cases
+ *   ran in both reports, so a PAIRED test on the discordant cases has
+ *   far more power than comparing two independent proportions - and it
+ *   is sample-size aware, which the fixed-tolerance regression gate is
+ *   not.
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * Arithmetic mean; `0` for an empty list.
+ *
+ * @stable
+ */
+export function mean(values: ReadonlyArray<number>): number {
+  if (values.length === 0) return 0;
+  let sum = 0;
+  for (const v of values) sum += v;
+  return sum / values.length;
+}
+
+/**
+ * Sample standard deviation (n-1 denominator); `0` for n < 2.
+ *
+ * @stable
+ */
+export function sampleStddev(values: ReadonlyArray<number>): number {
+  if (values.length < 2) return 0;
+  const m = mean(values);
+  let acc = 0;
+  for (const v of values) acc += (v - m) * (v - m);
+  return Math.sqrt(acc / (values.length - 1));
+}
+
+/**
+ * Wilson score interval for a binomial proportion.
+ *
+ * @param passed number of successes
+ * @param total  number of trials
+ * @param z      normal quantile (default 1.96 = 95% two-sided)
+ * @returns `{ lo, hi }` clamped to [0, 1]; `{ lo: 0, hi: 1 }` when `total` is 0
+ *
+ * @stable
+ */
+export function wilsonInterval(
+  passed: number,
+  total: number,
+  z = 1.96,
+): { readonly lo: number; readonly hi: number } {
+  if (total <= 0) return { lo: 0, hi: 1 };
+  const p = passed / total;
+  const z2 = z * z;
+  const denom = 1 + z2 / total;
+  const centre = p + z2 / (2 * total);
+  const margin = z * Math.sqrt((p * (1 - p)) / total + z2 / (4 * total * total));
+  return {
+    lo: Math.max(0, (centre - margin) / denom),
+    hi: Math.min(1, (centre + margin) / denom),
+  };
+}
+
+const ITERATION_SUFFIX_RE = /-iter-\d+$/;
+
+/**
+ * Strip the `-iter-N` disambiguation suffix the runner appends under
+ * `iterations > 1` (EB-6), recovering the base case id.
+ *
+ * @stable
+ */
+export function stripIterationSuffix(caseId: string): string {
+  return caseId.replace(ITERATION_SUFFIX_RE, '');
+}
+
+/**
+ * pass^k over per-iteration case outcomes: group by base case id and
+ * report the fraction of base cases whose EVERY iteration passed.
+ *
+ * @param outcomes one entry per executed case iteration
+ * @returns `k` = the largest group size observed, `baseCases` = number of
+ *   distinct base cases, `value` = the pass^k fraction (0 when no cases)
+ *
+ * @stable
+ */
+export function passHatK(
+  outcomes: ReadonlyArray<{ readonly caseId: string; readonly pass: boolean }>,
+): { readonly k: number; readonly baseCases: number; readonly value: number } {
+  const groups = new Map<string, { total: number; passed: number }>();
+  for (const o of outcomes) {
+    const base = stripIterationSuffix(o.caseId);
+    const g = groups.get(base) ?? { total: 0, passed: 0 };
+    g.total += 1;
+    if (o.pass) g.passed += 1;
+    groups.set(base, g);
+  }
+  if (groups.size === 0) return { k: 0, baseCases: 0, value: 0 };
+  let k = 0;
+  let allPass = 0;
+  for (const g of groups.values()) {
+    if (g.total > k) k = g.total;
+    if (g.passed === g.total) allPass += 1;
+  }
+  return { k, baseCases: groups.size, value: allPass / groups.size };
+}
+
+/**
+ * Result of {@link pairedPassSignificance}.
+ *
+ * @stable
+ */
+export interface PairedSignificance {
+  /** Cases present in BOTH runs (the paired sample). */
+  readonly pairs: number;
+  /** baseline-pass -> current-fail count (the regressions). */
+  readonly regressed: number;
+  /** baseline-fail -> current-pass count (the improvements). */
+  readonly improved: number;
+  /**
+   * Two-sided p-value from McNemar's test (continuity-corrected normal
+   * approximation over the discordant pairs). `1` when there are no
+   * discordant pairs - identical outcomes are never "significant".
+   */
+  readonly pValue: number;
+}
+
+/**
+ * McNemar's paired test on per-case pass outcomes of two eval runs.
+ * Only cases present in both maps are compared (by base case id).
+ *
+ * @stable
+ */
+export function pairedPassSignificance(
+  current: ReadonlyMap<string, boolean>,
+  baseline: ReadonlyMap<string, boolean>,
+): PairedSignificance {
+  let pairs = 0;
+  let regressed = 0;
+  let improved = 0;
+  for (const [caseId, basePass] of baseline) {
+    const curPass = current.get(caseId);
+    if (curPass === undefined) continue;
+    pairs += 1;
+    if (basePass && !curPass) regressed += 1;
+    else if (!basePass && curPass) improved += 1;
+  }
+  const discordant = regressed + improved;
+  if (discordant === 0) return { pairs, regressed, improved, pValue: 1 };
+  // Continuity-corrected normal approximation to the binomial(discordant, 0.5).
+  const z = (Math.abs(regressed - improved) - 1) / Math.sqrt(discordant);
+  const pValue = z <= 0 ? 1 : Math.min(1, 2 * (1 - normalCdf(z)));
+  return { pairs, regressed, improved, pValue };
+}
+
+/**
+ * Collapse per-iteration outcomes to per-base-case pass (a base case
+ * passes when EVERY iteration passed) - the paired unit for
+ * {@link pairedPassSignificance}.
+ *
+ * @stable
+ */
+export function passByBaseCase(
+  outcomes: ReadonlyArray<{ readonly caseId: string; readonly pass: boolean }>,
+): ReadonlyMap<string, boolean> {
+  const out = new Map<string, boolean>();
+  for (const o of outcomes) {
+    const base = stripIterationSuffix(o.caseId);
+    const prev = out.get(base);
+    out.set(base, prev === undefined ? o.pass : prev && o.pass);
+  }
+  return out;
+}
+
+/** Standard normal CDF via the Abramowitz-Stegun 7.1.26 erf approximation. */
+function normalCdf(x: number): number {
+  const t = 1 / (1 + 0.3275911 * (Math.abs(x) / Math.SQRT2));
+  const erf =
+    1 -
+    ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) *
+      t *
+      Math.exp(-(x * x) / 2);
+  return x >= 0 ? 0.5 * (1 + erf) : 0.5 * (1 - erf);
+}

package/src/types.ts

@@ -0,0 +1,135 @@
+/**
+ * Public types for the full eval framework. Re-exports the
+ * {@link Case} / {@link Dataset} / {@link EvalReport} primitives from
+ * `@graphorin/observability/eval` so consumers do not need to depend
+ * on both packages, then layers on the framework-specific concepts:
+ * agent shape, parallel runner config, regression policy, baseline
+ * loaders.
+ *
+ * @packageDocumentation
+ */
+
+export type {
+  Case,
+  Dataset,
+  EvalCaseResult,
+  EvalReport,
+  RunEvalOptions,
+  ScoreResult,
+  Scorer,
+} from '@graphorin/observability/eval';
+
+import type { EvalReport, Scorer } from '@graphorin/observability/eval';
+
+/**
+ * Agent shape consumed by the runner. Anything with a `run(input)`
+ * method satisfies the contract - the framework's own `Agent` type
+ * matches by structural typing.
+ *
+ * @stable
+ */
+export interface AgentLike<I, O> {
+  run(input: I, ctx?: { signal?: AbortSignal }): Promise<O>;
+}
+
+/**
+ * Options accepted by the parallel runner.
+ *
+ * @stable
+ */
+export interface RunOptions<I, O> {
+  readonly agent: AgentLike<I, O>;
+  readonly dataset: {
+    readonly cases: ReadonlyArray<{
+      readonly id?: string;
+      readonly input: I;
+      readonly expected?: O;
+      readonly metadata?: Readonly<Record<string, unknown>>;
+    }>;
+  };
+  readonly scorers: ReadonlyArray<Scorer<I, O>>;
+  /** Default `1`. */
+  readonly iterations?: number;
+  /** Default `1` (sequential). Set higher for parallel evaluation. */
+  readonly concurrency?: number;
+  readonly signal?: AbortSignal;
+  /**
+   * How to handle `signal` abort. Default (`false`): stop dispatching and
+   * resolve with a **partial** {@link EvalReport} (`aborted: true`) covering
+   * the cases that finished - a long judged run shouldn't lose all completed
+   * work to a Ctrl+C. Set `true` to throw the abort reason instead.
+   */
+  readonly throwOnAbort?: boolean;
+  /**
+   * Optional progress hook invoked after every case. Useful for
+   * terminal reporters that want a per-case heartbeat.
+   */
+  readonly onProgress?: (event: ProgressEvent) => void;
+}
+
+/** @stable */
+export interface ProgressEvent {
+  readonly index: number;
+  readonly total: number;
+  readonly caseId: string;
+  readonly durationMs: number;
+  readonly passed: boolean;
+}
+
+/**
+ * Regression-detection options.
+ *
+ * @stable
+ */
+export interface RegressionOptions {
+  /** Minimum drop in pass-rate (in percentage points) that counts as a regression. */
+  readonly maxPassRateDropPct?: number;
+  /** Minimum drop in average score per scorer that counts as a regression. */
+  readonly maxAvgScoreDrop?: number;
+  /**
+   * Maximum allowed increase in `avgDurationMs` before it counts as a
+   * regression. **Opt-in: defaults to `Infinity` (gate off)** because absolute
+   * wall-clock budgets are environment-sensitive (workstation baseline vs CI
+   * runner, real LLM-latency jitter). Pass a finite ms budget to enable an
+   * absolute duration gate; leave unset to ignore duration entirely.
+   */
+  readonly maxAvgDurationIncreaseMs?: number;
+  /**
+   * E8 (evals-05/08): when `true`, a `pass-rate-drop` finding is only kept if
+   * McNemar's paired test over the shared cases rejects "no real change" at
+   * {@link significanceAlpha} - a fixed percentage tolerance is blind to
+   * sample size (a 5pp drop is one case in a 20-case suite). Off by default
+   * so existing gates keep their exact behavior; the computed `pValue` is
+   * attached to the finding either way whenever both reports carry per-case
+   * results.
+   */
+  readonly requireSignificance?: boolean;
+  /** Significance level for {@link requireSignificance}. Default `0.05`. */
+  readonly significanceAlpha?: number;
+}
+
+/**
+ * Result of {@link detectRegressions}.
+ *
+ * @stable
+ */
+export interface RegressionReport<I, O> {
+  readonly hasRegressions: boolean;
+  readonly findings: ReadonlyArray<RegressionFinding>;
+  readonly current: EvalReport<I, O>;
+  readonly baseline: EvalReport<I, O>;
+}
+
+/** @stable */
+export interface RegressionFinding {
+  readonly kind: 'pass-rate-drop' | 'avg-score-drop' | 'avg-duration-increase' | 'scorer-removed';
+  readonly scorer?: string;
+  readonly message: string;
+  readonly delta: number;
+  /**
+   * Two-sided McNemar p-value over the cases shared by both reports
+   * (E8; only on `pass-rate-drop` findings, and only when both reports
+   * carry per-case results to pair on).
+   */
+  readonly pValue?: number;
+}