@graphorin/evals 0.6.1 → 0.7.0

package/src/loaders/locomo.ts

@@ -0,0 +1,217 @@
+/**
+ * Real LOCOMO dataset loader (arXiv:2402.17753 -
+ * https://snap-research.github.io/locomo/). LOCOMO ships as a JSON
+ * **array** of samples; each sample bundles one multi-session
+ * conversation with many QA pairs over it:
+ *
+ * ```jsonc
+ * {
+ *   "sample_id": "conv-26",
+ *   "conversation": {
+ *     "speaker_a": "Caroline",
+ *     "speaker_b": "Melanie",
+ *     "session_1_date_time": "1:56 pm on 8 May, 2023",
+ *     "session_1": [ { "speaker": "Caroline", "dia_id": "D1:1", "text": "..." }, ... ],
+ *     "session_2_date_time": "...",
+ *     "session_2": [ ... ]
+ *   },
+ *   "qa": [
+ *     { "question": "...", "answer": "...", "evidence": ["D1:1"], "category": 1 },
+ *     { "question": "...", "category": 5, "adversarial_answer": "Not mentioned ..." }
+ *   ]
+ * }
+ * ```
+ *
+ * Each QA pair becomes one {@link MemoryEvalInput} case that shares the
+ * sample's sessions. LOCOMO categories map to abilities as:
+ * `1`→multi-session (multi-hop), `2`→temporal, `5`→abstention
+ * (adversarial), everything else (`3` open-domain, `4` single-hop)
+ * →info-extraction.
+ *
+ * Answer handling (W-022): LOCOMO answers may be numbers (e.g. `2022`)
+ * - they are stringified, never silently coerced to `''`. A QA pair
+ * with NO answer at all (neither `answer` nor `adversarial_answer`) is
+ * SKIPPED rather than emitted with an empty `expected`: the LLM judge
+ * grading against an empty reference produces meaningless verdicts, so
+ * a missing reference must reduce the case count, not poison scores.
+ * Turns carry the dataset-native `speaker` name alongside the
+ * two-role mapping (~99 percent of LOCOMO questions reference speakers
+ * by name).
+ *
+ * @packageDocumentation
+ */
+
+import { readFile } from 'node:fs/promises';
+
+import type { Case, Dataset } from '@graphorin/observability/eval';
+
+import type {
+  MemoryEvalAbility,
+  MemoryEvalInput,
+  MemoryEvalSession,
+  MemoryEvalTurn,
+} from './memory-eval.js';
+
+/** @stable */
+export interface LoadLocomoOptions {
+  /** Local path to the LOCOMO JSON (under `benchmarks/.datasets/`). */
+  readonly path: string;
+  /** Optional dataset name surfaced in `Dataset.metadata.name`. */
+  readonly name?: string;
+  /** Optional description surfaced in `Dataset.metadata.description`. */
+  readonly description?: string;
+}
+
+/**
+ * Read a LOCOMO JSON file and return a fully-materialised
+ * {@link Dataset} - one case per QA pair, scored against the reference
+ * answer string (LLM-judge "J").
+ *
+ * @stable
+ */
+export async function loadLocomoDataset(
+  options: LoadLocomoOptions,
+): Promise<Dataset<MemoryEvalInput, string>> {
+  const text = await readFile(options.path, 'utf8');
+  const cases = parseLocomo(text);
+  return {
+    cases,
+    metadata: {
+      name: options.name ?? 'locomo',
+      ...(options.description !== undefined ? { description: options.description } : {}),
+      createdAt: new Date(),
+    },
+  };
+}
+
+/**
+ * Pure parser. Exported so tests can exercise the mapping without
+ * touching the filesystem.
+ *
+ * @stable
+ */
+export function parseLocomo(text: string): ReadonlyArray<Case<MemoryEvalInput, string>> {
+  const parsed: unknown = JSON.parse(text);
+  if (!Array.isArray(parsed)) {
+    throw new Error('[graphorin/evals] LOCOMO dataset must be a JSON array of samples.');
+  }
+  const samples = parsed as ReadonlyArray<unknown>;
+  const out: Case<MemoryEvalInput, string>[] = [];
+  for (let i = 0; i < samples.length; i++) {
+    const sample = samples[i];
+    if (sample === null || typeof sample !== 'object') {
+      throw new Error(`[graphorin/evals] LOCOMO sample ${i} is not an object.`);
+    }
+    appendSampleCases(sample as Record<string, unknown>, i, out);
+  }
+  return out;
+}
+
+function appendSampleCases(
+  sample: Record<string, unknown>,
+  index: number,
+  out: Case<MemoryEvalInput, string>[],
+): void {
+  const sampleId = asString(sample.sample_id) ?? `locomo-${index}`;
+  const conversation = isRecord(sample.conversation) ? sample.conversation : {};
+  const sessions = extractSessions(conversation);
+  const qa = asArray(sample.qa);
+  for (let q = 0; q < qa.length; q++) {
+    const item = qa[q];
+    if (!isRecord(item)) continue;
+    const question = asString(item.question);
+    if (question === undefined) continue;
+    const category = typeof item.category === 'number' ? item.category : undefined;
+    const ability = mapCategory(category);
+    const answer = answerToText(item.answer) ?? answerToText(item.adversarial_answer);
+    // No reference answer at all: skip the pair (see module docs, W-022).
+    if (answer === undefined) continue;
+    const evidence = asArray(item.evidence).filter((e): e is string => typeof e === 'string');
+    out.push({
+      id: `${sampleId}-q${q}`,
+      input: { haystackSessions: sessions, question, ability },
+      expected: answer,
+      metadata: {
+        datasetName: 'locomo',
+        sampleId,
+        ability,
+        ...(category !== undefined ? { category } : {}),
+        evidence,
+      },
+    });
+  }
+}
+
+function extractSessions(conversation: Record<string, unknown>): MemoryEvalSession[] {
+  const speakerA = asString(conversation.speaker_a);
+  const indices: number[] = [];
+  for (const key of Object.keys(conversation)) {
+    const m = /^session_(\d+)$/.exec(key);
+    if (m?.[1] !== undefined) indices.push(Number.parseInt(m[1], 10));
+  }
+  indices.sort((a, b) => a - b);
+  const sessions: MemoryEvalSession[] = [];
+  for (const n of indices) {
+    const raw = asArray(conversation[`session_${n}`]);
+    const date = asString(conversation[`session_${n}_date_time`]);
+    const turns: MemoryEvalTurn[] = [];
+    for (const t of raw) {
+      if (!isRecord(t)) continue;
+      const content = asString(t.text) ?? asString(t.content);
+      if (content === undefined) continue;
+      const speaker = asString(t.speaker);
+      const role: 'user' | 'assistant' =
+        speakerA !== undefined && speaker !== undefined && speaker !== speakerA
+          ? 'assistant'
+          : 'user';
+      turns.push({
+        role,
+        content,
+        ...(date !== undefined ? { timestamp: date } : {}),
+        // W-022: keep the dataset-native name; the role mapping stays
+        // for cross-dataset machinery compatibility.
+        ...(speaker !== undefined ? { speaker } : {}),
+      });
+    }
+    sessions.push({ id: `session_${n}`, turns });
+  }
+  return sessions;
+}
+
+/** Maps a LOCOMO numeric `category` onto a {@link MemoryEvalAbility}. */
+function mapCategory(category: number | undefined): MemoryEvalAbility {
+  switch (category) {
+    case 1:
+      return 'multi-session';
+    case 2:
+      return 'temporal';
+    case 5:
+      return 'abstention';
+    default:
+      return 'info-extraction';
+  }
+}
+
+function asString(value: unknown): string | undefined {
+  return typeof value === 'string' ? value : undefined;
+}
+
+/**
+ * LOCOMO reference answers may be strings OR bare numbers (6 of the
+ * 1986 QA pairs ship e.g. `2022`). Stringify finite numbers instead of
+ * dropping them to `''` (W-022); a blank string counts as "no answer"
+ * so the judge never grades against an empty reference.
+ */
+function answerToText(value: unknown): string | undefined {
+  if (typeof value === 'string') return value.trim().length > 0 ? value : undefined;
+  if (typeof value === 'number' && Number.isFinite(value)) return String(value);
+  return undefined;
+}
+
+function asArray(value: unknown): ReadonlyArray<unknown> {
+  return Array.isArray(value) ? (value as ReadonlyArray<unknown>) : [];
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}

package/src/loaders/longmemeval.ts

@@ -0,0 +1,164 @@
+/**
+ * LongMemEval dataset loader (ICLR 2025, arXiv:2410.10813 -
+ * https://github.com/xiaowu0162/LongMemEval). LongMemEval ships as a
+ * JSON **array** of question objects:
+ *
+ * ```jsonc
+ * {
+ *   "question_id": "abc_abs",          // `_abs` suffix marks abstention
+ *   "question_type": "temporal-reasoning",
+ *   "question": "Where did I live in 2021?",
+ *   "answer": "Berlin",
+ *   "question_date": "2023/05/20 (Sat) 14:00",
+ *   "haystack_session_ids": ["s1", "s2"],
+ *   "haystack_dates": ["2021/03/01 ...", "2023/01/05 ..."],
+ *   "haystack_sessions": [
+ *     [ { "role": "user", "content": "..." }, { "role": "assistant", "content": "...", "has_answer": true } ],
+ *     [ ... ]
+ *   ]
+ * }
+ * ```
+ *
+ * The three parallel arrays (`haystack_session_ids` / `haystack_dates`
+ * / `haystack_sessions`) are zipped by index; one date applies to every
+ * turn in its session.
+ *
+ * @packageDocumentation
+ */
+
+import { readFile } from 'node:fs/promises';
+
+import type { Case, Dataset } from '@graphorin/observability/eval';
+
+import type {
+  MemoryEvalAbility,
+  MemoryEvalInput,
+  MemoryEvalSession,
+  MemoryEvalTurn,
+} from './memory-eval.js';
+
+/** @stable */
+export interface LoadLongMemEvalOptions {
+  /** Local path to the dataset JSON (under `benchmarks/.datasets/`). */
+  readonly path: string;
+  /** Which release this file is. `'S'` (~115 sessions) is the default target. */
+  readonly variant?: 'S' | 'M';
+  /** When set, keep only cases whose mapped ability is in this list. */
+  readonly abilities?: ReadonlyArray<MemoryEvalAbility>;
+  /** Optional dataset name surfaced in `Dataset.metadata.name`. */
+  readonly name?: string;
+  /** Optional description surfaced in `Dataset.metadata.description`. */
+  readonly description?: string;
+}
+
+/**
+ * Read a LongMemEval JSON file and return a fully-materialised
+ * {@link Dataset} of {@link MemoryEvalInput} cases scored against the
+ * reference `answer` string.
+ *
+ * @stable
+ */
+export async function loadLongMemEvalDataset(
+  options: LoadLongMemEvalOptions,
+): Promise<Dataset<MemoryEvalInput, string>> {
+  const text = await readFile(options.path, 'utf8');
+  const cases = parseLongMemEval(text, options.abilities);
+  const variantSuffix = options.variant !== undefined ? `_${options.variant.toLowerCase()}` : '';
+  return {
+    cases,
+    metadata: {
+      name: options.name ?? `longmemeval${variantSuffix}`,
+      ...(options.description !== undefined ? { description: options.description } : {}),
+      createdAt: new Date(),
+    },
+  };
+}
+
+/**
+ * Pure parser. Exported so tests can exercise the mapping without
+ * touching the filesystem.
+ *
+ * @stable
+ */
+export function parseLongMemEval(
+  text: string,
+  abilities?: ReadonlyArray<MemoryEvalAbility>,
+): ReadonlyArray<Case<MemoryEvalInput, string>> {
+  const parsed: unknown = JSON.parse(text);
+  if (!Array.isArray(parsed)) {
+    throw new Error('[graphorin/evals] LongMemEval dataset must be a JSON array of questions.');
+  }
+  const rows = parsed as ReadonlyArray<unknown>;
+  const out: Case<MemoryEvalInput, string>[] = [];
+  for (let i = 0; i < rows.length; i++) {
+    const row = rows[i];
+    if (row === null || typeof row !== 'object') {
+      throw new Error(`[graphorin/evals] LongMemEval row ${i} is not an object.`);
+    }
+    const mapped = mapRow(row as Record<string, unknown>, i);
+    if (abilities !== undefined && !abilities.includes(mapped.input.ability ?? 'info-extraction')) {
+      continue;
+    }
+    out.push(mapped);
+  }
+  return out;
+}
+
+function mapRow(row: Record<string, unknown>, index: number): Case<MemoryEvalInput, string> {
+  const question = asString(row.question);
+  if (question === undefined) {
+    throw new Error(`[graphorin/evals] LongMemEval row ${index} is missing a 'question'.`);
+  }
+  const questionId = asString(row.question_id) ?? `longmemeval-${index}`;
+  const questionType = asString(row.question_type) ?? '';
+  const askedAt = asString(row.question_date);
+  const ability = mapAbility(questionType, questionId);
+
+  const rawSessions = asArray(row.haystack_sessions);
+  const sessionIds = asArray(row.haystack_session_ids);
+  const sessionDates = asArray(row.haystack_dates);
+  const haystackSessions: MemoryEvalSession[] = [];
+  for (let s = 0; s < rawSessions.length; s++) {
+    const rawTurns = asArray(rawSessions[s]);
+    const date = asString(sessionDates[s]);
+    const turns: MemoryEvalTurn[] = [];
+    for (const t of rawTurns) {
+      if (t === null || typeof t !== 'object') continue;
+      const content = asString((t as Record<string, unknown>).content);
+      if (content === undefined) continue;
+      const role = (t as Record<string, unknown>).role === 'assistant' ? 'assistant' : 'user';
+      turns.push({ role, content, ...(date !== undefined ? { timestamp: date } : {}) });
+    }
+    haystackSessions.push({ id: asString(sessionIds[s]) ?? `session-${s}`, turns });
+  }
+
+  return {
+    id: questionId,
+    input: {
+      haystackSessions,
+      question,
+      ...(askedAt !== undefined ? { askedAt } : {}),
+      ability,
+    },
+    expected: asString(row.answer) ?? '',
+    metadata: { datasetName: 'longmemeval', ability, questionType },
+  };
+}
+
+/** Maps a LongMemEval `question_type` / `_abs` suffix onto a {@link MemoryEvalAbility}. */
+function mapAbility(questionType: string, questionId: string): MemoryEvalAbility {
+  const qt = questionType.toLowerCase();
+  if (questionId.endsWith('_abs') || qt.includes('abstention')) return 'abstention';
+  if (qt.includes('temporal')) return 'temporal';
+  if (qt.includes('knowledge-update') || qt.includes('knowledge_update')) return 'knowledge-update';
+  if (qt.includes('multi-session') || qt.includes('multi_session')) return 'multi-session';
+  return 'info-extraction';
+}
+
+function asString(value: unknown): string | undefined {
+  return typeof value === 'string' ? value : undefined;
+}
+
+function asArray(value: unknown): ReadonlyArray<unknown> {
+  return Array.isArray(value) ? (value as ReadonlyArray<unknown>) : [];
+}

package/src/loaders/memory-eval.ts

@@ -0,0 +1,80 @@
+/**
+ * Shared "memory system under test" contract used by the long-term
+ * memory eval loaders ({@link loadLongMemEvalDataset},
+ * {@link loadLocomoDataset}, {@link loadDmrDataset}) and the benchmark
+ * runner. A loader turns a dataset's native JSON into
+ * `Case<MemoryEvalInput, string>`: the input carries the haystack of
+ * prior sessions plus the question; the expected output is the
+ * reference answer string, graded by the `llmJudge` "J" scorer.
+ *
+ * This module is intentionally **type-only** and carries no dependency
+ * on `@graphorin/memory` - the concrete system-under-test agent lives
+ * in the benchmark package so `@graphorin/evals` stays a generic
+ * harness.
+ *
+ * @packageDocumentation
+ */
+
+/**
+ * One conversational turn inside a haystack session.
+ *
+ * @stable
+ */
+export interface MemoryEvalTurn {
+  readonly role: 'user' | 'assistant';
+  readonly content: string;
+  /** Dataset-native (often ISO-8601) timestamp, when the dataset provides one. */
+  readonly timestamp?: string;
+  /**
+   * Dataset-native speaker NAME (e.g. LOCOMO's `"Melanie"`), when the
+   * dataset provides one (W-022). Distinct from `role`: two-speaker
+   * datasets map onto user/assistant for machinery compatibility, but
+   * most LOCOMO questions reference the speakers by name, so the
+   * system under test must see the names in the ingested text.
+   */
+  readonly speaker?: string;
+}
+
+/**
+ * One prior session in the haystack the memory system must ingest
+ * before the question is asked.
+ *
+ * @stable
+ */
+export interface MemoryEvalSession {
+  readonly id: string;
+  readonly turns: ReadonlyArray<MemoryEvalTurn>;
+}
+
+/**
+ * The five LongMemEval abilities. Every loader maps its dataset-native
+ * category onto one of these so per-ability scoring is comparable
+ * across datasets; the raw category is preserved in `Case.metadata`.
+ *
+ * @stable
+ */
+export type MemoryEvalAbility =
+  | 'info-extraction'
+  | 'multi-session'
+  | 'temporal'
+  | 'knowledge-update'
+  | 'abstention';
+
+/**
+ * Input handed to the memory system under test for one eval case.
+ *
+ * @stable
+ */
+export interface MemoryEvalInput {
+  /** Prior sessions to ingest before the question is asked. */
+  readonly haystackSessions: ReadonlyArray<MemoryEvalSession>;
+  /** The question to answer from memory. */
+  readonly question: string;
+  /**
+   * When the question is asked. Drives temporal / knowledge-update
+   * reasoning; dataset-native string (not necessarily ISO-8601).
+   */
+  readonly askedAt?: string;
+  /** Mapped ability ({@link MemoryEvalAbility}); the raw category lives in metadata. */
+  readonly ability?: MemoryEvalAbility;
+}

package/src/regression.ts

@@ -0,0 +1,133 @@
+/**
+ * Compares two `EvalReport`s - typically the freshly-produced report
+ * vs a stored baseline - and lists every metric that regressed beyond
+ * the configured tolerance.
+ *
+ * @packageDocumentation
+ */
+
+import { pairedPassSignificance, passByBaseCase } from './stats.js';
+import type {
+  EvalReport,
+  RegressionFinding,
+  RegressionOptions,
+  RegressionReport,
+} from './types.js';
+
+const DEFAULT_PASS_RATE_DROP_PCT = 5;
+const DEFAULT_AVG_SCORE_DROP = 0.05;
+// EB-4: the avg-duration gate is OPT-IN (default off). Absolute wall-clock
+// budgets are environment-sensitive - a baseline recorded on a fast
+// workstation vs a loaded CI runner, or real LLM-latency jitter of whole
+// seconds, would flag spurious "regressions". Callers that genuinely want a
+// duration budget pass an explicit finite `maxAvgDurationIncreaseMs`.
+const DEFAULT_AVG_DURATION_INCREASE_MS = Number.POSITIVE_INFINITY;
+
+/**
+ * Detect regressions between `current` and `baseline` reports.
+ *
+ * @stable
+ */
+export function detectRegressions<I, O>(
+  current: EvalReport<I, O>,
+  baseline: EvalReport<I, O>,
+  options: RegressionOptions = {},
+): RegressionReport<I, O> {
+  const maxPassRateDropPct = options.maxPassRateDropPct ?? DEFAULT_PASS_RATE_DROP_PCT;
+  const maxAvgScoreDrop = options.maxAvgScoreDrop ?? DEFAULT_AVG_SCORE_DROP;
+  const maxAvgDurationIncreaseMs =
+    options.maxAvgDurationIncreaseMs ?? DEFAULT_AVG_DURATION_INCREASE_MS;
+  const findings: RegressionFinding[] = [];
+
+  const currentPassRate = total(current) === 0 ? 0 : current.summary.passed / total(current);
+  const baselinePassRate = total(baseline) === 0 ? 0 : baseline.summary.passed / total(baseline);
+  const passRateDropPct = (baselinePassRate - currentPassRate) * 100;
+  if (passRateDropPct > maxPassRateDropPct) {
+    // E8 (evals-05/08): the fixed tolerance is sample-size blind, so pair the
+    // shared cases and run McNemar's test. The p-value is always attached for
+    // the report; it VETOES the finding only under opt-in requireSignificance.
+    const significance = pairedPassSignificance(
+      passByBaseCase(caseOutcomes(current)),
+      passByBaseCase(caseOutcomes(baseline)),
+    );
+    const havePairs = significance.pairs > 0;
+    const alpha = options.significanceAlpha ?? 0.05;
+    const vetoed =
+      options.requireSignificance === true && havePairs && significance.pValue >= alpha;
+    if (!vetoed) {
+      findings.push({
+        kind: 'pass-rate-drop',
+        message:
+          `pass rate dropped by ${passRateDropPct.toFixed(2)}% ` +
+          `(${(currentPassRate * 100).toFixed(2)}% < baseline ${(baselinePassRate * 100).toFixed(2)}% ` +
+          `- tolerance ${maxPassRateDropPct.toFixed(2)}%)` +
+          (havePairs
+            ? ` [paired: ${significance.regressed} regressed / ${significance.improved} improved over ${significance.pairs} shared case(s), McNemar p=${significance.pValue.toFixed(4)}]`
+            : '') +
+          '.',
+        delta: -passRateDropPct,
+        ...(havePairs ? { pValue: significance.pValue } : {}),
+      });
+    }
+  }
+
+  const durationDelta = current.summary.avgDurationMs - baseline.summary.avgDurationMs;
+  if (durationDelta > maxAvgDurationIncreaseMs) {
+    findings.push({
+      kind: 'avg-duration-increase',
+      message:
+        `avg duration increased by ${durationDelta.toFixed(2)} ms ` +
+        `(${current.summary.avgDurationMs.toFixed(2)} ms > baseline ` +
+        `${baseline.summary.avgDurationMs.toFixed(2)} ms - tolerance ${maxAvgDurationIncreaseMs} ms).`,
+      delta: durationDelta,
+    });
+  }
+
+  for (const [scorer, baselineRow] of Object.entries(baseline.summary.byScorer)) {
+    const currentRow = current.summary.byScorer[scorer];
+    if (currentRow === undefined) {
+      findings.push({
+        kind: 'scorer-removed',
+        scorer,
+        message: `scorer '${scorer}' is in the baseline but missing from the current run.`,
+        delta: Number.NaN,
+      });
+      continue;
+    }
+    if (typeof baselineRow.avgScore === 'number' && typeof currentRow.avgScore === 'number') {
+      const drop = baselineRow.avgScore - currentRow.avgScore;
+      if (drop > maxAvgScoreDrop) {
+        findings.push({
+          kind: 'avg-score-drop',
+          scorer,
+          message:
+            `'${scorer}' avg score dropped by ${drop.toFixed(4)} ` +
+            `(${currentRow.avgScore.toFixed(4)} < baseline ${baselineRow.avgScore.toFixed(4)} ` +
+            `- tolerance ${maxAvgScoreDrop}).`,
+          delta: -drop,
+        });
+      }
+    }
+  }
+
+  return {
+    hasRegressions: findings.length > 0,
+    findings,
+    current,
+    baseline,
+  };
+}
+
+function total<I, O>(r: EvalReport<I, O>): number {
+  return r.summary.total;
+}
+
+/** Per-iteration case outcomes (a case passes when every scorer passed). */
+function caseOutcomes<I, O>(
+  r: EvalReport<I, O>,
+): ReadonlyArray<{ readonly caseId: string; readonly pass: boolean }> {
+  return r.results.map((c) => ({
+    caseId: c.caseId,
+    pass: c.scores.every((s) => s.result.pass),
+  }));
+}

package/src/reporters/html.ts

@@ -0,0 +1,83 @@
+/**
+ * HTML reporter. Renders an {@link EvalReport} as a self-contained
+ * HTML document - no external CSS / JS - so artifact viewers can
+ * render it directly. Designed to fit on a single page; for richer
+ * dashboards consume the JSON reporter instead.
+ *
+ * @packageDocumentation
+ */
+
+import type { EvalReport } from '@graphorin/observability/eval';
+
+/** @stable */
+export function renderHtmlReport<I, O>(
+  report: EvalReport<I, O>,
+  options: { readonly title?: string } = {},
+): string {
+  const title = escapeHtml(options.title ?? 'graphorin/evals report');
+  const summary = report.summary;
+  const passRate = summary.total === 0 ? 0 : (summary.passed / summary.total) * 100;
+  const scorerRows = Object.entries(summary.byScorer)
+    .map(([scorer, row]) => {
+      const avg = row.avgScore === null ? 'n/a' : row.avgScore.toFixed(4);
+      return `<tr><td><code>${escapeHtml(scorer)}</code></td><td>${row.passed}</td><td>${row.failed}</td><td>${avg}</td></tr>`;
+    })
+    .join('');
+  const failureRows = report.results
+    .flatMap((r) => {
+      const failures = r.scores.filter((s) => !s.result.pass);
+      return failures.map(
+        (f) =>
+          `<tr><td><code>${escapeHtml(r.caseId)}</code></td><td><code>${escapeHtml(f.scorer)}</code></td><td>${escapeHtml(f.result.reason ?? '')}</td></tr>`,
+      );
+    })
+    .join('');
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<title>${title}</title>
+<style>
+  body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; margin: 2rem; max-width: 64rem; color: #1a1a1a; }
+  h1 { margin-top: 0; }
+  table { border-collapse: collapse; width: 100%; margin: 1rem 0; }
+  th, td { padding: 0.4rem 0.6rem; text-align: left; border-bottom: 1px solid #e5e5e5; }
+  th { background: #f5f5f5; }
+  .summary-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(8rem, 1fr)); gap: 0.6rem; }
+  .summary-card { background: #f5f5f5; padding: 0.8rem; border-radius: 6px; }
+  .summary-card .label { font-size: 0.8rem; text-transform: uppercase; color: #666; letter-spacing: 0.04em; }
+  .summary-card .value { font-size: 1.4rem; font-weight: 600; }
+  code { font-family: 'SFMono-Regular', Consolas, 'Liberation Mono', Menlo, monospace; font-size: 0.9rem; }
+</style>
+</head>
+<body>
+<h1>${title}</h1>
+<div class="summary-grid">
+  <div class="summary-card"><div class="label">Total</div><div class="value">${summary.total}</div></div>
+  <div class="summary-card"><div class="label">Passed</div><div class="value">${summary.passed}</div></div>
+  <div class="summary-card"><div class="label">Failed</div><div class="value">${summary.failed}</div></div>
+  <div class="summary-card"><div class="label">Pass rate</div><div class="value">${passRate.toFixed(1)}%</div></div>
+  <div class="summary-card"><div class="label">Avg ms</div><div class="value">${summary.avgDurationMs.toFixed(0)}</div></div>
+</div>
+<h2>Per-scorer</h2>
+<table>
+  <thead><tr><th>Scorer</th><th>Pass</th><th>Fail</th><th>Avg score</th></tr></thead>
+  <tbody>${scorerRows || '<tr><td colspan="4">No scorers.</td></tr>'}</tbody>
+</table>
+<h2>Failures</h2>
+<table>
+  <thead><tr><th>Case</th><th>Scorer</th><th>Reason</th></tr></thead>
+  <tbody>${failureRows || '<tr><td colspan="3">No failures.</td></tr>'}</tbody>
+</table>
+</body>
+</html>`;
+}
+
+function escapeHtml(value: string): string {
+  return value
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&#39;');
+}

package/src/reporters/index.ts

@@ -0,0 +1,14 @@
+/**
+ * Barrel export for every shipped reporter. Each renderer takes an
+ * `EvalReport` and returns the canonical text representation; the
+ * caller decides where to write it (`writeFile`, `process.stdout`,
+ * GitHub Actions step summary, etc.).
+ *
+ * @packageDocumentation
+ */
+
+export { renderHtmlReport } from './html.js';
+export { renderJsonReport } from './json.js';
+export { renderJunitReport } from './junit.js';
+export { renderMarkdownReport } from './markdown.js';
+export { renderTerminalReport } from './terminal.js';