@cogitator-ai/evals 0.12.0 → 0.13.0

package/dist/reporters/html-format.js

@@ -0,0 +1,111 @@
+/** Escapes text for HTML content and attribute values */
+export function escapeHtml(value) {
+    return value
+        .replace(/&/g, '&')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&#39;');
+}
+/** Text that may be long or span lines: as it is when short, folded in `<details>` otherwise */
+export function foldable(value, max = 120) {
+    const line = value.replace(/\s+/g, ' ').trim();
+    if (line.length <= max && !value.includes('\n'))
+        return escapeHtml(value);
+    const summary = [...line]
+        .slice(0, max - 3)
+        .join('')
+        .trimEnd();
+    return `<details><summary>${escapeHtml(summary)}...</summary><pre>${escapeHtml(value)}</pre></details>`;
+}
+const STYLE = `
+:root {
+  color-scheme: light dark;
+  --bg: #ffffff;
+  --surface: #f5f6f8;
+  --text: #1c1f24;
+  --muted: #5b6370;
+  --border: #d9dde3;
+  --pass: #1a7f37;
+  --fail: #cf222e;
+  --warn: #9a6700;
+}
+@media (prefers-color-scheme: dark) {
+  :root {
+    --bg: #0f1115;
+    --surface: #181b21;
+    --text: #e6e8eb;
+    --muted: #9aa3ae;
+    --border: #2c313a;
+    --pass: #3fb950;
+    --fail: #f85149;
+    --warn: #d29922;
+  }
+}
+* { box-sizing: border-box; }
+body {
+  margin: 0;
+  padding: 24px 16px 48px;
+  background: var(--bg);
+  color: var(--text);
+  font: 14px/1.5 system-ui, -apple-system, 'Segoe UI', sans-serif;
+}
+main { max-width: 1100px; margin: 0 auto; }
+h1 { font-size: 22px; margin: 0 0 4px; }
+h2 { font-size: 16px; margin: 32px 0 8px; }
+.sub { color: var(--muted); margin: 0 0 16px; }
+.tiles { display: flex; flex-wrap: wrap; gap: 8px; }
+.tile { background: var(--surface); border: 1px solid var(--border); border-radius: 8px; padding: 8px 12px; min-width: 110px; }
+.tile b { display: block; font-size: 18px; }
+.tile span { color: var(--muted); font-size: 12px; }
+.scroll { overflow-x: auto; }
+table { border-collapse: collapse; width: 100%; }
+th, td { border-bottom: 1px solid var(--border); padding: 6px 8px; text-align: left; vertical-align: top; }
+th { color: var(--muted); font-weight: 600; font-size: 12px; }
+td.n, th.n { text-align: right; font-variant-numeric: tabular-nums; white-space: nowrap; }
+.pass { color: var(--pass); font-weight: 600; }
+.fail { color: var(--fail); font-weight: 600; }
+.warn { color: var(--warn); }
+.muted { color: var(--muted); }
+code { font: 12px ui-monospace, SFMono-Regular, Menlo, monospace; }
+pre { white-space: pre-wrap; word-break: break-word; margin: 6px 0 0; font: 12px ui-monospace, SFMono-Regular, Menlo, monospace; }
+details summary { cursor: pointer; }
+ul { padding-left: 20px; }
+`;
+/** A self-contained HTML page: inline CSS, no scripts and no external assets */
+export function htmlDocument(title, body) {
+    return [
+        '<!doctype html>',
+        '<html lang="en">',
+        '<head>',
+        '<meta charset="utf-8">',
+        '<meta name="viewport" content="width=device-width, initial-scale=1">',
+        `<title>${escapeHtml(title)}</title>`,
+        `<style>${STYLE}</style>`,
+        '</head>',
+        '<body>',
+        '<main>',
+        body,
+        '</main>',
+        '</body>',
+        '</html>',
+        '',
+    ].join('\n');
+}
+/** A table, its cells already HTML; `numeric` marks the columns aligned right */
+export function htmlTable(headers, rows, numeric = new Set()) {
+    const head = headers
+        .map((h, i) => `<th${numeric.has(i) ? ' class="n"' : ''}>${escapeHtml(h)}</th>`)
+        .join('');
+    const body = rows
+        .map((row) => `<tr>${row.map((c, i) => `<td${numeric.has(i) ? ' class="n"' : ''}>${c}</td>`).join('')}</tr>`)
+        .join('\n');
+    return `<div class="scroll"><table><thead><tr>${head}</tr></thead><tbody>\n${body}\n</tbody></table></div>`;
+}
+/** A row of figures, each value already HTML */
+export function htmlTiles(tiles) {
+    return `<div class="tiles">${tiles
+        .map(([label, value]) => `<div class="tile"><b>${value}</b><span>${escapeHtml(label)}</span></div>`)
+        .join('')}</div>`;
+}
+//# sourceMappingURL=html-format.js.map

package/dist/reporters/html-format.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html-format.js","sourceRoot":"","sources":["../../src/reporters/html-format.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,MAAM,UAAU,UAAU,CAAC,KAAa;IACtC,OAAO,KAAK;SACT,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAC5B,CAAC;AAED,gGAAgG;AAChG,MAAM,UAAU,QAAQ,CAAC,KAAa,EAAE,MAAc,GAAG;IACvD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;IAC1E,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC;SACtB,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC;SACjB,IAAI,CAAC,EAAE,CAAC;SACR,OAAO,EAAE,CAAC;IACb,OAAO,qBAAqB,UAAU,CAAC,OAAO,CAAC,qBAAqB,UAAU,CAAC,KAAK,CAAC,kBAAkB,CAAC;AAC1G,CAAC;AAED,MAAM,KAAK,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqDb,CAAC;AAEF,gFAAgF;AAChF,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,IAAY;IACtD,OAAO;QACL,iBAAiB;QACjB,kBAAkB;QAClB,QAAQ;QACR,wBAAwB;QACxB,sEAAsE;QACtE,UAAU,UAAU,CAAC,KAAK,CAAC,UAAU;QACrC,UAAU,KAAK,UAAU;QACzB,SAAS;QACT,QAAQ;QACR,QAAQ;QACR,IAAI;QACJ,SAAS;QACT,SAAS;QACT,SAAS;QACT,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,SAAS,CACvB,OAA0B,EAC1B,IAAoC,EACpC,UAA+B,IAAI,GAAG,EAAE;IAExC,MAAM,IAAI,GAAG,OAAO;SACjB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,IAAI,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC;SAC/E,IAAI,CAAC,EAAE,CAAC,CAAC;IACZ,MAAM,IAAI,GAAG,IAAI;SACd,GAAG,CACF,CAAC,GAAG,EAAE,EAAE,CACN,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,CACjG;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,OAAO,yCAAyC,IAAI,yBAAyB,IAAI,0BAA0B,CAAC;AAC9G,CAAC;AAED,gDAAgD;AAChD,MAAM,UAAU,SAAS,CAAC,KAA6D;IACrF,OAAO,sBAAsB,KAAK;SAC/B,GAAG,CACF,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,wBAAwB,KAAK,aAAa,UAAU,CAAC,KAAK,CAAC,eAAe,CAC/F;SACA,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC;AACtB,CAAC"}

package/dist/reporters/html.d.ts

@@ -0,0 +1,12 @@
+import type { EvalSuiteResult } from './index.js';
+/**
+ * A self-contained HTML page of a suite run: the totals, the aggregated metrics, the assertions
+ * and every case with its scores and the failures of each of its trials. Inline CSS for light and
+ * dark color schemes, no scripts, no external assets, and every value from the run escaped
+ */
+export declare function htmlPage(result: EvalSuiteResult): string;
+/** Writes the HTML page of a suite run */
+export declare function htmlReport(result: EvalSuiteResult, options: {
+    path: string;
+}): void;
+//# sourceMappingURL=html.d.ts.map

package/dist/reporters/html.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"html.d.ts","sourceRoot":"","sources":["../../src/reporters/html.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAgB,MAAM,SAAS,CAAC;AAmD7D;;;;GAIG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CAqFxD;AAED,0CAA0C;AAC1C,wBAAgB,UAAU,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAEnF"}

package/dist/reporters/html.js

@@ -0,0 +1,119 @@
+import { writeFileSync } from 'node:fs';
+import { caseFlags, caseProblems, num, trialsPerCase } from './format.js';
+import { escapeHtml, foldable, htmlDocument, htmlTable, htmlTiles } from './html-format.js';
+function scoresCell(r) {
+    if (r.scores.length === 0)
+        return '<span class="muted">-</span>';
+    return r.scores
+        .map((s) => {
+        const failed = s.error !== undefined ? ` <span class="fail">${escapeHtml(s.error)}</span>` : '';
+        return `<code>${escapeHtml(s.name)}</code> ${escapeHtml(num(s.score))}${failed}`;
+    })
+        .join('<br>');
+}
+function statusCell(r) {
+    const parts = [];
+    if (r.error !== undefined)
+        parts.push(`<span class="fail">error</span> ${foldable(r.error, 80)}`);
+    else
+        parts.push('<span class="pass">answered</span>');
+    for (const flag of caseFlags(r))
+        parts.push(`<span class="warn">${escapeHtml(flag)}</span>`);
+    if (r.trials && r.trials.length > 1) {
+        const answered = r.trials.filter((t) => t.error === undefined).length;
+        parts.push(`<span class="muted">${answered} of ${r.trials.length} trials answered</span>`);
+        for (const problem of caseProblems(r)) {
+            parts.push(`<span class="fail">${foldable(problem, 80)}</span>`);
+        }
+    }
+    return parts.join('<br>');
+}
+function outputCell(r) {
+    if (!r.turns || r.turns.length === 0)
+        return foldable(r.output);
+    const conversation = r.turns
+        .map((turn) => `user: ${turn.input}\nagent: ${turn.error !== undefined ? `(error) ${turn.error}` : turn.output}`)
+        .join('\n\n');
+    return `<details><summary>${r.turns.length} turns</summary><pre>${escapeHtml(conversation)}</pre></details>`;
+}
+function runCell(r) {
+    const parts = [];
+    if (r.traceId !== undefined)
+        parts.push(`<code>${escapeHtml(r.traceId)}</code>`);
+    if (r.prompt) {
+        const version = r.prompt.version !== undefined ? ` v${r.prompt.version}` : '';
+        parts.push(`<span class="muted">${escapeHtml(r.prompt.key + version)}</span>`);
+    }
+    return parts.length > 0 ? parts.join('<br>') : '<span class="muted">-</span>';
+}
+/**
+ * A self-contained HTML page of a suite run: the totals, the aggregated metrics, the assertions
+ * and every case with its scores and the failures of each of its trials. Inline CSS for light and
+ * dark color schemes, no scripts, no external assets, and every value from the run escaped
+ */
+export function htmlPage(result) {
+    const { stats } = result;
+    const title = result.name === undefined ? 'Eval results' : `Eval results: ${result.name}`;
+    const passed = result.assertions.filter((a) => a.passed).length;
+    const failed = result.assertions.length - passed;
+    const trials = trialsPerCase(result);
+    const sections = [
+        `<h1>${escapeHtml(title)}</h1>`,
+        ...(result.runId !== undefined
+            ? [`<p class="sub">Run <code>${escapeHtml(result.runId)}</code></p>`]
+            : []),
+        htmlTiles([
+            ['cases', escapeHtml(String(stats.total))],
+            ...(trials > 1 ? [['trials per case', escapeHtml(String(trials))]] : []),
+            ['errored', escapeHtml(String(stats.errors ?? 0))],
+            ...(stats.metricErrors
+                ? [['metric errors', escapeHtml(String(stats.metricErrors))]]
+                : []),
+            ['duration', `${escapeHtml(String(stats.duration))} ms`],
+            ['cost', `${escapeHtml(num(stats.cost))} USD`],
+            ...(result.assertions.length > 0
+                ? [
+                    [
+                        'assertions',
+                        `<span class="${failed > 0 ? 'fail' : 'pass'}">${passed} / ${result.assertions.length}</span>`,
+                    ],
+                ]
+                : []),
+        ]),
+    ];
+    const metrics = Object.values(result.aggregated);
+    if (metrics.length > 0) {
+        sections.push('<h2>Metrics</h2>', htmlTable(['Metric', 'Mean', 'P50', 'P95', 'Min', 'Max', 'Cases'], metrics.map((m) => [
+            `<code>${escapeHtml(m.name)}</code>`,
+            ...[m.mean, m.p50, m.p95, m.min, m.max].map((v) => escapeHtml(num(v))),
+            m.count === undefined ? '-' : escapeHtml(String(m.count)),
+        ]), new Set([1, 2, 3, 4, 5, 6])));
+    }
+    if (result.assertions.length > 0) {
+        sections.push('<h2>Assertions</h2>', htmlTable(['Result', 'Assertion', 'Details'], result.assertions.map((a) => [
+            a.passed ? '<span class="pass">pass</span>' : '<span class="fail">fail</span>',
+            `<code>${escapeHtml(a.name)}</code>`,
+            escapeHtml(a.message),
+        ])));
+    }
+    if (result.results.length > 0) {
+        const showRun = result.results.some((r) => r.traceId !== undefined || r.prompt !== undefined);
+        sections.push('<h2>Cases</h2>', htmlTable(['#', 'Input', 'Expected', 'Output', 'Scores', 'Status', ...(showRun ? ['Trace'] : [])], result.results.map((r, i) => [
+            String(i + 1),
+            foldable(r.case.input),
+            r.case.expected === undefined
+                ? '<span class="muted">-</span>'
+                : foldable(r.case.expected),
+            outputCell(r),
+            scoresCell(r),
+            statusCell(r),
+            ...(showRun ? [runCell(r)] : []),
+        ]), new Set([0])));
+    }
+    return htmlDocument(title, sections.join('\n'));
+}
+/** Writes the HTML page of a suite run */
+export function htmlReport(result, options) {
+    writeFileSync(options.path, htmlPage(result), 'utf-8');
+}
+//# sourceMappingURL=html.js.map

package/dist/reporters/html.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"html.js","sourceRoot":"","sources":["../../src/reporters/html.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACvE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAEzF,SAAS,UAAU,CAAC,CAAe;IACjC,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,8BAA8B,CAAC;IACjE,OAAO,CAAC,CAAC,MAAM;SACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACT,MAAM,MAAM,GACV,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,uBAAuB,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QACnF,OAAO,SAAS,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,EAAE,CAAC;IACnF,CAAC,CAAC;SACD,IAAI,CAAC,MAAM,CAAC,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,CAAe;IACjC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,mCAAmC,QAAQ,CAAC,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;;QAC7F,KAAK,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;IACtD,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,CAAC,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,sBAAsB,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC7F,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;QACtE,KAAK,CAAC,IAAI,CAAC,uBAAuB,QAAQ,OAAO,CAAC,CAAC,MAAM,CAAC,MAAM,yBAAyB,CAAC,CAAC;QAC3F,KAAK,MAAM,OAAO,IAAI,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC,sBAAsB,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,UAAU,CAAC,CAAe;IACjC,IAAI,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK;SACzB,GAAG,CACF,CAAC,IAAI,EAAE,EAAE,CACP,SAAS,IAAI,CAAC,KAAK,YAAY,IAAI,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,CACpG;SACA,IAAI,CAAC,MAAM,CAAC,CAAC;IAChB,OAAO,qBAAqB,CAAC,CAAC,KAAK,CAAC,MAAM,wBAAwB,UAAU,CAAC,YAAY,CAAC,kBAAkB,CAAC;AAC/G,CAAC;AAED,SAAS,OAAO,CAAC,CAAe;IAC9B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,CAAC,CAAC,OAAO,KAAK,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,SAAS,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACjF,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9E,KAAK,CAAC,IAAI,CAAC,uBAAuB,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,8BAA8B,CAAC;AAChF,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,QAAQ,CAAC,MAAuB;IAC9C,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,iBAAiB,MAAM,CAAC,IAAI,EAAE,CAAC;IAC1F,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;IACjD,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAErC,MAAM,QAAQ,GAAa;QACzB,OAAO,UAAU,CAAC,KAAK,CAAC,OAAO;QAC/B,GAAG,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS;YAC5B,CAAC,CAAC,CAAC,4BAA4B,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC;YACrE,CAAC,CAAC,EAAE,CAAC;QACP,SAAS,CAAC;YACR,CAAC,OAAO,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC1C,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,EAAE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACjF,CAAC,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;YAClD,GAAG,CAAC,KAAK,CAAC,YAAY;gBACpB,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAU,CAAC;gBACtE,CAAC,CAAC,EAAE,CAAC;YACP,CAAC,UAAU,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC;YACxD,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YAC9C,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;gBAC9B,CAAC,CAAC;oBACE;wBACE,YAAY;wBACZ,gBAAgB,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,SAAS;qBACtF;iBACX;gBACH,CAAC,CAAC,EAAE,CAAC;SACR,CAAC;KACH,CAAC;IAEF,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,QAAQ,CAAC,IAAI,CACX,kBAAkB,EAClB,SAAS,CACP,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,EACvD,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACjB,SAAS,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS;YACpC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACtE,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;SAC1D,CAAC,EACF,IAAI,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAC5B,CACF,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,QAAQ,CAAC,IAAI,CACX,qBAAqB,EACrB,SAAS,CACP,CAAC,QAAQ,EAAE,WAAW,EAAE,SAAS,CAAC,EAClC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YAC3B,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,gCAAgC,CAAC,CAAC,CAAC,gCAAgC;YAC9E,SAAS,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS;YACpC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC;SACtB,CAAC,CACH,CACF,CAAC;IACJ,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;QAC9F,QAAQ,CAAC,IAAI,CACX,gBAAgB,EAChB,SAAS,CACP,CAAC,GAAG,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EACvF,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC;YACb,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YACtB,CAAC,CAAC,IAAI,CAAC,QAAQ,KAAK,SAAS;gBAC3B,CAAC,CAAC,8BAA8B;gBAChC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;YAC7B,UAAU,CAAC,CAAC,CAAC;YACb,UAAU,CAAC,CAAC,CAAC;YACb,UAAU,CAAC,CAAC,CAAC;YACb,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SACjC,CAAC,EACF,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CACb,CACF,CAAC;IACJ,CAAC;IAED,OAAO,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,0CAA0C;AAC1C,MAAM,UAAU,UAAU,CAAC,MAAuB,EAAE,OAAyB;IAC3E,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AACzD,CAAC"}

package/dist/reporters/index.d.ts

@@ -1,5 +1,5 @@
 import type { EvalStats } from '../assertions/index.js';
-import type { EvalToolCall, EvalToolResult, EvalTurnResult, Passage } from '../metrics/types.js';
+import type { EvalPrompt, EvalToolCall, EvalToolResult, EvalTurnResult, Passage } from '../metrics/types.js';
 export interface AggregatedMetric {
     name: string;
     mean: number;
@@ -46,6 +46,10 @@ export interface ReportedCase {
     retrieved?: readonly Passage[];
     /** Each turn of a multi-turn case */
     turns?: readonly EvalTurnResult[];
+    /** The trace of the agent run, the last turn's for a multi-turn case */
+    traceId?: string;
+    /** The versioned instructions the agent run used, the last turn's for a multi-turn case */
+    prompt?: EvalPrompt;
     scores: Array<{
         name: string;
         score: number;
@@ -56,21 +60,29 @@ export interface ReportedCase {
     trials?: readonly ReportedCase[];
 }
 export interface EvalSuiteResult {
+    /** The suite's `name`, the title of the markdown and html reports */
+    name?: string;
+    /** The id of the run */
+    runId?: string;
     results: ReportedCase[];
     aggregated: Record<string, AggregatedMetric>;
     assertions: AssertionResult[];
     stats: EvalStats;
 }
-export type ReporterType = 'console' | 'json' | 'csv' | 'ci' | 'junit' | 'github';
+export type ReporterType = 'console' | 'json' | 'csv' | 'ci' | 'junit' | 'github' | 'markdown' | 'html';
 export type ReporterOptions = {
     /**
-     * Base path of file reports. Each file reporter writes it with its own
-     * extension, so `./reports/eval` gives `eval.json`, `eval.csv` and `eval.xml`.
+     * Base path of file reports. Each file reporter writes it with its own extension, so
+     * `./reports/eval` gives `eval.json`, `eval.csv`, `eval.xml`, `eval.md` and `eval.html`.
      */
     path?: string;
 };
-/** The file a reporter writes: the base path with the reporter's extension. */
-export declare function reportPath(base: string | undefined, extension: 'json' | 'csv' | 'xml'): string;
+/** The extension of a file report */
+export type ReportExtension = 'json' | 'csv' | 'xml' | 'md' | 'html';
+/** The file a reporter writes: the base path, without a report extension, with the reporter's */
+export declare function reportPath(base: string | undefined, extension: ReportExtension, fallback?: string): string;
+/** Creates the directory of a report file, returning the path */
+export declare function prepare(path: string): string;
 /**
  * Runs the reporters in the order given, except `ci`, which runs last so its verdict is the
  * final line of the output. `ci` sets `process.exitCode` to 1 when an assertion failed instead
@@ -83,4 +95,6 @@ export { csvReport } from './csv.js';
 export { ciReport } from './ci.js';
 export { junitReport } from './junit.js';
 export { githubReport } from './github.js';
+export { markdownReport, markdownSummary } from './markdown.js';
+export { htmlReport, htmlPage } from './html.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/reporters/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE9F,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,oEAAoE;IACpE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,oDAAoD;IACpD,WAAW,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IACxC,qCAAqC;IACrC,SAAS,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC/B,qCAAqC;IACrC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjF,+FAA+F;IAC/F,MAAM,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,GAAG,OAAO,GAAG,QAAQ,CAAC;AAClF,MAAM,MAAM,eAAe,GAAG;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,+EAA+E;AAC/E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAG9F;AAOD;;;;GAIG;AACH,wBAAgB,MAAM,CACpB,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EACnC,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CA0BN;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,KAAK,EACV,UAAU,EACV,YAAY,EACZ,cAAc,EACd,cAAc,EACd,OAAO,EACR,MAAM,kBAAkB,CAAC;AAE1B,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,oEAAoE;IACpE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,oDAAoD;IACpD,WAAW,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IACxC,qCAAqC;IACrC,SAAS,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC/B,qCAAqC;IACrC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,2FAA2F;IAC3F,MAAM,CAAC,EAAE,UAAU,CAAC;IACpB,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjF,+FAA+F;IAC/F,MAAM,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,qEAAqE;IACrE,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,wBAAwB;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,MAAM,YAAY,GACtB,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,GAAG,OAAO,GAAG,QAAQ,GAAG,UAAU,GAAG,MAAM,CAAC;AAC/E,MAAM,MAAM,eAAe,GAAG;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,qCAAqC;AACrC,MAAM,MAAM,eAAe,GAAG,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,IAAI,GAAG,MAAM,CAAC;AAIrE,iGAAiG;AACjG,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,GAAG,SAAS,EACxB,SAAS,EAAE,eAAe,EAC1B,QAAQ,GAAE,MAAqB,GAC9B,MAAM,CAGR;AAED,iEAAiE;AACjE,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAG5C;AAED;;;;GAIG;AACH,wBAAgB,MAAM,CACpB,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EACnC,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CAgCN;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC"}

package/dist/reporters/index.js

@@ -6,13 +6,16 @@ import { csvReport } from './csv.js';
 import { ciReport } from './ci.js';
 import { junitReport } from './junit.js';
 import { githubReport } from './github.js';
+import { markdownReport } from './markdown.js';
+import { htmlReport } from './html.js';
 const DEFAULT_BASE = 'eval-report';
-/** The file a reporter writes: the base path with the reporter's extension. */
-export function reportPath(base, extension) {
-    const stem = (base ?? DEFAULT_BASE).replace(/\.(json|csv|xml)$/i, '');
+/** The file a reporter writes: the base path, without a report extension, with the reporter's */
+export function reportPath(base, extension, fallback = DEFAULT_BASE) {
+    const stem = (base ?? fallback).replace(/\.(json|csv|xml|md|html)$/i, '');
     return `${stem}.${extension}`;
 }
-function prepare(path) {
+/** Creates the directory of a report file, returning the path */
+export function prepare(path) {
     mkdirSync(dirname(path), { recursive: true });
     return path;
 }
@@ -41,6 +44,12 @@ export function report(result, type, options) {
             case 'github':
                 githubReport(result);
                 break;
+            case 'markdown':
+                markdownReport(result, { path: prepare(reportPath(options?.path, 'md')) });
+                break;
+            case 'html':
+                htmlReport(result, { path: prepare(reportPath(options?.path, 'html')) });
+                break;
             case 'ci':
                 ciReport(result);
                 break;
@@ -53,4 +62,6 @@ export { csvReport } from './csv.js';
 export { ciReport } from './ci.js';
 export { junitReport } from './junit.js';
 export { githubReport } from './github.js';
+export { markdownReport, markdownSummary } from './markdown.js';
+export { htmlReport, htmlPage } from './html.js';
 //# sourceMappingURL=index.js.map

package/dist/reporters/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAsExC,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC,+EAA+E;AAC/E,MAAM,UAAU,UAAU,CAAC,IAAwB,EAAE,SAAiC;IACpF,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;IACtE,OAAO,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,MAAM,CACpB,MAAuB,EACvB,IAAmC,EACnC,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAEzF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,aAAa,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,KAAK;gBACR,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,OAAO;gBACV,WAAW,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,QAAQ;gBACX,YAAY,CAAC,MAAM,CAAC,CAAC;gBACrB,MAAM;YACR,KAAK,IAAI;gBACP,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACjB,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAwFpC,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC,iGAAiG;AACjG,MAAM,UAAU,UAAU,CACxB,IAAwB,EACxB,SAA0B,EAC1B,WAAmB,YAAY;IAE/B,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,QAAQ,CAAC,CAAC,OAAO,CAAC,4BAA4B,EAAE,EAAE,CAAC,CAAC;IAC1E,OAAO,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,MAAM,CACpB,MAAuB,EACvB,IAAmC,EACnC,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAEzF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,aAAa,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,KAAK;gBACR,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,OAAO;gBACV,WAAW,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,QAAQ;gBACX,YAAY,CAAC,MAAM,CAAC,CAAC;gBACrB,MAAM;YACR,KAAK,UAAU;gBACb,cAAc,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC3E,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,IAAI;gBACP,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACjB,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AACxC,OAAO,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC"}

package/dist/reporters/markdown.d.ts

@@ -0,0 +1,15 @@
+import type { EvalSuiteResult } from './index.js';
+/** Most failures the summary lists before it counts the rest */
+export declare const MARKDOWN_MAX_FAILURES = 25;
+/**
+ * A compact Markdown summary of a suite run for a pull request comment: the totals, the mean of
+ * each metric with the cases it covers, the assertions and the failures of every case and trial.
+ * Every value from the run is escaped, so no answer can break the tables or hide the rest of the
+ * comment
+ */
+export declare function markdownSummary(result: EvalSuiteResult): string;
+/** Writes the Markdown summary of a suite run */
+export declare function markdownReport(result: EvalSuiteResult, options: {
+    path: string;
+}): void;
+//# sourceMappingURL=markdown.d.ts.map

package/dist/reporters/markdown.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/reporters/markdown.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAG/C,gEAAgE;AAChE,eAAO,MAAM,qBAAqB,KAAK,CAAC;AAUxC;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CAkD/D;AAED,iDAAiD;AACjD,wBAAgB,cAAc,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAEvF"}

package/dist/reporters/markdown.js

@@ -0,0 +1,67 @@
+import { writeFileSync } from 'node:fs';
+import { caseLabel, caseProblems, cell, num, trialsPerCase } from './format.js';
+/** Most failures the summary lists before it counts the rest */
+export const MARKDOWN_MAX_FAILURES = 25;
+/** The failures of the cases, of every trial: errored runs, then metrics that failed */
+function failures(result) {
+    return result.results.flatMap((r, i) => {
+        const label = caseLabel(i, r.case.input);
+        return caseProblems(r).map((problem) => ({ label, problem }));
+    });
+}
+/**
+ * A compact Markdown summary of a suite run for a pull request comment: the totals, the mean of
+ * each metric with the cases it covers, the assertions and the failures of every case and trial.
+ * Every value from the run is escaped, so no answer can break the tables or hide the rest of the
+ * comment
+ */
+export function markdownSummary(result) {
+    const { stats } = result;
+    const passed = result.assertions.filter((a) => a.passed).length;
+    const failed = result.assertions.length - passed;
+    const trials = trialsPerCase(result);
+    const totals = [
+        `**${stats.total}** cases${trials > 1 ? ` x **${trials}** trials` : ''}`,
+        `**${stats.errors ?? 0}** errored`,
+        ...(stats.metricErrors ? [`**${stats.metricErrors}** metric errors`] : []),
+        `${stats.duration} ms`,
+        `${num(stats.cost)} USD`,
+        ...(result.assertions.length > 0
+            ? [`assertions: **${passed}** passed, **${failed}** failed`]
+            : []),
+    ];
+    const title = result.name === undefined ? 'Eval results' : `Eval results: ${cell(result.name)}`;
+    const lines = [`### ${title}`, '', totals.join(' | '), ''];
+    const metrics = Object.values(result.aggregated);
+    if (metrics.length > 0) {
+        lines.push('| Metric | Mean | Cases |', '| --- | ---: | ---: |');
+        for (const m of metrics) {
+            lines.push(`| ${cell(m.name)} | ${num(m.mean)} | ${m.count === undefined ? '-' : m.count} |`);
+        }
+        lines.push('');
+    }
+    if (result.assertions.length > 0) {
+        lines.push('| Assertion | Result | Details |', '| --- | --- | --- |');
+        for (const a of result.assertions) {
+            lines.push(`| ${cell(a.name)} | ${a.passed ? 'pass' : '**fail**'} | ${cell(a.message)} |`);
+        }
+        lines.push('');
+    }
+    const failedCases = failures(result);
+    if (failedCases.length > 0) {
+        lines.push(`**Failures** (${failedCases.length})`, '', '| Case | Problem |', '| --- | --- |');
+        for (const { label, problem } of failedCases.slice(0, MARKDOWN_MAX_FAILURES)) {
+            lines.push(`| ${cell(label)} | ${cell(problem)} |`);
+        }
+        if (failedCases.length > MARKDOWN_MAX_FAILURES) {
+            lines.push('', `and ${failedCases.length - MARKDOWN_MAX_FAILURES} more`);
+        }
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+/** Writes the Markdown summary of a suite run */
+export function markdownReport(result, options) {
+    writeFileSync(options.path, markdownSummary(result), 'utf-8');
+}
+//# sourceMappingURL=markdown.js.map

package/dist/reporters/markdown.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/reporters/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,IAAI,EAAE,GAAG,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE7E,gEAAgE;AAChE,MAAM,CAAC,MAAM,qBAAqB,GAAG,EAAE,CAAC;AAExC,wFAAwF;AACxF,SAAS,QAAQ,CAAC,MAAuB;IACvC,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACrC,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,eAAe,CAAC,MAAuB;IACrD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;IACjD,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IAErC,MAAM,MAAM,GAAG;QACb,KAAK,KAAK,CAAC,KAAK,WAAW,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,MAAM,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE;QACxE,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,YAAY;QAClC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,YAAY,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,KAAK,CAAC,QAAQ,KAAK;QACtB,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM;QACxB,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;YAC9B,CAAC,CAAC,CAAC,iBAAiB,MAAM,gBAAgB,MAAM,WAAW,CAAC;YAC5D,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;IAEF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,iBAAiB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;IAChG,MAAM,KAAK,GAAG,CAAC,OAAO,KAAK,EAAE,EAAE,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;IAE3D,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CAAC,2BAA2B,EAAE,uBAAuB,CAAC,CAAC;QACjE,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;QAChG,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,kCAAkC,EAAE,qBAAqB,CAAC,CAAC;QACtE,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,MAAM,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;IACrC,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,iBAAiB,WAAW,CAAC,MAAM,GAAG,EAAE,EAAE,EAAE,oBAAoB,EAAE,eAAe,CAAC,CAAC;QAC9F,KAAK,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC,EAAE,CAAC;YAC7E,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACtD,CAAC;QACD,IAAI,WAAW,CAAC,MAAM,GAAG,qBAAqB,EAAE,CAAC;YAC/C,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,WAAW,CAAC,MAAM,GAAG,qBAAqB,OAAO,CAAC,CAAC;QAC3E,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,iDAAiD;AACjD,MAAM,UAAU,cAAc,CAAC,MAAuB,EAAE,OAAyB;IAC/E,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC"}

package/dist/runs/compare.d.ts

@@ -0,0 +1,103 @@
+import { z } from 'zod';
+import type { ComparisonTest } from '../comparison-stats.js';
+import type { EvalRunSource } from './run.js';
+export declare const CompareRunsOptionsSchema: z.ZodObject<{
+    alpha: z.ZodDefault<z.ZodNumber>;
+    test: z.ZodDefault<z.ZodEnum<{
+        t: "t";
+        wilcoxon: "wilcoxon";
+    }>>;
+    seed: z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>>;
+    metrics: z.ZodOptional<z.ZodArray<z.ZodString>>;
+}, z.core.$strip>;
+export interface CompareRunsOptions {
+    /**
+     * Significance level, 0.05 by default. A metric is significant when its p-value, adjusted with
+     * Holm's correction across the compared metrics, is below it, and its `ci` covers `1 - alpha`
+     */
+    alpha?: number;
+    /** Test of metrics that are not binary: `t` (paired t-test, the default) or `wilcoxon` */
+    test?: 't' | 'wilcoxon';
+    /** Seeds the bootstrap of each metric's `ci`, so the intervals are the same on every call */
+    seed?: number | string;
+    /**
+     * The metrics to compare. Default: every per-case metric both runs have and scored on at least
+     * one matched case, the others named in `warnings`, so a metric without data neither shows two
+     * means of 0 nor weakens the Holm correction of the others
+     */
+    metrics?: string[];
+}
+export type RunWinner = 'base' | 'head' | 'tie';
+/** How the score of a case moved from the base run to the head run */
+export interface RunCaseChange {
+    caseKey: string;
+    /** The case input on one line, shortened */
+    input: string;
+    base: number;
+    head: number;
+    /** `head` minus `base` */
+    delta: number;
+}
+export interface RunMetricComparison {
+    /** Mean score of the base run over the compared cases, NaN when `n` is 0 */
+    base: number;
+    head: number;
+    /** Cases compared: matched cases at least one run scored, an errored side counting as a 0 */
+    n: number;
+    /** Mean of the paired differences, head minus base */
+    difference: number;
+    /** Percentile bootstrap confidence interval of `difference` at `1 - alpha` */
+    ci: {
+        low: number;
+        high: number;
+    };
+    /** Cohen's d of the paired differences (dz), NaN with fewer than 2 cases */
+    effectSize: number;
+    test: ComparisonTest;
+    pValue: number;
+    /** `pValue` after Holm's correction across the compared metrics */
+    adjustedPValue: number;
+    significant: boolean;
+    /** The higher run when significant, by the test's direction */
+    winner: RunWinner;
+    /** Cases whose score rose, the largest rise first */
+    improved: RunCaseChange[];
+    /** Cases whose score fell, the largest fall first */
+    regressed: RunCaseChange[];
+}
+/** Which run a side of a comparison is */
+export interface RunRef {
+    runId: string;
+    name?: string;
+    startedAt: string;
+}
+/** A case only one of the runs has */
+export interface RunCaseRef {
+    caseKey: string;
+    input: string;
+}
+export interface RunComparison {
+    base: RunRef;
+    head: RunRef;
+    /** The run more metrics name as their winner, a tie when as many name each */
+    winner: RunWinner;
+    metrics: Record<string, RunMetricComparison>;
+    /** Cases both runs have, matched by `caseKey` */
+    matched: number;
+    /** Cases only the head run has */
+    added: RunCaseRef[];
+    /** Cases only the base run has */
+    removed: RunCaseRef[];
+    /**
+     * What makes the runs harder to compare: other cases, other prompts, other metrics or judge,
+     * and metrics left out because no matched case was scored on them
+     */
+    warnings: string[];
+}
+/**
+ * Compares two saved runs, or suite results, case by case: cases are matched by their `caseKey`,
+ * and each metric both runs have is compared as `EvalComparison` compares its sides, the base run
+ * in the baseline's place and the head run in the challenger's
+ */
+export declare function compareRuns(baseSource: EvalRunSource, headSource: EvalRunSource, options?: CompareRunsOptions): RunComparison;
+//# sourceMappingURL=compare.d.ts.map

package/dist/runs/compare.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compare.d.ts","sourceRoot":"","sources":["../../src/runs/compare.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AASxB,OAAO,KAAK,EAAE,cAAc,EAAoB,MAAM,qBAAqB,CAAC;AAI5E,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,OAAO,CAAC;AAG3C,eAAO,MAAM,wBAAwB;;;;;;;;iBAKnC,CAAC;AAEH,MAAM,WAAW,kBAAkB;IACjC;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0FAA0F;IAC1F,IAAI,CAAC,EAAE,GAAG,GAAG,UAAU,CAAC;IACxB,6FAA6F;IAC7F,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IACvB;;;;OAIG;IACH,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,MAAM,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,KAAK,CAAC;AAEhD,sEAAsE;AACtE,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,4CAA4C;IAC5C,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,0BAA0B;IAC1B,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,mBAAmB;IAClC,4EAA4E;IAC5E,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,6FAA6F;IAC7F,CAAC,EAAE,MAAM,CAAC;IACV,sDAAsD;IACtD,UAAU,EAAE,MAAM,CAAC;IACnB,8EAA8E;IAC9E,EAAE,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAClC,4EAA4E;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,cAAc,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,OAAO,CAAC;IACrB,+DAA+D;IAC/D,MAAM,EAAE,SAAS,CAAC;IAClB,qDAAqD;IACrD,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,qDAAqD;IACrD,SAAS,EAAE,aAAa,EAAE,CAAC;CAC5B;AAED,0CAA0C;AAC1C,MAAM,WAAW,MAAM;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,sCAAsC;AACtC,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,8EAA8E;IAC9E,MAAM,EAAE,SAAS,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;IAC7C,iDAAiD;IACjD,OAAO,EAAE,MAAM,CAAC;IAChB,kCAAkC;IAClC,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,kCAAkC;IAClC,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB;;;OAGG;IACH,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AA8JD;;;;GAIG;AACH,wBAAgB,WAAW,CACzB,UAAU,EAAE,aAAa,EACzB,UAAU,EAAE,aAAa,EACzB,OAAO,GAAE,kBAAuB,GAC/B,aAAa,CAyDf"}