@cogitator-ai/evals 0.8.0 → 0.10.0

package/dist/reporters/github.js

@@ -0,0 +1,95 @@
+import { appendFileSync } from 'node:fs';
+import { caseFlags, caseLabel, trialsPerCase } from './format.js';
+/** Escapes the message of a workflow command */
+function escapeData(value) {
+    return value.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A');
+}
+/** Escapes a workflow command property, where `:` and `,` are separators */
+function escapeProperty(value) {
+    return escapeData(value).replace(/:/g, '%3A').replace(/,/g, '%2C');
+}
+function command(kind, title, message) {
+    return `::${kind} title=${escapeProperty(title)}::${escapeData(message)}`;
+}
+/** The workflow commands that annotate the run: failures as errors, metric errors as warnings */
+export function githubAnnotations(result) {
+    const lines = [];
+    result.results.forEach((r, i) => {
+        const label = caseLabel(i, r.case.input);
+        if (r.error !== undefined) {
+            lines.push(command('error', `Eval case ${label}`, r.error));
+        }
+        for (const score of r.scores) {
+            if (score.error !== undefined) {
+                lines.push(command('warning', `Eval metric ${score.name} on case ${label}`, score.error));
+            }
+        }
+    });
+    for (const a of result.assertions) {
+        if (!a.passed)
+            lines.push(command('error', `Eval assertion ${a.name}`, a.message));
+    }
+    return lines;
+}
+/**
+ * A table cell of the step summary: any line break (a lone `\r` too) becomes `<br>` so the row
+ * holds, and `<` and `&` are escaped so no tag or comment in an answer hides the rest
+ */
+function cell(value) {
+    return value
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/\\/g, '\\\\')
+        .replace(/\|/g, '\\|')
+        .replace(/\r\n|\r|\n/g, '<br>');
+}
+function num(n) {
+    return Number.isInteger(n) ? String(n) : n.toFixed(4);
+}
+/** The Markdown summary of the run for `$GITHUB_STEP_SUMMARY` */
+export function githubSummary(result) {
+    const { stats } = result;
+    const passed = result.assertions.filter((a) => a.passed).length;
+    const failed = result.assertions.length - passed;
+    const flagged = result.results.filter((r) => caseFlags(r).length > 0).length;
+    const trials = trialsPerCase(result);
+    const totals = [
+        `**${stats.total}** cases${trials > 1 ? ` x **${trials}** trials` : ''}`,
+        `**${stats.errors ?? 0}** errored`,
+        ...(stats.metricErrors ? [`**${stats.metricErrors}** metric errors`] : []),
+        ...(flagged > 0 ? [`**${flagged}** truncated, paused or at the iteration limit`] : []),
+        `${stats.duration} ms`,
+        `${num(stats.cost)} USD`,
+        `assertions: **${passed}** passed, **${failed}** failed`,
+    ];
+    const lines = ['## Eval results', '', totals.join(' | '), ''];
+    const metrics = Object.values(result.aggregated);
+    if (metrics.length > 0) {
+        lines.push('### Metrics', '', '| Metric | Mean | P50 | P95 | Cases |', '| --- | ---: | ---: | ---: | ---: |');
+        for (const m of metrics) {
+            const count = m.count === undefined ? '-' : String(m.count);
+            lines.push(`| ${cell(m.name)} | ${num(m.mean)} | ${num(m.p50)} | ${num(m.p95)} | ${count} |`);
+        }
+        lines.push('');
+    }
+    if (result.assertions.length > 0) {
+        lines.push('### Assertions', '', '| Result | Assertion | Details |', '| --- | --- | --- |');
+        for (const a of result.assertions) {
+            lines.push(`| ${a.passed ? 'pass' : '**fail**'} | ${cell(a.name)} | ${cell(a.message)} |`);
+        }
+        lines.push('');
+    }
+    return lines.join('\n') + '\n';
+}
+/**
+ * Prints GitHub Actions annotations to stdout, and appends a Markdown summary to the file in
+ * `GITHUB_STEP_SUMMARY` when the variable is set
+ */
+export function githubReport(result) {
+    for (const line of githubAnnotations(result))
+        console.log(line);
+    const summaryPath = process.env.GITHUB_STEP_SUMMARY;
+    if (summaryPath)
+        appendFileSync(summaryPath, githubSummary(result), 'utf-8');
+}
+//# sourceMappingURL=github.js.map

package/dist/reporters/github.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAE/D,gDAAgD;AAChD,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAChF,CAAC;AAED,4EAA4E;AAC5E,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,OAAO,CAAC,IAAyB,EAAE,KAAa,EAAE,OAAe;IACxE,OAAO,KAAK,IAAI,UAAU,cAAc,CAAC,KAAK,CAAC,KAAK,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,iBAAiB,CAAC,MAAuB;IACvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,aAAa,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YAC7B,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,eAAe,KAAK,CAAC,IAAI,YAAY,KAAK,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,IAAI,CAAC,KAAa;IACzB,OAAO,KAAK;SACT,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;SACtB,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;SACrB,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;IACjD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAE7E,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;IACrC,MAAM,MAAM,GAAG;QACb,KAAK,KAAK,CAAC,KAAK,WAAW,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,MAAM,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE;QACxE,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,YAAY;QAClC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,YAAY,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,gDAAgD,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtF,GAAG,KAAK,CAAC,QAAQ,KAAK;QACtB,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM;QACxB,iBAAiB,MAAM,gBAAgB,MAAM,WAAW;KACzD,CAAC;IAEF,MAAM,KAAK,GAAG,CAAC,iBAAiB,EAAE,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;IAE9D,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CACR,aAAa,EACb,EAAE,EACF,uCAAuC,EACvC,qCAAqC,CACtC,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;QAChG,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,EAAE,kCAAkC,EAAE,qBAAqB,CAAC,CAAC;QAC5F,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,MAAM,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAuB;IAClD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEhE,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IACpD,IAAI,WAAW;QAAE,cAAc,CAAC,WAAW,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AAC/E,CAAC"}

package/dist/reporters/index.d.ts

@@ -1,4 +1,5 @@
 import type { EvalStats } from '../assertions/index.js';
+import type { EvalToolCall, EvalToolResult, EvalTurnResult, Passage } from '../metrics/types.js';
 export interface AggregatedMetric {
     name: string;
     mean: number;
@@ -9,6 +10,8 @@ export interface AggregatedMetric {
     p50: number;
     p95: number;
     p99: number;
+    /** Cases the metric was scored on: errored cases are not scored */
+    count?: number;
     /** Extra figures a statistical metric reports, such as the total cost */
     metadata?: Record<string, unknown>;
 }
@@ -19,44 +22,65 @@ export interface AssertionResult {
     actual?: number;
     expected?: number;
 }
-export interface EvalSuiteResult {
-    results: Array<{
-        case: {
-            input: string;
-            expected?: string;
-        };
-        output: string;
-        duration: number;
-        /** Why the case has no output: every attempt failed or timed out */
+/** A case result as the reporters get it */
+export interface ReportedCase {
+    case: {
+        input: string;
+        expected?: string;
+    };
+    output: string;
+    duration: number;
+    /** Why the case has no output: every attempt failed or timed out */
+    error?: string;
+    /** The agent's answer stopped at its output token limit */
+    truncated?: boolean;
+    /** Tool calls used up the agent's `maxIterations` before it answered */
+    iterationLimitReached?: boolean;
+    /** The run paused for tool approvals, so `output` holds no answer */
+    paused?: boolean;
+    /** The tool calls of the run, over every turn */
+    toolCalls?: readonly EvalToolCall[];
+    /** What the tool calls returned, over every turn */
+    toolResults?: readonly EvalToolResult[];
+    /** The passages the run retrieved */
+    retrieved?: readonly Passage[];
+    /** Each turn of a multi-turn case */
+    turns?: readonly EvalTurnResult[];
+    scores: Array<{
+        name: string;
+        score: number;
+        details?: string;
         error?: string;
-        scores: Array<{
-            name: string;
-            score: number;
-            details?: string;
-            error?: string;
-        }>;
     }>;
+    /** Every run of the case when the suite ran several trials, the first being the case itself */
+    trials?: readonly ReportedCase[];
+}
+export interface EvalSuiteResult {
+    results: ReportedCase[];
     aggregated: Record<string, AggregatedMetric>;
     assertions: AssertionResult[];
     stats: EvalStats;
 }
-export type ReporterType = 'console' | 'json' | 'csv' | 'ci';
+export type ReporterType = 'console' | 'json' | 'csv' | 'ci' | 'junit' | 'github';
 export type ReporterOptions = {
     /**
      * Base path of file reports. Each file reporter writes it with its own
-     * extension, so `./reports/eval` gives `eval.json` and `eval.csv`.
+     * extension, so `./reports/eval` gives `eval.json`, `eval.csv` and `eval.xml`.
      */
     path?: string;
 };
 /** The file a reporter writes: the base path with the reporter's extension. */
-export declare function reportPath(base: string | undefined, extension: 'json' | 'csv'): string;
+export declare function reportPath(base: string | undefined, extension: 'json' | 'csv' | 'xml'): string;
 /**
- * Runs the reporters in the order given, except `ci`, which runs last because
- * it ends the process when an assertion failed.
+ * Runs the reporters in the order given, except `ci`, which runs last so its verdict is the
+ * final line of the output. `ci` sets `process.exitCode` to 1 when an assertion failed instead
+ * of ending the process, so output still flushes and later hooks still run.
  */
 export declare function report(result: EvalSuiteResult, type: ReporterType | ReporterType[], options?: ReporterOptions): void;
 export { consoleReport } from './console.js';
 export { jsonReport } from './json.js';
 export { csvReport } from './csv.js';
 export { ciReport } from './ci.js';
+export { junitReport } from './junit.js';
+export { githubReport } from './github.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/reporters/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAE/C,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3C,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,oEAAoE;QACpE,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAClF,CAAC,CAAC;IACH,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC;AAC7D,MAAM,MAAM,eAAe,GAAG;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,+EAA+E;AAC/E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAGtF;AAOD;;;GAGG;AACH,wBAAgB,MAAM,CACpB,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EACnC,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CAoBN;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC/C,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAE,cAAc,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE9F,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,oEAAoE;IACpE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,2DAA2D;IAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,wEAAwE;IACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,iDAAiD;IACjD,SAAS,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;IACpC,oDAAoD;IACpD,WAAW,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IACxC,qCAAqC;IACrC,SAAS,CAAC,EAAE,SAAS,OAAO,EAAE,CAAC;IAC/B,qCAAqC;IACrC,KAAK,CAAC,EAAE,SAAS,cAAc,EAAE,CAAC;IAClC,MAAM,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjF,+FAA+F;IAC/F,MAAM,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,GAAG,OAAO,GAAG,QAAQ,CAAC;AAClF,MAAM,MAAM,eAAe,GAAG;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,+EAA+E;AAC/E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAG9F;AAOD;;;;GAIG;AACH,wBAAgB,MAAM,CACpB,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EACnC,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CA0BN;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}

package/dist/reporters/index.js

@@ -4,10 +4,12 @@ import { consoleReport } from './console.js';
 import { jsonReport } from './json.js';
 import { csvReport } from './csv.js';
 import { ciReport } from './ci.js';
+import { junitReport } from './junit.js';
+import { githubReport } from './github.js';
 const DEFAULT_BASE = 'eval-report';
 /** The file a reporter writes: the base path with the reporter's extension. */
 export function reportPath(base, extension) {
-    const stem = (base ?? DEFAULT_BASE).replace(/\.(json|csv)$/i, '');
+    const stem = (base ?? DEFAULT_BASE).replace(/\.(json|csv|xml)$/i, '');
     return `${stem}.${extension}`;
 }
 function prepare(path) {
@@ -15,8 +17,9 @@ function prepare(path) {
     return path;
 }
 /**
- * Runs the reporters in the order given, except `ci`, which runs last because
- * it ends the process when an assertion failed.
+ * Runs the reporters in the order given, except `ci`, which runs last so its verdict is the
+ * final line of the output. `ci` sets `process.exitCode` to 1 when an assertion failed instead
+ * of ending the process, so output still flushes and later hooks still run.
  */
 export function report(result, type, options) {
     const types = Array.isArray(type) ? type : [type];
@@ -32,6 +35,12 @@ export function report(result, type, options) {
             case 'csv':
                 csvReport(result, { path: prepare(reportPath(options?.path, 'csv')) });
                 break;
+            case 'junit':
+                junitReport(result, { path: prepare(reportPath(options?.path, 'xml')) });
+                break;
+            case 'github':
+                githubReport(result);
+                break;
             case 'ci':
                 ciReport(result);
                 break;
@@ -42,4 +51,6 @@ export { consoleReport } from './console.js';
 export { jsonReport } from './json.js';
 export { csvReport } from './csv.js';
 export { ciReport } from './ci.js';
+export { junitReport } from './junit.js';
+export { githubReport } from './github.js';
 //# sourceMappingURL=index.js.map

package/dist/reporters/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAgDhC,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC,+EAA+E;AAC/E,MAAM,UAAU,UAAU,CAAC,IAAwB,EAAE,SAAyB;IAC5E,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;IAClE,OAAO,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,MAAM,CACpB,MAAuB,EACvB,IAAmC,EACnC,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAEzF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,aAAa,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,KAAK;gBACR,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,IAAI;gBACP,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACjB,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAsExC,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC,+EAA+E;AAC/E,MAAM,UAAU,UAAU,CAAC,IAAwB,EAAE,SAAiC;IACpF,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;IACtE,OAAO,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,MAAM,CACpB,MAAuB,EACvB,IAAmC,EACnC,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAEzF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,aAAa,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,KAAK;gBACR,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,OAAO;gBACV,WAAW,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,QAAQ;gBACX,YAAY,CAAC,MAAM,CAAC,CAAC;gBACrB,MAAM;YACR,KAAK,IAAI;gBACP,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACjB,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}

package/dist/reporters/junit.d.ts

@@ -0,0 +1,8 @@
+import type { EvalSuiteResult } from './index.js';
+/** The JUnit XML document of a suite run: one testcase per case and one per assertion */
+export declare function junitXml(result: EvalSuiteResult): string;
+/** Writes the suite run as JUnit XML, which CI systems show as a test report */
+export declare function junitReport(result: EvalSuiteResult, options: {
+    path: string;
+}): void;
+//# sourceMappingURL=junit.d.ts.map

package/dist/reporters/junit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"junit.d.ts","sourceRoot":"","sources":["../../src/reporters/junit.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAiG/C,yFAAyF;AACzF,wBAAgB,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CAqBxD;AAED,gFAAgF;AAChF,wBAAgB,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAEpF"}

package/dist/reporters/junit.js

@@ -0,0 +1,107 @@
+import { writeFileSync } from 'node:fs';
+import { caseFlags, caseLabel } from './format.js';
+const SUITE_NAME = 'eval';
+function isXmlChar(code) {
+    return (code === 0x9 ||
+        code === 0xa ||
+        code === 0xd ||
+        (code >= 0x20 && code <= 0xd7ff) ||
+        (code >= 0xe000 && code <= 0xfffd) ||
+        (code >= 0x10000 && code <= 0x10ffff));
+}
+/** Replaces the characters XML 1.0 does not allow, lone surrogates included, with U+FFFD */
+function xmlSafe(value) {
+    let out = '';
+    for (const ch of value) {
+        const code = ch.codePointAt(0) ?? 0;
+        out += isXmlChar(code) ? ch : '�';
+    }
+    return out;
+}
+/** Escapes text content, `\r` kept as a reference since parsers fold a raw one into `\n` */
+function escapeText(value) {
+    return xmlSafe(value)
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/\r/g, '&#13;');
+}
+/** Escapes an attribute value, keeping tabs and line breaks that a parser would fold to spaces */
+function escapeAttr(value) {
+    return escapeText(value)
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;')
+        .replace(/\t/g, '&#9;')
+        .replace(/\n/g, '&#10;');
+}
+function attrs(values) {
+    return Object.entries(values)
+        .map(([key, value]) => ` ${key}="${escapeAttr(String(value))}"`)
+        .join('');
+}
+function seconds(ms) {
+    return (Math.max(0, ms) / 1000).toFixed(3);
+}
+function firstLine(text) {
+    return text.split(/\r?\n/, 1)[0] ?? '';
+}
+function caseOutput(result) {
+    const lines = result.scores.map((s) => {
+        const details = s.details ? ` (${s.details})` : '';
+        const error = s.error !== undefined ? ` [error: ${s.error}]` : '';
+        return `${s.name}: ${s.score}${details}${error}`;
+    });
+    for (const flag of caseFlags(result))
+        lines.push(`${flag}: true`);
+    return lines.join('\n');
+}
+function caseTestcase(result, index) {
+    const head = `    <testcase${attrs({
+        name: caseLabel(index, result.case.input),
+        classname: `${SUITE_NAME}.cases`,
+        time: seconds(result.duration),
+    })}`;
+    const body = [];
+    if (result.error !== undefined) {
+        body.push(`      <failure${attrs({ message: firstLine(result.error), type: 'CaseError' })}>${escapeText(result.error)}</failure>`);
+    }
+    const output = caseOutput(result);
+    if (output !== '')
+        body.push(`      <system-out>${escapeText(output)}</system-out>`);
+    return body.length === 0 ? `${head}/>` : `${head}>\n${body.join('\n')}\n    </testcase>`;
+}
+function assertionTestcase(assertion) {
+    const head = `    <testcase${attrs({
+        name: assertion.name,
+        classname: `${SUITE_NAME}.assertions`,
+        time: seconds(0),
+    })}`;
+    if (assertion.passed)
+        return `${head}/>`;
+    return `${head}>\n      <failure${attrs({ message: firstLine(assertion.message), type: 'AssertionFailure' })}>${escapeText(assertion.message)}</failure>\n    </testcase>`;
+}
+/** The JUnit XML document of a suite run: one testcase per case and one per assertion */
+export function junitXml(result) {
+    const tests = result.results.length + result.assertions.length;
+    const failures = result.results.filter((r) => r.error !== undefined).length +
+        result.assertions.filter((a) => !a.passed).length;
+    const time = seconds(result.stats.duration);
+    const counts = { tests, failures, errors: 0, skipped: 0, time };
+    const testcases = [
+        ...result.results.map((r, i) => caseTestcase(r, i)),
+        ...result.assertions.map((a) => assertionTestcase(a)),
+    ];
+    return [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        `<testsuites${attrs({ name: SUITE_NAME, tests, failures, errors: 0, time })}>`,
+        `  <testsuite${attrs({ name: SUITE_NAME, ...counts })}${testcases.length === 0 ? '/>' : '>'}`,
+        ...(testcases.length === 0 ? [] : [...testcases, '  </testsuite>']),
+        '</testsuites>',
+        '',
+    ].join('\n');
+}
+/** Writes the suite run as JUnit XML, which CI systems show as a test report */
+export function junitReport(result, options) {
+    writeFileSync(options.path, junitXml(result), 'utf-8');
+}
+//# sourceMappingURL=junit.js.map

package/dist/reporters/junit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"junit.js","sourceRoot":"","sources":["../../src/reporters/junit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAEhD,MAAM,UAAU,GAAG,MAAM,CAAC;AAE1B,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,CACL,IAAI,KAAK,GAAG;QACZ,IAAI,KAAK,GAAG;QACZ,IAAI,KAAK,GAAG;QACZ,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,CAAC;QAChC,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC;QAClC,CAAC,IAAI,IAAI,OAAO,IAAI,IAAI,IAAI,QAAQ,CAAC,CACtC,CAAC;AACJ,CAAC;AAED,4FAA4F;AAC5F,SAAS,OAAO,CAAC,KAAa;IAC5B,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACpC,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IACpC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4FAA4F;AAC5F,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,OAAO,CAAC,KAAK,CAAC;SAClB,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,kGAAkG;AAClG,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,UAAU,CAAC,KAAK,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;SACtB,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,KAAK,CAAC,MAAuC;IACpD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;SAC1B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC;SAC/D,IAAI,CAAC,EAAE,CAAC,CAAC;AACd,CAAC;AAED,SAAS,OAAO,CAAC,EAAU;IACzB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACzC,CAAC;AAID,SAAS,UAAU,CAAC,MAAkB;IACpC,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACpC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,OAAO,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,GAAG,OAAO,GAAG,KAAK,EAAE,CAAC;IACnD,CAAC,CAAC,CAAC;IACH,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,QAAQ,CAAC,CAAC;IAClE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,YAAY,CAAC,MAAkB,EAAE,KAAa;IACrD,MAAM,IAAI,GAAG,gBAAgB,KAAK,CAAC;QACjC,IAAI,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QACzC,SAAS,EAAE,GAAG,UAAU,QAAQ;QAChC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;KAC/B,CAAC,EAAE,CAAC;IACL,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CACP,iBAAiB,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CACxH,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAClC,IAAI,MAAM,KAAK,EAAE;QAAE,IAAI,CAAC,IAAI,CAAC,qBAAqB,UAAU,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IACrF,OAAO,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC;AAC3F,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAgD;IACzE,MAAM,IAAI,GAAG,gBAAgB,KAAK,CAAC;QACjC,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,SAAS,EAAE,GAAG,UAAU,aAAa;QACrC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;KACjB,CAAC,EAAE,CAAC;IACL,IAAI,SAAS,CAAC,MAAM;QAAE,OAAO,GAAG,IAAI,IAAI,CAAC;IACzC,OAAO,GAAG,IAAI,oBAAoB,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC,IAAI,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,6BAA6B,CAAC;AAC7K,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,QAAQ,CAAC,MAAuB;IAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;IAC/D,MAAM,QAAQ,GACZ,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;QAC1D,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACpD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC;IAEhE,MAAM,SAAS,GAAG;QAChB,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnD,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;KACtD,CAAC;IAEF,OAAO;QACL,wCAAwC;QACxC,cAAc,KAAK,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG;QAC9E,eAAe,KAAK,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE;QAC7F,GAAG,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,EAAE,gBAAgB,CAAC,CAAC;QACnE,eAAe;QACf,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,WAAW,CAAC,MAAuB,EAAE,OAAyB;IAC5E,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AACzD,CAAC"}

package/dist/schema.d.ts

@@ -1,14 +1,26 @@
 import { z } from 'zod';
+/** A tool call a case expects: the tool's name and, optionally, arguments to compare */
+export declare const ExpectedToolCallSchema: z.ZodObject<{
+    name: z.ZodString;
+    arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+}, z.core.$strip>;
 export declare const EvalCaseSchema: z.ZodObject<{
     input: z.ZodString;
     expected: z.ZodOptional<z.ZodString>;
     context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    turns: z.ZodOptional<z.ZodArray<z.ZodString>>;
+    expectedToolCalls: z.ZodOptional<z.ZodArray<z.ZodObject<{
+        name: z.ZodString;
+        arguments: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strip>>>;
+    expectedPassages: z.ZodOptional<z.ZodArray<z.ZodString>>;
 }, z.core.$strip>;
 export declare const EvalSuiteConfigSchema: z.ZodObject<{
     concurrency: z.ZodDefault<z.ZodNumber>;
     timeout: z.ZodDefault<z.ZodNumber>;
     retries: z.ZodDefault<z.ZodNumber>;
+    trials: z.ZodDefault<z.ZodNumber>;
 }, z.core.$strip>;
 /**
  * The part of a Cogitator runtime the judge needs: running an agent. The judge's `usage`, when
@@ -26,17 +38,30 @@ export interface JudgeCogitator {
         };
     }>;
 }
+/**
+ * Where the judge keeps the verdicts it already gave, by a key of its model, settings and the
+ * whole prompt: a case judged again with the same answer reuses the verdict and costs nothing.
+ * A store of your own (a file, Redis) keeps verdicts across runs
+ */
+export interface JudgeCache {
+    get(key: string): string | undefined | Promise<string | undefined>;
+    set(key: string, verdict: string): void | Promise<void>;
+}
 export declare const JudgeConfigSchema: z.ZodObject<{
     model: z.ZodString;
     temperature: z.ZodDefault<z.ZodNumber>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
     cogitator: z.ZodOptional<z.ZodCustom<JudgeCogitator, JudgeCogitator>>;
+    cache: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<JudgeCache, JudgeCache>]>>;
+    concurrency: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 export declare const EvalComparisonConfigSchema: z.ZodObject<{
     concurrency: z.ZodDefault<z.ZodNumber>;
     timeout: z.ZodDefault<z.ZodNumber>;
     retries: z.ZodDefault<z.ZodNumber>;
+    trials: z.ZodDefault<z.ZodNumber>;
 }, z.core.$strip>;
+export type ExpectedToolCall = z.output<typeof ExpectedToolCallSchema>;
 export type EvalCase = z.output<typeof EvalCaseSchema>;
 export type EvalCaseInput = z.input<typeof EvalCaseSchema>;
 export type EvalSuiteConfig = z.output<typeof EvalSuiteConfigSchema>;

package/dist/schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,cAAc;;;;;iBAKzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;CAChF;AAUD,eAAO,MAAM,iBAAiB;;;;;iBAM5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;iBAIrC,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wFAAwF;AACxF,eAAO,MAAM,sBAAsB;;;iBAGjC,CAAC;AAEH,eAAO,MAAM,cAAc;;;;;;;;;;;iBAWzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;;iBAKhC,CAAC;AAEH;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;CAChF;AAED;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACnE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzD;AAmBD,eAAO,MAAM,iBAAiB;;;;;;;iBAgB5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;iBAKrC,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,sBAAsB,CAAC,CAAC;AACvE,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}

package/dist/schema.js

@@ -1,15 +1,33 @@
 import { z } from 'zod';
+/** A tool call a case expects: the tool's name and, optionally, arguments to compare */
+export const ExpectedToolCallSchema = z.object({
+    name: z.string().min(1),
+    arguments: z.record(z.string(), z.unknown()).optional(),
+});
 export const EvalCaseSchema = z.object({
     input: z.string(),
     expected: z.string().optional(),
     context: z.record(z.string(), z.unknown()).optional(),
     metadata: z.record(z.string(), z.unknown()).optional(),
+    /** Further user turns after `input`, run in order on the same conversation */
+    turns: z.array(z.string()).optional(),
+    /** The tool calls the agent should make over the whole conversation, in order */
+    expectedToolCalls: z.array(ExpectedToolCallSchema).optional(),
+    /** Ids or texts of the passages the retrieval should surface */
+    expectedPassages: z.array(z.string().min(1)).optional(),
 });
 export const EvalSuiteConfigSchema = z.object({
     concurrency: z.number().int().min(1).default(5),
     timeout: z.number().int().min(1000).default(30000),
     retries: z.number().int().min(0).max(10).default(0),
+    trials: z.number().int().min(1).default(1),
 });
+function isJudgeCache(value) {
+    return (typeof value === 'object' &&
+        value !== null &&
+        typeof value.get === 'function' &&
+        typeof value.set === 'function');
+}
 function isJudgeCogitator(value) {
     return (typeof value === 'object' &&
         value !== null &&
@@ -21,10 +39,21 @@ export const JudgeConfigSchema = z.object({
     maxTokens: z.number().int().positive().optional(),
     /** Runtime that runs the judge; defaults to the `cogitator` of an agent target */
     cogitator: z.custom(isJudgeCogitator).optional(),
+    /**
+     * Reuse verdicts: `true` keeps them in memory for the suite, a `JudgeCache` of your own keeps
+     * them where it likes. Default: every case is judged afresh
+     */
+    cache: z.union([z.boolean(), z.custom(isJudgeCache)]).optional(),
+    /**
+     * Most judge calls at once across the run, below the suite's own concurrency times its judge
+     * metrics. Default: no limit of its own
+     */
+    concurrency: z.number().int().positive().optional(),
 });
 export const EvalComparisonConfigSchema = z.object({
     concurrency: z.number().int().min(1).default(5),
     timeout: z.number().int().min(1000).default(30000),
     retries: z.number().int().min(0).max(10).default(0),
+    trials: z.number().int().min(1).default(1),
 });
 //# sourceMappingURL=schema.js.map

package/dist/schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC;AAaH,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;CACjE,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC"}
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,wFAAwF;AACxF,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC7C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACvB,SAAS,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACtD,8EAA8E;IAC9E,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrC,iFAAiF;IACjF,iBAAiB,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE;IAC7D,gEAAgE;IAChE,gBAAgB,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;CACxD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CAC3C,CAAC,CAAC;AAuBH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU;QACtD,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;IAChE;;;OAGG;IACH,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAa,YAAY,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5E;;;OAGG;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACnD,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CAC3C,CAAC,CAAC"}

package/dist/vitest.d.ts

@@ -0,0 +1,35 @@
+import type { EvalRunOptions, EvalSuite, EvalSuiteResult } from './eval-suite.js';
+export interface DescribeEvalOptions {
+    /** The suite to run */
+    suite: EvalSuite;
+    /** Passed to `suite.run()`: `maxCases` also decides which cases get a test */
+    run?: EvalRunOptions;
+    /**
+     * Milliseconds the suite run may take, 10 minutes by default. When it runs out the run is
+     * cancelled and every test fails with the timeout.
+     */
+    timeout?: number;
+}
+export interface DescribeEvalHandle {
+    /**
+     * The result of the suite run. Call it inside a test or an `afterAll` hook of the same
+     * describe block: it throws while the run has not finished, or with the cause when it failed.
+     */
+    result: () => EvalSuiteResult;
+}
+/**
+ * Registers a Vitest describe block for an eval suite. The suite runs once in `beforeAll`, each
+ * case becomes a test that fails when the case errored, and an `assertions` test fails with
+ * every assertion that did not pass.
+ *
+ * @example
+ * ```ts
+ * import { describeEval } from '@cogitator-ai/evals/vitest';
+ *
+ * const qa = describeEval('qa agent', { suite, run: { maxCases: 50 } });
+ *
+ * afterAll(() => qa.result().report('junit', { path: './reports/qa' }));
+ * ```
+ */
+export declare function describeEval(name: string, options: DescribeEvalOptions): DescribeEvalHandle;
+//# sourceMappingURL=vitest.d.ts.map

package/dist/vitest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vitest.d.ts","sourceRoot":"","sources":["../src/vitest.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAS/E,MAAM,WAAW,mBAAmB;IAClC,uBAAuB;IACvB,KAAK,EAAE,SAAS,CAAC;IACjB,8EAA8E;IAC9E,GAAG,CAAC,EAAE,cAAc,CAAC;IACrB;;;OAGG;IACH,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,kBAAkB;IACjC;;;OAGG;IACH,MAAM,EAAE,MAAM,eAAe,CAAC;CAC/B;AAWD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,mBAAmB,GAAG,kBAAkB,CAsE3F"}