@cogitator-ai/evals 0.8.0 → 0.9.0

package/dist/reporters/ci.js.map

@@ -1 +1 @@
-{"version":3,"file":"ci.js","sourceRoot":"","sources":["../../src/reporters/ci.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,MAAM,UAAU,QAAQ,CAAC,MAAuB;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IAExC,OAAO,CAAC,GAAG,CACT,SAAS,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CACpI,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,YAAY,MAAM,SAAS,CAAC,CAAC;IAE1D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC"}
+{"version":3,"file":"ci.js","sourceRoot":"","sources":["../../src/reporters/ci.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB;;;GAGG;AACH,MAAM,UAAU,QAAQ,CAAC,MAAuB;IAC9C,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IAExC,OAAO,CAAC,GAAG,CACT,SAAS,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,MAAM,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CACpI,CAAC;IAEF,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAClD,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,OAAO,KAAK,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,YAAY,MAAM,SAAS,CAAC,CAAC;IAE1D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QACf,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;IACvB,CAAC;AACH,CAAC"}

package/dist/reporters/console.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"console.d.ts","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAgB/C,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CA8C3D"}
+{"version":3,"file":"console.d.ts","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAgB/C,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAiD3D"}

package/dist/reporters/console.js

@@ -12,13 +12,15 @@ function fmt(n) {
 export function consoleReport(result) {
     const metrics = Object.values(result.aggregated);
     if (metrics.length > 0) {
-        const cols = { metric: 14, mean: 10, median: 10, p95: 10, min: 10, max: 10 };
+        const cols = { metric: 14, mean: 10, median: 10, p95: 10, min: 10, max: 10, cases: 8 };
+        const showCount = metrics.some((m) => m.count !== undefined);
         const header = pad('Metric', cols.metric) +
             pad('Mean', cols.mean) +
             pad('Median', cols.median) +
             pad('P95', cols.p95) +
             pad('Min', cols.min) +
-            pad('Max', cols.max);
+            pad('Max', cols.max) +
+            (showCount ? pad('Cases', cols.cases) : '');
         console.log(`\n${BOLD}${header}${RESET}`);
         console.log(DIM + '─'.repeat(header.length) + RESET);
         for (const m of metrics) {
@@ -27,7 +29,8 @@ export function consoleReport(result) {
                 pad(fmt(m.median), cols.median) +
                 pad(fmt(m.p95), cols.p95) +
                 pad(fmt(m.min), cols.min) +
-                pad(fmt(m.max), cols.max);
+                pad(fmt(m.max), cols.max) +
+                (showCount ? pad(m.count === undefined ? '-' : String(m.count), cols.cases) : '');
             console.log(row);
         }
     }

package/dist/reporters/console.js.map

@@ -1 +1 @@
-{"version":3,"file":"console.js","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,IAAI,GAAG,SAAS,CAAC;AACvB,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,SAAS,GAAG,CAAC,GAAW,EAAE,GAAW;IACnC,OAAO,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,CAAC;QAC7E,MAAM,MAAM,GACV,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC;YACtB,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;QAEvB,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QAErD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,GAAG,GACP,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;gBACxB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC;gBAC3B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC;gBAC/B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAC5B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,aAAa,KAAK,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,KAAK,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,MAAM,WAAW,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAEzE,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,UAAU,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,SAAS,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,KAAK,GAAG,MAAM,UAAU,KAAK,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,MAAM,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAClN,CAAC;AACJ,CAAC"}
+{"version":3,"file":"console.js","sourceRoot":"","sources":["../../src/reporters/console.ts"],"names":[],"mappings":"AAEA,MAAM,KAAK,GAAG,UAAU,CAAC;AACzB,MAAM,GAAG,GAAG,UAAU,CAAC;AACvB,MAAM,IAAI,GAAG,SAAS,CAAC;AACvB,MAAM,GAAG,GAAG,SAAS,CAAC;AACtB,MAAM,KAAK,GAAG,SAAS,CAAC;AAExB,SAAS,GAAG,CAAC,GAAW,EAAE,GAAW;IACnC,OAAO,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAEjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACvF,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;QAC7D,MAAM,MAAM,GACV,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC;YACtB,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC;YAC1B,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC;YACpB,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAE9C,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,GAAG,MAAM,GAAG,KAAK,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC;QAErD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,GAAG,GACP,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC;gBACxB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC;gBAC3B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC;gBAC/B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC;gBACzB,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACpF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,aAAa,KAAK,EAAE,CAAC,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;YAChE,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,KAAK,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC;QAChF,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAEjE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;IACxC,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,MAAM,WAAW,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAEzE,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,UAAU,KAAK,KAAK,MAAM,CAAC,KAAK,CAAC,KAAK,SAAS,SAAS,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,SAAS,MAAM,CAAC,KAAK,CAAC,IAAI,MAAM,KAAK,GAAG,MAAM,UAAU,KAAK,IAAI,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,MAAM,UAAU,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAClN,CAAC;AACJ,CAAC"}

package/dist/reporters/csv.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAS/C,wBAAgB,SAAS,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAyBlF"}
+{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAU/C,wBAAgB,SAAS,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CA2BlF"}

package/dist/reporters/csv.js

@@ -1,6 +1,7 @@
 import { writeFileSync } from 'node:fs';
+import { CASE_FLAGS } from './format.js';
 function escapeField(value) {
-    if (value.includes(',') || value.includes('"') || value.includes('\n')) {
+    if (/[",\r\n]/.test(value)) {
         return `"${value.replace(/"/g, '""')}"`;
     }
     return value;
@@ -13,7 +14,8 @@ export function csvReport(result, options) {
         }
     }
     const metrics = [...metricNames];
-    const headers = ['input', 'expected', 'output', 'duration', ...metrics];
+    const flags = CASE_FLAGS.filter((flag) => result.results.some((r) => r[flag] === true));
+    const headers = ['input', 'expected', 'output', 'duration', ...flags, ...metrics];
     const lines = [headers.join(',')];
     for (const r of result.results) {
         const scoreMap = new Map(r.scores.map((s) => [s.name, s.score]));
@@ -22,6 +24,7 @@ export function csvReport(result, options) {
             escapeField(r.case.expected ?? ''),
             escapeField(r.output),
             String(r.duration),
+            ...flags.map((flag) => (r[flag] === true ? 'true' : '')),
             ...metrics.map((m) => String(scoreMap.get(m) ?? '')),
         ];
         lines.push(row.join(','));

package/dist/reporters/csv.js.map

@@ -1 +1 @@
-{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAGxC,SAAS,WAAW,CAAC,KAAa;IAChC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QACvE,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAuB,EAAE,OAAyB;IAC1E,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACzB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;IAEjC,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC,CAAC;IACxE,MAAM,KAAK,GAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,GAAG,GAAG;YACV,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAClC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;YACrB,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClB,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SACrD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC"}
+{"version":3,"file":"csv.js","sourceRoot":"","sources":["../../src/reporters/csv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAEtC,SAAS,WAAW,CAAC,KAAa;IAChC,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAuB,EAAE,OAAyB;IAC1E,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACzB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,GAAG,WAAW,CAAC,CAAC;IACjC,MAAM,KAAK,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAExF,MAAM,OAAO,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,EAAE,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC,CAAC;IAClF,MAAM,KAAK,GAAa,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAE5C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjE,MAAM,GAAG,GAAG;YACV,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YACzB,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;YAClC,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC;YACrB,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;YAClB,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACxD,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;SACrD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,CAAC;IAED,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC"}

package/dist/reporters/format.d.ts

@@ -0,0 +1,13 @@
+/** The run flags a case result can carry, in the order reports list them */
+export declare const CASE_FLAGS: readonly ["truncated", "iterationLimitReached", "paused"];
+export type CaseFlag = (typeof CASE_FLAGS)[number];
+/** The flags set on a case result, in `CASE_FLAGS` order */
+export declare function caseFlags(result: Partial<Record<CaseFlag, boolean>>): CaseFlag[];
+/**
+ * A one-line label for a case input: whitespace collapsed and cut to `max` characters with an
+ * ellipsis, so test names and annotation titles stay readable
+ */
+export declare function truncateInput(input: string, max?: number): string;
+/** The label of the case at `index` (zero based) in a report or test name */
+export declare function caseLabel(index: number, input: string): string;
+//# sourceMappingURL=format.d.ts.map

package/dist/reporters/format.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,eAAO,MAAM,UAAU,2DAA4D,CAAC;AACpF,MAAM,MAAM,QAAQ,GAAG,CAAC,OAAO,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AAEnD,4DAA4D;AAC5D,wBAAgB,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,GAAG,QAAQ,EAAE,CAEhF;AAID;;;GAGG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,GAAE,MAA6B,GAAG,MAAM,CAQvF;AAED,6EAA6E;AAC7E,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CAG9D"}

package/dist/reporters/format.js

@@ -0,0 +1,27 @@
+/** The run flags a case result can carry, in the order reports list them */
+export const CASE_FLAGS = ['truncated', 'iterationLimitReached', 'paused'];
+/** The flags set on a case result, in `CASE_FLAGS` order */
+export function caseFlags(result) {
+    return CASE_FLAGS.filter((flag) => result[flag] === true);
+}
+const DEFAULT_LABEL_LENGTH = 60;
+/**
+ * A one-line label for a case input: whitespace collapsed and cut to `max` characters with an
+ * ellipsis, so test names and annotation titles stay readable
+ */
+export function truncateInput(input, max = DEFAULT_LABEL_LENGTH) {
+    const line = input.replace(/\s+/g, ' ').trim();
+    const chars = [...line];
+    if (chars.length <= max)
+        return line;
+    return `${chars
+        .slice(0, max - 3)
+        .join('')
+        .trimEnd()}...`;
+}
+/** The label of the case at `index` (zero based) in a report or test name */
+export function caseLabel(index, input) {
+    const text = truncateInput(input);
+    return `[${index + 1}] ${text === '' ? '(empty input)' : text}`;
+}
+//# sourceMappingURL=format.js.map

package/dist/reporters/format.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.js","sourceRoot":"","sources":["../../src/reporters/format.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAC5E,MAAM,CAAC,MAAM,UAAU,GAAG,CAAC,WAAW,EAAE,uBAAuB,EAAE,QAAQ,CAAU,CAAC;AAGpF,4DAA4D;AAC5D,MAAM,UAAU,SAAS,CAAC,MAA0C;IAClE,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,KAAa,EAAE,MAAc,oBAAoB;IAC7E,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC;IACxB,IAAI,KAAK,CAAC,MAAM,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC;IACrC,OAAO,GAAG,KAAK;SACZ,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC;SACjB,IAAI,CAAC,EAAE,CAAC;SACR,OAAO,EAAE,KAAK,CAAC;AACpB,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,SAAS,CAAC,KAAa,EAAE,KAAa;IACpD,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IAClC,OAAO,IAAI,KAAK,GAAG,CAAC,KAAK,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AAClE,CAAC"}

package/dist/reporters/github.d.ts

@@ -0,0 +1,11 @@
+import type { EvalSuiteResult } from './index.js';
+/** The workflow commands that annotate the run: failures as errors, metric errors as warnings */
+export declare function githubAnnotations(result: EvalSuiteResult): string[];
+/** The Markdown summary of the run for `$GITHUB_STEP_SUMMARY` */
+export declare function githubSummary(result: EvalSuiteResult): string;
+/**
+ * Prints GitHub Actions annotations to stdout, and appends a Markdown summary to the file in
+ * `GITHUB_STEP_SUMMARY` when the variable is set
+ */
+export declare function githubReport(result: EvalSuiteResult): void;
+//# sourceMappingURL=github.d.ts.map

package/dist/reporters/github.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAiB/C,iGAAiG;AACjG,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,EAAE,CAoBnE;AAmBD,iEAAiE;AACjE,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CA0C7D;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,eAAe,GAAG,IAAI,CAK1D"}

package/dist/reporters/github.js

@@ -0,0 +1,94 @@
+import { appendFileSync } from 'node:fs';
+import { caseFlags, caseLabel } from './format.js';
+/** Escapes the message of a workflow command */
+function escapeData(value) {
+    return value.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A');
+}
+/** Escapes a workflow command property, where `:` and `,` are separators */
+function escapeProperty(value) {
+    return escapeData(value).replace(/:/g, '%3A').replace(/,/g, '%2C');
+}
+function command(kind, title, message) {
+    return `::${kind} title=${escapeProperty(title)}::${escapeData(message)}`;
+}
+/** The workflow commands that annotate the run: failures as errors, metric errors as warnings */
+export function githubAnnotations(result) {
+    const lines = [];
+    result.results.forEach((r, i) => {
+        const label = caseLabel(i, r.case.input);
+        if (r.error !== undefined) {
+            lines.push(command('error', `Eval case ${label}`, r.error));
+        }
+        for (const score of r.scores) {
+            if (score.error !== undefined) {
+                lines.push(command('warning', `Eval metric ${score.name} on case ${label}`, score.error));
+            }
+        }
+    });
+    for (const a of result.assertions) {
+        if (!a.passed)
+            lines.push(command('error', `Eval assertion ${a.name}`, a.message));
+    }
+    return lines;
+}
+/**
+ * A table cell of the step summary: any line break (a lone `\r` too) becomes `<br>` so the row
+ * holds, and `<` and `&` are escaped so no tag or comment in an answer hides the rest
+ */
+function cell(value) {
+    return value
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/\\/g, '\\\\')
+        .replace(/\|/g, '\\|')
+        .replace(/\r\n|\r|\n/g, '<br>');
+}
+function num(n) {
+    return Number.isInteger(n) ? String(n) : n.toFixed(4);
+}
+/** The Markdown summary of the run for `$GITHUB_STEP_SUMMARY` */
+export function githubSummary(result) {
+    const { stats } = result;
+    const passed = result.assertions.filter((a) => a.passed).length;
+    const failed = result.assertions.length - passed;
+    const flagged = result.results.filter((r) => caseFlags(r).length > 0).length;
+    const totals = [
+        `**${stats.total}** cases`,
+        `**${stats.errors ?? 0}** errored`,
+        ...(stats.metricErrors ? [`**${stats.metricErrors}** metric errors`] : []),
+        ...(flagged > 0 ? [`**${flagged}** truncated, paused or at the iteration limit`] : []),
+        `${stats.duration} ms`,
+        `${num(stats.cost)} USD`,
+        `assertions: **${passed}** passed, **${failed}** failed`,
+    ];
+    const lines = ['## Eval results', '', totals.join(' | '), ''];
+    const metrics = Object.values(result.aggregated);
+    if (metrics.length > 0) {
+        lines.push('### Metrics', '', '| Metric | Mean | P50 | P95 | Cases |', '| --- | ---: | ---: | ---: | ---: |');
+        for (const m of metrics) {
+            const count = m.count === undefined ? '-' : String(m.count);
+            lines.push(`| ${cell(m.name)} | ${num(m.mean)} | ${num(m.p50)} | ${num(m.p95)} | ${count} |`);
+        }
+        lines.push('');
+    }
+    if (result.assertions.length > 0) {
+        lines.push('### Assertions', '', '| Result | Assertion | Details |', '| --- | --- | --- |');
+        for (const a of result.assertions) {
+            lines.push(`| ${a.passed ? 'pass' : '**fail**'} | ${cell(a.name)} | ${cell(a.message)} |`);
+        }
+        lines.push('');
+    }
+    return lines.join('\n') + '\n';
+}
+/**
+ * Prints GitHub Actions annotations to stdout, and appends a Markdown summary to the file in
+ * `GITHUB_STEP_SUMMARY` when the variable is set
+ */
+export function githubReport(result) {
+    for (const line of githubAnnotations(result))
+        console.log(line);
+    const summaryPath = process.env.GITHUB_STEP_SUMMARY;
+    if (summaryPath)
+        appendFileSync(summaryPath, githubSummary(result), 'utf-8');
+}
+//# sourceMappingURL=github.js.map

package/dist/reporters/github.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.js","sourceRoot":"","sources":["../../src/reporters/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAEhD,gDAAgD;AAChD,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;AAChF,CAAC;AAED,4EAA4E;AAC5E,SAAS,cAAc,CAAC,KAAa;IACnC,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,OAAO,CAAC,IAAyB,EAAE,KAAa,EAAE,OAAe;IACxE,OAAO,KAAK,IAAI,UAAU,cAAc,CAAC,KAAK,CAAC,KAAK,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;AAC5E,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,iBAAiB,CAAC,MAAuB;IACvD,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,aAAa,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9D,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YAC7B,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;gBAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,eAAe,KAAK,CAAC,IAAI,YAAY,KAAK,EAAE,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5F,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,IAAI,CAAC,CAAC,CAAC,MAAM;YAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IACrF,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,IAAI,CAAC,KAAa;IACzB,OAAO,KAAK;SACT,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;SACtB,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC;SACrB,OAAO,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACxD,CAAC;AAED,iEAAiE;AACjE,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,CAAC;IACzB,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,MAAM,CAAC;IACjD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAE7E,MAAM,MAAM,GAAG;QACb,KAAK,KAAK,CAAC,KAAK,UAAU;QAC1B,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,YAAY;QAClC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,YAAY,kBAAkB,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC1E,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,gDAAgD,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACtF,GAAG,KAAK,CAAC,QAAQ,KAAK;QACtB,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM;QACxB,iBAAiB,MAAM,gBAAgB,MAAM,WAAW;KACzD,CAAC;IAEF,MAAM,KAAK,GAAG,CAAC,iBAAiB,EAAE,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC;IAE9D,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CACR,aAAa,EACb,EAAE,EACF,uCAAuC,EACvC,qCAAqC,CACtC,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC;QAChG,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,gBAAgB,EAAE,EAAE,EAAE,kCAAkC,EAAE,qBAAqB,CAAC,CAAC;QAC5F,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,MAAM,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACjC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,MAAuB;IAClD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAEhE,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;IACpD,IAAI,WAAW;QAAE,cAAc,CAAC,WAAW,EAAE,aAAa,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AAC/E,CAAC"}

package/dist/reporters/index.d.ts

@@ -9,6 +9,8 @@ export interface AggregatedMetric {
     p50: number;
     p95: number;
     p99: number;
+    /** Cases the metric was scored on: errored cases are not scored */
+    count?: number;
     /** Extra figures a statistical metric reports, such as the total cost */
     metadata?: Record<string, unknown>;
 }
@@ -29,6 +31,12 @@ export interface EvalSuiteResult {
         duration: number;
         /** Why the case has no output: every attempt failed or timed out */
         error?: string;
+        /** The agent's answer stopped at its output token limit */
+        truncated?: boolean;
+        /** Tool calls used up the agent's `maxIterations` before it answered */
+        iterationLimitReached?: boolean;
+        /** The run paused for tool approvals, so `output` holds no answer */
+        paused?: boolean;
         scores: Array<{
             name: string;
             score: number;
@@ -40,23 +48,26 @@ export interface EvalSuiteResult {
     assertions: AssertionResult[];
     stats: EvalStats;
 }
-export type ReporterType = 'console' | 'json' | 'csv' | 'ci';
+export type ReporterType = 'console' | 'json' | 'csv' | 'ci' | 'junit' | 'github';
 export type ReporterOptions = {
     /**
      * Base path of file reports. Each file reporter writes it with its own
-     * extension, so `./reports/eval` gives `eval.json` and `eval.csv`.
+     * extension, so `./reports/eval` gives `eval.json`, `eval.csv` and `eval.xml`.
      */
     path?: string;
 };
 /** The file a reporter writes: the base path with the reporter's extension. */
-export declare function reportPath(base: string | undefined, extension: 'json' | 'csv'): string;
+export declare function reportPath(base: string | undefined, extension: 'json' | 'csv' | 'xml'): string;
 /**
- * Runs the reporters in the order given, except `ci`, which runs last because
- * it ends the process when an assertion failed.
+ * Runs the reporters in the order given, except `ci`, which runs last so its verdict is the
+ * final line of the output. `ci` sets `process.exitCode` to 1 when an assertion failed instead
+ * of ending the process, so output still flushes and later hooks still run.
  */
 export declare function report(result: EvalSuiteResult, type: ReporterType | ReporterType[], options?: ReporterOptions): void;
 export { consoleReport } from './console.js';
 export { jsonReport } from './json.js';
 export { csvReport } from './csv.js';
 export { ciReport } from './ci.js';
+export { junitReport } from './junit.js';
+export { githubReport } from './github.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/reporters/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAE/C,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3C,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,oEAAoE;QACpE,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAClF,CAAC,CAAC;IACH,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC;AAC7D,MAAM,MAAM,eAAe,GAAG;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,+EAA+E;AAC/E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAGtF;AAOD;;;GAGG;AACH,wBAAgB,MAAM,CACpB,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EACnC,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CAoBN;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAE/C,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yEAAyE;IACzE,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE;YAAE,KAAK,EAAE,MAAM,CAAC;YAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC3C,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,EAAE,MAAM,CAAC;QACjB,oEAAoE;QACpE,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,2DAA2D;QAC3D,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,wEAAwE;QACxE,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAChC,qEAAqE;QACrE,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,MAAM,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,OAAO,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC,CAAC;KAClF,CAAC,CAAC;IACH,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE,SAAS,CAAC;CAClB;AAED,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,GAAG,IAAI,GAAG,OAAO,GAAG,QAAQ,CAAC;AAClF,MAAM,MAAM,eAAe,GAAG;IAC5B;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAIF,+EAA+E;AAC/E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,KAAK,GAAG,KAAK,GAAG,MAAM,CAG9F;AAOD;;;;GAIG;AACH,wBAAgB,MAAM,CACpB,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EACnC,OAAO,CAAC,EAAE,eAAe,GACxB,IAAI,CA0BN;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}

package/dist/reporters/index.js

@@ -4,10 +4,12 @@ import { consoleReport } from './console.js';
 import { jsonReport } from './json.js';
 import { csvReport } from './csv.js';
 import { ciReport } from './ci.js';
+import { junitReport } from './junit.js';
+import { githubReport } from './github.js';
 const DEFAULT_BASE = 'eval-report';
 /** The file a reporter writes: the base path with the reporter's extension. */
 export function reportPath(base, extension) {
-    const stem = (base ?? DEFAULT_BASE).replace(/\.(json|csv)$/i, '');
+    const stem = (base ?? DEFAULT_BASE).replace(/\.(json|csv|xml)$/i, '');
     return `${stem}.${extension}`;
 }
 function prepare(path) {
@@ -15,8 +17,9 @@ function prepare(path) {
     return path;
 }
 /**
- * Runs the reporters in the order given, except `ci`, which runs last because
- * it ends the process when an assertion failed.
+ * Runs the reporters in the order given, except `ci`, which runs last so its verdict is the
+ * final line of the output. `ci` sets `process.exitCode` to 1 when an assertion failed instead
+ * of ending the process, so output still flushes and later hooks still run.
  */
 export function report(result, type, options) {
     const types = Array.isArray(type) ? type : [type];
@@ -32,6 +35,12 @@ export function report(result, type, options) {
             case 'csv':
                 csvReport(result, { path: prepare(reportPath(options?.path, 'csv')) });
                 break;
+            case 'junit':
+                junitReport(result, { path: prepare(reportPath(options?.path, 'xml')) });
+                break;
+            case 'github':
+                githubReport(result);
+                break;
             case 'ci':
                 ciReport(result);
                 break;
@@ -42,4 +51,6 @@ export { consoleReport } from './console.js';
 export { jsonReport } from './json.js';
 export { csvReport } from './csv.js';
 export { ciReport } from './ci.js';
+export { junitReport } from './junit.js';
+export { githubReport } from './github.js';
 //# sourceMappingURL=index.js.map

package/dist/reporters/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAgDhC,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC,+EAA+E;AAC/E,MAAM,UAAU,UAAU,CAAC,IAAwB,EAAE,SAAyB;IAC5E,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;IAClE,OAAO,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,MAAM,CACpB,MAAuB,EACvB,IAAmC,EACnC,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAEzF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,aAAa,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,KAAK;gBACR,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,IAAI;gBACP,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACjB,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/reporters/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC;AAwDxC,MAAM,YAAY,GAAG,aAAa,CAAC;AAEnC,+EAA+E;AAC/E,MAAM,UAAU,UAAU,CAAC,IAAwB,EAAE,SAAiC;IACpF,MAAM,IAAI,GAAG,CAAC,IAAI,IAAI,YAAY,CAAC,CAAC,OAAO,CAAC,oBAAoB,EAAE,EAAE,CAAC,CAAC;IACtE,OAAO,GAAG,IAAI,IAAI,SAAS,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,OAAO,CAAC,IAAY;IAC3B,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,MAAM,CACpB,MAAuB,EACvB,IAAmC,EACnC,OAAyB;IAEzB,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;IAEzF,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,QAAQ,CAAC,EAAE,CAAC;YACV,KAAK,SAAS;gBACZ,aAAa,CAAC,MAAM,CAAC,CAAC;gBACtB,MAAM;YACR,KAAK,MAAM;gBACT,UAAU,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,KAAK;gBACR,SAAS,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACvE,MAAM;YACR,KAAK,OAAO;gBACV,WAAW,CAAC,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;gBACzE,MAAM;YACR,KAAK,QAAQ;gBACX,YAAY,CAAC,MAAM,CAAC,CAAC;gBACrB,MAAM;YACR,KAAK,IAAI;gBACP,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACjB,MAAM;QACV,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,OAAO,CAAC;AAClC,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,UAAU,CAAC"}

package/dist/reporters/junit.d.ts

@@ -0,0 +1,8 @@
+import type { EvalSuiteResult } from './index.js';
+/** The JUnit XML document of a suite run: one testcase per case and one per assertion */
+export declare function junitXml(result: EvalSuiteResult): string;
+/** Writes the suite run as JUnit XML, which CI systems show as a test report */
+export declare function junitReport(result: EvalSuiteResult, options: {
+    path: string;
+}): void;
+//# sourceMappingURL=junit.d.ts.map

package/dist/reporters/junit.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"junit.d.ts","sourceRoot":"","sources":["../../src/reporters/junit.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAiG/C,yFAAyF;AACzF,wBAAgB,QAAQ,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CAqBxD;AAED,gFAAgF;AAChF,wBAAgB,WAAW,CAAC,MAAM,EAAE,eAAe,EAAE,OAAO,EAAE;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAAG,IAAI,CAEpF"}

package/dist/reporters/junit.js

@@ -0,0 +1,107 @@
+import { writeFileSync } from 'node:fs';
+import { caseFlags, caseLabel } from './format.js';
+const SUITE_NAME = 'eval';
+function isXmlChar(code) {
+    return (code === 0x9 ||
+        code === 0xa ||
+        code === 0xd ||
+        (code >= 0x20 && code <= 0xd7ff) ||
+        (code >= 0xe000 && code <= 0xfffd) ||
+        (code >= 0x10000 && code <= 0x10ffff));
+}
+/** Replaces the characters XML 1.0 does not allow, lone surrogates included, with U+FFFD */
+function xmlSafe(value) {
+    let out = '';
+    for (const ch of value) {
+        const code = ch.codePointAt(0) ?? 0;
+        out += isXmlChar(code) ? ch : '�';
+    }
+    return out;
+}
+/** Escapes text content, `\r` kept as a reference since parsers fold a raw one into `\n` */
+function escapeText(value) {
+    return xmlSafe(value)
+        .replace(/&/g, '&amp;')
+        .replace(/</g, '&lt;')
+        .replace(/>/g, '&gt;')
+        .replace(/\r/g, '&#13;');
+}
+/** Escapes an attribute value, keeping tabs and line breaks that a parser would fold to spaces */
+function escapeAttr(value) {
+    return escapeText(value)
+        .replace(/"/g, '&quot;')
+        .replace(/'/g, '&apos;')
+        .replace(/\t/g, '&#9;')
+        .replace(/\n/g, '&#10;');
+}
+function attrs(values) {
+    return Object.entries(values)
+        .map(([key, value]) => ` ${key}="${escapeAttr(String(value))}"`)
+        .join('');
+}
+function seconds(ms) {
+    return (Math.max(0, ms) / 1000).toFixed(3);
+}
+function firstLine(text) {
+    return text.split(/\r?\n/, 1)[0] ?? '';
+}
+function caseOutput(result) {
+    const lines = result.scores.map((s) => {
+        const details = s.details ? ` (${s.details})` : '';
+        const error = s.error !== undefined ? ` [error: ${s.error}]` : '';
+        return `${s.name}: ${s.score}${details}${error}`;
+    });
+    for (const flag of caseFlags(result))
+        lines.push(`${flag}: true`);
+    return lines.join('\n');
+}
+function caseTestcase(result, index) {
+    const head = `    <testcase${attrs({
+        name: caseLabel(index, result.case.input),
+        classname: `${SUITE_NAME}.cases`,
+        time: seconds(result.duration),
+    })}`;
+    const body = [];
+    if (result.error !== undefined) {
+        body.push(`      <failure${attrs({ message: firstLine(result.error), type: 'CaseError' })}>${escapeText(result.error)}</failure>`);
+    }
+    const output = caseOutput(result);
+    if (output !== '')
+        body.push(`      <system-out>${escapeText(output)}</system-out>`);
+    return body.length === 0 ? `${head}/>` : `${head}>\n${body.join('\n')}\n    </testcase>`;
+}
+function assertionTestcase(assertion) {
+    const head = `    <testcase${attrs({
+        name: assertion.name,
+        classname: `${SUITE_NAME}.assertions`,
+        time: seconds(0),
+    })}`;
+    if (assertion.passed)
+        return `${head}/>`;
+    return `${head}>\n      <failure${attrs({ message: firstLine(assertion.message), type: 'AssertionFailure' })}>${escapeText(assertion.message)}</failure>\n    </testcase>`;
+}
+/** The JUnit XML document of a suite run: one testcase per case and one per assertion */
+export function junitXml(result) {
+    const tests = result.results.length + result.assertions.length;
+    const failures = result.results.filter((r) => r.error !== undefined).length +
+        result.assertions.filter((a) => !a.passed).length;
+    const time = seconds(result.stats.duration);
+    const counts = { tests, failures, errors: 0, skipped: 0, time };
+    const testcases = [
+        ...result.results.map((r, i) => caseTestcase(r, i)),
+        ...result.assertions.map((a) => assertionTestcase(a)),
+    ];
+    return [
+        '<?xml version="1.0" encoding="UTF-8"?>',
+        `<testsuites${attrs({ name: SUITE_NAME, tests, failures, errors: 0, time })}>`,
+        `  <testsuite${attrs({ name: SUITE_NAME, ...counts })}${testcases.length === 0 ? '/>' : '>'}`,
+        ...(testcases.length === 0 ? [] : [...testcases, '  </testsuite>']),
+        '</testsuites>',
+        '',
+    ].join('\n');
+}
+/** Writes the suite run as JUnit XML, which CI systems show as a test report */
+export function junitReport(result, options) {
+    writeFileSync(options.path, junitXml(result), 'utf-8');
+}
+//# sourceMappingURL=junit.js.map

package/dist/reporters/junit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"junit.js","sourceRoot":"","sources":["../../src/reporters/junit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAExC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAEhD,MAAM,UAAU,GAAG,MAAM,CAAC;AAE1B,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,CACL,IAAI,KAAK,GAAG;QACZ,IAAI,KAAK,GAAG;QACZ,IAAI,KAAK,GAAG;QACZ,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,MAAM,CAAC;QAChC,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC;QAClC,CAAC,IAAI,IAAI,OAAO,IAAI,IAAI,IAAI,QAAQ,CAAC,CACtC,CAAC;AACJ,CAAC;AAED,4FAA4F;AAC5F,SAAS,OAAO,CAAC,KAAa;IAC5B,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,KAAK,MAAM,EAAE,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACpC,GAAG,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IACpC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4FAA4F;AAC5F,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,OAAO,CAAC,KAAK,CAAC;SAClB,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,kGAAkG;AAClG,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,UAAU,CAAC,KAAK,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC;SACtB,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,KAAK,CAAC,MAAuC;IACpD,OAAO,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;SAC1B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC;SAC/D,IAAI,CAAC,EAAE,CAAC,CAAC;AACd,CAAC;AAED,SAAS,OAAO,CAAC,EAAU;IACzB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACzC,CAAC;AAID,SAAS,UAAU,CAAC,MAAkB;IACpC,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACpC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACnD,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAClE,OAAO,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,GAAG,OAAO,GAAG,KAAK,EAAE,CAAC;IACnD,CAAC,CAAC,CAAC;IACH,KAAK,MAAM,IAAI,IAAI,SAAS,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,QAAQ,CAAC,CAAC;IAClE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,YAAY,CAAC,MAAkB,EAAE,KAAa;IACrD,MAAM,IAAI,GAAG,gBAAgB,KAAK,CAAC;QACjC,IAAI,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QACzC,SAAS,EAAE,GAAG,UAAU,QAAQ;QAChC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;KAC/B,CAAC,EAAE,CAAC;IACL,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,MAAM,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/B,IAAI,CAAC,IAAI,CACP,iBAAiB,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,IAAI,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CACxH,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IAClC,IAAI,MAAM,KAAK,EAAE;QAAE,IAAI,CAAC,IAAI,CAAC,qBAAqB,UAAU,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IACrF,OAAO,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC;AAC3F,CAAC;AAED,SAAS,iBAAiB,CAAC,SAAgD;IACzE,MAAM,IAAI,GAAG,gBAAgB,KAAK,CAAC;QACjC,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,SAAS,EAAE,GAAG,UAAU,aAAa;QACrC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;KACjB,CAAC,EAAE,CAAC;IACL,IAAI,SAAS,CAAC,MAAM;QAAE,OAAO,GAAG,IAAI,IAAI,CAAC;IACzC,OAAO,GAAG,IAAI,oBAAoB,KAAK,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,CAAC,IAAI,UAAU,CAAC,SAAS,CAAC,OAAO,CAAC,6BAA6B,CAAC;AAC7K,CAAC;AAED,yFAAyF;AACzF,MAAM,UAAU,QAAQ,CAAC,MAAuB;IAC9C,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;IAC/D,MAAM,QAAQ,GACZ,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,MAAM;QAC1D,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACpD,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC5C,MAAM,MAAM,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC;IAEhE,MAAM,SAAS,GAAG;QAChB,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnD,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC;KACtD,CAAC;IAEF,OAAO;QACL,wCAAwC;QACxC,cAAc,KAAK,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,GAAG;QAC9E,eAAe,KAAK,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,MAAM,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE;QAC7F,GAAG,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,EAAE,gBAAgB,CAAC,CAAC;QACnE,eAAe;QACf,EAAE;KACH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,WAAW,CAAC,MAAuB,EAAE,OAAyB;IAC5E,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;AACzD,CAAC"}

package/dist/schema.d.ts

@@ -26,11 +26,22 @@ export interface JudgeCogitator {
         };
     }>;
 }
+/**
+ * Where the judge keeps the verdicts it already gave, by a key of its model, settings and the
+ * whole prompt: a case judged again with the same answer reuses the verdict and costs nothing.
+ * A store of your own (a file, Redis) keeps verdicts across runs
+ */
+export interface JudgeCache {
+    get(key: string): string | undefined | Promise<string | undefined>;
+    set(key: string, verdict: string): void | Promise<void>;
+}
 export declare const JudgeConfigSchema: z.ZodObject<{
     model: z.ZodString;
     temperature: z.ZodDefault<z.ZodNumber>;
     maxTokens: z.ZodOptional<z.ZodNumber>;
     cogitator: z.ZodOptional<z.ZodCustom<JudgeCogitator, JudgeCogitator>>;
+    cache: z.ZodOptional<z.ZodUnion<readonly [z.ZodBoolean, z.ZodCustom<JudgeCache, JudgeCache>]>>;
+    concurrency: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 export declare const EvalComparisonConfigSchema: z.ZodObject<{
     concurrency: z.ZodDefault<z.ZodNumber>;

package/dist/schema.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,cAAc;;;;;iBAKzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;CAChF;AAUD,eAAO,MAAM,iBAAiB;;;;;iBAM5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;iBAIrC,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,cAAc;;;;;iBAKzB,CAAC;AAEH,eAAO,MAAM,qBAAqB;;;;iBAIhC,CAAC;AAEH;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,GAAG,CACD,KAAK,EAAE,OAAO,EACd,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAC9C,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAC,KAAK,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAAE,CAAC,CAAC;CAChF;AAED;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;IACnE,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACzD;AAmBD,eAAO,MAAM,iBAAiB;;;;;;;iBAgB5B,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;iBAIrC,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC;AACvD,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAC3D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACrE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACzE,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAC7D,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,iBAAiB,CAAC,CAAC;AACjE,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,0BAA0B,CAAC,CAAC"}

package/dist/schema.js

@@ -10,6 +10,12 @@ export const EvalSuiteConfigSchema = z.object({
     timeout: z.number().int().min(1000).default(30000),
     retries: z.number().int().min(0).max(10).default(0),
 });
+function isJudgeCache(value) {
+    return (typeof value === 'object' &&
+        value !== null &&
+        typeof value.get === 'function' &&
+        typeof value.set === 'function');
+}
 function isJudgeCogitator(value) {
     return (typeof value === 'object' &&
         value !== null &&
@@ -21,6 +27,16 @@ export const JudgeConfigSchema = z.object({
     maxTokens: z.number().int().positive().optional(),
     /** Runtime that runs the judge; defaults to the `cogitator` of an agent target */
     cogitator: z.custom(isJudgeCogitator).optional(),
+    /**
+     * Reuse verdicts: `true` keeps them in memory for the suite, a `JudgeCache` of your own keeps
+     * them where it likes. Default: every case is judged afresh
+     */
+    cache: z.union([z.boolean(), z.custom(isJudgeCache)]).optional(),
+    /**
+     * Most judge calls at once across the run, below the suite's own concurrency times its judge
+     * metrics. Default: no limit of its own
+     */
+    concurrency: z.number().int().positive().optional(),
 });
 export const EvalComparisonConfigSchema = z.object({
     concurrency: z.number().int().min(1).default(5),

package/dist/schema.js.map

@@ -1 +1 @@
-{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC;AAaH,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;CACjE,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC"}
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../src/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;IACrD,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CACvD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC;AAuBH,SAAS,YAAY,CAAC,KAAc;IAClC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU;QACtD,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,CACL,OAAO,KAAK,KAAK,QAAQ;QACzB,KAAK,KAAK,IAAI;QACd,OAAQ,KAA2B,CAAC,GAAG,KAAK,UAAU,CACvD,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC;IACxC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACjD,kFAAkF;IAClF,SAAS,EAAE,CAAC,CAAC,MAAM,CAAiB,gBAAgB,CAAC,CAAC,QAAQ,EAAE;IAChE;;;OAGG;IACH,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,MAAM,CAAa,YAAY,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC5E;;;OAGG;IACH,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CACpD,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,CAAC,MAAM,CAAC;IACjD,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC/C,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC;IAClD,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;CACpD,CAAC,CAAC"}