@cogitator-ai/evals 0.8.0 → 0.10.0

package/dist/datasets/csv-loader.js

@@ -1,26 +1,74 @@
 import { readFile } from 'node:fs/promises';
 import { EvalCaseSchema } from '../schema.js';
+import { describeIssues } from './issues.js';
+/** Columns whose cells hold JSON, such as `["follow-up question"]` for `turns` */
+const JSON_FIELDS = ['turns', 'expectedToolCalls', 'expectedPassages'];
+/**
+ * Cases from a CSV file with an `input` column, an optional `expected` one, `metadata.<key>` and
+ * `context.<key>` columns, and `turns`, `expectedToolCalls` and `expectedPassages` columns whose
+ * cells hold JSON arrays (an empty cell leaves the field out)
+ */
 export async function loadCsv(path) {
     let Papa;
     try {
-        Papa = await import('papaparse');
+        const loaded = (await import('papaparse'));
+        Papa = loaded.default ?? loaded;
     }
     catch {
         throw new Error('papaparse is required for CSV loading. Install it with: npm install papaparse');
     }
-    const content = await readFile(path, 'utf-8');
-    const { data, meta } = Papa.parse(content, {
-        header: true,
+    const content = (await readFile(path, 'utf-8')).replace(/^\uFEFF/, '');
+    const lineOf = lineFinder(content);
+    let fields;
+    const rows = [];
+    let rowStart = 0;
+    let malformed;
+    Papa.parse(content, {
+        delimiter: ',',
         skipEmptyLines: true,
+        step: (result, parser) => {
+            const line = lineOf(firstContent(content, rowStart));
+            rowStart = result.meta.cursor;
+            const [error] = result.errors;
+            if (error) {
+                malformed = `Invalid CSV at line ${line}: ${error.message}`;
+                parser.abort();
+                return;
+            }
+            if (!fields) {
+                fields = result.data;
+                return;
+            }
+            const row = {};
+            fields.forEach((field, at) => {
+                const value = result.data[at];
+                if (value !== undefined)
+                    row[field] = value;
+            });
+            rows.push({ row, line });
+        },
     });
-    if (!meta.fields?.includes('input')) {
+    if (malformed)
+        throw new Error(malformed);
+    if (!fields?.includes('input')) {
         throw new Error('CSV must have an "input" column');
     }
-    return data.map((row) => {
+    return rows.map(({ row, line }) => {
         const evalCase = { input: row.input };
         if (row.expected !== undefined && row.expected !== '') {
             evalCase.expected = row.expected;
         }
+        for (const field of JSON_FIELDS) {
+            const cell = row[field];
+            if (cell === undefined || cell.trim() === '')
+                continue;
+            try {
+                evalCase[field] = JSON.parse(cell);
+            }
+            catch {
+                throw new Error(`Invalid case at line ${line}: ${field}: not valid JSON`);
+            }
+        }
         const metadata = {};
         const context = {};
         for (const [key, value] of Object.entries(row)) {
@@ -37,7 +85,41 @@ export async function loadCsv(path) {
             evalCase.metadata = metadata;
         if (Object.keys(context).length > 0)
             evalCase.context = context;
-        return EvalCaseSchema.parse(evalCase);
+        const checked = EvalCaseSchema.safeParse(evalCase);
+        if (!checked.success) {
+            throw new Error(`Invalid case at line ${line}: ${describeIssues(checked.error)}`);
+        }
+        return checked.data;
     });
 }
+/** Where the text after `offset` starts, past the line breaks of blank lines before it */
+function firstContent(content, offset) {
+    let at = offset;
+    while (at < content.length && (content[at] === '\n' || content[at] === '\r'))
+        at++;
+    return at;
+}
+/** The 1-based line of an offset in `content`, a line ending at `\r\n`, `\r` or `\n` */
+function lineFinder(content) {
+    const starts = [0];
+    for (let at = 0; at < content.length; at++) {
+        const char = content[at];
+        if (char === '\r' && content[at + 1] === '\n')
+            continue;
+        if (char === '\n' || char === '\r')
+            starts.push(at + 1);
+    }
+    return (offset) => {
+        let low = 0;
+        let high = starts.length - 1;
+        while (low < high) {
+            const mid = (low + high + 1) >> 1;
+            if (starts[mid] <= offset)
+                low = mid;
+            else
+                high = mid - 1;
+        }
+        return low + 1;
+    };
+}
 //# sourceMappingURL=csv-loader.js.map

package/dist/datasets/csv-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../../src/datasets/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAG3C,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAY;IACxC,IAAI,IAAgC,CAAC;IACrC,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,+EAA+E,CAChF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,KAAK,CAAyB,OAAO,EAAE;QACjE,MAAM,EAAE,IAAI;QACZ,cAAc,EAAE,IAAI;KACrB,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACtB,MAAM,QAAQ,GAA4B,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;QAE/D,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,EAAE,CAAC;YACtD,QAAQ,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;QACnC,CAAC;QAED,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,MAAM,OAAO,GAA4B,EAAE,CAAC;QAE5C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,UAAU;gBAAE,SAAS;YAEpD,IAAI,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;gBAChC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC;YAClD,CAAC;iBAAM,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC;YAChD,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACnE,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,QAAQ,CAAC,OAAO,GAAG,OAAO,CAAC;QAEhE,OAAO,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../../src/datasets/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAG1C,kFAAkF;AAClF,MAAM,WAAW,GAAG,CAAC,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,CAAU,CAAC;AAEhF;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAY;IACxC,IAAI,IAAgC,CAAC;IACrC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,WAAW,CAAC,CAExC,CAAC;QACF,IAAI,GAAG,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,+EAA+E,CAChF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;IACvE,MAAM,MAAM,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;IACnC,IAAI,MAA4B,CAAC;IACjC,MAAM,IAAI,GAAyD,EAAE,CAAC;IACtE,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,SAA6B,CAAC;IAClC,IAAI,CAAC,KAAK,CAAW,OAAO,EAAE;QAC5B,SAAS,EAAE,GAAG;QACd,cAAc,EAAE,IAAI;QACpB,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE;YACvB,MAAM,IAAI,GAAG,MAAM,CAAC,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;YACrD,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;YAC9B,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;YAC9B,IAAI,KAAK,EAAE,CAAC;gBACV,SAAS,GAAG,uBAAuB,IAAI,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC5D,MAAM,CAAC,KAAK,EAAE,CAAC;gBACf,OAAO;YACT,CAAC;YACD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC;gBACrB,OAAO;YACT,CAAC;YACD,MAAM,GAAG,GAA2B,EAAE,CAAC;YACvC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC9B,IAAI,KAAK,KAAK,SAAS;oBAAE,GAAG,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC;YAC9C,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3B,CAAC;KACF,CAAC,CAAC;IAEH,IAAI,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,SAAS,CAAC,CAAC;IAC1C,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE;QAChC,MAAM,QAAQ,GAA4B,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;QAE/D,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,EAAE,CAAC;YACtD,QAAQ,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;QACnC,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YACxB,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;gBAAE,SAAS;YACvD,IAAI,CAAC;gBACH,QAAQ,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAY,CAAC;YAChD,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,KAAK,KAAK,kBAAkB,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,MAAM,OAAO,GAA4B,EAAE,CAAC;QAE5C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,UAAU;gBAAE,SAAS;YAEpD,IAAI,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;gBAChC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC;YAClD,CAAC;iBAAM,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC;YAChD,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACnE,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,QAAQ,CAAC,OAAO,GAAG,OAAO,CAAC;QAEhE,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACnD,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,wBAAwB,IAAI,KAAK,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACpF,CAAC;QACD,OAAO,OAAO,CAAC,IAAI,CAAC;IACtB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,0FAA0F;AAC1F,SAAS,YAAY,CAAC,OAAe,EAAE,MAAc;IACnD,IAAI,EAAE,GAAG,MAAM,CAAC;IAChB,OAAO,EAAE,GAAG,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC;QAAE,EAAE,EAAE,CAAC;IACnF,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,wFAAwF;AACxF,SAAS,UAAU,CAAC,OAAe;IACjC,MAAM,MAAM,GAAa,CAAC,CAAC,CAAC,CAAC;IAC7B,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC;QACzB,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC,KAAK,IAAI;YAAE,SAAS;QACxD,IAAI,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAC1D,CAAC;IACD,OAAO,CAAC,MAAM,EAAE,EAAE;QAChB,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,IAAI,IAAI,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;QAC7B,OAAO,GAAG,GAAG,IAAI,EAAE,CAAC;YAClB,MAAM,GAAG,GAAG,CAAC,GAAG,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;YAClC,IAAI,MAAM,CAAC,GAAG,CAAE,IAAI,MAAM;gBAAE,GAAG,GAAG,GAAG,CAAC;;gBACjC,IAAI,GAAG,GAAG,GAAG,CAAC,CAAC;QACtB,CAAC;QACD,OAAO,GAAG,GAAG,CAAC,CAAC;IACjB,CAAC,CAAC;AACJ,CAAC"}

package/dist/datasets/dataset.d.ts

@@ -9,7 +9,16 @@ export declare class Dataset {
     get cases(): readonly EvalCase[];
     [Symbol.iterator](): Iterator<EvalCase>;
     filter(fn: (c: EvalCase) => boolean): Dataset;
-    sample(n: number): Dataset;
-    shuffle(): Dataset;
+    /**
+     * `n` cases picked at random, all of them when `n` is at least the size. With `seed` the pick
+     * is the same on every run, so a sampled eval compares like with like
+     */
+    sample(n: number, options?: {
+        seed?: number | string;
+    }): Dataset;
+    /** The cases in random order, the same order on every run with `seed` */
+    shuffle(options?: {
+        seed?: number | string;
+    }): Dataset;
 }
 //# sourceMappingURL=dataset.d.ts.map

package/dist/datasets/dataset.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/datasets/dataset.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI1C,qBAAa,OAAO;IAClB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAE7C,OAAO;IAIP,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO;WAI1B,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAKzC,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKpD,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,KAAK,IAAI,SAAS,QAAQ,EAAE,CAE/B;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC;IAavC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,KAAK,OAAO,GAAG,OAAO;IAI7C,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO;IAa1B,OAAO,IAAI,OAAO;CAQnB"}
+{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/datasets/dataset.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI1C,qBAAa,OAAO;IAClB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAE7C,OAAO;IAIP,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO;WAI1B,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAKzC,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKpD,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,KAAK,IAAI,SAAS,QAAQ,EAAE,CAE/B;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC;IAavC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,KAAK,OAAO,GAAG,OAAO;IAI7C;;;OAGG;IACH,MAAM,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAO,GAAG,OAAO;IAYpE,yEAAyE;IACzE,OAAO,CAAC,OAAO,GAAE;QAAE,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;KAAO,GAAG,OAAO;CAG3D"}

package/dist/datasets/dataset.js

@@ -38,25 +38,54 @@ export class Dataset {
     filter(fn) {
         return new Dataset(this._cases.filter(fn));
     }
-    sample(n) {
+    /**
+     * `n` cases picked at random, all of them when `n` is at least the size. With `seed` the pick
+     * is the same on every run, so a sampled eval compares like with like
+     */
+    sample(n, options = {}) {
         if (n <= 0 || this._cases.length === 0) {
             return new Dataset([]);
         }
         const count = Math.min(n, this._cases.length);
-        const indices = Array.from({ length: this._cases.length }, (_, i) => i);
-        for (let i = indices.length - 1; i > 0; i--) {
-            const j = Math.floor(Math.random() * (i + 1));
-            [indices[i], indices[j]] = [indices[j], indices[i]];
-        }
+        const indices = shuffled(Array.from({ length: this._cases.length }, (_, i) => i), randomOf(options.seed));
         return new Dataset(indices.slice(0, count).map((i) => this._cases[i]));
     }
-    shuffle() {
-        const arr = [...this._cases];
-        for (let i = arr.length - 1; i > 0; i--) {
-            const j = Math.floor(Math.random() * (i + 1));
-            [arr[i], arr[j]] = [arr[j], arr[i]];
-        }
-        return new Dataset(arr);
+    /** The cases in random order, the same order on every run with `seed` */
+    shuffle(options = {}) {
+        return new Dataset(shuffled([...this._cases], randomOf(options.seed)));
+    }
+}
+/** Shuffles `items` in place (Fisher-Yates) with the random numbers `random` gives */
+function shuffled(items, random) {
+    for (let i = items.length - 1; i > 0; i--) {
+        const j = Math.floor(random() * (i + 1));
+        [items[i], items[j]] = [items[j], items[i]];
+    }
+    return items;
+}
+/**
+ * `Math.random`, or a mulberry32 generator seeded by the hash of `seed`, so every number and
+ * string seeds its own sequence (`1.2` and `1.7`, `-1` and `2 ** 32 - 1` alike)
+ */
+function randomOf(seed) {
+    if (seed === undefined)
+        return Math.random;
+    let state = hashSeed(String(seed));
+    return () => {
+        state = (state + 0x6d2b79f5) >>> 0;
+        let t = state;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+/** FNV-1a of a string, to seed the generator */
+function hashSeed(seed) {
+    let hash = 0x811c9dc5;
+    for (let i = 0; i < seed.length; i++) {
+        hash ^= seed.charCodeAt(i);
+        hash = Math.imul(hash, 0x01000193) >>> 0;
     }
+    return hash;
 }
 //# sourceMappingURL=dataset.js.map

package/dist/datasets/dataset.js.map

@@ -1 +1 @@
-{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/datasets/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,OAAO,OAAO;IACD,MAAM,CAAsB;IAE7C,YAAoB,KAAiB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,KAAiB;QAC3B,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAY;QACjC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAY;QAC/B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC;QACf,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,OAAO;YACL,IAAI;gBACF,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;oBACzB,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;gBAChD,CAAC;gBACD,OAAO,EAAE,KAAK,EAAE,SAAkB,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACnD,CAAC;SACF,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,EAA4B;QACjC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,CAAC,CAAS;QACd,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QACxE,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,OAAO;QACL,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAe,CAAC;QAC3C,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;CACF"}
+{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/datasets/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,OAAO,OAAO;IACD,MAAM,CAAsB;IAE7C,YAAoB,KAAiB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,KAAiB;QAC3B,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAY;QACjC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAY;QAC/B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC;QACf,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,OAAO;YACL,IAAI;gBACF,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;oBACzB,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;gBAChD,CAAC;gBACD,OAAO,EAAE,KAAK,EAAE,SAAkB,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACnD,CAAC;SACF,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,EAA4B;QACjC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7C,CAAC;IAED;;;OAGG;IACH,MAAM,CAAC,CAAS,EAAE,UAAsC,EAAE;QACxD,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,QAAQ,CACtB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EACvD,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CACvB,CAAC;QACF,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,yEAAyE;IACzE,OAAO,CAAC,UAAsC,EAAE;QAC9C,OAAO,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;CACF;AAED,sFAAsF;AACtF,SAAS,QAAQ,CAAI,KAAU,EAAE,MAAoB;IACnD,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACzC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAE,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAS,QAAQ,CAAC,IAAiC;IACjD,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC,MAAM,CAAC;IAC3C,IAAI,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IACnC,OAAO,GAAG,EAAE;QACV,KAAK,GAAG,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC;QACnC,IAAI,CAAC,GAAG,KAAK,CAAC;QACd,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC;QAC1C,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAC;IAC/C,CAAC,CAAC;AACJ,CAAC;AAED,gDAAgD;AAChD,SAAS,QAAQ,CAAC,IAAY;IAC5B,IAAI,IAAI,GAAG,UAAU,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IAC3C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/datasets/issues.d.ts

@@ -0,0 +1,4 @@
+import type { z } from 'zod';
+/** What a case breaks of the case schema, one `field: problem` per issue */
+export declare function describeIssues(error: z.ZodError): string;
+//# sourceMappingURL=issues.d.ts.map

package/dist/datasets/issues.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"issues.d.ts","sourceRoot":"","sources":["../../src/datasets/issues.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAE7B,4EAA4E;AAC5E,wBAAgB,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC,QAAQ,GAAG,MAAM,CAIxD"}

package/dist/datasets/issues.js

@@ -0,0 +1,7 @@
+/** What a case breaks of the case schema, one `field: problem` per issue */
+export function describeIssues(error) {
+    return error.issues
+        .map((issue) => `${issue.path.length > 0 ? issue.path.join('.') : '(case)'}: ${issue.message}`)
+        .join('; ');
+}
+//# sourceMappingURL=issues.js.map

package/dist/datasets/issues.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"issues.js","sourceRoot":"","sources":["../../src/datasets/issues.ts"],"names":[],"mappings":"AAEA,4EAA4E;AAC5E,MAAM,UAAU,cAAc,CAAC,KAAiB;IAC9C,OAAO,KAAK,CAAC,MAAM;SAChB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC;SAC9F,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC"}

package/dist/datasets/jsonl-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl-loader.d.ts","sourceRoot":"","sources":["../../src/datasets/jsonl-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAyBjE"}
+{"version":3,"file":"jsonl-loader.d.ts","sourceRoot":"","sources":["../../src/datasets/jsonl-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CA6BjE"}

package/dist/datasets/jsonl-loader.js

@@ -1,6 +1,7 @@
 import { createReadStream } from 'node:fs';
 import { createInterface } from 'node:readline';
 import { EvalCaseSchema } from '../schema.js';
+import { describeIssues } from './issues.js';
 export async function loadJsonl(path) {
     const cases = [];
     let lineNumber = 0;
@@ -20,7 +21,11 @@ export async function loadJsonl(path) {
         catch {
             throw new Error(`Invalid JSON at line ${lineNumber}: ${trimmed}`);
         }
-        cases.push(EvalCaseSchema.parse(parsed));
+        const checked = EvalCaseSchema.safeParse(parsed);
+        if (!checked.success) {
+            throw new Error(`Invalid case at line ${lineNumber}: ${describeIssues(checked.error)}`);
+        }
+        cases.push(checked.data);
     }
     return cases;
 }

package/dist/datasets/jsonl-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl-loader.js","sourceRoot":"","sources":["../../src/datasets/jsonl-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAG3C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,EAAE,GAAG,eAAe,CAAC;QACzB,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QACpD,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC5B,UAAU,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,wBAAwB,UAAU,KAAK,OAAO,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}
+{"version":3,"file":"jsonl-loader.js","sourceRoot":"","sources":["../../src/datasets/jsonl-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAG1C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,EAAE,GAAG,eAAe,CAAC;QACzB,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QACpD,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC5B,UAAU,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,wBAAwB,UAAU,KAAK,OAAO,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACjD,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,wBAAwB,UAAU,KAAK,cAAc,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC1F,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3B,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/eval-builder.d.ts

@@ -14,6 +14,7 @@ export declare class EvalBuilder {
     private _concurrency?;
     private _timeout?;
     private _retries?;
+    private _trials?;
     private _onProgress?;
     withDataset(dataset: Dataset): this;
     withTarget(target: EvalTarget): this;
@@ -24,6 +25,8 @@ export declare class EvalBuilder {
     withConcurrency(n: number): this;
     withTimeout(ms: number): this;
     withRetries(n: number): this;
+    /** Runs each case `n` times, see `EvalSuiteOptions.trials` */
+    withTrials(n: number): this;
     onProgress(fn: EvalSuiteOptions['onProgress']): this;
     build(): EvalSuite;
 }

package/dist/eval-builder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-builder.d.ts","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAe,MAAM,cAAc,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,CAAU;IAC3B,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,mBAAmB,CAA6B;IACxD,OAAO,CAAC,MAAM,CAAC,CAAmB;IAClC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAC,CAAiC;IAErD,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKnC,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAKpC,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,IAAI;IAKtC,sBAAsB,CAAC,OAAO,EAAE,mBAAmB,EAAE,GAAG,IAAI;IAK5D,SAAS,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAKzC,cAAc,CAAC,UAAU,EAAE,WAAW,EAAE,GAAG,IAAI;IAK/C,eAAe,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAKhC,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAK7B,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAK5B,UAAU,CAAC,EAAE,EAAE,gBAAgB,CAAC,YAAY,CAAC,GAAG,IAAI;IAKpD,KAAK,IAAI,SAAS;CA8BnB"}
+{"version":3,"file":"eval-builder.d.ts","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAe,MAAM,cAAc,CAAC;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AACjD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEhD,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,CAAU;IAC3B,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,mBAAmB,CAA6B;IACxD,OAAO,CAAC,MAAM,CAAC,CAAmB;IAClC,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,OAAO,CAAC,CAAS;IACzB,OAAO,CAAC,WAAW,CAAC,CAAiC;IAErD,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKnC,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAKpC,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,IAAI;IAKtC,sBAAsB,CAAC,OAAO,EAAE,mBAAmB,EAAE,GAAG,IAAI;IAK5D,SAAS,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAKzC,cAAc,CAAC,UAAU,EAAE,WAAW,EAAE,GAAG,IAAI;IAK/C,eAAe,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAKhC,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAK7B,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAK5B,8DAA8D;IAC9D,UAAU,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAK3B,UAAU,CAAC,EAAE,EAAE,gBAAgB,CAAC,YAAY,CAAC,GAAG,IAAI;IAKpD,KAAK,IAAI,SAAS;CA+BnB"}

package/dist/eval-builder.js

@@ -9,6 +9,7 @@ export class EvalBuilder {
     _concurrency;
     _timeout;
     _retries;
+    _trials;
     _onProgress;
     withDataset(dataset) {
         this._dataset = dataset;
@@ -46,6 +47,11 @@ export class EvalBuilder {
         this._retries = n;
         return this;
     }
+    /** Runs each case `n` times, see `EvalSuiteOptions.trials` */
+    withTrials(n) {
+        this._trials = n;
+        return this;
+    }
     onProgress(fn) {
         this._onProgress = fn;
         return this;
@@ -71,6 +77,7 @@ export class EvalBuilder {
             concurrency: this._concurrency,
             timeout: this._timeout,
             retries: this._retries,
+            trials: this._trials,
             onProgress: this._onProgress,
         };
         return new EvalSuite(opts);

package/dist/eval-builder.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-builder.js","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAOtD,MAAM,OAAO,WAAW;IACd,QAAQ,CAAW;IACnB,OAAO,CAAc;IACrB,QAAQ,GAAe,EAAE,CAAC;IAC1B,mBAAmB,GAA0B,EAAE,CAAC;IAChD,MAAM,CAAoB;IAC1B,WAAW,GAAkB,EAAE,CAAC;IAChC,YAAY,CAAU;IACtB,QAAQ,CAAU;IAClB,QAAQ,CAAU;IAClB,WAAW,CAAkC;IAErD,WAAW,CAAC,OAAgB;QAC1B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,MAAkB;QAC3B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,OAAmB;QAC7B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sBAAsB,CAAC,OAA8B;QACnD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,SAAS,CAAC,MAAwB;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,cAAc,CAAC,UAAyB;QACtC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,eAAe,CAAC,CAAS;QACvB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,EAAU;QACpB,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,CAAS;QACnB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,EAAkC;QAC3C,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,MAAM,IAAI,GAAqB;YAC7B,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YAC7D,kBAAkB,EAChB,IAAI,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,SAAS;YAC5E,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;YACtE,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,CAAC;QAEF,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACF"}
+{"version":3,"file":"eval-builder.js","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAOtD,MAAM,OAAO,WAAW;IACd,QAAQ,CAAW;IACnB,OAAO,CAAc;IACrB,QAAQ,GAAe,EAAE,CAAC;IAC1B,mBAAmB,GAA0B,EAAE,CAAC;IAChD,MAAM,CAAoB;IAC1B,WAAW,GAAkB,EAAE,CAAC;IAChC,YAAY,CAAU;IACtB,QAAQ,CAAU;IAClB,QAAQ,CAAU;IAClB,OAAO,CAAU;IACjB,WAAW,CAAkC;IAErD,WAAW,CAAC,OAAgB;QAC1B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,MAAkB;QAC3B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,OAAmB;QAC7B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sBAAsB,CAAC,OAA8B;QACnD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,SAAS,CAAC,MAAwB;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,cAAc,CAAC,UAAyB;QACtC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,eAAe,CAAC,CAAS;QACvB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,EAAU;QACpB,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,CAAS;QACnB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,8DAA8D;IAC9D,UAAU,CAAC,CAAS;QAClB,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;QACjB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,EAAkC;QAC3C,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,MAAM,IAAI,GAAqB;YAC7B,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YAC7D,kBAAkB,EAChB,IAAI,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,SAAS;YAC5E,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;YACtE,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,CAAC;QAEF,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACF"}

package/dist/eval-comparison.d.ts

@@ -14,6 +14,11 @@ export interface EvalComparisonOptions {
     concurrency?: number;
     timeout?: number;
     retries?: number;
+    /**
+     * Runs of each case on each side, as the suite's `trials`. A metric is compared on each case's
+     * mean over its scored trials, and `passAtK()` statistical metrics see every trial
+     */
+    trials?: number;
     onProgress?: (progress: {
         target: string;
         completed: number;
@@ -27,6 +32,8 @@ export interface EvalComparisonRunOptions {
 export interface MetricComparison {
     baseline: number;
     challenger: number;
+    /** Cases compared: those at least one side scored, an errored side counting as a 0 */
+    n: number;
     pValue: number;
     significant: boolean;
     winner: 'baseline' | 'challenger' | 'tie';

package/dist/eval-comparison.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-comparison.d.ts","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAe,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAKrC,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE;QACP,QAAQ,EAAE,UAAU,CAAC;QACrB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACvF;AAED,MAAM,WAAW,wBAAwB;IACvC,kEAAkE;IAClE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE;QACP,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;QAC1C,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;KAC3C,CAAC;IACF,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,eAAe,CAAC;CAC7B;AAkED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAwB;gBAEjC,IAAI,EAAE,qBAAqB;IASjC,GAAG,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAoE7E"}
+{"version":3,"file":"eval-comparison.d.ts","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAiC,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACpG,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAKrC,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE;QACP,QAAQ,EAAE,UAAU,CAAC;QACrB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,gBAAgB,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACvF;AAED,MAAM,WAAW,wBAAwB;IACvC,kEAAkE;IAClE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,sFAAsF;IACtF,CAAC,EAAE,MAAM,CAAC;IACV,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE;QACP,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;QAC1C,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;KAC3C,CAAC;IACF,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,eAAe,CAAC;CAC7B;AAmED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAwB;gBAEjC,IAAI,EAAE,qBAAqB;IAUjC,GAAG,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,gBAAgB,CAAC;CA2E7E"}

package/dist/eval-comparison.js

@@ -14,6 +14,7 @@ function compareMetric(baselineScores, challengerScores) {
         return {
             baseline: baselineMean,
             challenger: challengerMean,
+            n,
             pValue: 1,
             significant: false,
             winner: 'tie',
@@ -44,7 +45,7 @@ function compareMetric(baselineScores, challengerScores) {
     if (significant) {
         winner = challengerMean > baselineMean ? 'challenger' : 'baseline';
     }
-    return { baseline: baselineMean, challenger: challengerMean, pValue, significant, winner };
+    return { baseline: baselineMean, challenger: challengerMean, n, pValue, significant, winner };
 }
 function determineOverallWinner(metrics) {
     let baselineWins = 0;
@@ -68,11 +69,12 @@ export class EvalComparison {
             concurrency: opts.concurrency,
             timeout: opts.timeout,
             retries: opts.retries,
+            trials: opts.trials,
         });
         this.opts = opts;
     }
     async run(options = {}) {
-        const { dataset, targets, metrics, statisticalMetrics, judge, concurrency, timeout, retries, onProgress, } = this.opts;
+        const { dataset, targets, metrics, statisticalMetrics, judge, concurrency, timeout, retries, trials, onProgress, } = this.opts;
         const sharedConfig = {
             dataset,
             metrics,
@@ -81,6 +83,7 @@ export class EvalComparison {
             concurrency,
             timeout,
             retries,
+            trials,
         };
         const baselineSuite = new EvalSuite({
             ...sharedConfig,
@@ -101,15 +104,17 @@ export class EvalComparison {
             challengerSuite.run({ signal: options.signal }),
         ]);
         const metricNames = new Set();
-        for (const r of baselineResult.results) {
-            for (const s of r.scores) {
-                metricNames.add(s.name);
+        for (const result of [baselineResult, challengerResult]) {
+            for (const r of result.results) {
+                for (const trial of r.trials ?? [r]) {
+                    for (const s of trial.scores)
+                        metricNames.add(s.name);
+                }
             }
         }
         const metricComparisons = {};
         for (const name of metricNames) {
-            const baselineScores = extractScores(baselineResult, name);
-            const challengerScores = extractScores(challengerResult, name);
+            const [baselineScores, challengerScores] = pairedScores(baselineResult, challengerResult, name);
             metricComparisons[name] = compareMetric(baselineScores, challengerScores);
         }
         return {
@@ -122,10 +127,35 @@ export class EvalComparison {
         };
     }
 }
-function extractScores(result, metricName) {
-    return result.results.map((r) => {
-        const score = r.scores.find((s) => s.name === metricName);
-        return score?.score ?? 0;
-    });
+/** A case's score of a metric: the mean over its trials the metric scored, none when it scored none */
+function caseScore(result, metricName) {
+    if (!result)
+        return undefined;
+    const trials = result.trials ?? [result];
+    if (trials.every((trial) => trial.error !== undefined))
+        return undefined;
+    const scores = trials.flatMap((trial) => trial.error !== undefined
+        ? [0]
+        : trial.scores.filter((s) => s.name === metricName).map((s) => s.score));
+    return scores.length === 0 ? undefined : mean(scores);
+}
+/**
+ * The scores of a metric for each case, side by side. A case one side errored on counts as a 0
+ * for that side, since a target that fails loses the case, and a case both sides errored on is
+ * left out, telling nothing about either
+ */
+function pairedScores(baseline, challenger, metricName) {
+    const scoreOf = (result, at) => caseScore(result.results[at], metricName);
+    const baselineScores = [];
+    const challengerScores = [];
+    for (let at = 0; at < baseline.results.length; at++) {
+        const b = scoreOf(baseline, at);
+        const c = scoreOf(challenger, at);
+        if (b === undefined && c === undefined)
+            continue;
+        baselineScores.push(b ?? 0);
+        challengerScores.push(c ?? 0);
+    }
+    return [baselineScores, challengerScores];
 }
 //# sourceMappingURL=eval-comparison.js.map

package/dist/eval-comparison.js.map

@@ -1 +1 @@
-{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIzC,OAAO,EAAE,0BAA0B,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAuC3C,SAAS,QAAQ,CAAC,MAAgB;IAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,aAAa,CAAC,cAAwB,EAAE,gBAA0B;IACzE,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9C,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC;IAEhC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,UAAU,EAAE,cAAc;YAC1B,MAAM,EAAE,CAAC;YACT,WAAW,EAAE,KAAK;YAClB,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IAEtE,IAAI,MAAc,CAAC;IACnB,IAAI,WAAoB,CAAC;IAEzB,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;YAChF,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;QAClF,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,mBAAmB,EAAE,mBAAmB,CAAC,CAAC;QACtE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC7D,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,MAAM,GAAsC,KAAK,CAAC;IACtD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,GAAG,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC;IACrE,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AAC7F,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAAyC;IAEzC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU;YAAE,YAAY,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,MAAM,KAAK,YAAY;YAAE,cAAc,EAAE,CAAC;IACnD,CAAC;IAED,IAAI,cAAc,GAAG,YAAY;QAAE,OAAO,YAAY,CAAC;IACvD,IAAI,YAAY,GAAG,cAAc;QAAE,OAAO,UAAU,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,OAAO,cAAc;IACR,IAAI,CAAwB;IAE7C,YAAY,IAA2B;QACrC,0BAA0B,CAAC,KAAK,CAAC;YAC/B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;SACtB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAAoC,EAAE;QAC9C,MAAM,EACJ,OAAO,EACP,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,OAAO,EACP,OAAO,EACP,UAAU,GACX,GAAG,IAAI,CAAC,IAAI,CAAC;QAEd,MAAM,YAAY,GAAG;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,KAAK;YACL,WAAW;YACX,OAAO;YACP,OAAO;SACR,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC;YAClC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;YACpC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,UAAU;YAC1B,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACrF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,aAAa,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC7C,eAAe,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SAChD,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QACtC,KAAK,MAAM,CAAC,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YACvC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,MAAM,iBAAiB,GAAqC,EAAE,CAAC;QAE/D,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,cAAc,GAAG,aAAa,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;YAC3D,MAAM,gBAAgB,GAAG,aAAa,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;YAC/D,iBAAiB,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO;YACL,OAAO,EAAE;gBACP,MAAM,EAAE,sBAAsB,CAAC,iBAAiB,CAAC;gBACjD,OAAO,EAAE,iBAAiB;aAC3B;YACD,QAAQ,EAAE,cAAc;YACxB,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;CACF;AAED,SAAS,aAAa,CAAC,MAAuB,EAAE,UAAkB;IAChE,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAc,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;QACvE,OAAO,KAAK,EAAE,KAAK,IAAI,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAIzC,OAAO,EAAE,0BAA0B,EAAE,MAAM,UAAU,CAAC;AAEtD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AA8C3C,SAAS,QAAQ,CAAC,MAAgB;IAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,aAAa,CAAC,cAAwB,EAAE,gBAA0B;IACzE,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9C,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC;IAEhC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,UAAU,EAAE,cAAc;YAC1B,CAAC;YACD,MAAM,EAAE,CAAC;YACT,WAAW,EAAE,KAAK;YAClB,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IAEtE,IAAI,MAAc,CAAC;IACnB,IAAI,WAAoB,CAAC;IAEzB,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;YAChF,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;QAClF,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,mBAAmB,EAAE,mBAAmB,CAAC,CAAC;QACtE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC7D,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,MAAM,GAAsC,KAAK,CAAC;IACtD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,GAAG,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC;IACrE,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AAChG,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAAyC;IAEzC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU;YAAE,YAAY,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,MAAM,KAAK,YAAY;YAAE,cAAc,EAAE,CAAC;IACnD,CAAC;IAED,IAAI,cAAc,GAAG,YAAY;QAAE,OAAO,YAAY,CAAC;IACvD,IAAI,YAAY,GAAG,cAAc;QAAE,OAAO,UAAU,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,OAAO,cAAc;IACR,IAAI,CAAwB;IAE7C,YAAY,IAA2B;QACrC,0BAA0B,CAAC,KAAK,CAAC;YAC/B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;QACH,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,UAAoC,EAAE;QAC9C,MAAM,EACJ,OAAO,EACP,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,OAAO,EACP,OAAO,EACP,MAAM,EACN,UAAU,GACX,GAAG,IAAI,CAAC,IAAI,CAAC;QAEd,MAAM,YAAY,GAAG;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,KAAK;YACL,WAAW;YACX,OAAO;YACP,OAAO;YACP,MAAM;SACP,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC;YAClC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;YACpC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,UAAU;YAC1B,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACrF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,aAAa,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAC7C,eAAe,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;SAChD,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QACtC,KAAK,MAAM,MAAM,IAAI,CAAC,cAAc,EAAE,gBAAgB,CAAC,EAAE,CAAC;YACxD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;gBAC/B,KAAK,MAAM,KAAK,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;oBACpC,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,MAAM;wBAAE,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;gBACxD,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,iBAAiB,GAAqC,EAAE,CAAC;QAE/D,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,YAAY,CACrD,cAAc,EACd,gBAAgB,EAChB,IAAI,CACL,CAAC;YACF,iBAAiB,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO;YACL,OAAO,EAAE;gBACP,MAAM,EAAE,sBAAsB,CAAC,iBAAiB,CAAC;gBACjD,OAAO,EAAE,iBAAiB;aAC3B;YACD,QAAQ,EAAE,cAAc;YACxB,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;CACF;AAED,uGAAuG;AACvG,SAAS,SAAS,CAAC,MAAoC,EAAE,UAAkB;IACzE,IAAI,CAAC,MAAM;QAAE,OAAO,SAAS,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IACzE,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACtC,KAAK,CAAC,KAAK,KAAK,SAAS;QACvB,CAAC,CAAC,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAc,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CACvF,CAAC;IACF,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACxD,CAAC;AAED;;;;GAIG;AACH,SAAS,YAAY,CACnB,QAAyB,EACzB,UAA2B,EAC3B,UAAkB;IAElB,MAAM,OAAO,GAAG,CAAC,MAAuB,EAAE,EAAU,EAAE,EAAE,CACtD,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QACpD,MAAM,CAAC,GAAG,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAChC,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,SAAS;YAAE,SAAS;QACjD,cAAc,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5B,gBAAgB,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAChC,CAAC;IACD,OAAO,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;AAC5C,CAAC"}