@cogitator-ai/evals 0.1.0

package/dist/datasets/csv-loader.js

@@ -0,0 +1,43 @@
+import { readFileSync } from 'node:fs';
+import { EvalCaseSchema } from '../schema';
+export async function loadCsv(path) {
+    let Papa;
+    try {
+        Papa = await import('papaparse');
+    }
+    catch {
+        throw new Error('papaparse is required for CSV loading. Install it with: npm install papaparse');
+    }
+    const content = readFileSync(path, 'utf-8');
+    const { data, meta } = Papa.parse(content, {
+        header: true,
+        skipEmptyLines: true,
+    });
+    if (!meta.fields?.includes('input')) {
+        throw new Error('CSV must have an "input" column');
+    }
+    return data.map((row) => {
+        const evalCase = { input: row.input };
+        if (row.expected !== undefined && row.expected !== '') {
+            evalCase.expected = row.expected;
+        }
+        const metadata = {};
+        const context = {};
+        for (const [key, value] of Object.entries(row)) {
+            if (key === 'input' || key === 'expected')
+                continue;
+            if (key.startsWith('metadata.')) {
+                metadata[key.slice('metadata.'.length)] = value;
+            }
+            else if (key.startsWith('context.')) {
+                context[key.slice('context.'.length)] = value;
+            }
+        }
+        if (Object.keys(metadata).length > 0)
+            evalCase.metadata = metadata;
+        if (Object.keys(context).length > 0)
+            evalCase.context = context;
+        return EvalCaseSchema.parse(evalCase);
+    });
+}
+//# sourceMappingURL=csv-loader.js.map

package/dist/datasets/csv-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../../src/datasets/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAG3C,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAY;IACxC,IAAI,IAAgC,CAAC;IACrC,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CACb,+EAA+E,CAChF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,KAAK,CAAyB,OAAO,EAAE;QACjE,MAAM,EAAE,IAAI;QACZ,cAAc,EAAE,IAAI;KACrB,CAAC,CAAC;IAEH,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACpC,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACtB,MAAM,QAAQ,GAA4B,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC;QAE/D,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,EAAE,CAAC;YACtD,QAAQ,CAAC,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC;QACnC,CAAC;QAED,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,MAAM,OAAO,GAA4B,EAAE,CAAC;QAE5C,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,IAAI,GAAG,KAAK,OAAO,IAAI,GAAG,KAAK,UAAU;gBAAE,SAAS;YAEpD,IAAI,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;gBAChC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC;YAClD,CAAC;iBAAM,IAAI,GAAG,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;gBACtC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,KAAK,CAAC;YAChD,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACnE,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,QAAQ,CAAC,OAAO,GAAG,OAAO,CAAC;QAEhE,OAAO,cAAc,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/datasets/dataset.d.ts

@@ -0,0 +1,15 @@
+import type { EvalCase } from '../schema';
+export declare class Dataset {
+    private readonly _cases;
+    private constructor();
+    static from(cases: EvalCase[]): Dataset;
+    static fromJsonl(path: string): Promise<Dataset>;
+    static fromCsv(path: string): Promise<Dataset>;
+    get length(): number;
+    get cases(): readonly EvalCase[];
+    [Symbol.iterator](): Iterator<EvalCase>;
+    filter(fn: (c: EvalCase) => boolean): Dataset;
+    sample(n: number): Dataset;
+    shuffle(): Dataset;
+}
+//# sourceMappingURL=dataset.d.ts.map

package/dist/datasets/dataset.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.d.ts","sourceRoot":"","sources":["../../src/datasets/dataset.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAI1C,qBAAa,OAAO;IAClB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAE7C,OAAO;IAIP,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,OAAO;WAI1B,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;WAKzC,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKpD,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,IAAI,KAAK,IAAI,SAAS,QAAQ,EAAE,CAE/B;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC;IAavC,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,KAAK,OAAO,GAAG,OAAO;IAI7C,MAAM,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO;IAa1B,OAAO,IAAI,OAAO;CAQnB"}

package/dist/datasets/dataset.js

@@ -0,0 +1,62 @@
+import { EvalCaseSchema } from '../schema';
+import { loadJsonl } from './jsonl-loader';
+import { loadCsv } from './csv-loader';
+export class Dataset {
+    _cases;
+    constructor(cases) {
+        this._cases = Object.freeze(cases.map((c) => EvalCaseSchema.parse(c)));
+    }
+    static from(cases) {
+        return new Dataset(cases);
+    }
+    static async fromJsonl(path) {
+        const cases = await loadJsonl(path);
+        return Dataset.from(cases);
+    }
+    static async fromCsv(path) {
+        const cases = await loadCsv(path);
+        return Dataset.from(cases);
+    }
+    get length() {
+        return this._cases.length;
+    }
+    get cases() {
+        return this._cases;
+    }
+    [Symbol.iterator]() {
+        let index = 0;
+        const cases = this._cases;
+        return {
+            next() {
+                if (index < cases.length) {
+                    return { value: cases[index++], done: false };
+                }
+                return { value: undefined, done: true };
+            },
+        };
+    }
+    filter(fn) {
+        return new Dataset(this._cases.filter(fn));
+    }
+    sample(n) {
+        if (n <= 0 || this._cases.length === 0) {
+            return new Dataset([]);
+        }
+        const count = Math.min(n, this._cases.length);
+        const indices = Array.from({ length: this._cases.length }, (_, i) => i);
+        for (let i = indices.length - 1; i > 0; i--) {
+            const j = Math.floor(Math.random() * (i + 1));
+            [indices[i], indices[j]] = [indices[j], indices[i]];
+        }
+        return new Dataset(indices.slice(0, count).map((i) => this._cases[i]));
+    }
+    shuffle() {
+        const arr = [...this._cases];
+        for (let i = arr.length - 1; i > 0; i--) {
+            const j = Math.floor(Math.random() * (i + 1));
+            [arr[i], arr[j]] = [arr[j], arr[i]];
+        }
+        return new Dataset(arr);
+    }
+}
+//# sourceMappingURL=dataset.js.map

package/dist/datasets/dataset.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/datasets/dataset.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAEvC,MAAM,OAAO,OAAO;IACD,MAAM,CAAsB;IAE7C,YAAoB,KAAiB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,KAAiB;QAC3B,OAAO,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,IAAY;QACjC,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;QACpC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAAY;QAC/B,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,CAAC,MAAM,CAAC,QAAQ,CAAC;QACf,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,OAAO;YACL,IAAI;gBACF,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;oBACzB,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;gBAChD,CAAC;gBACD,OAAO,EAAE,KAAK,EAAE,SAAkB,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YACnD,CAAC;SACF,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,EAA4B;QACjC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAe,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,CAAC,CAAS;QACd,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,OAAO,IAAI,OAAO,CAAC,EAAE,CAAC,CAAC;QACzB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QACxE,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAe,CAAC,CAAC;IACvF,CAAC;IAED,OAAO;QACL,MAAM,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAe,CAAC;QAC3C,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;QACD,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC;IAC1B,CAAC;CACF"}

package/dist/datasets/index.d.ts

@@ -0,0 +1,4 @@
+export { Dataset } from './dataset';
+export { loadJsonl } from './jsonl-loader';
+export { loadCsv } from './csv-loader';
+//# sourceMappingURL=index.d.ts.map

package/dist/datasets/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/datasets/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC"}

package/dist/datasets/index.js

@@ -0,0 +1,4 @@
+export { Dataset } from './dataset';
+export { loadJsonl } from './jsonl-loader';
+export { loadCsv } from './csv-loader';
+//# sourceMappingURL=index.js.map

package/dist/datasets/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/datasets/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC"}

package/dist/datasets/jsonl-loader.d.ts

@@ -0,0 +1,3 @@
+import type { EvalCase } from '../schema';
+export declare function loadJsonl(path: string): Promise<EvalCase[]>;
+//# sourceMappingURL=jsonl-loader.d.ts.map

package/dist/datasets/jsonl-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonl-loader.d.ts","sourceRoot":"","sources":["../../src/datasets/jsonl-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAE1C,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAyBjE"}

package/dist/datasets/jsonl-loader.js

@@ -0,0 +1,27 @@
+import { createReadStream } from 'node:fs';
+import { createInterface } from 'node:readline';
+import { EvalCaseSchema } from '../schema';
+export async function loadJsonl(path) {
+    const cases = [];
+    let lineNumber = 0;
+    const rl = createInterface({
+        input: createReadStream(path, { encoding: 'utf-8' }),
+        crlfDelay: Infinity,
+    });
+    for await (const line of rl) {
+        lineNumber++;
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        let parsed;
+        try {
+            parsed = JSON.parse(trimmed);
+        }
+        catch {
+            throw new Error(`Invalid JSON at line ${lineNumber}: ${trimmed}`);
+        }
+        cases.push(EvalCaseSchema.parse(parsed));
+    }
+    return cases;
+}
+//# sourceMappingURL=jsonl-loader.js.map

package/dist/datasets/jsonl-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonl-loader.js","sourceRoot":"","sources":["../../src/datasets/jsonl-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,WAAW,CAAC;AAG3C,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,MAAM,EAAE,GAAG,eAAe,CAAC;QACzB,KAAK,EAAE,gBAAgB,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC;QACpD,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC5B,UAAU,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,IAAI,KAAK,CAAC,wBAAwB,UAAU,KAAK,OAAO,EAAE,CAAC,CAAC;QACpE,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/eval-builder.d.ts

@@ -0,0 +1,30 @@
+import { EvalSuite } from './eval-suite';
+import type { EvalTarget, EvalSuiteOptions } from './eval-suite';
+import { Dataset } from './datasets';
+import type { MetricFn, StatisticalMetricFn } from './metrics/types';
+import type { JudgeConfig } from './schema';
+import type { AssertionFn } from './assertions';
+export declare class EvalBuilder {
+    private _dataset?;
+    private _target?;
+    private _metrics;
+    private _statisticalMetrics;
+    private _judge?;
+    private _assertions;
+    private _concurrency?;
+    private _timeout?;
+    private _retries?;
+    private _onProgress?;
+    withDataset(dataset: Dataset): this;
+    withTarget(target: EvalTarget): this;
+    withMetrics(metrics: MetricFn[]): this;
+    withStatisticalMetrics(metrics: StatisticalMetricFn[]): this;
+    withJudge(config: JudgeConfig): this;
+    withAssertions(assertions: AssertionFn[]): this;
+    withConcurrency(n: number): this;
+    withTimeout(ms: number): this;
+    withRetries(n: number): this;
+    onProgress(fn: EvalSuiteOptions['onProgress']): this;
+    build(): EvalSuite;
+}
+//# sourceMappingURL=eval-builder.d.ts.map

package/dist/eval-builder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-builder.d.ts","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACjE,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AACrE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAOhD,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,CAAU;IAC3B,OAAO,CAAC,OAAO,CAAC,CAAa;IAC7B,OAAO,CAAC,QAAQ,CAAkB;IAClC,OAAO,CAAC,mBAAmB,CAA6B;IACxD,OAAO,CAAC,MAAM,CAAC,CAAc;IAC7B,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,CAAS;IAC1B,OAAO,CAAC,WAAW,CAAC,CAAiC;IAErD,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAKnC,UAAU,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAKpC,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,GAAG,IAAI;IAKtC,sBAAsB,CAAC,OAAO,EAAE,mBAAmB,EAAE,GAAG,IAAI;IAK5D,SAAS,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI;IAKpC,cAAc,CAAC,UAAU,EAAE,WAAW,EAAE,GAAG,IAAI;IAK/C,eAAe,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAKhC,WAAW,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI;IAK7B,WAAW,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI;IAK5B,UAAU,CAAC,EAAE,EAAE,gBAAgB,CAAC,YAAY,CAAC,GAAG,IAAI;IAKpD,KAAK,IAAI,SAAS;CA8BnB"}

package/dist/eval-builder.js

@@ -0,0 +1,82 @@
+import { EvalSuite } from './eval-suite';
+function isLLMMetric(m) {
+    return 'requiresJudge' in m && m.requiresJudge === true;
+}
+export class EvalBuilder {
+    _dataset;
+    _target;
+    _metrics = [];
+    _statisticalMetrics = [];
+    _judge;
+    _assertions = [];
+    _concurrency;
+    _timeout;
+    _retries;
+    _onProgress;
+    withDataset(dataset) {
+        this._dataset = dataset;
+        return this;
+    }
+    withTarget(target) {
+        this._target = target;
+        return this;
+    }
+    withMetrics(metrics) {
+        this._metrics = metrics;
+        return this;
+    }
+    withStatisticalMetrics(metrics) {
+        this._statisticalMetrics = metrics;
+        return this;
+    }
+    withJudge(config) {
+        this._judge = config;
+        return this;
+    }
+    withAssertions(assertions) {
+        this._assertions = assertions;
+        return this;
+    }
+    withConcurrency(n) {
+        this._concurrency = n;
+        return this;
+    }
+    withTimeout(ms) {
+        this._timeout = ms;
+        return this;
+    }
+    withRetries(n) {
+        this._retries = n;
+        return this;
+    }
+    onProgress(fn) {
+        this._onProgress = fn;
+        return this;
+    }
+    build() {
+        if (!this._dataset) {
+            throw new Error('Dataset is required. Use .withDataset()');
+        }
+        if (!this._target) {
+            throw new Error('Target is required. Use .withTarget()');
+        }
+        const hasLLMMetrics = this._metrics.some(isLLMMetric);
+        if (hasLLMMetrics && !this._judge) {
+            throw new Error('Judge config required for LLM metrics. Use .withJudge()');
+        }
+        const opts = {
+            dataset: this._dataset,
+            target: this._target,
+            metrics: this._metrics.length > 0 ? this._metrics : undefined,
+            statisticalMetrics: this._statisticalMetrics.length > 0 ? this._statisticalMetrics : undefined,
+            judge: this._judge,
+            assertions: this._assertions.length > 0 ? this._assertions : undefined,
+            concurrency: this._concurrency,
+            timeout: this._timeout,
+            retries: this._retries,
+            onProgress: this._onProgress,
+        };
+        return new EvalSuite(opts);
+    }
+}
+//# sourceMappingURL=eval-builder.js.map

package/dist/eval-builder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-builder.js","sourceRoot":"","sources":["../src/eval-builder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAQzC,SAAS,WAAW,CAAC,CAAW;IAC9B,OAAO,eAAe,IAAI,CAAC,IAAK,CAAiB,CAAC,aAAa,KAAK,IAAI,CAAC;AAC3E,CAAC;AAED,MAAM,OAAO,WAAW;IACd,QAAQ,CAAW;IACnB,OAAO,CAAc;IACrB,QAAQ,GAAe,EAAE,CAAC;IAC1B,mBAAmB,GAA0B,EAAE,CAAC;IAChD,MAAM,CAAe;IACrB,WAAW,GAAkB,EAAE,CAAC;IAChC,YAAY,CAAU;IACtB,QAAQ,CAAU;IAClB,QAAQ,CAAU;IAClB,WAAW,CAAkC;IAErD,WAAW,CAAC,OAAgB;QAC1B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,MAAkB;QAC3B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,OAAmB;QAC7B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,sBAAsB,CAAC,OAA8B;QACnD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,SAAS,CAAC,MAAmB;QAC3B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,cAAc,CAAC,UAAyB;QACtC,IAAI,CAAC,WAAW,GAAG,UAAU,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,eAAe,CAAC,CAAS;QACvB,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,EAAU;QACpB,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,CAAS;QACnB,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,EAAkC;QAC3C,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC3D,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QACtD,IAAI,aAAa,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;QAC7E,CAAC;QAED,MAAM,IAAI,GAAqB;YAC7B,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,MAAM,EAAE,IAAI,CAAC,OAAO;YACpB,OAAO,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS;YAC7D,kBAAkB,EAChB,IAAI,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,CAAC,SAAS;YAC5E,KAAK,EAAE,IAAI,CAAC,MAAM;YAClB,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;YACtE,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,UAAU,EAAE,IAAI,CAAC,WAAW;SAC7B,CAAC;QAEF,OAAO,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;CACF"}

package/dist/eval-comparison.d.ts

@@ -0,0 +1,43 @@
+import type { EvalTarget, EvalSuiteResult } from './eval-suite';
+import type { MetricFn, StatisticalMetricFn } from './metrics/types';
+import type { JudgeConfig } from './schema';
+import { Dataset } from './datasets';
+export interface EvalComparisonOptions {
+    dataset: Dataset;
+    targets: {
+        baseline: EvalTarget;
+        challenger: EvalTarget;
+    };
+    metrics?: MetricFn[];
+    statisticalMetrics?: StatisticalMetricFn[];
+    judge?: JudgeConfig;
+    concurrency?: number;
+    timeout?: number;
+    retries?: number;
+    onProgress?: (progress: {
+        target: string;
+        completed: number;
+        total: number;
+    }) => void;
+}
+export interface MetricComparison {
+    baseline: number;
+    challenger: number;
+    pValue: number;
+    significant: boolean;
+    winner: 'baseline' | 'challenger' | 'tie';
+}
+export interface ComparisonResult {
+    summary: {
+        winner: 'baseline' | 'challenger' | 'tie';
+        metrics: Record<string, MetricComparison>;
+    };
+    baseline: EvalSuiteResult;
+    challenger: EvalSuiteResult;
+}
+export declare class EvalComparison {
+    private readonly opts;
+    constructor(opts: EvalComparisonOptions);
+    run(): Promise<ComparisonResult>;
+}
+//# sourceMappingURL=eval-comparison.d.ts.map

package/dist/eval-comparison.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-comparison.d.ts","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EAAE,QAAQ,EAAe,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAKrC,MAAM,WAAW,qBAAqB;IACpC,OAAO,EAAE,OAAO,CAAC;IACjB,OAAO,EAAE;QACP,QAAQ,EAAE,UAAU,CAAC;QACrB,UAAU,EAAE,UAAU,CAAC;KACxB,CAAC;IACF,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,KAAK,IAAI,CAAC;CACvF;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,OAAO,CAAC;IACrB,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;CAC3C;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE;QACP,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,KAAK,CAAC;QAC1C,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;KAC3C,CAAC;IACF,QAAQ,EAAE,eAAe,CAAC;IAC1B,UAAU,EAAE,eAAe,CAAC;CAC7B;AAkED,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAwB;gBAEjC,IAAI,EAAE,qBAAqB;IAIjC,GAAG,IAAI,OAAO,CAAC,gBAAgB,CAAC;CAoEvC"}

package/dist/eval-comparison.js

@@ -0,0 +1,125 @@
+import { EvalSuite } from './eval-suite';
+import { pairedTTest } from './stats/t-test';
+import { mcnemarsTest } from './stats/mcnemar';
+import { mean } from './stats/percentiles';
+function isBinary(scores) {
+    return scores.every((s) => s === 0 || s === 1);
+}
+function compareMetric(baselineScores, challengerScores) {
+    const baselineMean = mean(baselineScores);
+    const challengerMean = mean(challengerScores);
+    const n = baselineScores.length;
+    if (n < 2) {
+        return {
+            baseline: baselineMean,
+            challenger: challengerMean,
+            pValue: 1,
+            significant: false,
+            winner: 'tie',
+        };
+    }
+    const binary = isBinary(baselineScores) && isBinary(challengerScores);
+    let pValue;
+    let significant;
+    if (binary) {
+        let bCorrect_cIncorrect = 0;
+        let bIncorrect_cCorrect = 0;
+        for (let i = 0; i < n; i++) {
+            if (baselineScores[i] === 1 && challengerScores[i] === 0)
+                bCorrect_cIncorrect++;
+            if (baselineScores[i] === 0 && challengerScores[i] === 1)
+                bIncorrect_cCorrect++;
+        }
+        const result = mcnemarsTest(bCorrect_cIncorrect, bIncorrect_cCorrect);
+        pValue = result.pValue;
+        significant = result.significant;
+    }
+    else {
+        const result = pairedTTest(baselineScores, challengerScores);
+        pValue = result.pValue;
+        significant = result.significant;
+    }
+    let winner = 'tie';
+    if (significant) {
+        winner = challengerMean > baselineMean ? 'challenger' : 'baseline';
+    }
+    return { baseline: baselineMean, challenger: challengerMean, pValue, significant, winner };
+}
+function determineOverallWinner(metrics) {
+    let baselineWins = 0;
+    let challengerWins = 0;
+    for (const mc of Object.values(metrics)) {
+        if (mc.winner === 'baseline')
+            baselineWins++;
+        if (mc.winner === 'challenger')
+            challengerWins++;
+    }
+    if (challengerWins > baselineWins)
+        return 'challenger';
+    if (baselineWins > challengerWins)
+        return 'baseline';
+    return 'tie';
+}
+export class EvalComparison {
+    opts;
+    constructor(opts) {
+        this.opts = opts;
+    }
+    async run() {
+        const { dataset, targets, metrics, statisticalMetrics, judge, concurrency, timeout, retries, onProgress, } = this.opts;
+        const sharedConfig = {
+            dataset,
+            metrics,
+            statisticalMetrics,
+            judge,
+            concurrency,
+            timeout,
+            retries,
+        };
+        const baselineSuite = new EvalSuite({
+            ...sharedConfig,
+            target: targets.baseline,
+            onProgress: onProgress
+                ? (p) => onProgress({ target: 'baseline', completed: p.completed, total: p.total })
+                : undefined,
+        });
+        const challengerSuite = new EvalSuite({
+            ...sharedConfig,
+            target: targets.challenger,
+            onProgress: onProgress
+                ? (p) => onProgress({ target: 'challenger', completed: p.completed, total: p.total })
+                : undefined,
+        });
+        const [baselineResult, challengerResult] = await Promise.all([
+            baselineSuite.run(),
+            challengerSuite.run(),
+        ]);
+        const metricNames = new Set();
+        for (const r of baselineResult.results) {
+            for (const s of r.scores) {
+                metricNames.add(s.name);
+            }
+        }
+        const metricComparisons = {};
+        for (const name of metricNames) {
+            const baselineScores = extractScores(baselineResult, name);
+            const challengerScores = extractScores(challengerResult, name);
+            metricComparisons[name] = compareMetric(baselineScores, challengerScores);
+        }
+        return {
+            summary: {
+                winner: determineOverallWinner(metricComparisons),
+                metrics: metricComparisons,
+            },
+            baseline: baselineResult,
+            challenger: challengerResult,
+        };
+    }
+}
+function extractScores(result, metricName) {
+    return result.results.map((r) => {
+        const score = r.scores.find((s) => s.name === metricName);
+        return score?.score ?? 0;
+    });
+}
+//# sourceMappingURL=eval-comparison.js.map

package/dist/eval-comparison.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-comparison.js","sourceRoot":"","sources":["../src/eval-comparison.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAKzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAkC3C,SAAS,QAAQ,CAAC,MAAgB;IAChC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,aAAa,CAAC,cAAwB,EAAE,gBAA0B;IACzE,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;IAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9C,MAAM,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC;IAEhC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,UAAU,EAAE,cAAc;YAC1B,MAAM,EAAE,CAAC;YACT,WAAW,EAAE,KAAK;YAClB,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,QAAQ,CAAC,cAAc,CAAC,IAAI,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IAEtE,IAAI,MAAc,CAAC;IACnB,IAAI,WAAoB,CAAC;IAEzB,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;YAChF,IAAI,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,gBAAgB,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,mBAAmB,EAAE,CAAC;QAClF,CAAC;QACD,MAAM,MAAM,GAAG,YAAY,CAAC,mBAAmB,EAAE,mBAAmB,CAAC,CAAC;QACtE,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,WAAW,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC7D,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QACvB,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,MAAM,GAAsC,KAAK,CAAC;IACtD,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,GAAG,cAAc,GAAG,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,UAAU,CAAC;IACrE,CAAC;IAED,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,CAAC;AAC7F,CAAC;AAED,SAAS,sBAAsB,CAC7B,OAAyC;IAEzC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,KAAK,MAAM,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;QACxC,IAAI,EAAE,CAAC,MAAM,KAAK,UAAU;YAAE,YAAY,EAAE,CAAC;QAC7C,IAAI,EAAE,CAAC,MAAM,KAAK,YAAY;YAAE,cAAc,EAAE,CAAC;IACnD,CAAC;IAED,IAAI,cAAc,GAAG,YAAY;QAAE,OAAO,YAAY,CAAC;IACvD,IAAI,YAAY,GAAG,cAAc;QAAE,OAAO,UAAU,CAAC;IACrD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,OAAO,cAAc;IACR,IAAI,CAAwB;IAE7C,YAAY,IAA2B;QACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,EACJ,OAAO,EACP,OAAO,EACP,OAAO,EACP,kBAAkB,EAClB,KAAK,EACL,WAAW,EACX,OAAO,EACP,OAAO,EACP,UAAU,GACX,GAAG,IAAI,CAAC,IAAI,CAAC;QAEd,MAAM,YAAY,GAAG;YACnB,OAAO;YACP,OAAO;YACP,kBAAkB;YAClB,KAAK;YACL,WAAW;YACX,OAAO;YACP,OAAO;SACR,CAAC;QAEF,MAAM,aAAa,GAAG,IAAI,SAAS,CAAC;YAClC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,QAAQ;YACxB,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,eAAe,GAAG,IAAI,SAAS,CAAC;YACpC,GAAG,YAAY;YACf,MAAM,EAAE,OAAO,CAAC,UAAU;YAC1B,UAAU,EAAE,UAAU;gBACpB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;gBACrF,CAAC,CAAC,SAAS;SACd,CAAC,CAAC;QAEH,MAAM,CAAC,cAAc,EAAE,gBAAgB,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC3D,aAAa,CAAC,GAAG,EAAE;YACnB,eAAe,CAAC,GAAG,EAAE;SACtB,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QACtC,KAAK,MAAM,CAAC,IAAI,cAAc,CAAC,OAAO,EAAE,CAAC;YACvC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,MAAM,iBAAiB,GAAqC,EAAE,CAAC;QAE/D,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;YAC/B,MAAM,cAAc,GAAG,aAAa,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;YAC3D,MAAM,gBAAgB,GAAG,aAAa,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;YAC/D,iBAAiB,CAAC,IAAI,CAAC,GAAG,aAAa,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;QAC5E,CAAC;QAED,OAAO;YACL,OAAO,EAAE;gBACP,MAAM,EAAE,sBAAsB,CAAC,iBAAiB,CAAC;gBACjD,OAAO,EAAE,iBAAiB;aAC3B;YACD,QAAQ,EAAE,cAAc;YACxB,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;CACF;AAED,SAAS,aAAa,CAAC,MAAuB,EAAE,UAAkB;IAChE,OAAO,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC9B,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAc,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;QACvE,OAAO,KAAK,EAAE,KAAK,IAAI,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/eval-suite.d.ts

@@ -0,0 +1,63 @@
+import { Dataset } from './datasets';
+import type { EvalCase } from './schema';
+import type { JudgeConfig } from './schema';
+import type { MetricFn, MetricScore, EvalCaseResult, StatisticalMetricFn } from './metrics/types';
+import type { AssertionFn, AssertionResult, AggregatedMetric } from './assertions';
+import type { ReporterType, ReporterOptions } from './reporters';
+export interface EvalTarget {
+    agent?: unknown;
+    cogitator?: unknown;
+    fn?: (input: string) => Promise<string>;
+}
+export interface EvalProgress {
+    completed: number;
+    total: number;
+    currentCase?: EvalCase;
+}
+export interface EvalSuiteOptions {
+    dataset: Dataset;
+    target: EvalTarget;
+    metrics?: MetricFn[];
+    statisticalMetrics?: StatisticalMetricFn[];
+    judge?: JudgeConfig;
+    assertions?: AssertionFn[];
+    concurrency?: number;
+    timeout?: number;
+    retries?: number;
+    onProgress?: (progress: EvalProgress) => void;
+}
+export interface EvalSuiteResult {
+    results: Array<EvalCaseResult & {
+        scores: MetricScore[];
+    }>;
+    aggregated: Record<string, AggregatedMetric>;
+    assertions: AssertionResult[];
+    stats: {
+        total: number;
+        duration: number;
+        cost: number;
+    };
+    report: (type: ReporterType | ReporterType[], options?: ReporterOptions) => void;
+    saveBaseline: (path: string) => void;
+}
+export declare class EvalSuite {
+    private readonly dataset;
+    private readonly target;
+    private readonly boundMetrics;
+    private readonly statisticalMetrics;
+    private readonly assertionFns;
+    private readonly concurrency;
+    private readonly timeout;
+    private readonly retries;
+    private readonly onProgress?;
+    constructor(opts: EvalSuiteOptions);
+    private validateTarget;
+    run(): Promise<EvalSuiteResult>;
+    private executeCase;
+    private executeCaseAttempt;
+    private executeFnTarget;
+    private executeAgentTarget;
+    private evaluateCaseMetrics;
+    private aggregateScores;
+}
+//# sourceMappingURL=eval-suite.d.ts.map

package/dist/eval-suite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-suite.d.ts","sourceRoot":"","sources":["../src/eval-suite.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AACrC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAGlG,OAAO,KAAK,EAAE,WAAW,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAGnF,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEjE,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,CAAC;IACpB,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACzC;AAED,MAAM,WAAW,YAAY;IAC3B,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,QAAQ,CAAC;CACxB;AAED,MAAM,WAAW,gBAAgB;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,MAAM,EAAE,UAAU,CAAC;IACnB,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC;IACrB,kBAAkB,CAAC,EAAE,mBAAmB,EAAE,CAAC;IAC3C,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,UAAU,CAAC,EAAE,WAAW,EAAE,CAAC;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;CAC/C;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,KAAK,CAAC,cAAc,GAAG;QAAE,MAAM,EAAE,WAAW,EAAE,CAAA;KAAE,CAAC,CAAC;IAC3D,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC7C,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACzD,MAAM,EAAE,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,KAAK,IAAI,CAAC;IACjF,YAAY,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CACtC;AAMD,qBAAa,SAAS;IACpB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAa;IAC1C,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAwB;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAgB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAmC;gBAEnD,IAAI,EAAE,gBAAgB;IAoClC,OAAO,CAAC,cAAc;IAuBhB,GAAG,IAAI,OAAO,CAAC,eAAe,CAAC;YA6FvB,WAAW;YAgBX,kBAAkB;YAyBlB,eAAe;YAMf,kBAAkB;YA+BlB,mBAAmB;IAKjC,OAAO,CAAC,eAAe;CAuBxB"}